This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch dev-1.0.0 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit dc1aaa9d0bd6f03d8319c2b53c75cc4552a3e9e6 Author: Mingyu Chen <[email protected]> AuthorDate: Sat Mar 5 14:41:04 2022 +0800 [refactor] Remove mysql-connector and replace org.json with com.googlecode.json-simple (#8319) 1. mysql-connector-java mysql-connector-java is under GLPv2 license, which is not compatible with APLv2, and Doris does not use it. 2. org.json org.json is under JSON license, which is not compatible with APLv2. I use `json-simple` to replace it. --- fe/fe-core/pom.xml | 10 +--- .../java/org/apache/doris/backup/Repository.java | 5 +- .../doris/deploy/impl/AmbariDeployManager.java | 19 +++---- .../external/elasticsearch/EsShardPartitions.java | 33 ++++++------ .../external/elasticsearch/EsShardRouting.java | 14 ++--- .../doris/external/elasticsearch/EsUtil.java | 10 ++-- .../doris/external/elasticsearch/MappingPhase.java | 50 +++++++++--------- .../doris/http/rest/StorageTypeCheckAction.java | 2 +- .../doris/http/rest/TableQueryPlanAction.java | 13 ++--- .../doris/httpv2/interceptor/AuthInterceptor.java | 8 --- .../doris/httpv2/rest/TableQueryPlanAction.java | 19 +++---- .../org/apache/doris/journal/bdbje/BDBTool.java | 10 ++-- .../apache/doris/persist/GlobalVarPersistInfo.java | 2 +- .../java/org/apache/doris/qe/SessionVariable.java | 20 +++---- .../main/java/org/apache/doris/qe/VariableMgr.java | 15 +++--- .../doris/external/elasticsearch/EsUtilTest.java | 23 ++++---- .../doris/http/TableQueryPlanActionTest.java | 61 ++++++++++------------ .../apache/doris/http/TableRowCountActionTest.java | 14 ++--- .../apache/doris/http/TableSchemaActionTest.java | 17 +++--- fe/pom.xml | 13 ++--- 20 files changed, 164 insertions(+), 194 deletions(-) diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 10aa192..f6798bd 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -215,10 +215,9 @@ under the License. <groupId>commons-io</groupId> <artifactId>commons-io</artifactId> </dependency> - <!-- https://mvnrepository.com/artifact/org.json/json --> <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> + <groupId>com.googlecode.json-simple</groupId> + <artifactId>json-simple</artifactId> </dependency> <!-- https://mvnrepository.com/artifact/junit/junit --> <dependency> @@ -255,11 +254,6 @@ under the License. <groupId>io.dropwizard.metrics</groupId> <artifactId>metrics-core</artifactId> </dependency> - <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java --> - <dependency> - <groupId>mysql</groupId> - <artifactId>mysql-connector-java</artifactId> - </dependency> <!-- https://mvnrepository.com/artifact/io.netty/netty-all --> <dependency> <groupId>io.netty</groupId> diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index df8230c..9078d17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -37,7 +37,8 @@ import com.google.common.collect.Lists; import org.apache.commons.codec.digest.DigestUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONObject; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import java.io.DataInput; import java.io.DataOutput; @@ -227,7 +228,7 @@ public class Repository implements Writable { byte[] bytes = Files.readAllBytes(Paths.get(localFilePath)); String json = new String(bytes, StandardCharsets.UTF_8); - JSONObject root = new JSONObject(json); + JSONObject root = (JSONObject) JSONValue.parse(json); name = (String) root.get("name"); createTime = TimeUtils.timeStringToLong((String) root.get("create_time")); if (createTime == -1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/deploy/impl/AmbariDeployManager.java b/fe/fe-core/src/main/java/org/apache/doris/deploy/impl/AmbariDeployManager.java index 65bc378..cf8691a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/deploy/impl/AmbariDeployManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/deploy/impl/AmbariDeployManager.java @@ -32,8 +32,9 @@ import com.google.common.collect.Maps; import org.apache.commons.codec.binary.Base64; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONArray; -import org.json.JSONObject; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import java.util.List; import java.util.Map; @@ -295,12 +296,12 @@ public class AmbariDeployManager extends DeployManager { } List<String> hostnames = Lists.newArrayList(); - JSONObject componentsObj = new JSONObject(componentsJson); - JSONArray componentsArray = componentsObj.getJSONArray(KEY_HOST_COMPONENTS); + JSONObject componentsObj = (JSONObject) JSONValue.parse(componentsJson); + JSONArray componentsArray = (JSONArray) componentsObj.get(KEY_HOST_COMPONENTS); for (Object component : componentsArray) { JSONObject componentObj = (JSONObject) component; try { - JSONObject roleObj = componentObj.getJSONObject(KEY_HOST_ROLES); + JSONObject roleObj = (JSONObject) componentObj.get(KEY_HOST_ROLES); String hostname = (String) roleObj.get(KEY_HOST_NAME); hostnames.add(hostname); } catch (Exception e) { @@ -314,13 +315,13 @@ public class AmbariDeployManager extends DeployManager { private String getPropertyFromBlueprint(String configNodeName, String propName) { Preconditions.checkNotNull(blueprintJson); String resProp = null; - JSONObject root = new JSONObject(blueprintJson); - JSONArray confArray = root.getJSONArray("configurations"); + JSONObject root = (JSONObject) JSONValue.parse(blueprintJson); + JSONArray confArray = (JSONArray) root.get("configurations"); for (Object object : confArray) { JSONObject jobj = (JSONObject) object; try { - JSONObject comNameObj = jobj.getJSONObject(configNodeName); - JSONObject propObj = comNameObj.getJSONObject("properties"); + JSONObject comNameObj = (JSONObject) jobj.get(configNodeName); + JSONObject propObj = (JSONObject) comNameObj.get("properties"); resProp = (String) propObj.get(propName); } catch (Exception e) { // nothing diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java index a7ceff2..710e6832 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java @@ -26,8 +26,9 @@ import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONArray; -import org.json.JSONObject; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import java.util.List; import java.util.Map; @@ -61,25 +62,25 @@ public class EsShardPartitions { public static EsShardPartitions findShardPartitions(String indexName, String searchShards) throws DorisEsException { EsShardPartitions partitions = new EsShardPartitions(indexName); - JSONObject jsonObject = new JSONObject(searchShards); - JSONArray shards = jsonObject.getJSONArray("shards"); - int length = shards.length(); - for (int i = 0; i < length; i++) { + JSONObject jsonObject = (JSONObject) JSONValue.parse(searchShards); + JSONArray shards = (JSONArray) jsonObject.get("shards"); + int size = shards.size(); + for (int i = 0; i < size; i++) { List<EsShardRouting> singleShardRouting = Lists.newArrayList(); - JSONArray shardsArray = shards.getJSONArray(i); - int arrayLength = shardsArray.length(); - for (int j = 0; j < arrayLength; j++) { - JSONObject indexShard = shardsArray.getJSONObject(j); - String shardState = indexShard.getString("state"); + JSONArray shardsArray = (JSONArray) shards.get(i); + int arraySize = shardsArray.size(); + for (int j = 0; j < arraySize; j++) { + JSONObject indexShard = (JSONObject) shardsArray.get(j); + String shardState = (String) indexShard.get("state"); if ("STARTED".equalsIgnoreCase(shardState) || "RELOCATING".equalsIgnoreCase(shardState)) { try { singleShardRouting.add( EsShardRouting.newSearchShard( - indexShard.getString("index"), - indexShard.getInt("shard"), - indexShard.getBoolean("primary"), - indexShard.getString("node"), - jsonObject.getJSONObject("nodes"))); + (String) indexShard.get("index"), + (Integer) indexShard.get("shard"), + (Boolean) indexShard.get("primary"), + (String) indexShard.get("node"), + (JSONObject) jsonObject.get("nodes"))); } catch (Exception e) { LOG.error("fetch index [{}] shard partitions failure", indexName, e); throw new DorisEsException("fetch [" + indexName + "] shard partitions failure [" + e.getMessage() + "]"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java index 980a62d..e54923d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java @@ -17,11 +17,11 @@ package org.apache.doris.external.elasticsearch; - import org.apache.doris.thrift.TNetworkAddress; -import org.apache.commons.lang.StringUtils; -import org.json.JSONObject; +import com.google.common.base.Strings; + +import org.json.simple.JSONObject; public class EsShardRouting { @@ -43,13 +43,13 @@ public class EsShardRouting { public static EsShardRouting newSearchShard(String indexName, int shardId, boolean isPrimary, String nodeId, JSONObject nodesMap) { - JSONObject nodeInfo = nodesMap.getJSONObject(nodeId); - String[] transportAddr = nodeInfo.getString("transport_address").split(":"); + JSONObject nodeInfo = (JSONObject) nodesMap.get(nodeId); + String[] transportAddr = ((String) nodeInfo.get("transport_address")).split(":"); // get thrift port from node info - String thriftPort = nodeInfo.getJSONObject("attributes").optString("thrift_port"); + String thriftPort = (String) ((JSONObject) nodeInfo.get("attributes")).get("thrift_port"); // In http transport mode, should ignore thrift_port, set address to null TNetworkAddress addr = null; - if (!StringUtils.isEmpty(thriftPort)) { + if (!Strings.isNullOrEmpty(thriftPort)) { addr = new TNetworkAddress(transportAddr[0], Integer.parseInt(thriftPort)); } return new EsShardRouting(indexName, shardId, isPrimary, addr, nodeId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java index 454fb5f..4dd0fcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java @@ -23,7 +23,7 @@ import org.apache.doris.analysis.RangePartitionDesc; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import org.json.JSONObject; +import org.json.simple.JSONObject; import java.util.Map; @@ -73,15 +73,15 @@ public class EsUtil { int firstOccr = key.indexOf('.', fromIndex); if (firstOccr == -1) { String token = key.substring(key.lastIndexOf('.') + 1); - if (jsonObject.has(token)) { - return jsonObject.getJSONObject(token); + if (jsonObject.containsKey(token)) { + return (JSONObject) jsonObject.get(token); } else { return null; } } String fieldName = key.substring(fromIndex, firstOccr); - if (jsonObject.has(fieldName)) { - return getJsonObject(jsonObject.getJSONObject(fieldName), key, firstOccr + 1); + if (jsonObject.containsKey(fieldName)) { + return getJsonObject((JSONObject) jsonObject.get(fieldName), key, firstOccr + 1); } else { return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java index 4b82e18..f736a9e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/MappingPhase.java @@ -21,7 +21,8 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.EsTable; import org.apache.commons.lang3.StringUtils; -import org.json.JSONObject; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import java.util.Iterator; @@ -69,13 +70,13 @@ public class MappingPhase implements SearchPhase { * @throws Exception */ public void resolveFields(SearchContext searchContext, String indexMapping) throws DorisEsException { - JSONObject jsonObject = new JSONObject(indexMapping); + JSONObject jsonObject = (JSONObject) JSONValue.parse(indexMapping); // the indexName use alias takes the first mapping - Iterator<String> keys = jsonObject.keys(); + Iterator<String> keys = jsonObject.keySet().iterator(); String docKey = keys.next(); - JSONObject docData = jsonObject.optJSONObject(docKey); - JSONObject mappings = docData.optJSONObject("mappings"); - JSONObject rootSchema = mappings.optJSONObject(searchContext.type()); + JSONObject docData = (JSONObject) jsonObject.get(docKey); + JSONObject mappings = (JSONObject) docData.get("mappings"); + JSONObject rootSchema = (JSONObject) mappings.get(searchContext.type()); JSONObject properties; // After (include) 7.x, type was removed from ES mapping, default type is `_doc` // https://www.elastic.co/guide/en/elasticsearch/reference/7.0/removal-of-types.html @@ -86,9 +87,9 @@ public class MappingPhase implements SearchPhase { + searchContext.type() + "], for table [" + searchContext.esTable().getName() + "]"); } - properties = mappings.optJSONObject("properties"); + properties = (JSONObject) mappings.get("properties"); } else { - properties = rootSchema.optJSONObject("properties"); + properties = (JSONObject) rootSchema.get("properties"); } if (properties == null) { throw new DorisEsException("index[" + searchContext.sourceIndex() + "] type[" + searchContext.type() + "] mapping not found for the ES Cluster"); @@ -96,11 +97,10 @@ public class MappingPhase implements SearchPhase { for (Column col : searchContext.columns()) { String colName = col.getName(); // if column exists in Doris Table but no found in ES's mapping, we choose to ignore this situation? - if (!properties.has(colName)) { + if (!properties.containsKey(colName)) { continue; } - JSONObject fieldObject = properties.optJSONObject(colName); - + JSONObject fieldObject = (JSONObject) properties.get(colName); resolveKeywordFields(searchContext, fieldObject, colName); resolveDocValuesFields(searchContext, fieldObject, colName); } @@ -108,16 +108,16 @@ public class MappingPhase implements SearchPhase { // get a field of keyword type in the fields private void resolveKeywordFields(SearchContext searchContext, JSONObject fieldObject, String colName) { - String fieldType = fieldObject.optString("type"); + String fieldType = (String) fieldObject.get("type"); // string-type field used keyword type to generate predicate // if text field type seen, we should use the `field` keyword type? if ("text".equals(fieldType)) { - JSONObject fieldsObject = fieldObject.optJSONObject("fields"); + JSONObject fieldsObject = (JSONObject) fieldObject.get("fields"); if (fieldsObject != null) { - for (String key : fieldsObject.keySet()) { - JSONObject innerTypeObject = fieldsObject.optJSONObject(key); + for (Object key : fieldsObject.keySet()) { + JSONObject innerTypeObject = (JSONObject) fieldsObject.get((String) key); // just for text type - if ("keyword".equals(innerTypeObject.optString("type"))) { + if ("keyword".equals((String) innerTypeObject.get("type"))) { searchContext.fetchFieldsContext().put(colName, colName + "." + key); } } @@ -126,18 +126,18 @@ public class MappingPhase implements SearchPhase { } private void resolveDocValuesFields(SearchContext searchContext, JSONObject fieldObject, String colName) { - String fieldType = fieldObject.optString("type"); + String fieldType = (String) fieldObject.get("type"); String docValueField = null; if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(fieldType)) { - JSONObject fieldsObject = fieldObject.optJSONObject("fields"); + JSONObject fieldsObject = (JSONObject) fieldObject.get("fields"); if (fieldsObject != null) { - for (String key : fieldsObject.keySet()) { - JSONObject innerTypeObject = fieldsObject.optJSONObject(key); - if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains(innerTypeObject.optString("type"))) { + for (Object key : fieldsObject.keySet()) { + JSONObject innerTypeObject = (JSONObject) fieldsObject.get((String) key); + if (EsTable.DEFAULT_DOCVALUE_DISABLED_FIELDS.contains((String) innerTypeObject.get("type"))) { continue; } - if (innerTypeObject.has("doc_values")) { - boolean docValue = innerTypeObject.getBoolean("doc_values"); + if (innerTypeObject.containsKey("doc_values")) { + boolean docValue = (Boolean) innerTypeObject.get("doc_values"); if (docValue) { docValueField = colName; } @@ -149,8 +149,8 @@ public class MappingPhase implements SearchPhase { } } else { // set doc_value = false manually - if (fieldObject.has("doc_values")) { - boolean docValue = fieldObject.optBoolean("doc_values"); + if (fieldObject.containsKey("doc_values")) { + Boolean docValue = (Boolean) fieldObject.get("doc_values"); if (!docValue) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java index f9968d2..bf6d271 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/http/rest/StorageTypeCheckAction.java @@ -34,7 +34,7 @@ import org.apache.doris.thrift.TStorageType; import com.google.common.base.Strings; -import org.json.JSONObject; +import org.json.simple.JSONObject; import java.util.List; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java index 92074ef..0778675 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java @@ -59,8 +59,8 @@ import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; import org.apache.thrift.TSerializer; import org.codehaus.jackson.map.ObjectMapper; -import org.json.JSONException; -import org.json.JSONObject; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import java.util.ArrayList; import java.util.Base64; @@ -107,17 +107,14 @@ public class TableQueryPlanAction extends RestBaseAction { || Strings.isNullOrEmpty(tableName)) { throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "{database}/{table} must be selected"); } - String sql; if (Strings.isNullOrEmpty(postContent)) { throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object"); } - JSONObject jsonObject; - try { - jsonObject = new JSONObject(postContent); - } catch (JSONException e) { + JSONObject jsonObject = (JSONObject) JSONValue.parse(postContent); + if (jsonObject == null) { throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "malformed json [ " + postContent + " ]"); } - sql = jsonObject.optString("sql"); + String sql = (String) jsonObject.get("sql"); if (Strings.isNullOrEmpty(sql)) { throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/interceptor/AuthInterceptor.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/interceptor/AuthInterceptor.java index 61f701f..554f003 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/interceptor/AuthInterceptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/interceptor/AuthInterceptor.java @@ -21,14 +21,11 @@ import org.apache.doris.httpv2.controller.BaseController; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONObject; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.servlet.HandlerInterceptor; import org.springframework.web.servlet.ModelAndView; -import java.util.Map; - import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -58,9 +55,4 @@ public class AuthInterceptor extends BaseController implements HandlerIntercepto @Override public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { } - - private String toJson(Map<String, Object> map) { - JSONObject root = new JSONObject(map); - return root.toString(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java index cf107df..f179c9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java @@ -17,8 +17,6 @@ package org.apache.doris.httpv2.rest; -import io.netty.handler.codec.http.HttpResponseStatus; - import org.apache.doris.analysis.InlineViewRef; import org.apache.doris.analysis.SelectStmt; import org.apache.doris.analysis.StatementBase; @@ -56,8 +54,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; import org.apache.thrift.TSerializer; -import org.json.JSONException; -import org.json.JSONObject; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -73,6 +71,8 @@ import java.util.UUID; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import io.netty.handler.codec.http.HttpResponseStatus; + /** * This class responsible for parse the sql and generate the query plan fragment for a (only one) table{@see OlapTable} * the related tablet maybe pruned by query planer according the `where` predicate. @@ -93,18 +93,15 @@ public class TableQueryPlanAction extends RestBaseController { String postContent = HttpUtil.getBody(request); try { // may be these common validate logic should be moved to one base class - String sql; if (Strings.isNullOrEmpty(postContent)) { return ResponseEntityBuilder.badRequest("POST body must contains [sql] root object"); } - JSONObject jsonObject; - try { - jsonObject = new JSONObject(postContent); - } catch (JSONException e) { - return ResponseEntityBuilder.badRequest("malformed json: " + e.getMessage()); + JSONObject jsonObject = (JSONObject) JSONValue.parse(postContent); + if (jsonObject == null) { + return ResponseEntityBuilder.badRequest("malformed json: " + postContent); } - sql = jsonObject.optString("sql"); + String sql = (String) jsonObject.get("sql"); if (Strings.isNullOrEmpty(sql)) { return ResponseEntityBuilder.badRequest("POST body must contains [sql] root object"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java index aa8b616..02c0550 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java @@ -34,8 +34,8 @@ import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.LockMode; import com.sleepycat.je.OperationStatus; -import org.json.JSONArray; -import org.json.JSONObject; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; import java.io.ByteArrayInputStream; import java.io.DataInputStream; @@ -74,8 +74,7 @@ public class BDBTool { if (options.isListDbs()) { // list all databases List<String> dbNames = env.getDatabaseNames(); - JSONArray jsonArray = new JSONArray(dbNames); - System.out.println(jsonArray.toString()); + System.out.println(JSONArray.toJSONString(dbNames)); return true; } else { // db operations @@ -90,8 +89,7 @@ public class BDBTool { // get db stat Map<String, String> statMap = Maps.newHashMap(); statMap.put("count", String.valueOf(db.count())); - JSONObject jsonObject = new JSONObject(statMap); - System.out.println(jsonObject.toString()); + System.out.println(JSONObject.toJSONString(statMap)); return true; } else { // set from key diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/GlobalVarPersistInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/GlobalVarPersistInfo.java index 4d1290d..2c5ce67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/GlobalVarPersistInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/GlobalVarPersistInfo.java @@ -27,7 +27,7 @@ import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONObject; +import org.json.simple.JSONObject; import java.io.DataInput; import java.io.DataOutput; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c49134f..1876611 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -28,6 +28,8 @@ import org.apache.doris.thrift.TResourceLimit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import java.io.DataInput; import java.io.DataOutput; @@ -37,8 +39,6 @@ import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; -import org.json.JSONObject; - // System variable public class SessionVariable implements Serializable, Writable { static final Logger LOG = LogManager.getLogger(SessionVariable.class); @@ -1008,7 +1008,7 @@ public class SessionVariable implements Serializable, Writable { private void readFromJson(DataInput in) throws IOException { String json = Text.readString(in); - JSONObject root = new JSONObject(json); + JSONObject root = (JSONObject) JSONValue.parse(json); try { for (Field field : SessionVariable.class.getDeclaredFields()) { VarAttr attr = field.getAnnotation(VarAttr.class); @@ -1016,28 +1016,28 @@ public class SessionVariable implements Serializable, Writable { continue; } - if (!root.has(attr.name())) { + if (!root.containsKey(attr.name())) { continue; } switch (field.getType().getSimpleName()) { case "boolean": - field.set(this, root.getBoolean(attr.name())); + field.set(this, root.get(attr.name())); break; case "int": - field.set(this, root.getInt(attr.name())); + field.set(this, root.get(attr.name())); break; case "long": - field.set(this, root.getLong(attr.name())); + field.set(this, root.get(attr.name())); break; case "float": - field.set(this, root.getFloat(attr.name())); + field.set(this, root.get(attr.name())); break; case "double": - field.set(this, root.getDouble(attr.name())); + field.set(this, root.get(attr.name())); break; case "String": - field.set(this, root.getString(attr.name())); + field.set(this, root.get(attr.name())); break; default: // Unsupported type variable. diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 46e73d9..7209d74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -38,7 +38,8 @@ import org.apache.commons.lang.SerializationUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; -import org.json.JSONObject; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -324,19 +325,19 @@ public class VariableMgr { wlock.lock(); try { String json = info.getPersistJsonString(); - JSONObject root = new JSONObject(json); - for (String varName : root.keySet()) { - VarContext varContext = ctxByVarName.get(varName); + JSONObject root = (JSONObject) JSONValue.parse(json); + for (Object varName : root.keySet()) { + VarContext varContext = ctxByVarName.get((String) varName); if (Catalog.isCheckpointThread()) { // If this is checkpoint thread, we should write value in `ctxByVarNameForCkpt` to the image // instead of `ctxByVarName`. - varContext = ctxByVarNameForCkpt.get(varName); + varContext = ctxByVarNameForCkpt.get((String) varName); } if (varContext == null) { - LOG.error("failed to get global variable {} when replaying", varName); + LOG.error("failed to get global variable {} when replaying", (String) varName); continue; } - setValue(varContext.getObj(), varContext.getField(), root.get(varName).toString()); + setValue(varContext.getObj(), varContext.getField(), root.get((String) varName).toString()); } } finally { wlock.unlock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java index 21d4909..28ed675 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java @@ -17,12 +17,9 @@ package org.apache.doris.external.elasticsearch; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import org.json.JSONException; -import org.json.JSONObject; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; +import org.junit.Assert; import org.junit.Test; public class EsUtilTest { @@ -47,21 +44,21 @@ public class EsUtilTest { @Test public void testGetJsonObject() { - JSONObject json = new JSONObject(jsonStr); + JSONObject json = (JSONObject) JSONValue.parse(jsonStr); JSONObject upperBoundSetting = EsUtil.getJsonObject(json, "settings.index.bpack.partition", 0); - assertTrue(upperBoundSetting.has("upperbound")); - assertEquals("12", upperBoundSetting.getString("upperbound")); + Assert.assertTrue(upperBoundSetting.containsKey("upperbound")); + Assert.assertEquals("12", (String) upperBoundSetting.get("upperbound")); JSONObject unExistKey = EsUtil.getJsonObject(json, "set", 0); - assertNull(unExistKey); + Assert.assertNull(unExistKey); JSONObject singleKey = EsUtil.getJsonObject(json, "settings", 0); - assertTrue(singleKey.has("index")); + Assert.assertTrue(singleKey.containsKey("index")); } - @Test(expected = JSONException.class) + @Test(expected = ClassCastException.class) public void testGetJsonObjectWithException() { - JSONObject json = new JSONObject(jsonStr); + JSONObject json = (JSONObject) JSONValue.parse(jsonStr); // only support json object could not get string value directly from this api, exception will be threw EsUtil.getJsonObject(json, "settings.index.bpack.partition.upperbound", 0); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java index 695c5f8..d3bb6ca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java @@ -21,7 +21,9 @@ import org.apache.doris.thrift.TQueryPlanInfo; import org.apache.thrift.TDeserializer; import org.apache.thrift.TException; -import org.json.JSONObject; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -54,22 +56,20 @@ public class TableQueryPlanActionTest extends DorisHttpTestCase { .build(); Response response = networkClient.newCall(request).execute(); String respStr = response.body().string(); - JSONObject jsonObject = new JSONObject(respStr); + JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); System.out.println(respStr); - Assert.assertEquals(200, jsonObject.getInt("status")); + Assert.assertEquals(200, (long) jsonObject.get("status")); - JSONObject partitionsObject = jsonObject.getJSONObject("partitions"); + JSONObject partitionsObject = (JSONObject) jsonObject.get("partitions"); Assert.assertNotNull(partitionsObject); - for (String tabletKey : partitionsObject.keySet()) { - JSONObject tabletObject = partitionsObject.getJSONObject(tabletKey); - Assert.assertNotNull(tabletObject.getJSONArray("routings")); - Assert.assertEquals(3, tabletObject.getJSONArray("routings").length()); - Assert.assertEquals(testStartVersion, tabletObject.getLong("version")); - Assert.assertEquals(testStartVersionHash, tabletObject.getLong("versionHash")); - Assert.assertEquals(testSchemaHash, tabletObject.getLong("schemaHash")); - + for (Object tabletKey : partitionsObject.keySet()) { + JSONObject tabletObject = (JSONObject) partitionsObject.get(tabletKey); + Assert.assertNotNull(tabletObject.get("routings")); + Assert.assertEquals(3, ((JSONArray) tabletObject.get("routings")).size()); + Assert.assertEquals(testStartVersion, (long) tabletObject.get("version")); + Assert.assertEquals(testSchemaHash, (long) tabletObject.get("schemaHash")); } - String queryPlan = jsonObject.getString("opaqued_query_plan"); + String queryPlan = (String) jsonObject.get("opaqued_query_plan"); Assert.assertNotNull(queryPlan); byte[] binaryPlanInfo = Base64.getDecoder().decode(queryPlan); TDeserializer deserializer = new TDeserializer(); @@ -91,10 +91,9 @@ public class TableQueryPlanActionTest extends DorisHttpTestCase { String respStr = response.body().string(); System.out.println(respStr); Assert.assertNotNull(respStr); - expectThrowsNoException(() -> new JSONObject(respStr)); - JSONObject jsonObject = new JSONObject(respStr); - Assert.assertEquals(400, jsonObject.getInt("status")); - String exception = jsonObject.getString("exception"); + JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); + Assert.assertEquals(400, (long) jsonObject.get("status")); + String exception = (String) jsonObject.get("exception"); Assert.assertNotNull(exception); Assert.assertEquals("POST body must contains [sql] root object", exception); } @@ -111,10 +110,9 @@ public class TableQueryPlanActionTest extends DorisHttpTestCase { String respStr = response.body().string(); System.out.println(respStr); Assert.assertNotNull(respStr); - expectThrowsNoException(() -> new JSONObject(respStr)); - JSONObject jsonObject = new JSONObject(respStr); - Assert.assertEquals(400, jsonObject.getInt("status")); - String exception = jsonObject.getString("exception"); + JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); + Assert.assertEquals(400, (long) jsonObject.get("status")); + String exception = (String) jsonObject.get("exception"); Assert.assertNotNull(exception); Assert.assertEquals("POST body must contains [sql] root object", exception); } @@ -131,10 +129,9 @@ public class TableQueryPlanActionTest extends DorisHttpTestCase { String respStr = response.body().string(); System.out.println(respStr); Assert.assertNotNull(respStr); - expectThrowsNoException(() -> new JSONObject(respStr)); - JSONObject jsonObject = new JSONObject(respStr); - Assert.assertEquals(400, jsonObject.getInt("status")); - String exception = jsonObject.getString("exception"); + JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); + Assert.assertEquals(400, (long) jsonObject.get("status")); + String exception = (String) jsonObject.get("exception"); Assert.assertNotNull(exception); Assert.assertTrue(exception.startsWith("requested database and table must consistent with sql")); } @@ -150,10 +147,9 @@ public class TableQueryPlanActionTest extends DorisHttpTestCase { Response response = networkClient.newCall(request).execute(); String respStr = response.body().string(); Assert.assertNotNull(respStr); - expectThrowsNoException(() -> new JSONObject(respStr)); - JSONObject jsonObject = new JSONObject(respStr); - Assert.assertEquals(400, jsonObject.getInt("status")); - String exception = jsonObject.getString("exception"); + JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); + Assert.assertEquals(400, (long) jsonObject.get("status")); + String exception = (String) jsonObject.get("exception"); Assert.assertNotNull(exception); Assert.assertTrue(exception.startsWith("malformed json")); } @@ -169,10 +165,9 @@ public class TableQueryPlanActionTest extends DorisHttpTestCase { Response response = networkClient.newCall(request).execute(); String respStr = response.body().string(); Assert.assertNotNull(respStr); - expectThrowsNoException(() -> new JSONObject(respStr)); - JSONObject jsonObject = new JSONObject(respStr); - Assert.assertEquals(403, jsonObject.getInt("status")); - String exception = jsonObject.getString("exception"); + JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr); + Assert.assertEquals(403, (long) jsonObject.get("status")); + String exception = (String) jsonObject.get("exception"); Assert.assertTrue(exception.contains("table type is not OLAP")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/TableRowCountActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/TableRowCountActionTest.java index 2a69bd6..d697c53 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/TableRowCountActionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/TableRowCountActionTest.java @@ -17,14 +17,16 @@ package org.apache.doris.http; -import okhttp3.Request; -import okhttp3.Response; -import org.json.JSONObject; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import org.junit.Assert; import org.junit.Test; import java.io.IOException; +import okhttp3.Request; +import okhttp3.Response; + public class TableRowCountActionTest extends DorisHttpTestCase { private static final String PATH_URI = "/_count"; @@ -38,8 +40,8 @@ public class TableRowCountActionTest extends DorisHttpTestCase { .build(); Response response = networkClient.newCall(request).execute(); - JSONObject jsonObject = new JSONObject(response.body().string()); - Assert.assertEquals(200, jsonObject.getInt("status")); - Assert.assertEquals(2000, jsonObject.getLong("size")); + JSONObject jsonObject = (JSONObject) JSONValue.parse(response.body().string()); + Assert.assertEquals(200, (long) jsonObject.get("status")); + Assert.assertEquals(2000, (long) jsonObject.get("size")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java index c883f76..cb96a10 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/TableSchemaActionTest.java @@ -17,15 +17,16 @@ package org.apache.doris.http; -import okhttp3.Request; -import okhttp3.Response; -import org.json.JSONArray; -import org.json.JSONObject; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.JSONValue; import org.junit.Assert; import org.junit.Test; import java.io.IOException; +import okhttp3.Request; +import okhttp3.Response; import static org.junit.Assert.assertTrue; public class TableSchemaActionTest extends DorisHttpTestCase { @@ -43,10 +44,10 @@ public class TableSchemaActionTest extends DorisHttpTestCase { assertTrue(response.isSuccessful()); String respStr = response.body().string(); Assert.assertNotNull(respStr); - JSONObject object = new JSONObject(respStr); - Assert.assertEquals(200, object.getInt("status")); - JSONArray propArray = object.getJSONArray("properties"); + JSONObject object = (JSONObject) JSONValue.parse(respStr); + Assert.assertEquals(200, (long) object.get("status")); + JSONArray propArray = (JSONArray) object.get("properties"); // k1, k2 - Assert.assertEquals(2, propArray.length()); + Assert.assertEquals(2, propArray.size()); } } diff --git a/fe/pom.xml b/fe/pom.xml index 8e62275..1eecd31 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -317,11 +317,10 @@ under the License. <artifactId>commons-io</artifactId> <version>2.6</version> </dependency> - <!-- https://mvnrepository.com/artifact/org.json/json --> <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <version>20171018</version> + <groupId>com.googlecode.json-simple</groupId> + <artifactId>json-simple</artifactId> + <version>1.1.1</version> </dependency> <!-- https://mvnrepository.com/artifact/junit/junit --> <dependency> @@ -371,12 +370,6 @@ under the License. <artifactId>metrics-core</artifactId> <version>4.0.2</version> </dependency> - <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java --> - <dependency> - <groupId>mysql</groupId> - <artifactId>mysql-connector-java</artifactId> - <version>5.1.41</version> - </dependency> <!-- https://mvnrepository.com/artifact/io.netty/netty-all --> <dependency> <groupId>io.netty</groupId> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
