github-advanced-security[bot] commented on code in PR #632: URL: https://github.com/apache/incubator-hugegraph-toolchain/pull/632#discussion_r2493912044
########## hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/langchain/LangChainController.java: ########## @@ -0,0 +1,678 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.controller.langchain; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.HashMap; + +import org.apache.hugegraph.controller.query.GremlinController; +import org.apache.hugegraph.driver.SchemaManager; +import org.apache.hugegraph.entity.query.GremlinQuery; +import org.apache.hugegraph.entity.query.JsonView; +import org.apache.hugegraph.service.query.QueryService; +import org.apache.hugegraph.structure.schema.EdgeLabel; +import org.apache.hugegraph.structure.schema.VertexLabel; +import org.apache.hugegraph.util.Ex; +import org.apache.hugegraph.util.JsonUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.compress.utils.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.hugegraph.util.E; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import org.apache.hugegraph.common.Constant; +import org.apache.hugegraph.controller.BaseController; +import org.apache.hugegraph.driver.HugeClient; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; + +/** + * langchain controller + */ +@Log4j2 +@RestController +@RequestMapping(Constant.API_VERSION + "graphspaces/{graphspace}/graphs/{graph}") +public class LangChainController extends BaseController { + + private static final String DEFAULT_PYTHON_FILE = "langchaincode/excute_langchain.py"; + + private static final String G_V = "g.v"; + private static final String G_E = "g.e"; + + private static final String WENXIN_4_MODEL = "wenxin4"; + private static final String GPT_4_MODEL = "gpt4"; + + private static final List<String> DEFAULT_MODEL = Arrays.asList(WENXIN_4_MODEL, GPT_4_MODEL); + + @Autowired + private QueryService queryService; + + @PostMapping("langchain") + public Object langchain(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}"); + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + @PostMapping("langchain/hubble") + public Object langchainHubble(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkParams(requestLangChainParams); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + + private ResponseLangChain langChainQuery(String graphSpace, String graph, + RequestLangChainParams requestLangChainParams) { + //uuapLoginController.tryLogin(graphSpace, graph, "admin", "S3#rd6(sg!"); //TODO C Deleted + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + String schema = JsonUtil.toJson(this.getBigModelSchema(vertexLabels, edgeLabels)); + log.info("langchain schema:{}", schema); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = this.excutePythonRuntime(requestLangChainParams.pythonPath, + filePath, requestLangChainParams.query, requestLangChainParams.openKey, schema, + requestLangChainParams.model, requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + + @PostMapping("langchain/schema") + public Object langchainSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + HashMap<String, Object> schema = this.getBigModelSchema(vertexLabels, edgeLabels); + return schema; + } + + @PostMapping("gremlin") + public Object gremlin(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + GremlinQuery query = new GremlinQuery(); + query.setContent(requestLangChainParams.query); + this.checkParamsValid(query); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + try { + HugeClient client = this.authClient(graphSpace, graph); + JsonView result = + this.queryService.executeSingleGremlinQuery(client, query); + return result.getData(); + } catch (Throwable e) { + throw e; + } + } + + @PostMapping("langchain_no_schema") + public Object langchainNoSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = + this.excutePythonByProcessBuilder( + requestLangChainParams.pythonPath, filePath, + requestLangChainParams.query, requestLangChainParams.openKey, + requestLangChainParams.graphSchema, requestLangChainParams.model, + requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + private void tryLogin(String graphSpace, String graph, + String username, String password) { + log.info("Attempting to login username:{}", username); + + E.checkNotNull(username, "username cannot be null"); + E.checkNotNull(password, "password cannot be null"); + String token = this.getToken(); + if (StringUtils.isNotEmpty(token)) { + log.info("Attempting to login token exist, username:{} token:{}", username, token); + return; + } + //uuapLoginController.loginVerifyUser(graphSpace, graph, username, password, null); + // TODO C Deleted + if (Objects.isNull(this.getToken())) { + log.error("Attempting to login failed, username:{}", username); + throw new IllegalStateException("login failed"); + } + } + + /** + * + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonRuntime(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + log.info("lang chain execute python command:\n [{}] \n", String.join(" ", args1)); + + // 执行Python文件，并传入参数 + List<String> lineList = new ArrayList<>(); + try { + Process proc = Runtime.getRuntime().exec(args1); + BufferedReader in = new BufferedReader(new InputStreamReader(proc.getInputStream())); + String line = null; + while ((line = in.readLine()) != null) { + lineList.add(line); + log.info("execute ret:{}", line); + } + if (!this.judgeResultSuccess(lineList, model)) { + this.calculateError(lineList, model); + log.error("excutePython lineList:{}", lineList); + lineList.clear(); + } else { + lineList = getGremlinResults(lineList, model); + } + in.close(); + proc.waitFor(); + } catch (Exception e) { + e.printStackTrace(); + log.error("excutePythonRuntime error", e); + } + return lineList; + } + + + /** + * 使用ProcessBuilder执行python脚本 + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonByProcessBuilder(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + + // 构造ProcessBuilder对象 + ProcessBuilder pb = new ProcessBuilder(args1); + + // 将python脚本的输出和错误输出合并 + pb.redirectErrorStream(false); + + List<String> lineList = new ArrayList<>(); + try { + // 启动进程 + Process process = pb.start(); + // 读取进程的输出 + BufferedReader in = new BufferedReader( + new InputStreamReader(process.getInputStream())); + String ret; + while ((ret = in.readLine()) != null) { + lineList.add(ret); + log.info("execute ret:{}", ret); + } + if (!this.judgeResultSuccess(lineList, model)) { + this.calculateError(lineList, model); + log.error("excutePython lineList:{}", lineList); + lineList.clear(); + } else { + lineList = getGremlinResults(lineList, model); + } + // 等待进程结束 + int exitCode = process.waitFor(); + log.info("Exited with error code : " + exitCode); + } catch (Exception e) { + log.error("excutePythonRuntime error", e); + } + return lineList; + } + + private void judgeFileExist(String filePath) { + File file = new File(filePath); + if (!file.exists()) { Review Comment: ## Uncontrolled data used in path expression This path depends on a [user-provided value](1). This path depends on a [user-provided value](2). This path depends on a [user-provided value](3). [Show more details](https://github.com/apache/incubator-hugegraph-toolchain/security/code-scanning/18) ########## hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/langchain/LangChainController.java: ########## @@ -0,0 +1,678 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.controller.langchain; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.HashMap; + +import org.apache.hugegraph.controller.query.GremlinController; +import org.apache.hugegraph.driver.SchemaManager; +import org.apache.hugegraph.entity.query.GremlinQuery; +import org.apache.hugegraph.entity.query.JsonView; +import org.apache.hugegraph.service.query.QueryService; +import org.apache.hugegraph.structure.schema.EdgeLabel; +import org.apache.hugegraph.structure.schema.VertexLabel; +import org.apache.hugegraph.util.Ex; +import org.apache.hugegraph.util.JsonUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.compress.utils.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.hugegraph.util.E; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import org.apache.hugegraph.common.Constant; +import org.apache.hugegraph.controller.BaseController; +import org.apache.hugegraph.driver.HugeClient; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; + +/** + * langchain controller + */ +@Log4j2 +@RestController +@RequestMapping(Constant.API_VERSION + "graphspaces/{graphspace}/graphs/{graph}") +public class LangChainController extends BaseController { + + private static final String DEFAULT_PYTHON_FILE = "langchaincode/excute_langchain.py"; + + private static final String G_V = "g.v"; + private static final String G_E = "g.e"; + + private static final String WENXIN_4_MODEL = "wenxin4"; + private static final String GPT_4_MODEL = "gpt4"; + + private static final List<String> DEFAULT_MODEL = Arrays.asList(WENXIN_4_MODEL, GPT_4_MODEL); + + @Autowired + private QueryService queryService; + + @PostMapping("langchain") + public Object langchain(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}"); + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + @PostMapping("langchain/hubble") + public Object langchainHubble(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkParams(requestLangChainParams); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + + private ResponseLangChain langChainQuery(String graphSpace, String graph, + RequestLangChainParams requestLangChainParams) { + //uuapLoginController.tryLogin(graphSpace, graph, "admin", "S3#rd6(sg!"); //TODO C Deleted + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + String schema = JsonUtil.toJson(this.getBigModelSchema(vertexLabels, edgeLabels)); + log.info("langchain schema:{}", schema); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = this.excutePythonRuntime(requestLangChainParams.pythonPath, + filePath, requestLangChainParams.query, requestLangChainParams.openKey, schema, + requestLangChainParams.model, requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + + @PostMapping("langchain/schema") + public Object langchainSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + HashMap<String, Object> schema = this.getBigModelSchema(vertexLabels, edgeLabels); + return schema; + } + + @PostMapping("gremlin") + public Object gremlin(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + GremlinQuery query = new GremlinQuery(); + query.setContent(requestLangChainParams.query); + this.checkParamsValid(query); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + try { + HugeClient client = this.authClient(graphSpace, graph); + JsonView result = + this.queryService.executeSingleGremlinQuery(client, query); + return result.getData(); + } catch (Throwable e) { + throw e; + } + } + + @PostMapping("langchain_no_schema") + public Object langchainNoSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = + this.excutePythonByProcessBuilder( + requestLangChainParams.pythonPath, filePath, + requestLangChainParams.query, requestLangChainParams.openKey, + requestLangChainParams.graphSchema, requestLangChainParams.model, + requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + private void tryLogin(String graphSpace, String graph, + String username, String password) { + log.info("Attempting to login username:{}", username); + + E.checkNotNull(username, "username cannot be null"); + E.checkNotNull(password, "password cannot be null"); + String token = this.getToken(); + if (StringUtils.isNotEmpty(token)) { + log.info("Attempting to login token exist, username:{} token:{}", username, token); + return; + } + //uuapLoginController.loginVerifyUser(graphSpace, graph, username, password, null); + // TODO C Deleted + if (Objects.isNull(this.getToken())) { + log.error("Attempting to login failed, username:{}", username); + throw new IllegalStateException("login failed"); + } + } + + /** + * + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonRuntime(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + log.info("lang chain execute python command:\n [{}] \n", String.join(" ", args1)); + + // 执行Python文件，并传入参数 + List<String> lineList = new ArrayList<>(); + try { + Process proc = Runtime.getRuntime().exec(args1); Review Comment: ## Uncontrolled command line This command line depends on a [user-provided value](1). This command line depends on a [user-provided value](2). [Show more details](https://github.com/apache/incubator-hugegraph-toolchain/security/code-scanning/16) ########## hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/langchain/LangChainController.java: ########## @@ -0,0 +1,678 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.controller.langchain; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.HashMap; + +import org.apache.hugegraph.controller.query.GremlinController; +import org.apache.hugegraph.driver.SchemaManager; +import org.apache.hugegraph.entity.query.GremlinQuery; +import org.apache.hugegraph.entity.query.JsonView; +import org.apache.hugegraph.service.query.QueryService; +import org.apache.hugegraph.structure.schema.EdgeLabel; +import org.apache.hugegraph.structure.schema.VertexLabel; +import org.apache.hugegraph.util.Ex; +import org.apache.hugegraph.util.JsonUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.compress.utils.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.hugegraph.util.E; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import org.apache.hugegraph.common.Constant; +import org.apache.hugegraph.controller.BaseController; +import org.apache.hugegraph.driver.HugeClient; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; + +/** + * langchain controller + */ +@Log4j2 +@RestController +@RequestMapping(Constant.API_VERSION + "graphspaces/{graphspace}/graphs/{graph}") +public class LangChainController extends BaseController { + + private static final String DEFAULT_PYTHON_FILE = "langchaincode/excute_langchain.py"; + + private static final String G_V = "g.v"; + private static final String G_E = "g.e"; + + private static final String WENXIN_4_MODEL = "wenxin4"; + private static final String GPT_4_MODEL = "gpt4"; + + private static final List<String> DEFAULT_MODEL = Arrays.asList(WENXIN_4_MODEL, GPT_4_MODEL); + + @Autowired + private QueryService queryService; + + @PostMapping("langchain") + public Object langchain(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}"); + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + @PostMapping("langchain/hubble") + public Object langchainHubble(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkParams(requestLangChainParams); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + + private ResponseLangChain langChainQuery(String graphSpace, String graph, + RequestLangChainParams requestLangChainParams) { + //uuapLoginController.tryLogin(graphSpace, graph, "admin", "S3#rd6(sg!"); //TODO C Deleted + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + String schema = JsonUtil.toJson(this.getBigModelSchema(vertexLabels, edgeLabels)); + log.info("langchain schema:{}", schema); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = this.excutePythonRuntime(requestLangChainParams.pythonPath, + filePath, requestLangChainParams.query, requestLangChainParams.openKey, schema, + requestLangChainParams.model, requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + + @PostMapping("langchain/schema") + public Object langchainSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + HashMap<String, Object> schema = this.getBigModelSchema(vertexLabels, edgeLabels); + return schema; + } + + @PostMapping("gremlin") + public Object gremlin(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + GremlinQuery query = new GremlinQuery(); + query.setContent(requestLangChainParams.query); + this.checkParamsValid(query); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + try { + HugeClient client = this.authClient(graphSpace, graph); + JsonView result = + this.queryService.executeSingleGremlinQuery(client, query); + return result.getData(); + } catch (Throwable e) { + throw e; + } + } + + @PostMapping("langchain_no_schema") + public Object langchainNoSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = + this.excutePythonByProcessBuilder( + requestLangChainParams.pythonPath, filePath, + requestLangChainParams.query, requestLangChainParams.openKey, + requestLangChainParams.graphSchema, requestLangChainParams.model, + requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + private void tryLogin(String graphSpace, String graph, + String username, String password) { + log.info("Attempting to login username:{}", username); + + E.checkNotNull(username, "username cannot be null"); + E.checkNotNull(password, "password cannot be null"); + String token = this.getToken(); + if (StringUtils.isNotEmpty(token)) { + log.info("Attempting to login token exist, username:{} token:{}", username, token); + return; + } + //uuapLoginController.loginVerifyUser(graphSpace, graph, username, password, null); + // TODO C Deleted + if (Objects.isNull(this.getToken())) { + log.error("Attempting to login failed, username:{}", username); + throw new IllegalStateException("login failed"); + } + } + + /** + * + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonRuntime(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + log.info("lang chain execute python command:\n [{}] \n", String.join(" ", args1)); + + // 执行Python文件，并传入参数 + List<String> lineList = new ArrayList<>(); + try { + Process proc = Runtime.getRuntime().exec(args1); + BufferedReader in = new BufferedReader(new InputStreamReader(proc.getInputStream())); + String line = null; + while ((line = in.readLine()) != null) { + lineList.add(line); + log.info("execute ret:{}", line); + } + if (!this.judgeResultSuccess(lineList, model)) { + this.calculateError(lineList, model); + log.error("excutePython lineList:{}", lineList); + lineList.clear(); + } else { + lineList = getGremlinResults(lineList, model); + } + in.close(); + proc.waitFor(); + } catch (Exception e) { + e.printStackTrace(); + log.error("excutePythonRuntime error", e); + } + return lineList; + } + + + /** + * 使用ProcessBuilder执行python脚本 + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonByProcessBuilder(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + + // 构造ProcessBuilder对象 + ProcessBuilder pb = new ProcessBuilder(args1); Review Comment: ## Uncontrolled command line This command line depends on a [user-provided value](1). [Show more details](https://github.com/apache/incubator-hugegraph-toolchain/security/code-scanning/17) ########## hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/schema/SchemaController.java: ########## @@ -18,147 +18,142 @@ package org.apache.hugegraph.controller.schema; -import java.util.ArrayList; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.util.Comparator; import java.util.Date; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.hugegraph.driver.HugeClient; +import org.apache.hugegraph.exception.ExternalException; +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.service.schema.SchemaService; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; + +import org.apache.hugegraph.util.Log; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + import org.apache.hugegraph.common.Constant; import org.apache.hugegraph.controller.BaseController; -import org.apache.hugegraph.entity.schema.EdgeLabelEntity; import org.apache.hugegraph.entity.schema.LabelUpdateEntity; import org.apache.hugegraph.entity.schema.Property; import org.apache.hugegraph.entity.schema.PropertyIndex; -import org.apache.hugegraph.entity.schema.PropertyKeyEntity; import org.apache.hugegraph.entity.schema.SchemaEntity; import org.apache.hugegraph.entity.schema.SchemaLabelEntity; import org.apache.hugegraph.entity.schema.Timefiable; -import org.apache.hugegraph.entity.schema.VertexLabelEntity; import org.apache.hugegraph.exception.InternalException; -import org.apache.hugegraph.service.schema.EdgeLabelService; import org.apache.hugegraph.service.schema.PropertyKeyService; -import org.apache.hugegraph.service.schema.VertexLabelService; -import org.apache.hugegraph.structure.constant.IdStrategy; import org.apache.hugegraph.util.Ex; import org.apache.hugegraph.util.PageUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.util.StringUtils; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - import com.baomidou.mybatisplus.core.metadata.IPage; -import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.servlet.http.HttpServletResponse; import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; @RestController -@RequestMapping(Constant.API_VERSION + "graph-connections/{connId}/schema") +@RequestMapping(Constant.API_VERSION + "graphspaces/{graphspace}/graphs" + + "/{graph}/schema") public class SchemaController extends BaseController { - @Autowired - private PropertyKeyService pkService; - @Autowired - private VertexLabelService vlService; - @Autowired - private EdgeLabelService elService; - - @GetMapping("graphview") - public SchemaView displayInSchemaView(@PathVariable("connId") int connId) { - List<PropertyKeyEntity> propertyKeys = this.pkService.list(connId); - List<VertexLabelEntity> vertexLabels = this.vlService.list(connId); - List<EdgeLabelEntity> edgeLabels = this.elService.list(connId); - - List<Map<String, Object>> vertices = new ArrayList<>(vertexLabels.size()); - for (VertexLabelEntity entity : vertexLabels) { - Map<String, Object> vertex = new LinkedHashMap<>(); - vertex.put("id", entity.getName()); - vertex.put("label", entity.getName()); - if (entity.getIdStrategy() == IdStrategy.PRIMARY_KEY) { - vertex.put("primary_keys", entity.getPrimaryKeys()); - } else { - vertex.put("primary_keys", new ArrayList<>()); - } - Map<String, String> properties = new LinkedHashMap<>(); - this.fillProperties(properties, entity, propertyKeys); - vertex.put("properties", properties); - vertex.put("~style", entity.getStyle()); - vertices.add(vertex); - } + public static Logger log = Log.logger(SchemaController.class); - List<Map<String, Object>> edges = new ArrayList<>(edgeLabels.size()); - for (EdgeLabelEntity entity : edgeLabels) { - Map<String, Object> edge = new LinkedHashMap<>(); - String edgeId = String.format( - "%s-%s->%s", entity.getSourceLabel(), - entity.getName(), entity.getTargetLabel()); - edge.put("id", edgeId); - edge.put("label", entity.getName()); - edge.put("source", entity.getSourceLabel()); - edge.put("target", entity.getTargetLabel()); - if (entity.isLinkMultiTimes()) { - edge.put("sort_keys", entity.getSortKeys()); - } else { - edge.put("sort_keys", new ArrayList<>()); - } - Map<String, String> properties = new LinkedHashMap<>(); - this.fillProperties(properties, entity, propertyKeys); - edge.put("properties", properties); - edge.put("~style", entity.getStyle()); - edges.add(edge); - } - return new SchemaView(vertices, edges); + @Autowired + private SchemaService schemaService; + @GetMapping("groovy") + public Object schemaGroovy(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph) { + HugeClient client = this.authClient(graphSpace, graph); + return ImmutableMap.of("schema", client.schema().getGroovySchema()); } - private void fillProperties(Map<String, String> properties, - SchemaLabelEntity entity, - List<PropertyKeyEntity> propertyKeys) { - for (Property property : entity.getProperties()) { - String name = property.getName(); - PropertyKeyEntity pkEntity = findPropertyKey(propertyKeys, name); - properties.put(name, pkEntity.getDataType().string()); + @PostMapping("groovy") + public Object addSchemaGroovy(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody SchemaGroovy schemaGroovy) { + HugeClient client = this.authClient(graphSpace, graph); + String content = schemaGroovy.getSchemaGroovy(); + log.info("Add schema groovy: {}", content); + checkSchemaGroovy(content); + try { + client.gremlin().gremlin(content).execute(); + }catch (Exception e) { + throw new HugeException( + "Add schema groovy failed. caused by" + e.getMessage()); } + return ImmutableMap.of("schema-groovy", content); } - private PropertyKeyEntity findPropertyKey(List<PropertyKeyEntity> entities, - String name) { - for (PropertyKeyEntity entity : entities) { - if (entity.getName().equals(name)) { - return entity; + private void checkSchemaGroovy(String content) { + String[] lines = content.split("\n|;"); + for (String line : lines) { + if (StringUtils.isEmpty(line)) { + continue; + } + if (!line.startsWith("graph.schema()")) { + throw new ExternalException( + "Schema Groovy each row must start with 'graph.schema" + + "().'"); } } - throw new InternalException("schema.propertykey.not-exist", name); } - @AllArgsConstructor - private static class SchemaView { + @GetMapping("groovy/export") + public void schemaGroovyExport(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + HttpServletResponse response) { + HugeClient client = this.authClient(graphSpace, graph); + String schema = client.schema().getGroovySchema(); - @JsonProperty("vertices") - private List<Map<String, Object>> vertices; + response.setCharacterEncoding("UTF-8"); + response.setContentType("text/html"); + String fileName = String.format("%s_%s.schema", graphSpace, graph); + response.setHeader("Content-Disposition", "attachment;fileName=" + fileName); Review Comment: ## HTTP response splitting This header depends on a [user-provided value](1), which may cause a response-splitting vulnerability. This header depends on a [user-provided value](2), which may cause a response-splitting vulnerability. [Show more details](https://github.com/apache/incubator-hugegraph-toolchain/security/code-scanning/15) ########## hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/langchain/LangChainController.java: ########## @@ -0,0 +1,678 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.controller.langchain; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.HashMap; + +import org.apache.hugegraph.controller.query.GremlinController; +import org.apache.hugegraph.driver.SchemaManager; +import org.apache.hugegraph.entity.query.GremlinQuery; +import org.apache.hugegraph.entity.query.JsonView; +import org.apache.hugegraph.service.query.QueryService; +import org.apache.hugegraph.structure.schema.EdgeLabel; +import org.apache.hugegraph.structure.schema.VertexLabel; +import org.apache.hugegraph.util.Ex; +import org.apache.hugegraph.util.JsonUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.compress.utils.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.hugegraph.util.E; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import org.apache.hugegraph.common.Constant; +import org.apache.hugegraph.controller.BaseController; +import org.apache.hugegraph.driver.HugeClient; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; + +/** + * langchain controller + */ +@Log4j2 +@RestController +@RequestMapping(Constant.API_VERSION + "graphspaces/{graphspace}/graphs/{graph}") +public class LangChainController extends BaseController { + + private static final String DEFAULT_PYTHON_FILE = "langchaincode/excute_langchain.py"; + + private static final String G_V = "g.v"; + private static final String G_E = "g.e"; + + private static final String WENXIN_4_MODEL = "wenxin4"; + private static final String GPT_4_MODEL = "gpt4"; + + private static final List<String> DEFAULT_MODEL = Arrays.asList(WENXIN_4_MODEL, GPT_4_MODEL); + + @Autowired + private QueryService queryService; + + @PostMapping("langchain") + public Object langchain(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}"); + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + @PostMapping("langchain/hubble") + public Object langchainHubble(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkParams(requestLangChainParams); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + + private ResponseLangChain langChainQuery(String graphSpace, String graph, + RequestLangChainParams requestLangChainParams) { + //uuapLoginController.tryLogin(graphSpace, graph, "admin", "S3#rd6(sg!"); //TODO C Deleted + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + String schema = JsonUtil.toJson(this.getBigModelSchema(vertexLabels, edgeLabels)); + log.info("langchain schema:{}", schema); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = this.excutePythonRuntime(requestLangChainParams.pythonPath, + filePath, requestLangChainParams.query, requestLangChainParams.openKey, schema, + requestLangChainParams.model, requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + + @PostMapping("langchain/schema") + public Object langchainSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + HashMap<String, Object> schema = this.getBigModelSchema(vertexLabels, edgeLabels); + return schema; + } + + @PostMapping("gremlin") + public Object gremlin(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + GremlinQuery query = new GremlinQuery(); + query.setContent(requestLangChainParams.query); + this.checkParamsValid(query); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + try { + HugeClient client = this.authClient(graphSpace, graph); + JsonView result = + this.queryService.executeSingleGremlinQuery(client, query); + return result.getData(); + } catch (Throwable e) { + throw e; + } + } + + @PostMapping("langchain_no_schema") + public Object langchainNoSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = + this.excutePythonByProcessBuilder( + requestLangChainParams.pythonPath, filePath, + requestLangChainParams.query, requestLangChainParams.openKey, + requestLangChainParams.graphSchema, requestLangChainParams.model, + requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + private void tryLogin(String graphSpace, String graph, + String username, String password) { + log.info("Attempting to login username:{}", username); + + E.checkNotNull(username, "username cannot be null"); + E.checkNotNull(password, "password cannot be null"); + String token = this.getToken(); + if (StringUtils.isNotEmpty(token)) { + log.info("Attempting to login token exist, username:{} token:{}", username, token); + return; + } + //uuapLoginController.loginVerifyUser(graphSpace, graph, username, password, null); + // TODO C Deleted + if (Objects.isNull(this.getToken())) { + log.error("Attempting to login failed, username:{}", username); + throw new IllegalStateException("login failed"); + } + } + + /** + * + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonRuntime(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + log.info("lang chain execute python command:\n [{}] \n", String.join(" ", args1)); + + // 执行Python文件，并传入参数 + List<String> lineList = new ArrayList<>(); + try { + Process proc = Runtime.getRuntime().exec(args1); + BufferedReader in = new BufferedReader(new InputStreamReader(proc.getInputStream())); + String line = null; + while ((line = in.readLine()) != null) { + lineList.add(line); + log.info("execute ret:{}", line); + } + if (!this.judgeResultSuccess(lineList, model)) { + this.calculateError(lineList, model); + log.error("excutePython lineList:{}", lineList); + lineList.clear(); + } else { + lineList = getGremlinResults(lineList, model); + } + in.close(); + proc.waitFor(); + } catch (Exception e) { + e.printStackTrace(); + log.error("excutePythonRuntime error", e); + } + return lineList; + } + + + /** + * 使用ProcessBuilder执行python脚本 + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonByProcessBuilder(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + + // 构造ProcessBuilder对象 + ProcessBuilder pb = new ProcessBuilder(args1); + + // 将python脚本的输出和错误输出合并 + pb.redirectErrorStream(false); + + List<String> lineList = new ArrayList<>(); + try { + // 启动进程 + Process process = pb.start(); + // 读取进程的输出 + BufferedReader in = new BufferedReader( + new InputStreamReader(process.getInputStream())); + String ret; + while ((ret = in.readLine()) != null) { + lineList.add(ret); + log.info("execute ret:{}", ret); + } + if (!this.judgeResultSuccess(lineList, model)) { + this.calculateError(lineList, model); + log.error("excutePython lineList:{}", lineList); + lineList.clear(); + } else { + lineList = getGremlinResults(lineList, model); + } + // 等待进程结束 + int exitCode = process.waitFor(); + log.info("Exited with error code : " + exitCode); + } catch (Exception e) { + log.error("excutePythonRuntime error", e); + } + return lineList; + } + + private void judgeFileExist(String filePath) { + File file = new File(filePath); + if (!file.exists()) { Review Comment: ## Uncontrolled data used in path expression This path depends on a [user-provided value](1). This path depends on a [user-provided value](2). This path depends on a [user-provided value](3). [Show more details](https://github.com/apache/incubator-hugegraph-toolchain/security/code-scanning/75) ########## hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/langchain/LangChainController.java: ########## @@ -0,0 +1,678 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.controller.langchain; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.HashMap; + +import org.apache.hugegraph.controller.query.GremlinController; +import org.apache.hugegraph.driver.SchemaManager; +import org.apache.hugegraph.entity.query.GremlinQuery; +import org.apache.hugegraph.entity.query.JsonView; +import org.apache.hugegraph.service.query.QueryService; +import org.apache.hugegraph.structure.schema.EdgeLabel; +import org.apache.hugegraph.structure.schema.VertexLabel; +import org.apache.hugegraph.util.Ex; +import org.apache.hugegraph.util.JsonUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.compress.utils.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.hugegraph.util.E; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import org.apache.hugegraph.common.Constant; +import org.apache.hugegraph.controller.BaseController; +import org.apache.hugegraph.driver.HugeClient; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; + +/** + * langchain controller + */ +@Log4j2 +@RestController +@RequestMapping(Constant.API_VERSION + "graphspaces/{graphspace}/graphs/{graph}") +public class LangChainController extends BaseController { + + private static final String DEFAULT_PYTHON_FILE = "langchaincode/excute_langchain.py"; + + private static final String G_V = "g.v"; + private static final String G_E = "g.e"; + + private static final String WENXIN_4_MODEL = "wenxin4"; + private static final String GPT_4_MODEL = "gpt4"; + + private static final List<String> DEFAULT_MODEL = Arrays.asList(WENXIN_4_MODEL, GPT_4_MODEL); + + @Autowired + private QueryService queryService; + + @PostMapping("langchain") + public Object langchain(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}"); + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + @PostMapping("langchain/hubble") + public Object langchainHubble(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkParams(requestLangChainParams); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + + private ResponseLangChain langChainQuery(String graphSpace, String graph, + RequestLangChainParams requestLangChainParams) { + //uuapLoginController.tryLogin(graphSpace, graph, "admin", "S3#rd6(sg!"); //TODO C Deleted + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + String schema = JsonUtil.toJson(this.getBigModelSchema(vertexLabels, edgeLabels)); + log.info("langchain schema:{}", schema); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = this.excutePythonRuntime(requestLangChainParams.pythonPath, + filePath, requestLangChainParams.query, requestLangChainParams.openKey, schema, + requestLangChainParams.model, requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + + @PostMapping("langchain/schema") + public Object langchainSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + HashMap<String, Object> schema = this.getBigModelSchema(vertexLabels, edgeLabels); + return schema; + } + + @PostMapping("gremlin") + public Object gremlin(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + GremlinQuery query = new GremlinQuery(); + query.setContent(requestLangChainParams.query); + this.checkParamsValid(query); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + try { + HugeClient client = this.authClient(graphSpace, graph); + JsonView result = + this.queryService.executeSingleGremlinQuery(client, query); + return result.getData(); + } catch (Throwable e) { + throw e; + } + } + + @PostMapping("langchain_no_schema") + public Object langchainNoSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = + this.excutePythonByProcessBuilder( + requestLangChainParams.pythonPath, filePath, + requestLangChainParams.query, requestLangChainParams.openKey, + requestLangChainParams.graphSchema, requestLangChainParams.model, + requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + private void tryLogin(String graphSpace, String graph, + String username, String password) { + log.info("Attempting to login username:{}", username); + + E.checkNotNull(username, "username cannot be null"); + E.checkNotNull(password, "password cannot be null"); + String token = this.getToken(); + if (StringUtils.isNotEmpty(token)) { + log.info("Attempting to login token exist, username:{} token:{}", username, token); + return; + } + //uuapLoginController.loginVerifyUser(graphSpace, graph, username, password, null); + // TODO C Deleted + if (Objects.isNull(this.getToken())) { + log.error("Attempting to login failed, username:{}", username); + throw new IllegalStateException("login failed"); + } + } + + /** + * + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonRuntime(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + log.info("lang chain execute python command:\n [{}] \n", String.join(" ", args1)); + + // 执行Python文件，并传入参数 + List<String> lineList = new ArrayList<>(); + try { + Process proc = Runtime.getRuntime().exec(args1); + BufferedReader in = new BufferedReader(new InputStreamReader(proc.getInputStream())); + String line = null; + while ((line = in.readLine()) != null) { + lineList.add(line); + log.info("execute ret:{}", line); + } + if (!this.judgeResultSuccess(lineList, model)) { + this.calculateError(lineList, model); + log.error("excutePython lineList:{}", lineList); + lineList.clear(); + } else { + lineList = getGremlinResults(lineList, model); + } + in.close(); + proc.waitFor(); + } catch (Exception e) { + e.printStackTrace(); + log.error("excutePythonRuntime error", e); + } + return lineList; + } + + + /** + * 使用ProcessBuilder执行python脚本 + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonByProcessBuilder(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + + // 构造ProcessBuilder对象 + ProcessBuilder pb = new ProcessBuilder(args1); Review Comment: ## Uncontrolled command line This command line depends on a [user-provided value](1). [Show more details](https://github.com/apache/incubator-hugegraph-toolchain/security/code-scanning/74) ########## hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/schema/SchemaController.java: ########## @@ -18,147 +18,142 @@ package org.apache.hugegraph.controller.schema; -import java.util.ArrayList; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.util.Comparator; import java.util.Date; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.hugegraph.driver.HugeClient; +import org.apache.hugegraph.exception.ExternalException; +import org.apache.hugegraph.exception.HugeException; +import org.apache.hugegraph.service.schema.SchemaService; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; + +import org.apache.hugegraph.util.Log; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.StringUtils; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + import org.apache.hugegraph.common.Constant; import org.apache.hugegraph.controller.BaseController; -import org.apache.hugegraph.entity.schema.EdgeLabelEntity; import org.apache.hugegraph.entity.schema.LabelUpdateEntity; import org.apache.hugegraph.entity.schema.Property; import org.apache.hugegraph.entity.schema.PropertyIndex; -import org.apache.hugegraph.entity.schema.PropertyKeyEntity; import org.apache.hugegraph.entity.schema.SchemaEntity; import org.apache.hugegraph.entity.schema.SchemaLabelEntity; import org.apache.hugegraph.entity.schema.Timefiable; -import org.apache.hugegraph.entity.schema.VertexLabelEntity; import org.apache.hugegraph.exception.InternalException; -import org.apache.hugegraph.service.schema.EdgeLabelService; import org.apache.hugegraph.service.schema.PropertyKeyService; -import org.apache.hugegraph.service.schema.VertexLabelService; -import org.apache.hugegraph.structure.constant.IdStrategy; import org.apache.hugegraph.util.Ex; import org.apache.hugegraph.util.PageUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.util.StringUtils; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - import com.baomidou.mybatisplus.core.metadata.IPage; -import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.servlet.http.HttpServletResponse; import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; @RestController -@RequestMapping(Constant.API_VERSION + "graph-connections/{connId}/schema") +@RequestMapping(Constant.API_VERSION + "graphspaces/{graphspace}/graphs" + + "/{graph}/schema") public class SchemaController extends BaseController { - @Autowired - private PropertyKeyService pkService; - @Autowired - private VertexLabelService vlService; - @Autowired - private EdgeLabelService elService; - - @GetMapping("graphview") - public SchemaView displayInSchemaView(@PathVariable("connId") int connId) { - List<PropertyKeyEntity> propertyKeys = this.pkService.list(connId); - List<VertexLabelEntity> vertexLabels = this.vlService.list(connId); - List<EdgeLabelEntity> edgeLabels = this.elService.list(connId); - - List<Map<String, Object>> vertices = new ArrayList<>(vertexLabels.size()); - for (VertexLabelEntity entity : vertexLabels) { - Map<String, Object> vertex = new LinkedHashMap<>(); - vertex.put("id", entity.getName()); - vertex.put("label", entity.getName()); - if (entity.getIdStrategy() == IdStrategy.PRIMARY_KEY) { - vertex.put("primary_keys", entity.getPrimaryKeys()); - } else { - vertex.put("primary_keys", new ArrayList<>()); - } - Map<String, String> properties = new LinkedHashMap<>(); - this.fillProperties(properties, entity, propertyKeys); - vertex.put("properties", properties); - vertex.put("~style", entity.getStyle()); - vertices.add(vertex); - } + public static Logger log = Log.logger(SchemaController.class); - List<Map<String, Object>> edges = new ArrayList<>(edgeLabels.size()); - for (EdgeLabelEntity entity : edgeLabels) { - Map<String, Object> edge = new LinkedHashMap<>(); - String edgeId = String.format( - "%s-%s->%s", entity.getSourceLabel(), - entity.getName(), entity.getTargetLabel()); - edge.put("id", edgeId); - edge.put("label", entity.getName()); - edge.put("source", entity.getSourceLabel()); - edge.put("target", entity.getTargetLabel()); - if (entity.isLinkMultiTimes()) { - edge.put("sort_keys", entity.getSortKeys()); - } else { - edge.put("sort_keys", new ArrayList<>()); - } - Map<String, String> properties = new LinkedHashMap<>(); - this.fillProperties(properties, entity, propertyKeys); - edge.put("properties", properties); - edge.put("~style", entity.getStyle()); - edges.add(edge); - } - return new SchemaView(vertices, edges); + @Autowired + private SchemaService schemaService; + @GetMapping("groovy") + public Object schemaGroovy(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph) { + HugeClient client = this.authClient(graphSpace, graph); + return ImmutableMap.of("schema", client.schema().getGroovySchema()); } - private void fillProperties(Map<String, String> properties, - SchemaLabelEntity entity, - List<PropertyKeyEntity> propertyKeys) { - for (Property property : entity.getProperties()) { - String name = property.getName(); - PropertyKeyEntity pkEntity = findPropertyKey(propertyKeys, name); - properties.put(name, pkEntity.getDataType().string()); + @PostMapping("groovy") + public Object addSchemaGroovy(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody SchemaGroovy schemaGroovy) { + HugeClient client = this.authClient(graphSpace, graph); + String content = schemaGroovy.getSchemaGroovy(); + log.info("Add schema groovy: {}", content); + checkSchemaGroovy(content); + try { + client.gremlin().gremlin(content).execute(); + }catch (Exception e) { + throw new HugeException( + "Add schema groovy failed. caused by" + e.getMessage()); } + return ImmutableMap.of("schema-groovy", content); } - private PropertyKeyEntity findPropertyKey(List<PropertyKeyEntity> entities, - String name) { - for (PropertyKeyEntity entity : entities) { - if (entity.getName().equals(name)) { - return entity; + private void checkSchemaGroovy(String content) { + String[] lines = content.split("\n|;"); + for (String line : lines) { + if (StringUtils.isEmpty(line)) { + continue; + } + if (!line.startsWith("graph.schema()")) { + throw new ExternalException( + "Schema Groovy each row must start with 'graph.schema" + + "().'"); } } - throw new InternalException("schema.propertykey.not-exist", name); } - @AllArgsConstructor - private static class SchemaView { + @GetMapping("groovy/export") + public void schemaGroovyExport(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + HttpServletResponse response) { + HugeClient client = this.authClient(graphSpace, graph); + String schema = client.schema().getGroovySchema(); - @JsonProperty("vertices") - private List<Map<String, Object>> vertices; + response.setCharacterEncoding("UTF-8"); + response.setContentType("text/html"); + String fileName = String.format("%s_%s.schema", graphSpace, graph); + response.setHeader("Content-Disposition", "attachment;fileName=" + fileName); Review Comment: ## HTTP response splitting This header depends on a [user-provided value](1), which may cause a response-splitting vulnerability. This header depends on a [user-provided value](2), which may cause a response-splitting vulnerability. [Show more details](https://github.com/apache/incubator-hugegraph-toolchain/security/code-scanning/72) ########## hugegraph-hubble/hubble-be/src/main/java/org/apache/hugegraph/controller/langchain/LangChainController.java: ########## @@ -0,0 +1,678 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.controller.langchain; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.HashMap; + +import org.apache.hugegraph.controller.query.GremlinController; +import org.apache.hugegraph.driver.SchemaManager; +import org.apache.hugegraph.entity.query.GremlinQuery; +import org.apache.hugegraph.entity.query.JsonView; +import org.apache.hugegraph.service.query.QueryService; +import org.apache.hugegraph.structure.schema.EdgeLabel; +import org.apache.hugegraph.structure.schema.VertexLabel; +import org.apache.hugegraph.util.Ex; +import org.apache.hugegraph.util.JsonUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.compress.utils.Lists; +import org.apache.commons.lang3.StringUtils; +import org.apache.hugegraph.util.E; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import org.apache.hugegraph.common.Constant; +import org.apache.hugegraph.controller.BaseController; +import org.apache.hugegraph.driver.HugeClient; +import com.fasterxml.jackson.annotation.JsonProperty; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; + +/** + * langchain controller + */ +@Log4j2 +@RestController +@RequestMapping(Constant.API_VERSION + "graphspaces/{graphspace}/graphs/{graph}") +public class LangChainController extends BaseController { + + private static final String DEFAULT_PYTHON_FILE = "langchaincode/excute_langchain.py"; + + private static final String G_V = "g.v"; + private static final String G_E = "g.e"; + + private static final String WENXIN_4_MODEL = "wenxin4"; + private static final String GPT_4_MODEL = "gpt4"; + + private static final List<String> DEFAULT_MODEL = Arrays.asList(WENXIN_4_MODEL, GPT_4_MODEL); + + @Autowired + private QueryService queryService; + + @PostMapping("langchain") + public Object langchain(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}"); + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + @PostMapping("langchain/hubble") + public Object langchainHubble(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkParams(requestLangChainParams); + + return this.langChainQuery(graphSpace, graph, requestLangChainParams); + } + + + private ResponseLangChain langChainQuery(String graphSpace, String graph, + RequestLangChainParams requestLangChainParams) { + //uuapLoginController.tryLogin(graphSpace, graph, "admin", "S3#rd6(sg!"); //TODO C Deleted + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + String schema = JsonUtil.toJson(this.getBigModelSchema(vertexLabels, edgeLabels)); + log.info("langchain schema:{}", schema); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = this.excutePythonRuntime(requestLangChainParams.pythonPath, + filePath, requestLangChainParams.query, requestLangChainParams.openKey, schema, + requestLangChainParams.model, requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + + @PostMapping("langchain/schema") + public Object langchainSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + HugeClient client = this.authClient(graphSpace, graph); + SchemaManager schemaManager = client.schema(); + List<VertexLabel> vertexLabels = schemaManager.getVertexLabels(); + List<EdgeLabel> edgeLabels = schemaManager.getEdgeLabels(); + HashMap<String, Object> schema = this.getBigModelSchema(vertexLabels, edgeLabels); + return schema; + } + + @PostMapping("gremlin") + public Object gremlin(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + GremlinQuery query = new GremlinQuery(); + query.setContent(requestLangChainParams.query); + this.checkParamsValid(query); + this.checkUserParam(requestLangChainParams); + + this.tryLogin(graphSpace, graph, + requestLangChainParams.userName, requestLangChainParams.password); + + try { + HugeClient client = this.authClient(graphSpace, graph); + JsonView result = + this.queryService.executeSingleGremlinQuery(client, query); + return result.getData(); + } catch (Throwable e) { + throw e; + } + } + + @PostMapping("langchain_no_schema") + public Object langchainNoSchema(@PathVariable("graphspace") String graphSpace, + @PathVariable("graph") String graph, + @RequestBody RequestLangChainParams requestLangChainParams) { + E.checkNotNull(requestLangChainParams, "params must not be null"); + log.info("LangChainController langchain params:{}", + requestLangChainParams); + + this.checkParams(requestLangChainParams); + this.checkModelParams(requestLangChainParams); + + String filePath; + if (StringUtils.isNotEmpty(requestLangChainParams.fileName)) { + URL url = LangChainController.class.getClassLoader().getResource(""); + filePath = String.format("%s%s", url.getPath(), requestLangChainParams.fileName); + this.judgeFileExist(filePath); + log.info("LangChainController filePath:{}", filePath); + } else { + throw new RuntimeException("fileName must not be null"); + } + + List<String> result = + this.excutePythonByProcessBuilder( + requestLangChainParams.pythonPath, filePath, + requestLangChainParams.query, requestLangChainParams.openKey, + requestLangChainParams.graphSchema, requestLangChainParams.model, + requestLangChainParams.ernieClientId, requestLangChainParams.ernieClientSecret); + if (CollectionUtils.isEmpty(result)) { + return this.generateResponseLangChain(requestLangChainParams.query, + "LangChain not generate gremlin"); + } else { + return this.generateResponseLangChain(requestLangChainParams.query, + result.get(result.size() - 1)); + } + } + + private void tryLogin(String graphSpace, String graph, + String username, String password) { + log.info("Attempting to login username:{}", username); + + E.checkNotNull(username, "username cannot be null"); + E.checkNotNull(password, "password cannot be null"); + String token = this.getToken(); + if (StringUtils.isNotEmpty(token)) { + log.info("Attempting to login token exist, username:{} token:{}", username, token); + return; + } + //uuapLoginController.loginVerifyUser(graphSpace, graph, username, password, null); + // TODO C Deleted + if (Objects.isNull(this.getToken())) { + log.error("Attempting to login failed, username:{}", username); + throw new IllegalStateException("login failed"); + } + } + + /** + * + * @param pythonPath + * @param pythonScriptPath + * @param query + * @param openKey + * @param graphSchema + * @return + */ + private List<String> excutePythonRuntime(String pythonPath, + String pythonScriptPath, + String query, + String openKey, + String graphSchema, + String model, + String ernieClientId, + String ernieClientSecret) { + String[] args1 = this.getExcuteArgs(pythonPath, pythonScriptPath, query, openKey, graphSchema, + model, ernieClientId, ernieClientSecret); + log.info("lang chain execute python command:\n [{}] \n", String.join(" ", args1)); + + // 执行Python文件，并传入参数 + List<String> lineList = new ArrayList<>(); + try { + Process proc = Runtime.getRuntime().exec(args1); Review Comment: ## Uncontrolled command line This command line depends on a [user-provided value](1). This command line depends on a [user-provided value](2). [Show more details](https://github.com/apache/incubator-hugegraph-toolchain/security/code-scanning/73) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
