ruanwenjun commented on code in PR #17165: URL: https://github.com/apache/dolphinscheduler/pull/17165#discussion_r2084822638
########## dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LocalLogClient.java: ########## @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.executor.logging; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.extract.base.client.Clients; +import org.apache.dolphinscheduler.extract.common.ILogService; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; + +import org.springframework.stereotype.Component; + +@Component +public class LocalLogClient { + + /** + * Download the complete log of a task instance. + * This method is used to retrieve all log information from the start to the end of a task instance, + * suitable for scenarios where a complete log record is required. + * + * @param taskInstance The task instance object, containing information needed to retrieve the log. + * @return The complete log file download response of the task instance, including log content and metadata. + */ + public TaskInstanceLogFileDownloadResponse getWholeLog(TaskInstance taskInstance) { + return getLocalWholeLog(taskInstance); + } + + /** + * Query a portion of the log of a task instance. + * This method is used to query log information of a task instance in a paginated manner, + * suitable for scenarios where the log content is large and needs to be retrieved in batches. + * + * @param taskInstance The task instance object, containing information needed to retrieve the log. + * @param skipLineNum The number of lines to skip, indicating from which line to start reading the log. + * @param limit The maximum number of lines to read, indicating the maximum number of lines to retrieve in this query. + * @return The partial log query response, including log content within the specified range and metadata. + */ + public TaskInstanceLogPageQueryResponse getPartLog(TaskInstance taskInstance, int skipLineNum, int limit) { + return getLocalPartLog(taskInstance, skipLineNum, limit); + } + + private static TaskInstanceLogFileDownloadResponse getLocalWholeLog(TaskInstance taskInstance) { + TaskInstanceLogFileDownloadRequest request = buildLogFileDownloadRequest(taskInstance); + return getProxyLogService(taskInstance).getTaskInstanceWholeLogFileBytes(request); + } + + private static TaskInstanceLogPageQueryResponse getLocalPartLog(TaskInstance taskInstance, int skipLineNum, + int limit) { + TaskInstanceLogPageQueryRequest request = buildLogPageQueryRequest(taskInstance, skipLineNum, limit); + return getProxyLogService(taskInstance).pageQueryTaskInstanceLog(request); + } + + private static TaskInstanceLogFileDownloadRequest buildLogFileDownloadRequest(TaskInstance taskInstance) { + return new TaskInstanceLogFileDownloadRequest( + taskInstance.getId(), + taskInstance.getLogPath()); + } + + private static TaskInstanceLogPageQueryRequest buildLogPageQueryRequest(TaskInstance taskInstance, int skipLineNum, + int limit) { + return TaskInstanceLogPageQueryRequest.builder() + .taskInstanceId(taskInstance.getId()) + .taskInstanceLogAbsolutePath(taskInstance.getLogPath()) + .skipLineNum(skipLineNum) + .limit(limit) + .build(); + } + + private static ILogService getProxyLogService(TaskInstance taskInstance) { + return Clients + .withService(ILogService.class) + .withHost(taskInstance.getHost()); + } Review Comment: Please don't use static here, and the method `buildLogFileDownloadRequest`, `buildLogFileDownloadRequest` seems meaningless, this is only use once. ########## dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/service/impl/LogServiceImpl.java: ########## @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.common.service.impl; + +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.LogUtils; +import org.apache.dolphinscheduler.extract.common.ILogService; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; + +import java.util.List; + +public class LogServiceImpl implements ILogService { + + /** + * Downloads the entire log file for a task instance. + * + * @param taskInstanceLogFileDownloadRequest Request object containing the path to the task instance log file. + * @return Response object containing the log file content in byte array form. + */ + @Override + public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { + final TaskInstanceLogFileDownloadResponse taskInstanceLogFileDownloadResponse = + new TaskInstanceLogFileDownloadResponse(); + try { + byte[] bytes = LogUtils + .getFileContentBytesFromLocal(taskInstanceLogFileDownloadRequest.getTaskInstanceLogAbsolutePath()); + taskInstanceLogFileDownloadResponse.setLogBytes(bytes); + } catch (Exception e) { + taskInstanceLogFileDownloadResponse.setCode(1); + taskInstanceLogFileDownloadResponse.setMessage(e.getMessage()); + } + return taskInstanceLogFileDownloadResponse; + } + + /** + * Performs paginated queries on task instance logs. + * + * @param taskInstanceLogPageQueryRequest Request object containing the path to the task instance log file, the number of lines to skip, and the maximum number of lines to read. + * @return Response object containing the log content. + */ + @Override + public TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest) { + final TaskInstanceLogPageQueryResponse taskInstanceLogPageQueryResponse = + new TaskInstanceLogPageQueryResponse(); + List<String> lines; + try { + lines = LogUtils.readPartFileContentFromLocal( + taskInstanceLogPageQueryRequest.getTaskInstanceLogAbsolutePath(), + taskInstanceLogPageQueryRequest.getSkipLineNum(), + taskInstanceLogPageQueryRequest.getLimit()); + taskInstanceLogPageQueryResponse.setLogContent(LogUtils.rollViewLogLines(lines)); + } catch (Exception e) { + taskInstanceLogPageQueryResponse.setCode(1); Review Comment: Use enum here please don't use 1. ########## dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/logging/LogClientDelegate.java: ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.executor.logging; + +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; +import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils; +import org.apache.dolphinscheduler.registry.api.RegistryClient; +import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class LogClientDelegate { + + @Autowired + private LocalLogClient localLogClient; + @Autowired + private RemoteLogClient remoteLogClient; + @Autowired + private RegistryClient registryClient; + + /** + * Retrieves a portion of the log string for a given task instance. + * This method first attempts to fetch the log from local storage; if unsuccessful, it tries to obtain the log from remote storage. + * + * @param taskInstance The task instance object, containing information needed for log retrieval. + * @param skipLineNum The number of log lines to skip from the beginning. + * @param limit The maximum number of log lines to retrieve. + * @return A string containing the specified portion of the log. + */ + public String getPartLogString(TaskInstance taskInstance, int skipLineNum, int limit) { + checkArgs(taskInstance); + checkNodeExists(taskInstance); Review Comment: If the node doesn't exist, then we should try to fetch the log from remote. ########## dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/service/impl/LogServiceImpl.java: ########## @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.common.service.impl; + +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.LogUtils; +import org.apache.dolphinscheduler.extract.common.ILogService; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadRequest; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest; +import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse; + +import java.util.List; + +public class LogServiceImpl implements ILogService { + + /** + * Downloads the entire log file for a task instance. + * + * @param taskInstanceLogFileDownloadRequest Request object containing the path to the task instance log file. + * @return Response object containing the log file content in byte array form. + */ + @Override + public TaskInstanceLogFileDownloadResponse getTaskInstanceWholeLogFileBytes(TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest) { + final TaskInstanceLogFileDownloadResponse taskInstanceLogFileDownloadResponse = + new TaskInstanceLogFileDownloadResponse(); + try { + byte[] bytes = LogUtils + .getFileContentBytesFromLocal(taskInstanceLogFileDownloadRequest.getTaskInstanceLogAbsolutePath()); + taskInstanceLogFileDownloadResponse.setLogBytes(bytes); + } catch (Exception e) { + taskInstanceLogFileDownloadResponse.setCode(1); + taskInstanceLogFileDownloadResponse.setMessage(e.getMessage()); + } + return taskInstanceLogFileDownloadResponse; + } + + /** + * Performs paginated queries on task instance logs. + * + * @param taskInstanceLogPageQueryRequest Request object containing the path to the task instance log file, the number of lines to skip, and the maximum number of lines to read. + * @return Response object containing the log content. + */ + @Override + public TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest) { + final TaskInstanceLogPageQueryResponse taskInstanceLogPageQueryResponse = + new TaskInstanceLogPageQueryResponse(); + List<String> lines; + try { + lines = LogUtils.readPartFileContentFromLocal( + taskInstanceLogPageQueryRequest.getTaskInstanceLogAbsolutePath(), + taskInstanceLogPageQueryRequest.getSkipLineNum(), + taskInstanceLogPageQueryRequest.getLimit()); + taskInstanceLogPageQueryResponse.setLogContent(LogUtils.rollViewLogLines(lines)); + } catch (Exception e) { + taskInstanceLogPageQueryResponse.setCode(1); + taskInstanceLogPageQueryResponse.setMessage(e.getMessage()); Review Comment: Use ExceptionUtils to get the root cause message. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
