doleyzi commented on code in PR #9145: URL: https://github.com/apache/inlong/pull/9145#discussion_r1375592818
########## inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/FileScanner.java: ########## @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.inlong.agent.plugin.task.filecollect; + +import org.apache.inlong.agent.conf.TaskProfile; +import org.apache.inlong.agent.constant.TaskConstants; +import org.apache.inlong.agent.plugin.utils.file.FilePathUtil; +import org.apache.inlong.agent.plugin.utils.file.FileTimeComparator; +import org.apache.inlong.agent.plugin.utils.file.Files; +import org.apache.inlong.agent.plugin.utils.file.NewDateUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/* + * This class is mainly used for scanning log file that we want to read. We use this class at + * tdagent recover process, the do and redo tasks and the current log file access when we deploy a + * new data source. + */ +public class FileScanner { + + public static class BasicFileInfo { + + public String fileName; + public String dataTime; + + public BasicFileInfo(String fileName, String dataTime) { + this.fileName = fileName; + this.dataTime = dataTime; + } + + } + + private static final Logger logger = LoggerFactory.getLogger(FileScanner.class); + + public static List<BasicFileInfo> scanTaskBetweenTimes(TaskProfile conf, String originPattern, long failTime, + long recoverTime, boolean isRetry) { + String cycleUnit = conf.getCycleUnit(); + if (!isRetry) { + failTime -= NewDateUtils.caclOffset(conf.getTimeOffset()); + recoverTime -= NewDateUtils.caclOffset(conf.getTimeOffset()); + } + + String startTime = NewDateUtils.millSecConvertToTimeStr(failTime, cycleUnit); + String endTime = NewDateUtils.millSecConvertToTimeStr(recoverTime, cycleUnit); + logger.info("task {} this scan time is between {} and {}.", + new Object[]{conf.getTaskId(), startTime, endTime}); + + return scanTaskBetweenTimes(conf, originPattern, startTime, endTime); + } + + /* Scan log files and create tasks between two times. */ + public static List<BasicFileInfo> scanTaskBetweenTimes(TaskProfile conf, String originPattern, String startTime, + String endTime) { + String cycleUnit = conf.getCycleUnit(); + int maxFileNum = conf.getInt(TaskConstants.FILE_MAX_NUM); + List<Long> dateRegion = NewDateUtils.getDateRegion(startTime, endTime, cycleUnit); + List<BasicFileInfo> infos = new ArrayList<BasicFileInfo>(); + for (Long time : dateRegion) { + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(time); + String filename = NewDateUtils.replaceDateExpression(calendar, originPattern); + ArrayList<String> allPaths = FilePathUtil.cutDirectory(filename); + String firstDir = allPaths.get(0); + String secondDir = allPaths.get(0) + File.separator + allPaths.get(1); + ArrayList<String> fileList = getUpdatedOrNewFiles(firstDir, secondDir, filename, 3, + maxFileNum); + for (String file : fileList) { + // TODO the time is not YYYYMMDDHH + String dataTime = NewDateUtils.millSecConvertToTimeStr(time, cycleUnit); + BasicFileInfo info = new BasicFileInfo(file, dataTime); + logger.info("scan new task fileName {} ,dataTime {}", file, + NewDateUtils.millSecConvertToTimeStr(time, cycleUnit)); + infos.add(info); + } + } + return infos; + } + + public static ArrayList<String> scanFile(TaskProfile conf, String originPattern, long dataTime) { + int maxFileNum = conf.getInt(TaskConstants.FILE_MAX_NUM); + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(dataTime); + + String filename = NewDateUtils.replaceDateExpression(calendar, originPattern); + ArrayList<String> allPaths = FilePathUtil.cutDirectory(filename); + String firstDir = allPaths.get(0); + String secondDir = allPaths.get(0) + File.separator + allPaths.get(1); + return getUpdatedOrNewFiles(firstDir, secondDir, filename, 3, maxFileNum); + } + + private static ArrayList<String> getUpdatedOrNewFiles(String firstDir, String secondDir, + String fileName, long depth, int maxFileNum) { + + // logger.info("getUpdatedOrNewFiles: firstdir: {}, seconddir: {} filename: {}", Review Comment: It is recommended to delete the commented code -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
