arpadboda commented on a change in pull request #529: MINIFICPP-792 - TailFile
processor should handle rotation of source file
URL: https://github.com/apache/nifi-minifi-cpp/pull/529#discussion_r270367067
##########
File path: libminifi/src/processors/TailFile.cpp
##########
@@ -179,58 +180,22 @@ void TailFile::checkRollOver(const std::string
&fileLocation, const std::string
std::size_t found = fileName.find_last_of(".");
if (found != std::string::npos)
pattern = fileName.substr(0, found);
-#ifndef WIN32
- DIR *d;
- d = opendir(fileLocation.c_str());
- if (!d)
- return;
- while (1) {
- struct dirent *entry;
- entry = readdir(d);
- if (!entry)
- break;
- std::string d_name = entry->d_name;
- if (!(entry->d_type & DT_DIR)) {
- std::string fileName = d_name;
- std::string fileFullName = fileLocation + "/" + d_name;
- if (fileFullName.find(pattern) != std::string::npos &&
stat(fileFullName.c_str(), &statbuf) == 0) {
- if (((uint64_t) (statbuf.st_mtime) * 1000) >=
modifiedTimeCurrentTailFile) {
- TailMatchedFileItem item;
- item.fileName = fileName;
- item.modifiedTime = ((uint64_t) (statbuf.st_mtime) * 1000);
- matchedFiles.push_back(item);
- }
+
+ auto lambda = [&](const std::string& path, const std::string& filename) ->
bool {
+ struct stat sb;
+ std::string fileFullName = path +
utils::file::FileUtils::get_separator() + filename;
+ if (fileFullName.find(pattern) != std::string::npos &&
stat(fileFullName.c_str(), &sb) == 0) {
+ if (((uint64_t) (sb.st_mtime) * 1000) >= modifiedTimeCurrentTailFile) {
+ TailMatchedFileItem item;
+ item.fileName = filename;
+ item.modifiedTime = ((uint64_t) (sb.st_mtime) * 1000);
+ matchedFiles.push_back(item);
}
}
- }
- closedir(d);
-#else
-
- HANDLE hFind;
- WIN32_FIND_DATA FindFileData;
-
- if ((hFind = FindFirstFile(fileLocation.c_str(), &FindFileData)) !=
INVALID_HANDLE_VALUE) {
- do {
- struct stat statbuf {};
- if (stat(FindFileData.cFileName, &statbuf) != 0) {
- logger_->log_warn("Failed to stat %s", FindFileData.cFileName);
- break;
- }
-
- std::string fileFullName = fileLocation + "/" + FindFileData.cFileName;
+ return true;
+ };
- if (fileFullName.find(pattern) != std::string::npos &&
stat(fileFullName.c_str(), &statbuf) == 0) {
- if (((uint64_t)(statbuf.st_mtime) * 1000) >=
modifiedTimeCurrentTailFile) {
- TailMatchedFileItem item;
- item.fileName = fileName;
- item.modifiedTime = ((uint64_t)(statbuf.st_mtime) * 1000);
- matchedFiles.push_back(item);
- }
- }
- }while (FindNextFile(hFind, &FindFileData));
- FindClose(hFind);
- }
-#endif
+ utils::file::FileUtils::list_dir(fileLocation, lambda, logger_, false);
// Sort the list based on modified time
std::sort(matchedFiles.begin(), matchedFiles.end(),
sortTailMatchedFileItem);
Review comment:
```
MiNiFi Config Version: 3
Flow Controller:
name: techops
comment: Created by MiNiFi C2 Flow Designer
Core Properties:
flow controller graceful shutdown period: 10 sec
flow service write delay interval: 500 ms
administrative yield duration: 30 sec
bored yield duration: 10 millis
max concurrent threads: 1
variable registry properties: ''
FlowFile Repository:
partitions: 256
checkpoint interval: 2 mins
always sync: false
Swap:
threshold: 20000
in period: 5 sec
in threads: 1
out period: 5 sec
out threads: 4
Content Repository:
content claim max appendable size: 10 MB
content claim max flow files: 100
always sync: false
Provenance Repository:
provenance rollover time: 1 min
implementation:
org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository
Component Status Repository:
buffer size: 1440
snapshot frequency: 1 min
Security Properties:
keystore: ''
keystore type: ''
keystore password: ''
key password: ''
truststore: ''
truststore type: ''
truststore password: ''
ssl protocol: ''
Sensitive Props:
key:
algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL
provider: BC
Processors:
- id: cb18aee6-b107-4603-98a3-689b515b2b10
name: TailFile
class: class org.apache.nifi.minifi.processors.TailFile
max concurrent tasks: 1
scheduling strategy: TIMER_DRIVEN
scheduling period: 1000 ms
penalization period: 30000 ms
yield period: 1000 ms
run duration nanos: 0
auto-terminated relationships list: []
Properties:
File to Tail:
/Users/aboda/work/shadow/minifi/nifi-minifi-cpp/logs/minifi-app.log
State File: TailFileState
Controller Services: []
Process Groups: []
Input Ports: []
Output Ports: []
Funnels: []
Connections:
- id: fa777b72-6263-41a0-ba9c-7f57e0467582
name: TailFile/success/c88192f5-0169-1000-d577-47f8091af222
source id: cb18aee6-b107-4603-98a3-689b515b2b10
source relationship names:
- success
destination id: 34948508-0166-1000-7a4c-194e61494bb1
max work queue size: 0
max work queue data size: 10000 B
flowfile expiration: 60 seconds
queue prioritizer class: ''
Remote Process Groups:
- id: 34762af9-0166-1000-1209-2a4e139b258b
name: http://localhost:8080/nifi
url: http://localhost:8080/nifi
comment: ''
timeout: 30 secs
yield period: 10 sec
transport protocol: RAW
proxy host: ''
proxy port: ''
proxy user: ''
proxy password: ''
local network interface: ''
Input Ports:
- id: 34948508-0166-1000-7a4c-194e61494bb1
name: MiNiFiData
comment: ''
max concurrent tasks: 1
use compression: true
Output Ports: []
NiFi Properties Overrides: {}
```
In nutshell, this tails minifi's own logfile and sends flowfile to NiFi.
In order to verify the change I also edited minifi-log.properties file to
reduce the max log file size, so rotation occurs more frequently.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
