This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 4478e2fb721a0107e30579136427fc2336c39d99 Author: Martin Zink <[email protected]> AuthorDate: Mon May 15 19:23:06 2023 +0200 MINIFICPP-2101 Compilation fix in PutSFTPTests Closes #1560 Signed-off-by: Marton Szasz <[email protected]> --- extensions/sftp/tests/PutSFTPTests.cpp | 2 +- .../tests/unit/ListFileTests.cpp | 37 ++++++++++---------- libminifi/include/utils/file/FileUtils.h | 17 ++-------- libminifi/src/utils/file/FileUtils.cpp | 32 ++++++++++++------ libminifi/test/unit/FileUtilsTests.cpp | 39 ++++++++++++++++++++++ 5 files changed, 83 insertions(+), 44 deletions(-) diff --git a/extensions/sftp/tests/PutSFTPTests.cpp b/extensions/sftp/tests/PutSFTPTests.cpp index 2eceb6fca..2d729e1bd 100644 --- a/extensions/sftp/tests/PutSFTPTests.cpp +++ b/extensions/sftp/tests/PutSFTPTests.cpp @@ -499,7 +499,7 @@ TEST_CASE_METHOD(PutSFTPTestsFixture, "PutSFTP set mtime", "[PutSFTP]") { testFile("nifi_test/tstFile1.ext", "content 1"); using namespace std::chrono; // NOLINT(build/namespaces) system_clock::time_point modification_time = date::sys_days(date::January / 24 / 2065) + 5h + 20min; - testModificationTime("nifi_test/tstFile1.ext", file_clock::from_sys(modification_time)); + testModificationTime("nifi_test/tstFile1.ext", utils::file::from_sys(modification_time)); } #ifndef WIN32 diff --git a/extensions/standard-processors/tests/unit/ListFileTests.cpp b/extensions/standard-processors/tests/unit/ListFileTests.cpp index 7428f4a7c..e58e179e6 100644 --- a/extensions/standard-processors/tests/unit/ListFileTests.cpp +++ b/extensions/standard-processors/tests/unit/ListFileTests.cpp @@ -48,7 +48,6 @@ using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; class ListFileTestFixture { public: - static const std::string FORMAT_STRING; ListFileTestFixture(); protected: @@ -63,8 +62,6 @@ class ListFileTestFixture { std::filesystem::path second_sub_file_abs_path_; }; -const std::string ListFileTestFixture::FORMAT_STRING = "%Y-%m-%dT%H:%M:%SZ"; - ListFileTestFixture::ListFileTestFixture() : plan_(test_controller_.createPlan()), input_dir_(test_controller_.createTempDirectory()) { @@ -107,8 +104,12 @@ TEST_CASE_METHOD(ListFileTestFixture, "Input Directory is empty", "[testListFile REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); } +std::string get_last_modified_time_formatted_string(const std::filesystem::path& path) { + return utils::timeutils::getDateTimeStr(std::chrono::time_point_cast<std::chrono::seconds>(utils::file::to_sys(*utils::file::last_write_time(path)))); +} + TEST_CASE_METHOD(ListFileTestFixture, "Test listing files only once with default parameters", "[testListFile]") { - test_controller_.runSession(plan_); + TestController::runSession(plan_); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:standard_file.log")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:empty_file.txt")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:sub_file_one.txt")); @@ -135,19 +136,19 @@ TEST_CASE_METHOD(ListFileTestFixture, "Test listing files only once with default if (auto owner = utils::file::FileUtils::get_file_owner(standard_file_abs_path_)) { REQUIRE(LogTestController::getInstance().countOccurrences("key:file.owner value:" + *owner) == 4); } - REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:file.lastModifiedTime value:" + *utils::file::FileUtils::get_last_modified_time_formatted_string(empty_file_abs_path_, FORMAT_STRING))); - REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:file.lastModifiedTime value:" + *utils::file::FileUtils::get_last_modified_time_formatted_string(standard_file_abs_path_, FORMAT_STRING))); - REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:file.lastModifiedTime value:" + *utils::file::FileUtils::get_last_modified_time_formatted_string(first_sub_file_abs_path_, FORMAT_STRING))); - REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:file.lastModifiedTime value:" + *utils::file::FileUtils::get_last_modified_time_formatted_string(second_sub_file_abs_path_, FORMAT_STRING))); + REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:file.lastModifiedTime value:" + get_last_modified_time_formatted_string(empty_file_abs_path_))); + REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:file.lastModifiedTime value:" + get_last_modified_time_formatted_string(standard_file_abs_path_))); + REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:file.lastModifiedTime value:" + get_last_modified_time_formatted_string(first_sub_file_abs_path_))); + REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:file.lastModifiedTime value:" + get_last_modified_time_formatted_string(second_sub_file_abs_path_))); plan_->reset(); LogTestController::getInstance().clear(); - test_controller_.runSession(plan_, true); + TestController::runSession(plan_, true); REQUIRE_FALSE(LogTestController::getInstance().contains("key:file.size", 0s, 0ms)); } TEST_CASE_METHOD(ListFileTestFixture, "Test turning off recursive file listing", "[testListFile]") { plan_->setProperty(list_file_processor_, "Recurse Subdirectories", "false"); - test_controller_.runSession(plan_); + TestController::runSession(plan_); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:standard_file.log")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:empty_file.txt")); REQUIRE_FALSE(LogTestController::getInstance().contains("key:filename value:sub_file_one.txt", 0s, 0ms)); @@ -156,7 +157,7 @@ TEST_CASE_METHOD(ListFileTestFixture, "Test turning off recursive file listing", TEST_CASE_METHOD(ListFileTestFixture, "Test listing files matching the File Filter pattern", "[testListFile]") { plan_->setProperty(list_file_processor_, "File Filter", "stand\\w+\\.log"); - test_controller_.runSession(plan_); + TestController::runSession(plan_); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:standard_file.log")); REQUIRE_FALSE(LogTestController::getInstance().contains("key:filename value:empty_file.txt", 0s, 0ms)); REQUIRE_FALSE(LogTestController::getInstance().contains("key:filename value:sub_file_one.txt", 0s, 0ms)); @@ -165,14 +166,14 @@ TEST_CASE_METHOD(ListFileTestFixture, "Test listing files matching the File Filt TEST_CASE_METHOD(ListFileTestFixture, "Test listing files matching the Path Filter pattern", "[testListFile]") { plan_->setProperty(list_file_processor_, "Path Filter", "first.*"); - test_controller_.runSession(plan_); + TestController::runSession(plan_); CHECK(verifyLogLinePresenceInPollTime(3s, "key:filename value:sub_file_one.txt")); CHECK(LogTestController::getInstance().countOccurrences("key:filename value:") == 1); } TEST_CASE_METHOD(ListFileTestFixture, "Test listing files matching the Path Filter pattern when the pattern also matches .", "[testListFile]") { plan_->setProperty(list_file_processor_, "Path Filter", "second.*|\\."); - test_controller_.runSession(plan_); + TestController::runSession(plan_); CHECK(verifyLogLinePresenceInPollTime(3s, "key:filename value:standard_file.log")); CHECK(verifyLogLinePresenceInPollTime(3s, "key:filename value:empty_file.txt")); CHECK(verifyLogLinePresenceInPollTime(3s, "key:filename value:sub_file_two.txt")); @@ -181,7 +182,7 @@ TEST_CASE_METHOD(ListFileTestFixture, "Test listing files matching the Path Filt TEST_CASE_METHOD(ListFileTestFixture, "Test listing files with restriction on the minimum file age", "[testListFile]") { plan_->setProperty(list_file_processor_, "Minimum File Age", "90 min"); - test_controller_.runSession(plan_); + TestController::runSession(plan_); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:sub_file_one.txt")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:sub_file_two.txt")); REQUIRE(LogTestController::getInstance().countOccurrences("key:filename value:empty_file.txt") == 0); @@ -190,7 +191,7 @@ TEST_CASE_METHOD(ListFileTestFixture, "Test listing files with restriction on th TEST_CASE_METHOD(ListFileTestFixture, "Test listing files with restriction on the maximum file age", "[testListFile]") { plan_->setProperty(list_file_processor_, "Maximum File Age", "90 min"); - test_controller_.runSession(plan_); + TestController::runSession(plan_); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:empty_file.txt")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:standard_file.log")); REQUIRE(LogTestController::getInstance().countOccurrences("key:filename value:sub_file_one.txt") == 0); @@ -199,7 +200,7 @@ TEST_CASE_METHOD(ListFileTestFixture, "Test listing files with restriction on th TEST_CASE_METHOD(ListFileTestFixture, "Test listing files with restriction on the minimum file size", "[testListFile]") { plan_->setProperty(list_file_processor_, "Minimum File Size", "4 B"); - test_controller_.runSession(plan_); + TestController::runSession(plan_); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:standard_file.log")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:sub_file_two.txt")); REQUIRE(LogTestController::getInstance().countOccurrences("key:filename value:empty_file.txt") == 0); @@ -208,7 +209,7 @@ TEST_CASE_METHOD(ListFileTestFixture, "Test listing files with restriction on th TEST_CASE_METHOD(ListFileTestFixture, "Test listing files with restriction on the maximum file size", "[testListFile]") { plan_->setProperty(list_file_processor_, "Maximum File Size", "4 B"); - test_controller_.runSession(plan_); + TestController::runSession(plan_); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:standard_file.log")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:empty_file.txt")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:sub_file_one.txt")); @@ -217,7 +218,7 @@ TEST_CASE_METHOD(ListFileTestFixture, "Test listing files with restriction on th TEST_CASE_METHOD(ListFileTestFixture, "Test listing hidden files", "[testListFile]") { plan_->setProperty(list_file_processor_, "Ignore Hidden Files", "false"); - test_controller_.runSession(plan_); + TestController::runSession(plan_); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:standard_file.log")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:empty_file.txt")); REQUIRE(verifyLogLinePresenceInPollTime(3s, "key:filename value:sub_file_one.txt")); diff --git a/libminifi/include/utils/file/FileUtils.h b/libminifi/include/utils/file/FileUtils.h index ba778f583..dfba2085b 100644 --- a/libminifi/include/utils/file/FileUtils.h +++ b/libminifi/include/utils/file/FileUtils.h @@ -83,9 +83,9 @@ namespace org::apache::nifi::minifi::utils::file { namespace FileUtils = ::org::apache::nifi::minifi::utils::file; -time_t to_time_t(std::filesystem::file_time_type time); +std::chrono::system_clock::time_point to_sys(std::filesystem::file_time_type file_time); -std::chrono::system_clock::time_point to_sys(std::filesystem::file_time_type time); +std::filesystem::file_time_type from_sys(std::chrono::system_clock::time_point sys_time); inline int64_t delete_dir(const std::filesystem::path& path, bool delete_files_recursively = true) { // Empty path is interpreted as the root of the current partition on Windows, which should not be allowed @@ -126,19 +126,6 @@ inline std::optional<std::filesystem::file_time_type> last_write_time(const std: return std::nullopt; } -inline std::optional<std::string> format_time(const std::filesystem::file_time_type& time, const std::string& format) { - auto last_write_time_t = to_time_t(time); - std::array<char, 128U> result{}; - if (std::strftime(result.data(), result.size(), format.c_str(), gmtime(&last_write_time_t)) != 0) { - return std::string(result.data()); - } - return std::nullopt; -} - -inline std::optional<std::string> get_last_modified_time_formatted_string(const std::filesystem::path& path, const std::string& format_string) { - return utils::file::last_write_time(path) | utils::flatMap([format_string](auto time) { return format_time(time, format_string); }); -} - inline bool set_last_write_time(const std::filesystem::path& path, std::filesystem::file_time_type new_time) { std::error_code ec; std::filesystem::last_write_time(path, new_time, ec); diff --git a/libminifi/src/utils/file/FileUtils.cpp b/libminifi/src/utils/file/FileUtils.cpp index 223a24704..0a8635a00 100644 --- a/libminifi/src/utils/file/FileUtils.cpp +++ b/libminifi/src/utils/file/FileUtils.cpp @@ -66,21 +66,33 @@ bool contains(const std::filesystem::path& file_path, std::string_view text_to_s return std::search(view.begin(), view.end(), searcher) != view.end(); } -time_t to_time_t(std::filesystem::file_time_type file_time) { -#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 14000 - return std::chrono::file_clock::to_time_t(file_time); +std::chrono::system_clock::time_point to_sys(std::filesystem::file_time_type file_time) { + using namespace std::chrono; // NOLINT(build/namespaces) +#if defined(WIN32) + // workaround for https://github.com/microsoft/STL/issues/2446 + // clock_cast can fail on older windows versions + constexpr file_clock::duration clock_epoch_diff{std::filesystem::__std_fs_file_time_epoch_adjustment}; + return system_clock::time_point(file_time.time_since_epoch() - clock_epoch_diff); +#elif(defined(_LIBCPP_VERSION) && (_LIBCPP_VERSION < 14000)) + // relies on file_clock and system_clock having the same epoch + return system_clock::time_point(duration_cast<system_clock::duration>(file_time.time_since_epoch())); #else - return std::chrono::system_clock::to_time_t(to_sys(file_time)); + return time_point_cast<system_clock::duration>(file_clock::to_sys(file_time)); #endif } -std::chrono::time_point<std::chrono::system_clock> to_sys(std::filesystem::file_time_type file_time) { -#if defined(WIN32) || defined(_LIBCPP_VERSION) && _LIBCPP_VERSION >= 14000 - return std::chrono::time_point_cast<std::chrono::system_clock::duration>(file_time - std::filesystem::file_time_type::clock::now() + std::chrono::system_clock::now()); -#elif defined(_LIBCPP_VERSION) - return std::chrono::system_clock::from_time_t(std::chrono::file_clock::to_time_t(file_time)); +std::filesystem::file_time_type from_sys(std::chrono::system_clock::time_point sys_time) { + using namespace std::chrono; // NOLINT(build/namespaces) +#if defined(WIN32) + // workaround for https://github.com/microsoft/STL/issues/2446 + // clock_cast can fail on older windows versions + constexpr file_clock::duration clock_epoch_diff{std::filesystem::__std_fs_file_time_epoch_adjustment}; + return file_clock::time_point(sys_time.time_since_epoch() + clock_epoch_diff); +#elif(defined(_LIBCPP_VERSION) && (_LIBCPP_VERSION < 14000)) + // relies on file_clock and system_clock having the same epoch + return file_clock::time_point(duration_cast<file_clock::duration>(sys_time.time_since_epoch())); #else - return std::chrono::file_clock::to_sys(file_time); + return time_point_cast<file_clock::duration>(file_clock::from_sys(sys_time)); #endif } diff --git a/libminifi/test/unit/FileUtilsTests.cpp b/libminifi/test/unit/FileUtilsTests.cpp index d537f6f27..28268f925 100644 --- a/libminifi/test/unit/FileUtilsTests.cpp +++ b/libminifi/test/unit/FileUtilsTests.cpp @@ -516,3 +516,42 @@ TEST_CASE("FileUtils::path_size", "[TestPathSize]") { REQUIRE(FileUtils::path_size(dir) == 12); } + +TEST_CASE("file_clock to system_clock conversion tests") { + using namespace std::chrono; + + static_assert(system_clock::period::num == file_clock::period::num); + constexpr auto lowest_den = std::min(file_clock::period::den, system_clock::period::den); + using LeastPreciseDurationType = duration<std::common_type_t<system_clock::duration::rep, file_clock::duration::rep>, std::ratio<system_clock::period::num, lowest_den>>; + + { + system_clock::time_point system_now = system_clock::now(); + file_clock::time_point converted_system_now = FileUtils::from_sys(system_now); + system_clock::time_point double_converted_system_now = FileUtils::to_sys(converted_system_now); + + CHECK(time_point_cast<LeastPreciseDurationType>(system_now).time_since_epoch().count() == time_point_cast<LeastPreciseDurationType>(double_converted_system_now).time_since_epoch().count()); + } + + { + file_clock::time_point file_now = file_clock ::now(); + system_clock::time_point converted_file_now = FileUtils::to_sys(file_now); + file_clock::time_point double_converted_file_now = FileUtils::from_sys(converted_file_now); + + CHECK(time_point_cast<LeastPreciseDurationType>(file_now).time_since_epoch().count() == time_point_cast<LeastPreciseDurationType>(double_converted_file_now).time_since_epoch().count()); + } + + { + // t0 <= t1 + auto sys_time_t0 = system_clock::now(); + auto file_time_t1 = file_clock ::now(); + + auto file_time_from_t0 = FileUtils::from_sys(sys_time_t0); + auto sys_time_from_t1 = FileUtils::to_sys(file_time_t1); + + CHECK(0ms <= sys_time_from_t1-sys_time_t0); + CHECK(sys_time_from_t1-sys_time_t0 < 10ms); + + CHECK(0ms <= file_time_t1-file_time_from_t0); + CHECK(file_time_t1-file_time_from_t0 < 10ms); + } +}
