This is an automated email from the ASF dual-hosted git repository. fgerlits pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit b41bed42305455e5006ce855cec86d47df344d0a Author: Martin Zink <[email protected]> AuthorDate: Thu Apr 14 09:32:07 2022 +0200 MINIFICPP-1461 Enable Expression Languaage tests on Windows Signed-off-by: Ferenc Gerlits <[email protected]> This closes #1337 --- CMakeLists.txt | 5 +- cmake/Date.cmake | 14 +++-- extensions/expression-language/Expression.cpp | 68 ++-------------------- extensions/expression-language/common/Value.h | 4 +- .../impl/expression/Expression.h | 13 ++--- .../expression-language/tests/CMakeLists.txt | 43 +++++++------- .../tests/ExpressionLanguageTests.cpp | 34 +++++------ .../tests/ProcessContextExprTests.cpp | 4 +- 8 files changed, 59 insertions(+), 126 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 871e061d3..a0830c18d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -646,10 +646,7 @@ set(CPACK_COMPONENTS_GROUPING "ALL_COMPONENTS_IN_ONE") list(APPEND CPACK_COMPONENTS_ALL bin) cpack_add_component(bin DISPLAY_NAME "MiNiFi C++ executables" REQUIRED) -if(WIN32) - list(APPEND CPACK_COMPONENTS_ALL tzdata) - cpack_add_component(tzdata DISPLAY_NAME "Timezone database for Expression Language") -else() +if(NOT WIN32) list(APPEND CPACK_COMPONENTS_ALL conf) cpack_add_component(conf DISPLAY_NAME "Default configuration files" REQUIRED) endif() diff --git a/cmake/Date.cmake b/cmake/Date.cmake index fdec3d1f7..9b6e24948 100644 --- a/cmake/Date.cmake +++ b/cmake/Date.cmake @@ -28,15 +28,21 @@ if (WIN32) FetchContent_Populate(tzdata) endif() + file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/tzdata) + + file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/cldr-common-38.1/common/supplemental/windowsZones.xml + DESTINATION ${CMAKE_BINARY_DIR}/tzdata) + + file(COPY ${tzdata_SOURCE_DIR}/ + DESTINATION ${CMAKE_BINARY_DIR}/tzdata) + install(DIRECTORY ${tzdata_SOURCE_DIR}/ DESTINATION tzdata - COMPONENT tzdata - ) + COMPONENT bin) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/cldr-common-38.1/common/supplemental/windowsZones.xml DESTINATION tzdata - COMPONENT tzdata - ) + COMPONENT bin) endif() FetchContent_Declare(date_src diff --git a/extensions/expression-language/Expression.cpp b/extensions/expression-language/Expression.cpp index fbce1b104..09a42c8f2 100644 --- a/extensions/expression-language/Expression.cpp +++ b/extensions/expression-language/Expression.cpp @@ -63,11 +63,7 @@ #include "Driver.h" -#ifdef EXPRESSION_LANGUAGE_USE_DATE #include "date/tz.h" -#else -#include <ctime> -#endif // EXPRESSION_LANGUAGE_USE_DATE namespace org { namespace apache { @@ -610,8 +606,6 @@ Value expr_escapeCsv(const std::vector<Value> &args) { return Value(result); } -#ifdef EXPRESSION_LANGUAGE_USE_DATE - Value expr_format(const std::vector<Value> &args) { std::chrono::milliseconds dur(args[0].asUnsignedLong()); std::chrono::system_clock::time_point dt(dur); @@ -640,52 +634,6 @@ Value expr_toDate(const std::vector<Value> &args) { return Value(int64_t{std::chrono::duration_cast<std::chrono::milliseconds>(zt.get_sys_time().time_since_epoch()).count()}); } -#else - -Value expr_format(const std::vector<Value>& args) { - const std::chrono::milliseconds dur(args.at(0).asUnsignedLong()); - const std::chrono::system_clock::time_point dt(dur); - const auto unix_time = std::chrono::system_clock::to_time_t(dt); - const auto zoned_time = [&args, unix_time] { - std::tm buf{}; - const auto requested_timezone = args.size() > 2 ? args[2].asString() : std::string{}; - if (requested_timezone == "UTC" || requested_timezone == "GMT") { -#ifdef WIN32 - const auto err = gmtime_s(&buf, &unix_time); - if (!err) { return buf; } - throw std::system_error{err, std::generic_category()}; -#else - tzset(); - const std::tm* const result = gmtime_r(&unix_time, &buf); - if (result) { return *result; } - throw std::system_error{errno, std::generic_category()}; -#endif /* WIN32 */ - } else if (!requested_timezone.empty()) { - throw std::domain_error{"format() with Non-UTC custom timezone is only supported when compiled with the date.h library"}; - } else { -#ifdef WIN32 - const auto err = localtime_s(&buf, &unix_time); - if (!err) { return buf; } - throw std::system_error{err, std::generic_category()}; -#else - tzset(); - const std::tm* const result = localtime_r(&unix_time, &buf); - if (result) { return *result; } - throw std::system_error{errno, std::generic_category()}; -#endif /* WIN32 */ - } - }(); - char result_buf[512] = {0}; - std::strftime(result_buf, 512, args.at(1).asString().c_str(), &zoned_time); - return Value(std::string(result_buf)); -} - -Value expr_toDate(const std::vector<Value>&) { - throw std::domain_error{"toDate() is only supported when compiled with the date.h library"}; -} - -#endif // EXPRESSION_LANGUAGE_USE_DATE - Value expr_now(const std::vector<Value>& /*args*/) { return Value(int64_t{std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count()}); } @@ -774,8 +722,6 @@ Value expr_base64Decode(const std::vector<Value> &args) { return Value(utils::StringUtils::from_base64(args[0].asString(), utils::as_string)); } -#ifdef EXPRESSION_LANGUAGE_USE_REGEX - Value expr_replace(const std::vector<Value> &args) { std::string result = args[0].asString(); const std::string &find = args[1].asString(); @@ -835,8 +781,6 @@ Value expr_find(const std::vector<Value> &args) { return Value(utils::regexSearch(subject, expr)); } -#endif // EXPRESSION_LANGUAGE_USE_REGEX - Value expr_trim(const std::vector<Value> &args) { return Value{utils::StringUtils::trim(args[0].asString())}; } @@ -1160,8 +1104,6 @@ Expression make_anyAttribute(const std::string &function_name, const std::vector return result; } -#ifdef EXPRESSION_LANGUAGE_USE_REGEX - Expression make_allMatchingAttributes(const std::string &function_name, const std::vector<Expression> &args) { if (args.size() < 1) { std::stringstream message_ss; @@ -1268,8 +1210,6 @@ Expression make_anyMatchingAttribute(const std::string &function_name, const std return result; } -#endif // EXPRESSION_LANGUAGE_USE_REGEX - Expression make_allDelineatedValues(const std::string &function_name, const std::vector<Expression> &args) { if (args.size() != 2) { std::stringstream message_ss; @@ -1457,7 +1397,6 @@ Expression make_dynamic_function(const std::string &function_name, const std::ve return make_dynamic_function_incomplete<expr_base64Encode>(function_name, args, 0); } else if (function_name == "base64Decode") { return make_dynamic_function_incomplete<expr_base64Decode>(function_name, args, 0); -#ifdef EXPRESSION_LANGUAGE_USE_REGEX } else if (function_name == "replace") { return make_dynamic_function_incomplete<expr_replace>(function_name, args, 2); } else if (function_name == "replaceFirst") { @@ -1476,7 +1415,6 @@ Expression make_dynamic_function(const std::string &function_name, const std::ve return make_allMatchingAttributes(function_name, args); } else if (function_name == "anyMatchingAttribute") { return make_anyMatchingAttribute(function_name, args); -#endif // EXPRESSION_LANGUAGE_USE_REGEX } else if (function_name == "trim") { return make_dynamic_function_incomplete<expr_trim>(function_name, args, 0); } else if (function_name == "append") { @@ -1665,6 +1603,12 @@ Expression Expression::make_aggregate(std::function<Value(const Parameters ¶ }); } +#ifdef WIN32 +void dateSetInstall(const std::string& install) { + date::set_install(install); +} +#endif + } /* namespace expression */ } /* namespace minifi */ } /* namespace nifi */ diff --git a/extensions/expression-language/common/Value.h b/extensions/expression-language/common/Value.h index 885775a2d..65d3cf586 100644 --- a/extensions/expression-language/common/Value.h +++ b/extensions/expression-language/common/Value.h @@ -180,7 +180,7 @@ class Value { if (is_unsigned_long_) { return unsigned_long_val_; } else if (is_string_) { - return string_val_.empty() ? 0 : std::stoul(string_val_); + return string_val_.empty() ? 0 : std::stoull(string_val_); } else if (is_signed_long_) { return signed_long_val_; } else if (is_long_double_) { @@ -196,7 +196,7 @@ class Value { } else if (is_unsigned_long_) { return unsigned_long_val_; } else if (is_string_) { - return string_val_.empty() ? 0 : std::stol(string_val_); + return string_val_.empty() ? 0 : std::stoll(string_val_); } else if (is_long_double_) { return static_cast<int64_t >(long_double_val_); } else { diff --git a/extensions/expression-language/impl/expression/Expression.h b/extensions/expression-language/impl/expression/Expression.h index 8e896087a..dd94a59fc 100644 --- a/extensions/expression-language/impl/expression/Expression.h +++ b/extensions/expression-language/impl/expression/Expression.h @@ -17,15 +17,6 @@ #pragma once -#define EXPRESSION_LANGUAGE_USE_REGEX - -// Disable regex in EL for incompatible compilers -#if !defined(WIN32) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 9)) -#undef EXPRESSION_LANGUAGE_USE_REGEX -#endif - -#define EXPRESSION_LANGUAGE_USE_DATE - #include <string> #include <memory> #include <functional> @@ -189,6 +180,10 @@ Expression make_dynamic_function(const std::string &function_name, const std::ve */ Expression make_function_composition(const Expression &arg, const std::vector<std::pair<std::string, std::vector<Expression>>> &chain); +#ifdef WIN32 +void dateSetInstall(const std::string& install); +#endif + } /* namespace expression */ } /* namespace minifi */ } /* namespace nifi */ diff --git a/extensions/expression-language/tests/CMakeLists.txt b/extensions/expression-language/tests/CMakeLists.txt index d2d2d8d2b..2214ef57b 100644 --- a/extensions/expression-language/tests/CMakeLists.txt +++ b/extensions/expression-language/tests/CMakeLists.txt @@ -21,33 +21,31 @@ file(GLOB EXPRESSION_LANGUAGE_TESTS "*.cpp") SET(EXTENSIONS_TEST_COUNT 0) -if(NOT WIN32) - FOREACH(testfile ${EXPRESSION_LANGUAGE_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_executable(${testfilename} "${testfile}") - target_include_directories(${testfilename} SYSTEM BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/catch") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") - createTests(${testfilename}) - target_link_libraries(${testfilename} ${CATCH_MAIN_LIB}) - if(NOT DISABLE_CURL) - target_link_libraries(${testfilename} CURL::libcurl) - endif() - target_link_libraries(${testfilename} minifi-expression-language-extensions) - target_link_libraries(${testfilename} minifi-standard-processors) +FOREACH(testfile ${EXPRESSION_LANGUAGE_TESTS}) + get_filename_component(testfilename "${testfile}" NAME_WE) + add_executable(${testfilename} "${testfile}") + target_include_directories(${testfilename} SYSTEM BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/catch") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") + createTests(${testfilename}) + target_link_libraries(${testfilename} ${CATCH_MAIN_LIB}) + if(NOT DISABLE_CURL) + target_link_libraries(${testfilename} CURL::libcurl) + endif() + target_link_libraries(${testfilename} minifi-expression-language-extensions) + target_link_libraries(${testfilename} minifi-standard-processors) + target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") - MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") - add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY ${TEST_DIR}) - ENDFOREACH() -endif() + MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") + add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY ${TEST_DIR}) +ENDFOREACH() ### integration tests -if(NOT WIN32) file(GLOB INT_EXPRESSION_LANGUAGE_TESTS "integration/*.cpp") SET(INT_EXTENSIONS_TEST_COUNT 0) @@ -74,4 +72,3 @@ ENDFOREACH() add_test(NAME UpdateAttributeIntegrationTest COMMAND UpdateAttributeIntegrationTest "${TEST_RESOURCES}/TestUpdateAttribute.yml" "${TEST_RESOURCES}/") message("-- Finished building ${EXTENSIONS_TEST_COUNT} expression language related test file(s)...") -endif() diff --git a/extensions/expression-language/tests/ExpressionLanguageTests.cpp b/extensions/expression-language/tests/ExpressionLanguageTests.cpp index 8e657f0af..7b03fb5d1 100644 --- a/extensions/expression-language/tests/ExpressionLanguageTests.cpp +++ b/extensions/expression-language/tests/ExpressionLanguageTests.cpp @@ -40,6 +40,7 @@ #include "TestBase.h" #include "Catch.h" #include "unit/ProvenanceTestHelper.h" +#include "date/tz.h" namespace expression = org::apache::nifi::minifi::expression; @@ -389,8 +390,6 @@ TEST_CASE("Substring After No Args", "[expressionLanguageSubstringAfterNoArgs]") REQUIRE_THROWS_WITH(expression::compile("${attr:substringAfter()}"), "Expression language function substringAfter called with 1 argument(s), but 2 are required"); } -#ifdef EXPRESSION_LANGUAGE_USE_REGEX - TEST_CASE("Replace", "[expressionLanguageReplace]") { auto expr = expression::compile("${attr:replace('.', '_')}"); @@ -599,8 +598,6 @@ TEST_CASE("LastIndexOf4", "[expressionLanguageLastIndexOf4]") { REQUIRE("11" == expr(expression::Parameters{ flow_file_a }).asString()); } -#endif // EXPRESSION_LANGUAGE_USE_REGEX - TEST_CASE("Plus Integer", "[expressionLanguagePlusInteger]") { auto expr = expression::compile("${attr:plus(13)}"); @@ -630,7 +627,7 @@ TEST_CASE("Plus Exponent 2", "[expressionLanguagePlusExponent2]") { auto flow_file_a = std::make_shared<core::FlowFile>(); flow_file_a->addAttribute("attr", "11.345678901234"); - REQUIRE("10000011.345678901234351" == expr(expression::Parameters{ flow_file_a }).asString()); + REQUIRE(10000011.345678901234 == Approx(expr(expression::Parameters{ flow_file_a }).asLongDouble())); } TEST_CASE("Minus Integer", "[expressionLanguageMinusInteger]") { @@ -662,7 +659,7 @@ TEST_CASE("Multiply Decimal", "[expressionLanguageMultiplyDecimal]") { auto flow_file_a = std::make_shared<core::FlowFile>(); flow_file_a->addAttribute("attr", "11.1"); - REQUIRE("-148.136937" == expr(expression::Parameters{ flow_file_a }).asString()); + REQUIRE(-148.136937 == Approx(expr(expression::Parameters{ flow_file_a }).asLongDouble())); } TEST_CASE("Divide Integer", "[expressionLanguageDivideInteger]") { @@ -1176,7 +1173,6 @@ TEST_CASE("Encode Decode CSV", "[expressionEncodeDecodeCSV]") { REQUIRE("Zero > One < \"two!\" & 'true'" == expr(expression::Parameters{ flow_file_a }).asString()); } -#ifndef WIN32 #ifndef DISABLE_CURL TEST_CASE("Encode URL", "[expressionEncodeURL]") { auto expr = expression::compile("${message:urlEncode()}"); @@ -1226,11 +1222,11 @@ TEST_CASE("Encode Decode URL", "[expressionEncodeDecodeURLExcept]") { REQUIRE_THROWS(expr(expression::Parameters{flow_file_a}).asString()); } #endif -#endif - -#ifdef EXPRESSION_LANGUAGE_USE_DATE TEST_CASE("Parse Date", "[expressionParseDate]") { +#ifdef WIN32 + expression::dateSetInstall(TZ_DATA_DIR); +#endif auto expr = expression::compile("${message:toDate('%Y/%m/%d', 'America/Los_Angeles')}"); auto flow_file_a = std::make_shared<core::FlowFile>(); @@ -1239,6 +1235,9 @@ TEST_CASE("Parse Date", "[expressionParseDate]") { } TEST_CASE("Reformat Date", "[expressionReformatDate]") { +#ifdef WIN32 + expression::dateSetInstall(TZ_DATA_DIR); +#endif auto expr = expression::compile("${message:toDate('%Y/%m/%d', 'GMT'):format('%m-%d-%Y', 'America/New_York')}"); auto flow_file_a = std::make_shared<core::FlowFile>(); @@ -1246,18 +1245,17 @@ TEST_CASE("Reformat Date", "[expressionReformatDate]") { REQUIRE("03-13-2014" == expr(expression::Parameters{ flow_file_a }).asString()); } -#endif // EXPRESSION_LANGUAGE_USE_DATE - TEST_CASE("Now Date", "[expressionNowDate]") { +#ifdef WIN32 + expression::dateSetInstall(TZ_DATA_DIR); +#endif auto expr = expression::compile("${now():format('%Y')}"); auto flow_file_a = std::make_shared<core::FlowFile>(); flow_file_a->addAttribute("message", "2014/03/14"); - time_t t = time(nullptr); - struct tm lt; - localtime_r(&t, <); + date::year_month_day date{std::chrono::floor<std::chrono::days>(std::chrono::system_clock::now())}; - REQUIRE(gsl::narrow<uint64_t>(lt.tm_year + 1900) == expr(expression::Parameters{ flow_file_a }).asUnsignedLong()); + REQUIRE(date.year().operator int() == expr(expression::Parameters{ flow_file_a }).asSignedLong()); } TEST_CASE("Format Date", "[expressionFormatDate]") { @@ -1383,8 +1381,6 @@ TEST_CASE("Any Contains 2", "[expressionAnyContains2]") { REQUIRE(!expr(expression::Parameters{ flow_file_a }).asBoolean()); } -#ifdef EXPRESSION_LANGUAGE_USE_REGEX - TEST_CASE("All Matching Contains", "[expressionAllMatchingContains]") { auto expr = expression::compile("${allMatchingAttributes('xyz_.*'):contains('hello')}"); @@ -1457,8 +1453,6 @@ TEST_CASE("Any Matching Contains 4", "[expressionAnyMatchingContains4]") { REQUIRE(!expr(expression::Parameters{ flow_file_a }).asBoolean()); } -#endif // EXPRESSION_LANGUAGE_USE_REGEX - TEST_CASE("All Delineated Contains", "[expressionAllDelineatedContains]") { auto expr = expression::compile("${allDelineatedValues(${word_list}, \",\"):contains('hello')}"); diff --git a/extensions/expression-language/tests/ProcessContextExprTests.cpp b/extensions/expression-language/tests/ProcessContextExprTests.cpp index 46da66fae..1cef4eabc 100644 --- a/extensions/expression-language/tests/ProcessContextExprTests.cpp +++ b/extensions/expression-language/tests/ProcessContextExprTests.cpp @@ -31,8 +31,8 @@ namespace org::apache::nifi::minifi { class DummyProcessor : public core::Processor { public: using core::Processor::Processor; - EXTENSIONAPI static core::Property SimpleProperty; - EXTENSIONAPI static core::Property ExpressionLanguageProperty; + static core::Property SimpleProperty; + static core::Property ExpressionLanguageProperty; void initialize() override { setSupportedProperties({SimpleProperty, ExpressionLanguageProperty}); } bool supportsDynamicProperties() override { return true; } };
