adamdebreceni commented on a change in pull request #1004:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1004#discussion_r603242347
##########
File path: extensions/sql/processors/QueryDatabaseTable.cpp
##########
@@ -75,361 +72,234 @@ const core::Property
QueryDatabaseTable::s_maxValueColumnNames(
"If no columns are provided, all rows from the table will be considered,
which could have a performance impact. "
"NOTE: It is important to use consistent max-value column names for a
given table for incremental fetch to work properly. "
"NOTE: Because of a limitation of database access library 'soci', which
doesn't support milliseconds in it's 'dt_date', "
- "there is a possibility that flowfiles might have duplicated records, if a
max-value column with 'dt_date' type has value with milliseconds.")->
- supportsExpressionLanguage(true)->build());
+ "there is a possibility that flowfiles might have duplicated records, if a
max-value column with 'dt_date' type has value with milliseconds.")
+ ->supportsExpressionLanguage(true)->build());
-const core::Property QueryDatabaseTable::s_whereClause(
-
core::PropertyBuilder::createProperty("db-fetch-where-clause")->isRequired(false)->withDescription(
- "A custom clause to be added in the WHERE condition when building SQL
queries.")->supportsExpressionLanguage(true)->build());
+const core::Property QueryDatabaseTable::WhereClause(
+ core::PropertyBuilder::createProperty("Where Clause")
+ ->isRequired(false)
+ ->withDescription("A custom clause to be added in the WHERE condition when
building SQL queries.")
+ ->supportsExpressionLanguage(true)->build());
-const core::Property QueryDatabaseTable::s_sqlQuery(
-
core::PropertyBuilder::createProperty("db-fetch-sql-query")->isRequired(false)->withDescription(
- "A custom SQL query used to retrieve data. Instead of building a SQL query
from other properties, this query will be wrapped as a sub-query. "
- "Query must have no ORDER BY
statement.")->supportsExpressionLanguage(true)->build());
+const std::string
QueryDatabaseTable::InitialMaxValueDynamicPropertyPrefix("initial.maxvalue.");
Review comment:
added
##########
File path: libminifi/test/sql-tests/QueryDatabaseTableTests.cpp
##########
@@ -0,0 +1,248 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#undef NDEBUG
+
+#include "../TestBase.h"
+#include "SQLTestController.h"
+#include "Utils.h"
+#include "FlowFileMatcher.h"
+
+TEST_CASE("QueryDatabaseTable queries the table and returns specified
columns", "[QueryDatabaseTable1]") {
+ SQLTestController controller;
+
+ auto plan = controller.createSQLPlan("QueryDatabaseTable", {{"success",
"d"}});
+ auto sql_proc = plan->getSQLProcessor();
+ sql_proc->setProperty(processors::QueryDatabaseTable::TableName.getName(),
"test_table");
+
sql_proc->setProperty(processors::QueryDatabaseTable::MaxValueColumnNames.getName(),
"int_col");
+ sql_proc->setProperty(processors::QueryDatabaseTable::ColumnNames.getName(),
"text_col");
+
+ controller.insertValues({
+ {101, "one"},
+ {102, "two"},
+ {103, "three"}
+ });
+
+ plan->run();
+
+ auto flow_files = plan->getOutputs({"success", "d"});
+ REQUIRE(flow_files.size() == 1);
+
+ std::string row_count;
+
flow_files[0]->getAttribute(processors::QueryDatabaseTable::RESULT_ROW_COUNT,
row_count);
+ REQUIRE(row_count == "3");
+ auto content = plan->getContent(flow_files[0]);
+ verifyJSON(content, R"(
+ [{"text_col": "one"}, {"text_col": "two"}, {"text_col": "three"}]
+ )", true);
+}
+
+TEST_CASE("QueryDatabaseTable requerying the table returns only new rows",
"[QueryDatabaseTable2]") {
+ SQLTestController controller;
+
+ auto plan = controller.createSQLPlan("QueryDatabaseTable", {{"success",
"d"}});
+ auto sql_proc = plan->getSQLProcessor();
+ sql_proc->setProperty(processors::QueryDatabaseTable::TableName.getName(),
"test_table");
+
sql_proc->setProperty(processors::QueryDatabaseTable::MaxValueColumnNames.getName(),
"int_col");
+ sql_proc->setProperty(processors::QueryDatabaseTable::ColumnNames.getName(),
"text_col");
+
+ controller.insertValues({
+ {101, "one"},
+ {102, "two"},
+ {103, "three"}
+ });
+
+ plan->run();
+
+ auto first_flow_files = plan->getOutputs({"success", "d"});
+ REQUIRE(first_flow_files.size() == 1);
+
+ controller.insertValues({
+ {104, "four"},
+ {105, "five"}
+ });
+
+ SECTION("Without schedule") {plan->run();}
+ SECTION("With schedule") {plan->run(true);}
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
