This is an automated email from the ASF dual-hosted git repository.
turcsanyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 4b0568e6fe NIFI-10242 Create QueryAirtableTable processor
4b0568e6fe is described below
commit 4b0568e6fe8cdf601042eb3820956939a566202a
Author: Robert Kalmar <[email protected]>
AuthorDate: Tue Jul 19 12:23:40 2022 +0200
NIFI-10242 Create QueryAirtableTable processor
This closes #6350.
Signed-off-by: Peter Turcsanyi <[email protected]>
---
nifi-assembly/pom.xml | 6 +
.../nifi-airtable-bundle/nifi-airtable-nar/pom.xml | 48 +++
.../src/main/resources/META-INF/LICENSE | 202 +++++++++++++
.../src/main/resources/META-INF/NOTICE | 45 +++
.../nifi-airtable-processors/pom.xml | 84 ++++++
.../processors/airtable/QueryAirtableTable.java | 330 +++++++++++++++++++++
.../parse/AirtableRecordSetFlowFileWriter.java | 63 ++++
.../airtable/parse/AirtableRetrievePageResult.java | 56 ++++
.../parse/AirtableRetrieveTableResult.java | 40 +++
.../airtable/parse/AirtableTableRetriever.java | 128 ++++++++
.../service/AirtableGetRecordsParameters.java | 132 +++++++++
.../airtable/service/AirtableRestService.java | 127 ++++++++
.../service/RateLimitExceededException.java | 25 ++
.../services/org.apache.nifi.processor.Processor | 16 +
.../additionalDetails.html | 63 ++++
.../airtable/TestQueryAirtableTable.java | 158 ++++++++++
nifi-nar-bundles/nifi-airtable-bundle/pom.xml | 35 +++
nifi-nar-bundles/pom.xml | 7 +
18 files changed, 1565 insertions(+)
diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml
index 2a90a924ac..c07683a50e 100644
--- a/nifi-assembly/pom.xml
+++ b/nifi-assembly/pom.xml
@@ -880,6 +880,12 @@ language governing permissions and limitations under the
License. -->
<version>1.18.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-airtable-nar</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
<!-- dependencies for jaxb/activation/annotation for running NiFi on
Java 11 -->
<!-- TODO: remove these once minimum Java version is 11 -->
<dependency>
diff --git a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-nar/pom.xml
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-nar/pom.xml
new file mode 100644
index 0000000000..a6a89c3bdf
--- /dev/null
+++ b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-nar/pom.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>nifi-airtable-bundle</artifactId>
+ <groupId>org.apache.nifi</groupId>
+ <version>1.18.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>nifi-airtable-nar</artifactId>
+ <packaging>nar</packaging>
+ <properties>
+ <maven.javadoc.skip>true</maven.javadoc.skip>
+ <source.skip>true</source.skip>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-airtable-processors</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-standard-services-api-nar</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-nar/src/main/resources/META-INF/LICENSE
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-nar/src/main/resources/META-INF/LICENSE
new file mode 100644
index 0000000000..d645695673
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-nar/src/main/resources/META-INF/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-nar/src/main/resources/META-INF/NOTICE
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-nar/src/main/resources/META-INF/NOTICE
new file mode 100644
index 0000000000..9aa168b42c
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-nar/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,45 @@
+nifi-airtable-nar
+Copyright 2014-2022 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+**************************
+Apache Software License v2
+**************************
+
+ (ASLv2) Apache Commons Lang
+ The following NOTICE information applies:
+ Apache Commons Lang
+ Copyright 2001-2015 The Apache Software Foundation
+
+ This product includes software from the Spring Framework,
+ under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+ (ASLv2) Apache Commons IO
+ The following NOTICE information applies:
+ Apache Commons IO
+ Copyright 2002-2016 The Apache Software Foundation
+
+ (ASLv2) Jackson JSON processor
+ The following NOTICE information applies:
+ # Jackson JSON processor
+
+ Jackson is a high-performance, Free/Open Source JSON processing library.
+ It was originally written by Tatu Saloranta ([email protected]), and
has
+ been in development since 2007.
+ It is currently developed by a community of developers, as well as
supported
+ commercially by FasterXML.com.
+
+ ## Licensing
+
+ Jackson core and extension components may licensed under different
licenses.
+ To find the details that apply to this artifact see the accompanying
LICENSE file.
+ For more information, including possible other licensing options, contact
+ FasterXML.com (http://fasterxml.com).
+
+ ## Credits
+
+ A list of contributors may be found from CREDITS file, which is included
+ in some artifacts (usually source distributions); but is always available
+ from the source code management (SCM) system project uses.
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/pom.xml
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/pom.xml
new file mode 100644
index 0000000000..207c4cd95c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>nifi-airtable-bundle</artifactId>
+ <groupId>org.apache.nifi</groupId>
+ <version>1.18.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>nifi-airtable-processors</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-web-client-provider-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-web-client-provider-service</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>mockwebserver</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-ssl-context-service-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-proxy-configuration-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/QueryAirtableTable.java
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/QueryAirtableTable.java
new file mode 100644
index 0000000000..ca5e877736
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/QueryAirtableTable.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.airtable;
+
+import static
org.apache.nifi.flowfile.attributes.FragmentAttributes.FRAGMENT_COUNT;
+import static
org.apache.nifi.flowfile.attributes.FragmentAttributes.FRAGMENT_ID;
+import static
org.apache.nifi.flowfile.attributes.FragmentAttributes.FRAGMENT_INDEX;
+import static
org.apache.nifi.processors.airtable.service.AirtableRestService.API_V0_BASE_URL;
+
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.PrimaryNodeOnly;
+import org.apache.nifi.annotation.behavior.Stateful;
+import org.apache.nifi.annotation.behavior.TriggerSerially;
+import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.configuration.DefaultSettings;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.state.Scope;
+import org.apache.nifi.components.state.StateMap;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.processors.airtable.parse.AirtableRetrieveTableResult;
+import org.apache.nifi.processors.airtable.parse.AirtableTableRetriever;
+import
org.apache.nifi.processors.airtable.service.AirtableGetRecordsParameters;
+import org.apache.nifi.processors.airtable.service.AirtableRestService;
+import org.apache.nifi.processors.airtable.service.RateLimitExceededException;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@PrimaryNodeOnly
+@InputRequirement(Requirement.INPUT_FORBIDDEN)
+@TriggerSerially
+@TriggerWhenEmpty
+@Tags({"airtable", "query", "database"})
+@CapabilityDescription("Query records from an Airtable table. Records are
incrementally retrieved based on the last modified time of the records."
+ + " Records can also be further filtered by setting the 'Custom
Filter' property which supports the formulas provided by the Airtable API."
+ + " This processor is intended to be run on the Primary Node only.")
+@Stateful(scopes = Scope.CLUSTER, description = "The last successful query's
time is stored in order to enable incremental loading."
+ + " The initial query returns all the records in the table and each
subsequent query filters the records by their last modified time."
+ + " In other words, if a record is updated after the last successful
query only the updated records will be returned in the next query."
+ + " State is stored across the cluster, so this Processor can run only
on the Primary Node and if a new Primary Node is selected,"
+ + " the new node can pick up where the previous one left off without
duplicating the data.")
+@WritesAttributes({
+ @WritesAttribute(attribute = "record.count", description = "Sets the
number of records in the FlowFile."),
+ @WritesAttribute(attribute = "fragment.identifier", description = "If
'Max Records Per FlowFile' is set then all FlowFiles from the same query result
set "
+ + "will have the same value for the fragment.identifier
attribute. This can then be used to correlate the results."),
+ @WritesAttribute(attribute = "fragment.count", description = "If 'Max
Records Per FlowFile' is set then this is the total number of "
+ + "FlowFiles produced by a single ResultSet. This can be used
in conjunction with the "
+ + "fragment.identifier attribute in order to know how many
FlowFiles belonged to the same incoming ResultSet."),
+ @WritesAttribute(attribute = "fragment.index", description = "If 'Max
Records Per FlowFile' is set then the position of this FlowFile in the list of "
+ + "outgoing FlowFiles that were all derived from the same
result set FlowFile. This can be "
+ + "used in conjunction with the fragment.identifier attribute
to know which FlowFiles originated from the same query result set and in what
order "
+ + "FlowFiles were produced"),
+})
+@DefaultSettings(yieldDuration = "15 sec")
+public class QueryAirtableTable extends AbstractProcessor {
+
+ static final PropertyDescriptor API_URL = new PropertyDescriptor.Builder()
+ .name("api-url")
+ .displayName("API URL")
+ .description("The URL for the Airtable REST API including the
domain and the path to the API (e.g. https://api.airtable.com/v0).")
+ .defaultValue(API_V0_BASE_URL)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .addValidator(StandardValidators.URL_VALIDATOR)
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .required(true)
+ .build();
+
+ static final PropertyDescriptor API_KEY = new PropertyDescriptor.Builder()
+ .name("api-key")
+ .displayName("API Key")
+ .description("The REST API key to use in queries. Should be
generated on Airtable's account page.")
+ .required(true)
+ .sensitive(true)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ static final PropertyDescriptor BASE_ID = new PropertyDescriptor.Builder()
+ .name("base-id")
+ .displayName("Base ID")
+ .description("The ID of the Airtable base to be queried.")
+ .required(true)
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ static final PropertyDescriptor TABLE_ID = new PropertyDescriptor.Builder()
+ .name("table-id")
+ .displayName("Table ID")
+ .description("The name or the ID of the Airtable table to be
queried.")
+ .required(true)
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ static final PropertyDescriptor FIELDS = new PropertyDescriptor.Builder()
+ .name("fields")
+ .displayName("Fields")
+ .description("Comma-separated list of fields to query from the
table. Both the field's name and ID can be used.")
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ static final PropertyDescriptor CUSTOM_FILTER = new
PropertyDescriptor.Builder()
+ .name("custom-filter")
+ .displayName("Custom Filter")
+ .description("Filter records by Airtable's formulas.")
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ static final PropertyDescriptor QUERY_TIME_WINDOW_LAG = new
PropertyDescriptor.Builder()
+ .name("query-time-window-lag")
+ .displayName("Query Time Window Lag")
+ .description("The amount of lag to be applied to the query time
window's end point. Set this property to avoid missing records when the clock
of your local machines"
+ + " and Airtable servers' clock are not in sync. Must be
greater than or equal to 1 second.")
+ .defaultValue("3 s")
+ .required(true)
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
+ .build();
+
+ static final PropertyDescriptor WEB_CLIENT_SERVICE_PROVIDER = new
PropertyDescriptor.Builder()
+ .name("web-client-service-provider")
+ .displayName("Web Client Service Provider")
+ .description("Web Client Service Provider to use for Airtable REST
API requests")
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .required(true)
+ .build();
+
+ static final PropertyDescriptor QUERY_PAGE_SIZE = new
PropertyDescriptor.Builder()
+ .name("query-page-size")
+ .displayName("Query Page Size")
+ .description("Number of records to be fetched in a page. Should be
between 1 and 100 inclusively.")
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.createLongValidator(1, 100, true))
+ .build();
+
+ static final PropertyDescriptor MAX_RECORDS_PER_FLOWFILE = new
PropertyDescriptor.Builder()
+ .name("max-records-per-flowfile")
+ .displayName("Max Records Per FlowFile")
+ .description("The maximum number of result records that will be
included in a single FlowFile. This will allow you to break up very large"
+ + " result sets into multiple FlowFiles. If no value
specified, then all records are returned in a single FlowFile.")
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
+ .build();
+
+ static final Relationship REL_SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("For FlowFiles created as a result of a successful
query.")
+ .build();
+
+ private static final List<PropertyDescriptor> PROPERTIES =
Collections.unmodifiableList(Arrays.asList(
+ API_URL,
+ API_KEY,
+ BASE_ID,
+ TABLE_ID,
+ FIELDS,
+ CUSTOM_FILTER,
+ QUERY_TIME_WINDOW_LAG,
+ WEB_CLIENT_SERVICE_PROVIDER,
+ QUERY_PAGE_SIZE,
+ MAX_RECORDS_PER_FLOWFILE
+ ));
+
+ private static final Set<Relationship> RELATIONSHIPS =
Collections.singleton(REL_SUCCESS);
+
+ private static final String LAST_QUERY_TIME_WINDOW_END =
"last_query_time_window_end";
+
+ private volatile AirtableRestService airtableRestService;
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void onScheduled(final ProcessContext context) {
+ final String apiUrl =
context.getProperty(API_URL).evaluateAttributeExpressions().getValue();
+ final String apiKey = context.getProperty(API_KEY).getValue();
+ final String baseId =
context.getProperty(BASE_ID).evaluateAttributeExpressions().getValue();
+ final String tableId =
context.getProperty(TABLE_ID).evaluateAttributeExpressions().getValue();
+ final WebClientServiceProvider webClientServiceProvider =
context.getProperty(WEB_CLIENT_SERVICE_PROVIDER).asControllerService(WebClientServiceProvider.class);
+ airtableRestService = new
AirtableRestService(webClientServiceProvider, apiUrl, apiKey, baseId, tableId);
+ }
+
+ @Override
+ public void onTrigger(final ProcessContext context, final ProcessSession
session) throws ProcessException {
+ final Integer maxRecordsPerFlowFile =
context.getProperty(MAX_RECORDS_PER_FLOWFILE).evaluateAttributeExpressions().asInteger();
+ final Long queryTimeWindowLagSeconds =
context.getProperty(QUERY_TIME_WINDOW_LAG).evaluateAttributeExpressions().asTimePeriod(TimeUnit.SECONDS);
+
+ final StateMap state;
+ try {
+ state = session.getState(Scope.CLUSTER);
+ } catch (IOException e) {
+ throw new ProcessException("Failed to get cluster state", e);
+ }
+
+ final String lastRecordFetchDateTime =
state.get(LAST_QUERY_TIME_WINDOW_END);
+ final String currentRecordFetchDateTime = OffsetDateTime.now()
+ .minusSeconds(queryTimeWindowLagSeconds)
+ .truncatedTo(ChronoUnit.SECONDS)
+ .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
+
+ final AirtableGetRecordsParameters getRecordsParameters =
buildGetRecordsParameters(context, lastRecordFetchDateTime,
currentRecordFetchDateTime);
+ final AirtableRetrieveTableResult retrieveTableResult;
+ try {
+ final AirtableTableRetriever tableRetriever = new
AirtableTableRetriever(airtableRestService, getRecordsParameters,
maxRecordsPerFlowFile);
+ retrieveTableResult = tableRetriever.retrieveAll(session);
+ } catch (IOException e) {
+ throw new ProcessException("Failed to read Airtable records", e);
+ } catch (RateLimitExceededException e) {
+ context.yield();
+ throw new ProcessException("Airtable REST API rate limit exceeded
while reading records", e);
+ }
+
+ final Map<String, String> newState = new HashMap<>(state.toMap());
+ newState.put(LAST_QUERY_TIME_WINDOW_END, currentRecordFetchDateTime);
+ try {
+ session.setState(newState, Scope.CLUSTER);
+ } catch (IOException e) {
+ throw new ProcessException("Failed to update cluster state", e);
+ }
+
+ final List<FlowFile> flowFiles = retrieveTableResult.getFlowFiles();
+ if (flowFiles.isEmpty()) {
+ context.yield();
+ return;
+ }
+
+ if (maxRecordsPerFlowFile != null) {
+ addFragmentAttributesToFlowFiles(session, flowFiles);
+ }
+ transferFlowFiles(session, flowFiles,
retrieveTableResult.getTotalRecordCount());
+ }
+
+ private AirtableGetRecordsParameters buildGetRecordsParameters(final
ProcessContext context,
+ final String lastRecordFetchTime,
+ final String nowDateTimeString) {
+ Objects.requireNonNull(context);
+ Objects.requireNonNull(nowDateTimeString);
+
+ final String fieldsProperty =
context.getProperty(FIELDS).evaluateAttributeExpressions().getValue();
+ final String customFilter =
context.getProperty(CUSTOM_FILTER).evaluateAttributeExpressions().getValue();
+ final Integer pageSize =
context.getProperty(QUERY_PAGE_SIZE).evaluateAttributeExpressions().asInteger();
+
+ final AirtableGetRecordsParameters.Builder getRecordsParametersBuilder
= new AirtableGetRecordsParameters.Builder();
+ if (lastRecordFetchTime != null) {
+ getRecordsParametersBuilder.modifiedAfter(lastRecordFetchTime);
+ }
+ getRecordsParametersBuilder.modifiedBefore(nowDateTimeString);
+ if (fieldsProperty != null) {
+
getRecordsParametersBuilder.fields(Arrays.stream(fieldsProperty.split(",")).map(String::trim).collect(Collectors.toList()));
+ }
+ getRecordsParametersBuilder.customFilter(customFilter);
+ if (pageSize != null) {
+ getRecordsParametersBuilder.pageSize(pageSize);
+ }
+
+ return getRecordsParametersBuilder.build();
+ }
+
+ private void addFragmentAttributesToFlowFiles(final ProcessSession
session, final List<FlowFile> flowFiles) {
+ final String fragmentIdentifier = UUID.randomUUID().toString();
+ for (int i = 0; i < flowFiles.size(); i++) {
+ final Map<String, String> fragmentAttributes = new HashMap<>();
+ fragmentAttributes.put(FRAGMENT_ID.key(), fragmentIdentifier);
+ fragmentAttributes.put(FRAGMENT_INDEX.key(), String.valueOf(i));
+ fragmentAttributes.put(FRAGMENT_COUNT.key(),
String.valueOf(flowFiles.size()));
+
+ flowFiles.set(i, session.putAllAttributes(flowFiles.get(i),
fragmentAttributes));
+ }
+ }
+
+ private void transferFlowFiles(final ProcessSession session, final
List<FlowFile> flowFiles, final int totalRecordCount) {
+ final String transitUri =
airtableRestService.createUriBuilder().build().toString();
+ for (final FlowFile flowFile : flowFiles) {
+ session.getProvenanceReporter().receive(flowFile, transitUri);
+ session.transfer(flowFile, REL_SUCCESS);
+ }
+ session.adjustCounter("Records Processed", totalRecordCount, false);
+ final String flowFilesAsString =
flowFiles.stream().map(FlowFile::toString).collect(Collectors.joining(", ",
"[", "]"));
+ getLogger().debug("Transferred FlowFiles [{}] Records [{}]",
flowFilesAsString, totalRecordCount);
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableRecordSetFlowFileWriter.java
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableRecordSetFlowFileWriter.java
new file mode 100644
index 0000000000..bb96315e55
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableRecordSetFlowFileWriter.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.airtable.parse;
+
+import static
org.apache.nifi.processors.airtable.parse.AirtableTableRetriever.JSON_FACTORY;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import java.io.IOException;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.ProcessSession;
+
+public class AirtableRecordSetFlowFileWriter {
+
+ private final FlowFile flowFile;
+ private final JsonGenerator jsonGenerator;
+ private int recordCount = 0;
+
+ private AirtableRecordSetFlowFileWriter(final FlowFile flowFile, final
JsonGenerator jsonGenerator) {
+ this.flowFile = flowFile;
+ this.jsonGenerator = jsonGenerator;
+ }
+
+ public static AirtableRecordSetFlowFileWriter startRecordSet(final
ProcessSession session) throws IOException {
+ final FlowFile flowFile = session.create();
+ final JsonGenerator jsonGenerator =
JSON_FACTORY.createGenerator(session.write(flowFile));
+ jsonGenerator.writeStartArray();
+ return new AirtableRecordSetFlowFileWriter(flowFile, jsonGenerator);
+ }
+
+ public void writeRecord(final JsonParser jsonParser) throws IOException {
+ recordCount++;
+ jsonGenerator.copyCurrentStructure(jsonParser);
+ }
+
+ public FlowFile closeRecordSet(final ProcessSession session) throws
IOException {
+ jsonGenerator.writeEndArray();
+ jsonGenerator.close();
+ FlowFile flowFileWithAttributes = session.putAttribute(flowFile,
"record.count", String.valueOf(recordCount));
+ flowFileWithAttributes = session.putAttribute(flowFileWithAttributes,
CoreAttributes.MIME_TYPE.key(), "application/json");
+ return flowFileWithAttributes;
+ }
+
+ public int getRecordCount() {
+ return recordCount;
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableRetrievePageResult.java
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableRetrievePageResult.java
new file mode 100644
index 0000000000..e279507348
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableRetrievePageResult.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.airtable.parse;
+
+import java.util.List;
+import java.util.Optional;
+import org.apache.nifi.flowfile.FlowFile;
+
+public class AirtableRetrievePageResult {
+
+ private final Optional<String> nextOffset;
+ private final List<FlowFile> flowFiles;
+ private final int parsedRecordCount;
+ private final Optional<AirtableRecordSetFlowFileWriter>
ongoingRecordSetFlowFileWriter;
+
+ public AirtableRetrievePageResult(final Optional<String> nextOffset,
+ final List<FlowFile> flowFiles,
+ final int parsedRecordCount,
+ final Optional<AirtableRecordSetFlowFileWriter>
ongoingRecordSetFlowFileWriter) {
+ this.nextOffset = nextOffset;
+ this.flowFiles = flowFiles;
+ this.parsedRecordCount = parsedRecordCount;
+ this.ongoingRecordSetFlowFileWriter = ongoingRecordSetFlowFileWriter;
+ }
+
+ public Optional<String> getNextOffset() {
+ return nextOffset;
+ }
+
+ public Optional<AirtableRecordSetFlowFileWriter>
getOngoingRecordSetFlowFileWriter() {
+ return ongoingRecordSetFlowFileWriter;
+ }
+
+ public List<FlowFile> getFlowFiles() {
+ return flowFiles;
+ }
+
+ public int getParsedRecordCount() {
+ return parsedRecordCount;
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableRetrieveTableResult.java
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableRetrieveTableResult.java
new file mode 100644
index 0000000000..427324f909
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableRetrieveTableResult.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.airtable.parse;
+
+import java.util.List;
+import org.apache.nifi.flowfile.FlowFile;
+
+public class AirtableRetrieveTableResult {
+
+ private final List<FlowFile> flowFiles;
+ private final int totalRecordCount;
+
+ public AirtableRetrieveTableResult(final List<FlowFile> flowFiles, final
int totalRecordCount) {
+ this.flowFiles = flowFiles;
+ this.totalRecordCount = totalRecordCount;
+ }
+
+ public List<FlowFile> getFlowFiles() {
+ return flowFiles;
+ }
+
+ public int getTotalRecordCount() {
+ return totalRecordCount;
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableTableRetriever.java
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableTableRetriever.java
new file mode 100644
index 0000000000..b2bfaa20e9
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/parse/AirtableTableRetriever.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.airtable.parse;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator.Feature;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.exception.ProcessException;
+import
org.apache.nifi.processors.airtable.service.AirtableGetRecordsParameters;
+import org.apache.nifi.processors.airtable.service.AirtableRestService;
+
+public class AirtableTableRetriever {
+
+ static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+ static final JsonFactory JSON_FACTORY = OBJECT_MAPPER.getFactory()
+ .configure(Feature.AUTO_CLOSE_JSON_CONTENT, false);
+
+ final AirtableRestService airtableRestService;
+ final AirtableGetRecordsParameters getRecordsParameters;
+ final Integer maxRecordsPerFlowFile;
+
+ public AirtableTableRetriever(final AirtableRestService
airtableRestService,
+ final AirtableGetRecordsParameters getRecordsParameters,
+ final Integer maxRecordsPerFlowFile) {
+ this.airtableRestService = airtableRestService;
+ this.getRecordsParameters = getRecordsParameters;
+ this.maxRecordsPerFlowFile = maxRecordsPerFlowFile;
+ }
+
+ public AirtableRetrieveTableResult retrieveAll(final ProcessSession
session) throws IOException {
+ int totalRecordCount = 0;
+ final List<FlowFile> flowFiles = new ArrayList<>();
+ AirtableRetrievePageResult retrievePageResult = null;
+ do {
+ retrievePageResult = retrieveNextPage(session,
Optional.ofNullable(retrievePageResult));
+ totalRecordCount += retrievePageResult.getParsedRecordCount();
+ flowFiles.addAll(retrievePageResult.getFlowFiles());
+ } while (retrievePageResult.getNextOffset().isPresent());
+
+ retrievePageResult.getOngoingRecordSetFlowFileWriter()
+ .map(writer -> {
+ try {
+ return writer.closeRecordSet(session);
+ } catch (IOException e) {
+ throw new ProcessException("Failed to close Airtable
record writer", e);
+ }
+ })
+ .ifPresent(flowFiles::add);
+ return new AirtableRetrieveTableResult(flowFiles, totalRecordCount);
+ }
+
+ private AirtableRetrievePageResult retrieveNextPage(final ProcessSession
session, final Optional<AirtableRetrievePageResult> previousPageResult) {
+ final AirtableGetRecordsParameters parameters =
previousPageResult.flatMap(AirtableRetrievePageResult::getNextOffset)
+ .map(getRecordsParameters::withOffset)
+ .orElse(getRecordsParameters);
+
+ return airtableRestService.getRecords(parameters, inputStream ->
parsePage(inputStream, session, previousPageResult));
+ }
+
+ private AirtableRetrievePageResult parsePage(final InputStream
inputStream, final ProcessSession session, final
Optional<AirtableRetrievePageResult> previousPageResult) {
+ final List<FlowFile> flowFiles = new ArrayList<>();
+ AirtableRecordSetFlowFileWriter flowFileWriter =
previousPageResult.flatMap(AirtableRetrievePageResult::getOngoingRecordSetFlowFileWriter)
+ .orElse(null);
+ int parsedRecordCount = 0;
+ String nextOffset = null;
+ try (final JsonParser jsonParser =
JSON_FACTORY.createParser(inputStream)) {
+ while (jsonParser.nextToken() != null) {
+ if (jsonParser.currentToken() != JsonToken.FIELD_NAME) {
+ continue;
+ }
+ switch (jsonParser.currentName()) {
+ case "records":
+ jsonParser.nextToken();
+ if (jsonParser.currentToken() !=
JsonToken.START_ARRAY) {
+ break;
+ }
+ while (jsonParser.nextToken() != null &&
jsonParser.currentToken() != JsonToken.END_ARRAY) {
+ if (jsonParser.currentToken() !=
JsonToken.START_OBJECT) {
+ continue;
+ }
+ if (flowFileWriter == null) {
+ flowFileWriter =
AirtableRecordSetFlowFileWriter.startRecordSet(session);
+ }
+ ++parsedRecordCount;
+ flowFileWriter.writeRecord(jsonParser);
+ if (maxRecordsPerFlowFile != null &&
maxRecordsPerFlowFile == flowFileWriter.getRecordCount()) {
+
flowFiles.add(flowFileWriter.closeRecordSet(session));
+ flowFileWriter = null;
+ }
+
+ }
+ break;
+ case "offset":
+ jsonParser.nextToken();
+ nextOffset = jsonParser.getValueAsString();
+ break;
+ }
+ }
+ } catch (final IOException e) {
+ throw new ProcessException("Failed to parse Airtable query table
response page", e);
+ }
+ return new AirtableRetrievePageResult(Optional.ofNullable(nextOffset),
flowFiles, parsedRecordCount, Optional.ofNullable(flowFileWriter));
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/service/AirtableGetRecordsParameters.java
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/service/AirtableGetRecordsParameters.java
new file mode 100644
index 0000000000..42fff1849b
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/service/AirtableGetRecordsParameters.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.airtable.service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalInt;
+
+public class AirtableGetRecordsParameters {
+
+ private final List<String> fields;
+ private final Optional<String> modifiedAfter;
+ private final Optional<String> modifiedBefore;
+ private final Optional<String> customFilter;
+ private final Optional<String> offset;
+ private final OptionalInt pageSize;
+
+ public AirtableGetRecordsParameters(final List<String> fields,
+ final Optional<String> modifiedAfter,
+ final Optional<String> modifiedBefore,
+ final Optional<String> customFilter,
+ final Optional<String> offset,
+ final OptionalInt pageSize) {
+ this.fields = Objects.requireNonNull(fields);
+ this.modifiedAfter = modifiedAfter;
+ this.modifiedBefore = modifiedBefore;
+ this.customFilter = customFilter;
+ this.offset = offset;
+ this.pageSize = pageSize;
+ }
+
+ public List<String> getFields() {
+ return fields;
+ }
+
+ public Optional<String> getModifiedAfter() {
+ return modifiedAfter;
+ }
+
+ public Optional<String> getModifiedBefore() {
+ return modifiedBefore;
+ }
+
+ public Optional<String> getCustomFilter() {
+ return customFilter;
+ }
+
+ public Optional<String> getOffset() {
+ return offset;
+ }
+
+ public OptionalInt getPageSize() {
+ return pageSize;
+ }
+
+ public AirtableGetRecordsParameters withOffset(final String offset) {
+ return new AirtableGetRecordsParameters(fields, modifiedAfter,
modifiedBefore, customFilter, Optional.of(offset), pageSize);
+ }
+
+ public static class Builder {
+ private List<String> fields;
+ private String modifiedAfter;
+ private String modifiedBefore;
+ private String customFilter;
+ private String offset;
+ private OptionalInt pageSize = OptionalInt.empty();
+
+ public Builder fields(final List<String> fields) {
+ this.fields = fields;
+ return this;
+ }
+
+ public Builder field(final String field) {
+ if (fields == null) {
+ fields = new ArrayList<>();
+ }
+ fields.add(field);
+ return this;
+ }
+
+ public Builder modifiedAfter(final String modifiedAfter) {
+ this.modifiedAfter = modifiedAfter;
+ return this;
+ }
+
+ public Builder modifiedBefore(final String modifiedBefore) {
+ this.modifiedBefore = modifiedBefore;
+ return this;
+ }
+
+ public Builder customFilter(final String customFilter) {
+ this.customFilter = customFilter;
+ return this;
+ }
+
+ public Builder offset(final String offset) {
+ this.offset = offset;
+ return this;
+ }
+
+ public Builder pageSize(final int pageSize) {
+ this.pageSize = OptionalInt.of(pageSize);
+ return this;
+ }
+
+ public AirtableGetRecordsParameters build() {
+ return new AirtableGetRecordsParameters(fields != null ? fields :
new ArrayList<>(),
+ Optional.ofNullable(modifiedAfter),
+ Optional.ofNullable(modifiedBefore),
+ Optional.ofNullable(customFilter),
+ Optional.ofNullable(offset),
+ pageSize);
+ }
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/service/AirtableRestService.java
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/service/AirtableRestService.java
new file mode 100644
index 0000000000..b3fb75577d
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/service/AirtableRestService.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.airtable.service;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.Range;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.HttpUriBuilder;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+public class AirtableRestService {
+
+ public static final String API_V0_BASE_URL = "https://api.airtable.com/v0";;
+
+ private static final int TOO_MANY_REQUESTS = 429;
+ private static final Range<Integer> SUCCESSFUL_RESPONSE_RANGE =
Range.between(200, 299);
+
+ private final WebClientServiceProvider webClientServiceProvider;
+ private final String apiUrl;
+ private final String apiKey;
+ private final String baseId;
+ private final String tableId;
+
+ public AirtableRestService(final WebClientServiceProvider
webClientServiceProvider,
+ final String apiUrl,
+ final String apiKey,
+ final String baseId,
+ final String tableId) {
+ this.webClientServiceProvider = webClientServiceProvider;
+ this.apiUrl = apiUrl;
+ this.apiKey = apiKey;
+ this.baseId = baseId;
+ this.tableId = tableId;
+ }
+
+ public <R> R getRecords(final AirtableGetRecordsParameters
getRecordsParameters, final Function<InputStream, R> callback) {
+ final URI uri = buildUri(getRecordsParameters);
+ try (final HttpResponseEntity response =
webClientServiceProvider.getWebClientService()
+ .get()
+ .uri(uri)
+ .header("Authorization", "Bearer " + apiKey)
+ .retrieve()) {
+
+ final InputStream bodyInputStream = response.body();
+ if (SUCCESSFUL_RESPONSE_RANGE.contains(response.statusCode())) {
+ return callback.apply(bodyInputStream);
+ }
+ if (response.statusCode() == TOO_MANY_REQUESTS) {
+ throw new RateLimitExceededException();
+ }
+ final StringBuilder exceptionMessageBuilder = new
StringBuilder("Error response. Code: " + response.statusCode());
+ final String bodyText = IOUtils.toString(bodyInputStream,
StandardCharsets.UTF_8);
+ if (bodyText != null) {
+ exceptionMessageBuilder.append(" Body: ").append(bodyText);
+ }
+
+ throw new ProcessException(exceptionMessageBuilder.toString());
+ } catch (IOException e) {
+ throw new ProcessException(String.format("Airtable HTTP request
failed [%s]", uri), e);
+ }
+ }
+
+ public HttpUriBuilder createUriBuilder() {
+ final URI uri = URI.create(apiUrl);
+ final HttpUriBuilder uriBuilder =
webClientServiceProvider.getHttpUriBuilder()
+ .scheme(uri.getScheme())
+ .host(uri.getHost())
+ .encodedPath(uri.getPath())
+ .addPathSegment(baseId)
+ .addPathSegment(tableId);
+ if (uri.getPort() != -1) {
+ uriBuilder.port(uri.getPort());
+ }
+ return uriBuilder;
+ }
+
+ private URI buildUri(AirtableGetRecordsParameters getRecordsParameters) {
+ final HttpUriBuilder uriBuilder = createUriBuilder();
+ for (final String field : getRecordsParameters.getFields()) {
+ uriBuilder.addQueryParameter("fields[]", field);
+ }
+
+ final List<String> filters = new ArrayList<>();
+ getRecordsParameters.getCustomFilter()
+ .ifPresent(filters::add);
+ getRecordsParameters.getModifiedAfter()
+ .map(modifiedAfter -> {
+ final String isSameFormula =
"IS_SAME(LAST_MODIFIED_TIME(),DATETIME_PARSE(\"" + modifiedAfter + "\"),
'second')";
+ final String isAfterFormula =
"IS_AFTER(LAST_MODIFIED_TIME(),DATETIME_PARSE(\"" + modifiedAfter + "\"))";
+ return "OR(" + isSameFormula + "," + isAfterFormula + ")";
+ })
+ .ifPresent(filters::add);
+ getRecordsParameters.getModifiedBefore()
+ .map(modifiedBefore ->
"IS_BEFORE(LAST_MODIFIED_TIME(),DATETIME_PARSE(\"" + modifiedBefore + "\"))")
+ .ifPresent(filters::add);
+ if (!filters.isEmpty()) {
+ uriBuilder.addQueryParameter("filterByFormula", "AND(" +
String.join(",", filters) + ")");
+ }
+ getRecordsParameters.getOffset().ifPresent(offset ->
uriBuilder.addQueryParameter("offset", offset));
+ getRecordsParameters.getPageSize().ifPresent(pageSize ->
uriBuilder.addQueryParameter("pageSize", String.valueOf(pageSize)));
+
+ return uriBuilder.build();
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/service/RateLimitExceededException.java
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/service/RateLimitExceededException.java
new file mode 100644
index 0000000000..f05e2430a7
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/java/org/apache/nifi/processors/airtable/service/RateLimitExceededException.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.airtable.service;
+
+public class RateLimitExceededException extends RuntimeException {
+
+ public RateLimitExceededException() {
+ super("Airtable REST API rate limit exceeded");
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
new file mode 100644
index 0000000000..cc78da9252
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.nifi.processors.airtable.QueryAirtableTable
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/resources/docs/org.apache.nifi.processors.airtable.QueryAirtableTable/additionalDetails.html
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/resources/docs/org.apache.nifi.processors.airtable.QueryAirtableTable/additionalDetails.html
new file mode 100644
index 0000000000..50086b57c2
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/main/resources/docs/org.apache.nifi.processors.airtable.QueryAirtableTable/additionalDetails.html
@@ -0,0 +1,63 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE html>
+<html lang="en" xmlns="http://www.w3.org/1999/html";>
+<head>
+ <meta charset="utf-8"/>
+ <title>QueryAirtableTable</title>
+ <link rel="stylesheet" href="../../../../../css/component-usage.css"
type="text/css"/>
+ <style>
+h2 {margin-top: 4em}
+h3 {margin-top: 3em}
+td {text-align: left}
+ </style>
+</head>
+
+<body>
+
+<h1>QueryAirtableTable</h1>
+
+<h3>Description</h3>
+<p>
+ Airtable is a spreadsheet-database hybrid. In Airtable an application is
called base and each base can have multiple tables.
+ A table consists of records (rows) and each record can have multiple
fields (columns).
+ The QueryAirtableTable processor can query records from a single base and
table via Airtable's REST API.
+ The processor utilizes streams to be able to handle a large number of
records.
+ It can also split large record sets to multiple FlowFiles just like a
database processor.
+</p>
+
+<h3>API Key</h3>
+<p>
+ Airtable REST API calls requires an API Key that needs to be passed in a
request. An Airtable account is required to generate an API Key.
+</p>
+
+<h3>API rate limit</h3>
+<p>
+ The Airtable REST API limits the number of requests that can be sent on a
per-base basis to avoid bottlenecks. Currently, this limit is 5 requests per
second per base.
+ If this limit is exceeded you can't make another request for 30 seconds.
+ It's your responsibility to handle this rate limit via configuring Yield
Duration and Run Schedule properly.
+ It is recommended to start off with the default settings and to increase
both parameters when rate limit issues occur.
+</p>
+
+<h3>Metadata API</h3>
+<p>
+ Currently the Metadata API of Airtable is unstable, and we don't provide a
way to use it.
+ Until it becomes stable you can set up a ConvertRecord or MergeRecord
processor with a JsonTreeReader to read the content and convert it into a
Record with schema.
+</p>
+</body>
+</html>
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/test/java/org/apache/nifi/processors/airtable/TestQueryAirtableTable.java
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/test/java/org/apache/nifi/processors/airtable/TestQueryAirtableTable.java
new file mode 100644
index 0000000000..dadd0cccb2
--- /dev/null
+++
b/nifi-nar-bundles/nifi-airtable-bundle/nifi-airtable-processors/src/test/java/org/apache/nifi/processors/airtable/TestQueryAirtableTable.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.airtable;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import okhttp3.HttpUrl;
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import org.apache.nifi.processor.Processor;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+import
org.apache.nifi.web.client.provider.service.StandardWebClientServiceProvider;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class TestQueryAirtableTable {
+
+ public static final String RECORDS_JSON_BODY = "{\"records\":[{"
+ + "\"id\":\"recabcdefghijklmn\","
+ + "\"createdTime\":\"1970-00-01T00:00:00.000Z\","
+ + "\"fields\":{\"foo\":\"bar\"}}]}";
+ public static final String RECORDS_WITH_OFFSET_JSON_BODY =
"{\"records\":[{"
+ + "\"id\":\"recabcdefghijklmn\","
+ + "\"createdTime\":\"1970-00-01T00:00:00.000Z\","
+ + "\"fields\":{\"foo\":\"bar\"}}],"
+ + "\"offset\":\"ofsabcdefghijklmn\"}";
+ public static final String EXPECTED_RECORD_CONTENT =
+
"{\"id\":\"recabcdefghijklmn\",\"createdTime\":\"1970-00-01T00:00:00.000Z\",\"fields\":{\"foo\":\"bar\"}}";
+ public static final String API_URL_PATH = "/v0/airtable";
+
+ private TestRunner runner;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ final Processor queryAirtableTable = new QueryAirtableTable();
+
+ runner = TestRunners.newTestRunner(queryAirtableTable);
+
+ final WebClientServiceProvider webClientServiceProvider = new
StandardWebClientServiceProvider();
+ runner.addControllerService("webClientService",
webClientServiceProvider);
+ runner.enableControllerService(webClientServiceProvider);
+
+ runner.setProperty(QueryAirtableTable.API_KEY, "???");
+ runner.setProperty(QueryAirtableTable.BASE_ID, "baseid");
+ runner.setProperty(QueryAirtableTable.TABLE_ID, "tableid");
+ runner.setProperty(QueryAirtableTable.WEB_CLIENT_SERVICE_PROVIDER,
webClientServiceProvider.getIdentifier());
+ }
+
+ @AfterEach
+ void tearDown() {
+ runner.shutdown();
+ }
+
+ @Test
+ void retrievesAndWritesRecords() throws Exception {
+ try (final MockWebServer server = new MockWebServer()) {
+ server.enqueue(new MockResponse().setBody(RECORDS_JSON_BODY));
+
+ server.start();
+ final HttpUrl httpUrl = server.url(API_URL_PATH);
+
+ runner.setProperty(QueryAirtableTable.API_URL, httpUrl.toString());
+ runner.run();
+
+ final List<MockFlowFile> results =
runner.getFlowFilesForRelationship(QueryAirtableTable.REL_SUCCESS);
+ assertEquals(1, results.size());
+ final MockFlowFile flowFile = results.get(0);
+ assertEquals("1", flowFile.getAttribute("record.count"));
+ final String content = flowFile.getContent();
+ assertEquals("[" + EXPECTED_RECORD_CONTENT + "]", content);
+ }
+ }
+
+ @Test
+ void retrievesAndWritesPagedRecords() throws Exception {
+ try (final MockWebServer server = new MockWebServer()) {
+ server.enqueue(new
MockResponse().setBody(RECORDS_WITH_OFFSET_JSON_BODY));
+ server.enqueue(new MockResponse().setBody(RECORDS_JSON_BODY));
+
+ server.start();
+ final HttpUrl httpUrl = server.url(API_URL_PATH);
+
+ runner.setProperty(QueryAirtableTable.API_URL, httpUrl.toString());
+ runner.run();
+
+ final List<MockFlowFile> results =
runner.getFlowFilesForRelationship(QueryAirtableTable.REL_SUCCESS);
+ assertEquals(1, results.size());
+ final MockFlowFile flowFile = results.get(0);
+ assertEquals("2", flowFile.getAttribute("record.count"));
+ final String content = flowFile.getContent();
+ assertEquals("[" + EXPECTED_RECORD_CONTENT + "," +
EXPECTED_RECORD_CONTENT + "]", content);
+ }
+ }
+
+ @Test
+ void retrievesAndWritesPagedRecordsInMultipleFlowFiles() throws Exception {
+ try (final MockWebServer server = new MockWebServer()) {
+ server.enqueue(new
MockResponse().setBody(RECORDS_WITH_OFFSET_JSON_BODY));
+ server.enqueue(new MockResponse().setBody(RECORDS_JSON_BODY));
+
+ server.start();
+ final HttpUrl httpUrl = server.url(API_URL_PATH);
+
+ runner.setProperty(QueryAirtableTable.MAX_RECORDS_PER_FLOWFILE,
"1");
+ runner.setProperty(QueryAirtableTable.API_URL, httpUrl.toString());
+ runner.run();
+
+ final List<MockFlowFile> results =
runner.getFlowFilesForRelationship(QueryAirtableTable.REL_SUCCESS);
+ assertEquals(2, results.size());
+ final MockFlowFile firstFlowFile = results.get(0);
+ assertEquals("1", firstFlowFile.getAttribute("record.count"));
+ final String firstContent = firstFlowFile.getContent();
+ assertEquals("[" + EXPECTED_RECORD_CONTENT + "]", firstContent);
+
+ final MockFlowFile secondFlowFile = results.get(1);
+ assertEquals("1", secondFlowFile.getAttribute("record.count"));
+ final String secondContent = secondFlowFile.getContent();
+ assertEquals("[" + EXPECTED_RECORD_CONTENT + "]", secondContent);
+ }
+ }
+
+ @Test
+ void doesNotWriteEmptyRecords() throws Exception {
+ try (final MockWebServer server = new MockWebServer()) {
+ server.enqueue(new MockResponse().setBody("{\"records\":[]}"));
+
+ server.start();
+ final HttpUrl httpUrl = server.url(API_URL_PATH);
+
+ runner.setProperty(QueryAirtableTable.API_URL, httpUrl.toString());
+ runner.run();
+
+ final List<MockFlowFile> results =
runner.getFlowFilesForRelationship(QueryAirtableTable.REL_SUCCESS);
+ assertTrue(results.isEmpty());
+ }
+ }
+}
diff --git a/nifi-nar-bundles/nifi-airtable-bundle/pom.xml
b/nifi-nar-bundles/nifi-airtable-bundle/pom.xml
new file mode 100644
index 0000000000..df6f17a1fe
--- /dev/null
+++ b/nifi-nar-bundles/nifi-airtable-bundle/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <parent>
+ <artifactId>nifi-nar-bundles</artifactId>
+ <groupId>org.apache.nifi</groupId>
+ <version>1.18.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>nifi-airtable-bundle</artifactId>
+ <packaging>pom</packaging>
+ <modules>
+ <module>nifi-airtable-processors</module>
+ <module>nifi-airtable-nar</module>
+ </modules>
+</project>
\ No newline at end of file
diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml
index 718be12d4c..37bc3640aa 100755
--- a/nifi-nar-bundles/pom.xml
+++ b/nifi-nar-bundles/pom.xml
@@ -112,6 +112,7 @@
<module>nifi-rocksdb-bundle</module>
<module>nifi-hubspot-bundle</module>
<module>nifi-dropbox-bundle</module>
+ <module>nifi-airtable-bundle</module>
</modules>
<build>
@@ -380,6 +381,12 @@
<version>1.18.0-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-web-client-provider-api</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
<!-- Managed dependency versions applicable to all modules -->
<dependency>
<groupId>org.apache.nifi</groupId>
