exceptionfactory commented on code in PR #10993: URL: https://github.com/apache/nifi/pull/10993#discussion_r2978886840
########## nifi-connectors/pom.xml: ########## @@ -0,0 +1,42 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns="http://maven.apache.org/POM/4.0.0"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi</artifactId> + <version>2.9.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-connectors</artifactId> + <packaging>pom</packaging> + + <modules> + <module>nifi-kafka-to-s3-bundle</module> + </modules> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-utils</artifactId> + <version>2.9.0-SNAPSHOT</version> Review Comment: Is this necessary? It seems like setting the dependency version here is duplicative. ########## nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockAuditService.java: ########## @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.mock.connector.server; + +import org.apache.nifi.action.Action; +import org.apache.nifi.admin.service.AuditService; +import org.apache.nifi.history.History; +import org.apache.nifi.history.HistoryQuery; +import org.apache.nifi.history.PreviousValue; + +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class MockAuditService implements AuditService { Review Comment: Does this need to be public or can it be package-private? ########## nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml: ########## @@ -0,0 +1,238 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-kafka-to-s3-bundle</artifactId> + <version>2.9.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-kafka-to-s3-integration-tests</artifactId> + + <repositories> + <repository> + <id>confluent</id> + <url>https://packages.confluent.io/maven/</url> + </repository> + </repositories> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock-api</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + + <!-- Test Containers for Kafka and S3 --> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers-kafka</artifactId> + <version>2.0.1</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>localstack</artifactId> + <version>1.20.4</version> + <scope>test</scope> + </dependency> + + <!-- Kafka client for producing test data --> + <!-- Override vulnerable kafka clients --> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <version>4.1.1</version> Review Comment: Given the relationship to the Kafka Processors, it seems like the Kafka version should be promoted to the root Maven configuration and shared with the Kafka component bundle to avoid mismatches. ########## nifi-commons/nifi-connector-utils/pom.xml: ########## @@ -0,0 +1,42 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-commons</artifactId> + <version>2.9.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-connector-utils</artifactId> Review Comment: Although some other modules in `nifi-common` have the `utils` suffix, what about `nifi-connector-shared` or `nifi-connector-common`, or `nifi-connector-support`? It looks like the utilities class supports construction of Connectors, so it is also to an API level contract, thus something a bit more concrete than `utils` seems better. ########## nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorTestRunner.java: ########## @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.mock.connector.server; + +import org.apache.nifi.components.DescribedValue; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.connector.AssetReference; +import org.apache.nifi.components.connector.ConnectorValueReference; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.SecretReference; +import org.apache.nifi.components.connector.StepConfiguration; +import org.apache.nifi.flow.VersionedExternalFlow; + +import java.io.Closeable; +import java.io.File; +import java.io.InputStream; +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public interface ConnectorTestRunner extends Closeable { Review Comment: This seems like a central interface for testing purposes, so it would be helpful to expand the documentation to cover most of the methods. ########## nifi-connectors/nifi-kafka-to-s3-bundle/pom.xml: ########## @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns="http://maven.apache.org/POM/4.0.0"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connectors</artifactId> + <version>2.9.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-kafka-to-s3-bundle</artifactId> + <packaging>pom</packaging> + + <modules> + <module>nifi-kafka-to-s3-connector</module> + <module>nifi-kafka-to-s3-nar</module> + <module>nifi-kafka-to-s3-integration-tests</module> + </modules> + + <dependencyManagement> + <dependencies> + </dependencies> + </dependencyManagement> Review Comment: This empty block can be removed. ########## nifi-connector-mock-bundle/nifi-connector-mock-server-nar/pom.xml: ########## @@ -0,0 +1,55 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock-bundle</artifactId> + <version>2.9.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-connector-mock-server-nar</artifactId> + <packaging>nar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock-server</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-framework-nar</artifactId> + <version>2.9.0-SNAPSHOT</version> + <type>nar</type> + </dependency> + </dependencies> + + <repositories> + <!-- Shibboleth Repository required for OpenSAML --> + <!-- This is required because nifi-framework-nar is the parent/NAR Dependency, andi t requires shibboleth. --> Review Comment: ```suggestion <!-- Required for nifi-framework-nar dependencies --> ``` ########## nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionMapper.java: ########## @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.mock.connector.server; + +import org.apache.nifi.flow.Bundle; +import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedProcessor; + +import java.util.HashMap; +import java.util.Map; + +public class MockExtensionMapper { + + private final Map<String, String> processorMocks = new HashMap<>(); + private final Map<String, String> controllerServiceMocks = new HashMap<>(); + + public void mockProcessor(final String processorType, final String mockProcessorClassName) { + processorMocks.put(processorType, mockProcessorClassName); + } + + public void mockControllerService(final String controllerServiceType, final String mockControllerServiceClassName) { + controllerServiceMocks.put(controllerServiceType, mockControllerServiceClassName); + } + + public void mapProcessor(final VersionedProcessor processor) { + final String type = processor.getType(); + final String implementationClassName = processorMocks.get(type); + if (implementationClassName == null) { + return; + } + + processor.setType(implementationClassName); + processor.setBundle(new Bundle("org.apache.nifi.mock", implementationClassName, "1.0.0")); Review Comment: It looks like the bundle package and version should be declared as static variables. ########## nifi-commons/nifi-write-ahead-log/src/main/java/org/apache/nifi/wali/HashMapSnapshot.java: ########## @@ -175,14 +175,18 @@ public void update(final Collection<T> records) { switch (updateType) { case DELETE: - recordMap.remove(recordId); + if (recordId != null) { Review Comment: Are these changes required for Connectors? It seems like they could be handled independently. ########## nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorAssetManager.java: ########## @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.mock.connector.server; + +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.asset.AssetManagerInitializationContext; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * A Mock implementation of AssetManager for use in ConnectorTestRunner tests. + * This implementation stores assets in a temporary directory structure. + */ +public class MockConnectorAssetManager implements AssetManager { + + private static final String ASSET_STORAGE_LOCATION_PROPERTY = "directory"; + private static final String DEFAULT_ASSET_STORAGE_LOCATION = "target/mock_connector_assets"; + + private final Map<String, Asset> assets = new ConcurrentHashMap<>(); + private volatile File assetStorageLocation; + + @Override + public void initialize(final AssetManagerInitializationContext context) { + final String storageLocation = context.getProperties().getOrDefault(ASSET_STORAGE_LOCATION_PROPERTY, DEFAULT_ASSET_STORAGE_LOCATION); + assetStorageLocation = new File(storageLocation); + + if (!assetStorageLocation.exists()) { + try { + Files.createDirectories(assetStorageLocation.toPath()); + } catch (final IOException e) { + throw new RuntimeException("Failed to create asset storage directory: " + storageLocation, e); + } + } + } + + @Override + public Asset createAsset(final String ownerId, final String assetName, final InputStream contents) throws IOException { + final String assetId = UUID.randomUUID().toString(); + return saveAsset(ownerId, assetId, assetName, contents); + } + + @Override + public Asset saveAsset(final String ownerId, final String assetId, final String assetName, final InputStream contents) throws IOException { + final File assetFile = getFile(ownerId, assetId, assetName); + final File parentDir = assetFile.getParentFile(); + + if (!parentDir.exists()) { + Files.createDirectories(parentDir.toPath()); + } + + Files.copy(contents, assetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + + final Asset asset = new MockAsset(assetId, ownerId, assetName, assetFile, null); + assets.put(assetId, asset); + return asset; + } + + @Override + public Optional<Asset> getAsset(final String id) { + return Optional.ofNullable(assets.get(id)); + } + + @Override + public List<Asset> getAssets(final String ownerId) { + final List<Asset> ownerAssets = new ArrayList<>(); + for (final Asset asset : assets.values()) { + if (asset.getOwnerIdentifier().equals(ownerId)) { + ownerAssets.add(asset); + } + } + return ownerAssets; + } + + @Override + public Asset createMissingAsset(final String ownerId, final String assetName) { + final String assetId = UUID.randomUUID().toString(); + final File file = getFile(ownerId, assetId, assetName); + final Asset asset = new MockAsset(assetId, ownerId, assetName, file, null); + assets.put(assetId, asset); + return asset; + } + + @Override + public Optional<Asset> deleteAsset(final String id) { + final Asset removed = assets.remove(id); + if (removed != null && removed.getFile().exists()) { + try { + Files.delete(removed.getFile().toPath()); + } catch (final IOException e) { + throw new RuntimeException("Failed to delete asset " + id + " from storage file " + removed.getFile().getAbsolutePath(), e); Review Comment: ```suggestion throw new UncheckedIOException("Failed to delete asset " + id + " from storage file " + removed.getFile().getAbsolutePath(), e); ``` ########## nifi-connector-mock-bundle/nifi-connector-mock-api/src/main/java/org/apache/nifi/mock/connector/server/ConnectorTestRunner.java: ########## @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.mock.connector.server; + +import org.apache.nifi.components.DescribedValue; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.connector.AssetReference; +import org.apache.nifi.components.connector.ConnectorValueReference; +import org.apache.nifi.components.connector.FlowUpdateException; +import org.apache.nifi.components.connector.SecretReference; +import org.apache.nifi.components.connector.StepConfiguration; +import org.apache.nifi.flow.VersionedExternalFlow; + +import java.io.Closeable; +import java.io.File; +import java.io.InputStream; +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public interface ConnectorTestRunner extends Closeable { + String SECRET_PROVIDER_ID = "TestRunnerSecretsManager"; + String SECRET_PROVIDER_NAME = "TestRunnerSecretsManager"; Review Comment: Do these constants belong on this interface? ########## nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockConnectorAssetManager.java: ########## @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.mock.connector.server; + +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.asset.AssetManagerInitializationContext; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * A Mock implementation of AssetManager for use in ConnectorTestRunner tests. + * This implementation stores assets in a temporary directory structure. + */ +public class MockConnectorAssetManager implements AssetManager { + + private static final String ASSET_STORAGE_LOCATION_PROPERTY = "directory"; + private static final String DEFAULT_ASSET_STORAGE_LOCATION = "target/mock_connector_assets"; + + private final Map<String, Asset> assets = new ConcurrentHashMap<>(); + private volatile File assetStorageLocation; + + @Override + public void initialize(final AssetManagerInitializationContext context) { + final String storageLocation = context.getProperties().getOrDefault(ASSET_STORAGE_LOCATION_PROPERTY, DEFAULT_ASSET_STORAGE_LOCATION); + assetStorageLocation = new File(storageLocation); + + if (!assetStorageLocation.exists()) { + try { + Files.createDirectories(assetStorageLocation.toPath()); + } catch (final IOException e) { + throw new RuntimeException("Failed to create asset storage directory: " + storageLocation, e); Review Comment: ```suggestion throw new UncheckedIOException("Failed to create asset storage directory: " + storageLocation, e); ``` ########## nifi-connector-mock-bundle/nifi-connector-mock-server/src/main/java/org/apache/nifi/mock/connector/server/MockExtensionDiscoveringManager.java: ########## @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.mock.connector.server; + +import org.apache.nifi.bundle.Bundle; +import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.bundle.BundleDetails; +import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.nar.InstanceClassLoader; +import org.apache.nifi.nar.StandardExtensionDiscoveringManager; +import org.apache.nifi.processor.Processor; + +import java.io.File; +import java.net.URL; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class MockExtensionDiscoveringManager extends StandardExtensionDiscoveringManager { + private final ConcurrentMap<String, ClassLoader> mockComponentClassLoaders = new ConcurrentHashMap<>(); + + public synchronized void addProcessor(final Class<? extends Processor> mockProcessorClass) { + final BundleDetails bundleDetails = new BundleDetails.Builder() + .workingDir(new File("target/work/extensions/mock-bundle")) Review Comment: Referencing the `target` directory is problematic from the perspective of a clean build environment over multiple runs. Using a JUnit temporary directory provides more independence from the build structure. It also seems like this class should have a configured base work directory, instead of repeating this directory. ########## nifi-connector-mock-bundle/nifi-connector-mock-test-bundle/nifi-connector-mock-test-connectors/src/main/resources/flows/Cron_Schedule_Connector.json: ########## @@ -0,0 +1,410 @@ +{ + "flowContents": { + "identifier": "1800c04e-f9b9-3293-bfc7-b35f43e0706c", + "instanceIdentifier": "4f8ca484-019c-1000-955f-68c5defeb22b", + "name": "Cron_Schedule_Connector", + "comments": "", + "position": { + "x": -1254.0, + "y": -437.70139741897583 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "4d4160b1-736c-3be4-bc3a-84fc6eb4ad2a", + "instanceIdentifier": "499f9781-71c5-367f-aa17-5441bff29de9", + "name": "UpdateAttribute", + "comments": "", + "position": { + "x": -360.0, + "y": 24.0 + }, + "type": "org.apache.nifi.processors.attributes.UpdateAttribute", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-update-attribute-nar", + "version": "2026.1.20.21-SNAPSHOT" Review Comment: As part of a consistency update, I recommend setting a version of either `2.9.0` or `2.9.0-SNAPSHOT` for all components in Flow JSON files. ########## nifi-connector-mock-bundle/nifi-connector-mock/src/main/resources/nifi.properties: ########## @@ -0,0 +1,165 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.flow.configuration.file=target/nifi-storage/conf/flow.json.gz +nifi.flow.configuration.archive.enabled=true +nifi.flow.configuration.archive.dir=target/nifi-storage/conf/archive/ +nifi.flow.configuration.archive.max.time=30 days +nifi.flow.configuration.archive.max.storage=500 MB +nifi.flow.configuration.archive.max.count= +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=3 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis +nifi.queue.backpressure.count=10000 +nifi.queue.backpressure.size=1 GB + +nifi.nar.library.directory=./lib +nifi.nar.library.autoload.directory=./extensions +nifi.nar.working.directory=./work/nar/ + +##################### +# Python Extensions # +##################### +# Uncomment in order to enable Python Extensions. +#nifi.python.command=python +nifi.python.framework.source.directory=./python/framework +nifi.python.extensions.source.directory.default=./python/extensions +nifi.python.working.directory=./work/python +nifi.python.max.processes=100 +nifi.python.max.processes.per.extension.type=10 + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=target/nifi-storage/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider + +# Database Settings +nifi.database.directory=target/nifi-storage/database_repository + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.wal.implementation=org.apache.nifi.wali.SequentialAccessWriteAheadLog +nifi.flowfile.repository.directory=target/nifi-storage/flowfile_repository +nifi.flowfile.repository.checkpoint.interval=1 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=50 KB +nifi.content.repository.directory.default=target/nifi-storage/content_repository +nifi.content.repository.archive.max.retention.period=12 hours +nifi.content.repository.archive.max.usage.percentage=90% +nifi.content.repository.archive.enabled=false +nifi.content.repository.always.sync=false + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.VolatileProvenanceRepository + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=1000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# NAR Persistence Provider Properties +nifi.nar.persistence.provider.properties.directory=target/nifi-storage/nar_repository + +# Asset Management +nifi.asset.manager.properties.directory=target/nifi-storage/assets + +# Connector Asset Manager +nifi.connector.asset.manager.implementation=org.apache.nifi.mock.connector.server.MockConnectorAssetManager + +# Secrets Manager +nifi.secrets.manager.implementation=org.apache.nifi.mock.connector.server.secrets.ConnectorTestRunnerSecretsManager + +# Site to Site properties +nifi.remote.input.host=localhost +nifi.remote.input.secure=true +nifi.remote.input.socket.port=7780 +nifi.remote.input.http.enabled=true +nifi.remote.input.http.transaction.ttl=30 sec +nifi.remote.contents.cache.expiration=30 secs + +# web properties # +nifi.web.war.directory=./lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.http.network.interface.default= +nifi.web.https.host=localhost +nifi.web.https.port=0 +nifi.web.https.network.interface.default= +nifi.web.jetty.working.directory=target/work/jetty +nifi.web.jetty.threads=200 +nifi.web.max.header.size=16 KB +nifi.web.proxy.context.path= +nifi.web.proxy.host= + +# security properties # +nifi.sensitive.props.key=nifi-connector-mock-tests +nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256 + +nifi.security.keystore= +nifi.security.keystoreType=PKCS12 +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType=PKCS12 +nifi.security.truststorePasswd= +nifi.security.user.authorizer=system-test-authorizer +nifi.security.user.login.identity.provider= + +# cluster common properties (all nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=true + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=10 +nifi.cluster.node.protocol.max.threads=50 +nifi.cluster.node.event.history.size=25 +nifi.cluster.node.connection.timeout=5 sec +nifi.cluster.node.read.timeout=5 sec +nifi.cluster.node.max.concurrent.requests=100 +nifi.cluster.firewall.file= +nifi.cluster.flow.election.max.wait.time=5 mins +nifi.cluster.flow.election.max.candidates= + +# cluster load balancing properties # +nifi.cluster.load.balance.host= +nifi.cluster.load.balance.port=6342 +nifi.cluster.load.balance.connections.per.node=4 +nifi.cluster.load.balance.max.thread.count=8 +nifi.cluster.load.balance.comms.timeout=30 sec Review Comment: Highlighting these cluster properties, but there appear to be many unnecessary properties in this file that could be removed so that defaults are used. ########## nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml: ########## @@ -0,0 +1,238 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-kafka-to-s3-bundle</artifactId> + <version>2.9.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-kafka-to-s3-integration-tests</artifactId> + + <repositories> + <repository> + <id>confluent</id> + <url>https://packages.confluent.io/maven/</url> + </repository> + </repositories> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock-api</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + + <!-- Test Containers for Kafka and S3 --> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers-kafka</artifactId> + <version>2.0.1</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>localstack</artifactId> + <version>1.20.4</version> + <scope>test</scope> + </dependency> + + <!-- Kafka client for producing test data --> + <!-- Override vulnerable kafka clients --> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <version>4.1.1</version> + <exclusions> + <!-- Excluded and replaced with at.yawk.lz4 version --> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>at.yawk.lz4</groupId> + <artifactId>lz4-java</artifactId> + <version>1.10.1</version> + </dependency> + + <!-- Confluent Schema Registry and Avro dependencies for testing --> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-avro-serializer</artifactId> + <version>7.8.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-schema-registry-client</artifactId> + <version>7.8.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <version>1.11.3</version> Review Comment: The Avro version should come from the parent Maven configuration ########## nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-connector/src/main/java/org/apache/nifi/connectors/kafkas3/S3Step.java: ########## @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.connectors.kafkas3; + +import org.apache.nifi.components.connector.ConfigurationStep; +import org.apache.nifi.components.connector.ConnectorPropertyDescriptor; +import org.apache.nifi.components.connector.ConnectorPropertyGroup; +import org.apache.nifi.components.connector.PropertyType; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.List; + +public class S3Step { + public static final String S3_STEP_NAME = "S3 Configuration"; + public static final String ACCESS_KEY_ID_SECRET_KEY = "Access Key ID and Secret Key"; + public static final String DEFAULT_CREDENTIALS = "Default AWS Credentials"; + + + public static final ConnectorPropertyDescriptor S3_BUCKET = new ConnectorPropertyDescriptor.Builder() + .name("S3 Bucket") + .description("The name of the S3 bucket to write data to.") + .required(true) + .build(); + + public static final ConnectorPropertyDescriptor S3_REGION = new ConnectorPropertyDescriptor.Builder() + .name("S3 Region") + .description("The AWS region where the S3 bucket is located.") + .allowableValuesFetchable(true) + .required(true) + .build(); + + public static final ConnectorPropertyDescriptor S3_DATA_FORMAT = new ConnectorPropertyDescriptor.Builder() + .name("S3 Data Format") + .description("The format to use when writing data to S3.") + .required(true) + .defaultValue("JSON") + .allowableValues("Avro", "JSON") + .build(); + + public static final ConnectorPropertyDescriptor S3_PREFIX = new ConnectorPropertyDescriptor.Builder() + .name("S3 Prefix") + .description("An optional prefix to prepend to all object keys written to the S3 bucket.") + .required(false) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final ConnectorPropertyDescriptor S3_ENDPOINT_OVERRIDE_URL = new ConnectorPropertyDescriptor.Builder() + .name("S3 Endpoint Override URL") + .description("An optional endpoint URL to use instead of the default AWS S3 endpoint. " + + "This can be used to connect to S3-compatible storage systems but should be left unset for connecting to S3.") + .required(false) + .addValidator(StandardValidators.URL_VALIDATOR) + .build(); + + public static final ConnectorPropertyDescriptor TARGET_OBJECT_SIZE = new ConnectorPropertyDescriptor.Builder() + .name("Target Object Size") + .description("The target size for each object written to S3. The connector will attempt to " + + "combine messages until this size is reached before writing an object to S3. Note that this size is approximate " + + "and may vary from object to object.") Review Comment: Descriptions can use multi-line strings. ########## nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml: ########## @@ -0,0 +1,238 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-kafka-to-s3-bundle</artifactId> + <version>2.9.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-kafka-to-s3-integration-tests</artifactId> + + <repositories> + <repository> + <id>confluent</id> + <url>https://packages.confluent.io/maven/</url> + </repository> + </repositories> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock-api</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + + <!-- Test Containers for Kafka and S3 --> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers-kafka</artifactId> + <version>2.0.1</version> Review Comment: The testcontainers version should come from the root Maven configuration. ########## nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml: ########## @@ -0,0 +1,238 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-kafka-to-s3-bundle</artifactId> + <version>2.9.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-kafka-to-s3-integration-tests</artifactId> + + <repositories> + <repository> + <id>confluent</id> + <url>https://packages.confluent.io/maven/</url> + </repository> + </repositories> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock-api</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + + <!-- Test Containers for Kafka and S3 --> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers-kafka</artifactId> + <version>2.0.1</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>localstack</artifactId> + <version>1.20.4</version> + <scope>test</scope> + </dependency> + + <!-- Kafka client for producing test data --> + <!-- Override vulnerable kafka clients --> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <version>4.1.1</version> + <exclusions> + <!-- Excluded and replaced with at.yawk.lz4 version --> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>at.yawk.lz4</groupId> + <artifactId>lz4-java</artifactId> + <version>1.10.1</version> + </dependency> + + <!-- Confluent Schema Registry and Avro dependencies for testing --> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-avro-serializer</artifactId> + <version>7.8.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-schema-registry-client</artifactId> + <version>7.8.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <version>1.11.3</version> + <scope>test</scope> + </dependency> + + <!-- AWS SDK for S3 verification in tests --> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>s3</artifactId> + <version>2.36.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>url-connection-client</artifactId> + <version>2.36.0</version> Review Comment: The AWS SDK client version should come from the parent Maven configuration ########## nifi-connectors/nifi-kafka-to-s3-bundle/nifi-kafka-to-s3-integration-tests/pom.xml: ########## @@ -0,0 +1,238 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-kafka-to-s3-bundle</artifactId> + <version>2.9.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-kafka-to-s3-integration-tests</artifactId> + + <repositories> + <repository> + <id>confluent</id> + <url>https://packages.confluent.io/maven/</url> + </repository> + </repositories> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock-api</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-connector-mock</artifactId> + <version>2.9.0-SNAPSHOT</version> + </dependency> + + <!-- Test Containers for Kafka and S3 --> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers-kafka</artifactId> + <version>2.0.1</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>localstack</artifactId> + <version>1.20.4</version> + <scope>test</scope> + </dependency> + + <!-- Kafka client for producing test data --> + <!-- Override vulnerable kafka clients --> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <version>4.1.1</version> + <exclusions> + <!-- Excluded and replaced with at.yawk.lz4 version --> + <exclusion> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>at.yawk.lz4</groupId> + <artifactId>lz4-java</artifactId> + <version>1.10.1</version> + </dependency> + + <!-- Confluent Schema Registry and Avro dependencies for testing --> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-avro-serializer</artifactId> + <version>7.8.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.confluent</groupId> + <artifactId>kafka-schema-registry-client</artifactId> + <version>7.8.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + <version>1.11.3</version> + <scope>test</scope> + </dependency> + + <!-- AWS SDK for S3 verification in tests --> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>s3</artifactId> + <version>2.36.0</version> Review Comment: The S3 version should come from the parent Maven configuration -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
