krisztina-zsihovszki commented on code in PR #6355: URL: https://github.com/apache/nifi/pull/6355#discussion_r966153514
########## nifi-nar-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/BoxFlowFileAttribute.java: ########## @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.box; + +import java.util.Optional; +import java.util.function.Function; + +public enum BoxFlowFileAttribute { + ID(BoxFileInfo.ID, BoxFileInfo::getId), + FILE_NAME(BoxFileInfo.FILENAME, BoxFileInfo::getName), Review Comment: Consider changing the enum names to FILENAME and TIMESTAMP. The Optional.ofNullable check is not necessary for size and timestamp since they have long value. ########## nifi-nar-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/AbstractBoxFilesIT.java: ########## @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.box; + +import com.box.sdk.BoxAPIConnection; +import com.box.sdk.BoxConfig; +import com.box.sdk.BoxDeveloperEditionAPIConnection; +import com.box.sdk.BoxFile; +import com.box.sdk.BoxFolder; +import org.apache.nifi.processor.Processor; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + +import java.io.ByteArrayInputStream; +import java.io.FileReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Set the following constants before running:<br /> + * <br /> + * FOLDER_ID - The ID of a Folder the test can use to create files and sub-folders within.<br /> + * USER_ID - The ID of the user owning the Folder.<br /> + * BOX_CONFIG_FILE - An App Settings Configuration JSON file. The app needs to have 'OAuth 2.0 with JSON Web Tokens (Server Authentication)' authentication method, + * have App + Enterprise access level and be able to read and write files and folders as well as generate user access tokens.<br /> + * <br /> + * Created files and folders are cleaned up, but it's advisable to dedicate a folder for this test so that it can be cleaned up easily should the test fail to do so. + */ +public abstract class AbstractBoxFilesIT<T extends BoxTrait & Processor> { + static final String FOLDER_ID = ""; + static final String USER_ID = ""; + static final String BOX_CONFIG_FILE = ""; + + protected static final String DEFAULT_FILE_CONTENT = "test_content"; + public static final String MAIN_FOLDER_NAME = "main"; + + protected T testSubject; + protected TestRunner testRunner; + + protected BoxAPIConnection boxAPIConnection; + + protected String targetFolderName; + protected String mainFolderId; + + protected abstract T createTestSubject(); + + @BeforeEach + protected void init() throws Exception { + testSubject = createTestSubject(); + testRunner = createTestRunner(); + + try ( + Reader reader = new FileReader(BOX_CONFIG_FILE); + ) { + BoxConfig boxConfig = BoxConfig.readFrom(reader); + boxAPIConnection = BoxDeveloperEditionAPIConnection.getAppEnterpriseConnection(boxConfig); + boxAPIConnection.asUser(USER_ID); + } + + targetFolderName = new BoxFolder(boxAPIConnection, FOLDER_ID).getInfo("name").getName(); + + BoxFolder.Info mainFolderInfo = createFolder(MAIN_FOLDER_NAME, FOLDER_ID); + mainFolderId = mainFolderInfo.getID(); + } + + @AfterEach + protected void tearDown() { + if (boxAPIConnection != null) { + BoxFolder folder = new BoxFolder(boxAPIConnection, mainFolderId); + folder.delete(true); + } + } + + protected TestRunner createTestRunner() { + TestRunner testRunner = TestRunners.newTestRunner(testSubject); Review Comment: Minor comment: the local variable is not necessary (same comment for createFolder, createFile and getCheckedAttributeNames) ########## nifi-nar-bundles/nifi-box-bundle/nifi-box-processors/pom.xml: ########## @@ -0,0 +1,121 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-box-bundle</artifactId> + <version>1.18.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-box-processors</artifactId> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>com.box</groupId> + <artifactId>box-java-sdk</artifactId> + <version>3.4.0</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-proxy-configuration-api</artifactId> + </dependency> + <dependency> Review Comment: Some of these dependencies are used only by the tests (e.g. nifi-record-serialization-service-api, nifi-schema-registry-service-api, nifi-distributed-cache-client-service-api). ########## nifi-nar-bundles/nifi-box-bundle/nifi-box-processors/src/test/java/org/apache/nifi/processors/box/ListBoxFilesSimpleTest.java: ########## @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.box; + +import com.box.sdk.BoxAPIConnection; +import com.box.sdk.BoxFolder; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.util.list.AbstractListProcessor; +import org.apache.nifi.proxy.ProxyConfiguration; +import org.apache.nifi.util.EqualsWrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +import static org.apache.nifi.util.EqualsWrapper.wrapList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; + +public class ListBoxFilesSimpleTest implements SimpleListBoxFileTestTrait { + private ListBoxFiles testSubject; + + private ProcessContext mockProcessContext; + private BoxAPIConnection mockBoxAPIConnection; + + private BoxFolder mockBoxFolder; + + @BeforeEach + void setUp() throws Exception { + mockProcessContext = mock(ProcessContext.class, RETURNS_DEEP_STUBS); + mockBoxAPIConnection = mock(BoxAPIConnection.class, RETURNS_DEEP_STUBS); + mockBoxFolder = mock(BoxFolder.class, RETURNS_DEEP_STUBS); + + testSubject = new ListBoxFiles() { + @Override + protected List<BoxFileInfo> performListing(ProcessContext context, Long minTimestamp, AbstractListProcessor.ListingMode ignoredListingMode) throws IOException { + return super.performListing(context, minTimestamp, ListingMode.EXECUTION); + } + + @Override + public BoxAPIConnection createBoxApiConnection(ProcessContext context, ProxyConfiguration proxyConfiguration) { + return mockBoxAPIConnection; + } + + @Override + BoxFolder getFolder(String folderId) { + return mockBoxFolder; + } + }; + + testSubject.onScheduled(mockProcessContext); + } + + @Test + void testCreatedListableEntityContainsCorrectData() throws Exception { + // GIVEN + Long minTimestamp = 0L; + + String id = "id_1"; + String filename = "file_name_1"; + List<String> pathParts = Arrays.asList("path", "to", "file"); + Long size = 125L; + long createdTime = 123456L; + long modifiedTime = 234567L; + + mockFetchedFileList(id, filename, pathParts, size, createdTime, modifiedTime); + + List<BoxFileInfo> expected = Arrays.asList( + new BoxFileInfo.Builder() + .id(id) + .fileName(filename) + .path("/path/to/file") + .size(size) + .createdTime(createdTime) + .modifiedTime(modifiedTime) + .build() + ); + + // WHEN + List<BoxFileInfo> actual = testSubject.performListing(mockProcessContext, minTimestamp, null); + + // THEN + List<Function<BoxFileInfo, Object>> propertyProviders = Arrays.asList( + BoxFileInfo::getId, + BoxFileInfo::getIdentifier, + BoxFileInfo::getName, + BoxFileInfo::getSize, Review Comment: getPath is missing from propertyProviders ########## nifi-nar-bundles/nifi-box-bundle/nifi-box-processors/src/main/java/org/apache/nifi/processors/box/FetchBoxFiles.java: ########## @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.box; + +import com.box.sdk.BoxAPIConnection; +import com.box.sdk.BoxAPIResponseException; +import com.box.sdk.BoxFile; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.proxy.ProxyConfiguration; +import org.apache.nifi.proxy.ProxySpec; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@Tags({"box", "storage", "fetch"}) +@CapabilityDescription("Fetches files from a Box Folder. Designed to be used in tandem with ListBoxFiles.") +@SeeAlso({ListBoxFiles.class}) +@WritesAttributes({ + @WritesAttribute(attribute = FetchBoxFiles.ERROR_CODE_ATTRIBUTE, description = "The error code returned by Box when the fetch of a file fails"), + @WritesAttribute(attribute = FetchBoxFiles.ERROR_MESSAGE_ATTRIBUTE, description = "The error message returned by Box when the fetch of a file fails") +}) +public class FetchBoxFiles extends AbstractProcessor implements BoxTrait { + public static final String ERROR_CODE_ATTRIBUTE = "error.code"; + public static final String ERROR_MESSAGE_ATTRIBUTE = "error.message"; + + public static final PropertyDescriptor FILE_ID = new PropertyDescriptor + .Builder().name("box-file-id") + .displayName("File ID") + .description("The ID of the File to fetch") + .required(true) + .defaultValue("${box.id}") + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final Relationship REL_SUCCESS = + new Relationship.Builder() + .name("success") + .description("A flowfile will be routed here for each successfully fetched File.") + .build(); + + public static final Relationship REL_FAILURE = + new Relationship.Builder().name("failure") + .description("A flowfile will be routed here for each File for which fetch was attempted but failed.") + .build(); + + private static final ProxySpec[] PROXY_SPECS = {ProxySpec.HTTP_AUTH}; + private static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList( + FILE_ID, + USER_ID, + BOX_CONFIG_FILE, + ProxyConfiguration.createProxyConfigPropertyDescriptor(false, PROXY_SPECS) + )); + + public static final Set<Relationship> relationships = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + REL_SUCCESS, + REL_FAILURE + ))); + + private volatile BoxAPIConnection boxAPIConnection; + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + public Set<Relationship> getRelationships() { + return relationships; + } + + @OnScheduled + public void onScheduled(final ProcessContext context) throws IOException { + final ProxyConfiguration proxyConfiguration = ProxyConfiguration.getConfiguration(context); + + boxAPIConnection = createBoxApiConnection(context, proxyConfiguration); + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + String fileId = context.getProperty(FILE_ID).evaluateAttributeExpressions(flowFile).getValue(); + FlowFile outFlowFile = flowFile; + try { + outFlowFile = fetchFile(fileId, session, outFlowFile); + + session.transfer(outFlowFile, REL_SUCCESS); + } catch (BoxAPIResponseException e) { + handleErrorResponse(session, fileId, flowFile, e); + } catch (Exception e) { + handleUnexpectedError(session, flowFile, fileId, e); + } + } + + FlowFile fetchFile(String fileId, ProcessSession session, FlowFile outFlowFile) { + BoxFile boxFile = new BoxFile(boxAPIConnection, fileId); + + outFlowFile = session.write(outFlowFile, outputStream -> boxFile.download(outputStream)); + + return outFlowFile; + } + + private void handleErrorResponse(ProcessSession session, String fileId, FlowFile outFlowFile, BoxAPIResponseException e) { + getLogger().error("Couldn't fetch file with id '{}'", fileId, e); + + session.putAttribute(outFlowFile, ERROR_CODE_ATTRIBUTE, "" + e.getResponseCode()); Review Comment: Please use the return value of putAttribute. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
