turcsanyip commented on code in PR #10467: URL: https://github.com/apache/nifi/pull/10467#discussion_r2747481946
########## nifi-extension-bundles/nifi-couchbase-bundle/nifi-couchbase-services/src/main/java/org/apache/nifi/services/couchbase/AbstractCouchbaseService.java: ########## @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.services.couchbase; + +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.services.couchbase.utils.CouchbaseContext; +import org.apache.nifi.services.couchbase.utils.DocumentType; + +import java.util.Set; + +public class AbstractCouchbaseService extends AbstractControllerService { + + protected static final String KEY = "key"; + protected static final Set<String> REQUIRED_KEYS = Set.of(KEY); + + private static final String DEFAULT_BUCKET = "default"; + private static final String DEFAULT_SCOPE = "_default"; + private static final String DEFAULT_COLLECTION = "_default"; + + protected CouchbaseClient couchbaseClient; Review Comment: The field should be `volatile` and please move it after the static property definitions below. Please also fix the same in `AbstractCouchbaseProcessor`. ########## nifi-extension-bundles/nifi-couchbase-bundle/nifi-couchbase-services/pom.xml: ########## @@ -0,0 +1,68 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-couchbase-bundle</artifactId> + <version>2.7.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-couchbase-services</artifactId> + <packaging>jar</packaging> + + <dependencies> + <!-- Internal dependencies --> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-couchbase-services-api</artifactId> + <version>2.7.0-SNAPSHOT</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-lookup-service-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-record-serialization-service-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-json-record-utils</artifactId> + <version>2.7.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-avro-record-utils</artifactId> + <version>2.7.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-distributed-cache-client-service-api</artifactId> + <scope>compile</scope> Review Comment: The service API dependency should be `provided` (no `scope` definition is needed but it can be inherited from dependency management). ########## nifi-extension-bundles/nifi-couchbase-bundle/nifi-couchbase-services/src/main/java/org/apache/nifi/services/couchbase/CouchbaseMapCacheClient.java: ########## @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.services.couchbase; + +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.distributed.cache.client.AtomicCacheEntry; +import org.apache.nifi.distributed.cache.client.AtomicDistributedMapCacheClient; +import org.apache.nifi.distributed.cache.client.Deserializer; +import org.apache.nifi.distributed.cache.client.Serializer; +import org.apache.nifi.services.couchbase.exception.CouchbaseException; +import org.apache.nifi.services.couchbase.utils.CouchbaseGetResult; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +@Tags({"distributed", "cache", "map", "cluster", "couchbase"}) +@CapabilityDescription("Provides the ability to communicate with a Couchbase Server cluster as a DistributedMapCacheServer." + + " This can be used in order to share a Map between nodes in a NiFi cluster." + + " Couchbase Server cluster can provide a high available and persistent cache storage.") +public class CouchbaseMapCacheClient extends AbstractCouchbaseService implements AtomicDistributedMapCacheClient<Long> { + + public static final PropertyDescriptor COUCHBASE_CONNECTION_SERVICE = new PropertyDescriptor.Builder() + .name("Couchbase Connection Service") + .description("A Couchbase Connection Service which manages connections to a Couchbase cluster.") + .required(true) + .identifiesControllerService(CouchbaseConnectionService.class) + .build(); + + private static final List<PropertyDescriptor> PROPERTIES = List.of( + COUCHBASE_CONNECTION_SERVICE, + BUCKET_NAME, + SCOPE_NAME, + COLLECTION_NAME + ); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + public <K, V> AtomicCacheEntry<K, V, Long> fetch(K key, Serializer<K> keySerializer, Deserializer<V> valueDeserializer) throws IOException { + final String documentId = serializeDocumentKey(key, keySerializer); + try { + final CouchbaseGetResult result = couchbaseClient.getDocument(documentId); + return new AtomicCacheEntry<>(key, deserializeDocument(valueDeserializer, result.resultContent()), result.cas()); + } catch (CouchbaseException e) { + return null; + } + } + + @Override + public <K, V> boolean replace(AtomicCacheEntry<K, V, Long> entry, Serializer<K> keySerializer, Serializer<V> valueSerializer) throws IOException { + final String documentId = serializeDocumentKey(entry.getKey(), keySerializer); + final byte[] document = serializeDocument(entry.getValue(), valueSerializer); + + try { + couchbaseClient.replaceDocument(documentId, document); Review Comment: `AtomicDistributedMapCacheClient` interface expects to apply optimistic locking so `AtomicCacheEntry.version` should be passed as the CAS value to Couchbase if it is available. ########## nifi-extension-bundles/nifi-couchbase-bundle/nifi-couchbase-services/src/main/java/org/apache/nifi/services/couchbase/CouchbaseMapCacheClient.java: ########## @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.services.couchbase; + +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.distributed.cache.client.AtomicCacheEntry; +import org.apache.nifi.distributed.cache.client.AtomicDistributedMapCacheClient; +import org.apache.nifi.distributed.cache.client.Deserializer; +import org.apache.nifi.distributed.cache.client.Serializer; +import org.apache.nifi.services.couchbase.exception.CouchbaseException; +import org.apache.nifi.services.couchbase.utils.CouchbaseGetResult; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +@Tags({"distributed", "cache", "map", "cluster", "couchbase"}) +@CapabilityDescription("Provides the ability to communicate with a Couchbase Server cluster as a DistributedMapCacheServer." + + " This can be used in order to share a Map between nodes in a NiFi cluster." + + " Couchbase Server cluster can provide a high available and persistent cache storage.") +public class CouchbaseMapCacheClient extends AbstractCouchbaseService implements AtomicDistributedMapCacheClient<Long> { + + public static final PropertyDescriptor COUCHBASE_CONNECTION_SERVICE = new PropertyDescriptor.Builder() + .name("Couchbase Connection Service") + .description("A Couchbase Connection Service which manages connections to a Couchbase cluster.") + .required(true) + .identifiesControllerService(CouchbaseConnectionService.class) + .build(); + + private static final List<PropertyDescriptor> PROPERTIES = List.of( + COUCHBASE_CONNECTION_SERVICE, + BUCKET_NAME, + SCOPE_NAME, + COLLECTION_NAME + ); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return PROPERTIES; + } + + @Override + public <K, V> AtomicCacheEntry<K, V, Long> fetch(K key, Serializer<K> keySerializer, Deserializer<V> valueDeserializer) throws IOException { + final String documentId = serializeDocumentKey(key, keySerializer); + try { + final CouchbaseGetResult result = couchbaseClient.getDocument(documentId); + return new AtomicCacheEntry<>(key, deserializeDocument(valueDeserializer, result.resultContent()), result.cas()); + } catch (CouchbaseException e) { + return null; + } + } + + @Override + public <K, V> boolean replace(AtomicCacheEntry<K, V, Long> entry, Serializer<K> keySerializer, Serializer<V> valueSerializer) throws IOException { + final String documentId = serializeDocumentKey(entry.getKey(), keySerializer); + final byte[] document = serializeDocument(entry.getValue(), valueSerializer); + + try { + couchbaseClient.replaceDocument(documentId, document); Review Comment: Though the API specification of the `replace()` method [says](https://github.com/apache/nifi/blob/29d5040714982a57e9d1ca622f6e0479762d5309/nifi-extension-bundles/nifi-standard-services/nifi-distributed-cache-client-service-api/src/main/java/org/apache/nifi/distributed/cache/client/AtomicDistributedMapCacheClient.java#L54) _"Replace an **existing key** with new value"_, the [only client](https://github.com/apache/nifi/blob/29d5040714982a57e9d1ca622f6e0479762d5309/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/WaitNotifyProtocol.java#L296) of the method expects insert logic as well. So we may consider adding that logic too (as it was in the old implementation). I suggest checking the other `replace()` implementations to get an idea how it should work. ########## nifi-extension-bundles/nifi-couchbase-bundle/nifi-couchbase-services/src/main/java/org/apache/nifi/services/couchbase/CouchbaseKeyValueLookupService.java: ########## @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.services.couchbase; + +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.lookup.LookupFailureException; +import org.apache.nifi.lookup.StringLookupService; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.services.couchbase.utils.CouchbaseLookupInResult; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +@Tags({"lookup", "enrich", "key", "value", "couchbase"}) +@CapabilityDescription("Lookup a string value from Couchbase Server associated with the specified key. The coordinates that are passed to the lookup must contain the key 'key'.") +public class CouchbaseKeyValueLookupService extends AbstractCouchbaseService implements StringLookupService { + + private volatile String subDocPath; Review Comment: Please move it after the static members. ########## nifi-extension-bundles/nifi-couchbase-bundle/nifi-couchbase-services/src/main/java/org/apache/nifi/services/couchbase/AbstractCouchbaseService.java: ########## @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.services.couchbase; + +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.services.couchbase.utils.CouchbaseContext; +import org.apache.nifi.services.couchbase.utils.DocumentType; + +import java.util.Set; + +public class AbstractCouchbaseService extends AbstractControllerService { + + protected static final String KEY = "key"; + protected static final Set<String> REQUIRED_KEYS = Set.of(KEY); + + private static final String DEFAULT_BUCKET = "default"; + private static final String DEFAULT_SCOPE = "_default"; + private static final String DEFAULT_COLLECTION = "_default"; + + protected CouchbaseClient couchbaseClient; + + public static final PropertyDescriptor COUCHBASE_CONNECTION_SERVICE = new PropertyDescriptor.Builder() + .name("Couchbase Connection Service") + .description("A Couchbase Connection Service which manages connections to a Couchbase cluster.") + .required(true) + .identifiesControllerService(CouchbaseConnectionService.class) + .build(); + + public static final PropertyDescriptor BUCKET_NAME = new PropertyDescriptor.Builder() + .name("Bucket Name") + .description("The name of the bucket where documents will be stored. Each bucket contains a hierarchy of scopes and collections to group keys and values logically.") + .required(true) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .defaultValue(DEFAULT_BUCKET) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .build(); + + public static final PropertyDescriptor SCOPE_NAME = new PropertyDescriptor.Builder() + .name("Scope Name") + .description("The name of the scope which is a logical namespace within a bucket, serving to categorize and organize related collections.") + .required(true) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .defaultValue(DEFAULT_SCOPE) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .build(); + + public static final PropertyDescriptor COLLECTION_NAME = new PropertyDescriptor.Builder() + .name("Collection Name") + .description("The name of collection which is a logical container within a scope, used to hold documents.") + .required(true) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .defaultValue(DEFAULT_COLLECTION) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .build(); + + @OnEnabled + public void onEnabled(final ConfigurationContext context) { + final CouchbaseConnectionService connectionService = context.getProperty(COUCHBASE_CONNECTION_SERVICE).asControllerService(CouchbaseConnectionService.class); + final CouchbaseContext couchbaseContext = getCouchbaseContext(context); + couchbaseClient = connectionService.getClient(couchbaseContext); + } + + public Set<String> getRequiredKeys() { + return REQUIRED_KEYS; + } + + private CouchbaseContext getCouchbaseContext(ConfigurationContext context) { + final String bucketName = context.getProperty(BUCKET_NAME).evaluateAttributeExpressions().getValue(); + final String scopeName = context.getProperty(SCOPE_NAME).evaluateAttributeExpressions().getValue(); + final String collectionName = context.getProperty(COLLECTION_NAME).evaluateAttributeExpressions().getValue(); + + return new CouchbaseContext(bucketName, scopeName, collectionName, DocumentType.JSON); Review Comment: Due to the fixed document type, `CouchbaseKeyValueLookupService` and `CouchbaseMapCacheClient` can only handle Json documents. They should also be able to process binary (raw) type too. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
