[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r73066145 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") --- End diff -- addressed --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r73065856 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") --- End diff -- completely missed this. my bad --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r73065633 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r73061861 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") --- End diff -- .name() and .displayName() in this class as well --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r73061690 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r73061305 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r73060999 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72882149 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72882132 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72842053 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72842097 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72597016 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72596817 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72596717 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72596621 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72595518 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/test/java/org/apache/nifi/processors/enrich/TestQueryDNS.java --- @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; + +import java.util.HashMap; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + + +import javax.naming.Context; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttribute; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; + +import static org.junit.Assert.assertTrue; + +public class TestQueryDNS { +private QueryDNS queryDNS; +private TestRunner queryDNSTestRunner; + +@Before +public void setupTest() throws Exception { +this.queryDNS = new QueryDNS(); +this.queryDNSTestRunner = TestRunners.newTestRunner(queryDNS); + +Hashtable env = new Hashtable(); +env.put(Context.INITIAL_CONTEXT_FACTORY, FakeDNSInitialDirContextFactory.class.getName()); + +this.queryDNS.initializeContext(env); + +final DirContext mockContext = FakeDNSInitialDirContextFactory.getLatestMockContext(); + +// Capture JNDI's getAttibutes method containing the (String) queryValue and (String[]) queryType +Mockito.when( mockContext.getAttributes(Mockito.anyString(), Mockito.any(String[].class))) +.thenAnswer(new Answer() { +public Object answer(InvocationOnMock invocation) throws Throwable { +// Craft a false DNS response +// Note the DNS response will not make use of any of the mocked +// query contents (all input is discarded and replies synthetically +// generated +return craftResponse(invocation); +} +}); +} + +@Test +public void testVanillaQueryWithoutSplit() { +queryDNSTestRunner.setProperty(QueryDNS.DNS_QUERY_TYPE, "PTR"); +queryDNSTestRunner.setProperty(QueryDNS.DNS_RETRIES, "1"); +queryDNSTestRunner.setProperty(QueryDNS.DNS_TIMEOUT, "1000"); +queryDNSTestRunner.setProperty(QueryDNS.QUERY_INPUT, "${ip_address:getDelimitedField(4, '.'):trim()}" + +".${ip_address:getDelimitedField(3, '.'):trim()}" + +".${ip_address:getDelimitedField(2, '.'):trim()}" + +".${ip_address:getDelimitedField(1, '.'):trim()}" + +".in-addr.arpa"); +queryDNSTestRunner.setProperty(QueryDNS.QUERY_PARSER, QueryDNS.NONE.getValue()); + +final Map attributeMap = new HashMap<>(); +attributeMap.put("ip_address", "123.123.123.123"); + +queryDNSTestRunner.enqueue(new byte[0], attributeMap); +queryDNSTestRunner.enqueue("teste teste teste chocolate".getBytes()); + +queryDNSTestRunner.run(1,true, false); + +List results = queryDNSTestRunner.getFlowFilesForRelationship(QueryDNS.REL_FOUND); +assertTrue(results.size() == 1); +String result = results.get(0).getAttribute("enrich.dns.record0.group0"); + +assertTrue(result.contains("apache.nifi.org")); + + +} + +@Test +public void testValidDataWithSplit() { +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72593682 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/AbstractEnrichProcessor.java --- @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractEnrichProcessor extends AbstractProcessor { +public static final PropertyDescriptor QUERY_INPUT = new PropertyDescriptor.Builder() +.name("Format the query should be to be executed ") +.required(true) +.description("The value that should be used to populate the query") +.expressionLanguageSupported(true) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final AllowableValue SPLIT= new AllowableValue("Split", "Split", +"String.splitUse a delimiter character or RegEx to split the results into attributes"); +public static final AllowableValue REGEX = new AllowableValue("RegEx", "RegEx", +"Use a regular expression to split the results into attributes "); +public static final AllowableValue NONE = new AllowableValue("None", "None", +"Do not split results"); + +public static final PropertyDescriptor QUERY_PARSER = new PropertyDescriptor.Builder() +.name("Results Parser") +.description("The method used to slice the results into attribute groups") +.allowableValues(SPLIT, REGEX, NONE) +.required(true) +.defaultValue(NONE.getValue()) +.build(); + +public static final PropertyDescriptor QUERY_PARSER_INPUT = new PropertyDescriptor.Builder() +.name("Parser RegEx") +.description("Choice between a splitter and regex matcher used to parse the results of the query into attribute groups") +.expressionLanguageSupported(false) +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + + +public static final Relationship REL_FOUND = new Relationship.Builder() +.name("found") +.description("Where to route flow files after successfully enriching attributes with data") +.build(); + +public static final Relationship REL_NOT_FOUND = new Relationship.Builder() +.name("not found") +.description("Where to route flow if data enrichment query rendered no results") +.build(); + + +@Override +public List customValidate(ValidationContext validationContext) { +final List results = new ArrayList<>(super.customValidate(validationContext)); + +final String chosenQUERY_PARSER = validationContext.getProperty(QUERY_PARSER).getValue(); +final boolean QUERY_PARSER_INPUT_isSet = validationContext.getProperty(QUERY_PARSER_INPUT).isSet(); + +if ((!chosenQUERY_PARSER.equals(NONE.getValue()) ) && ( !QUERY_PARSER_INPUT_isSet )) { +results.add(new ValidationResult.Builder().input("Parser RegEx") +.explanation("Split and Regex parsers require a valid Regular Expression") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72592665 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/AbstractEnrichProcessor.java --- @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractEnrichProcessor extends AbstractProcessor { +public static final PropertyDescriptor QUERY_INPUT = new PropertyDescriptor.Builder() +.name("Format the query should be to be executed ") +.required(true) +.description("The value that should be used to populate the query") +.expressionLanguageSupported(true) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final AllowableValue SPLIT= new AllowableValue("Split", "Split", +"String.splitUse a delimiter character or RegEx to split the results into attributes"); --- End diff -- addressed --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72592679 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/AbstractEnrichProcessor.java --- @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractEnrichProcessor extends AbstractProcessor { +public static final PropertyDescriptor QUERY_INPUT = new PropertyDescriptor.Builder() +.name("Format the query should be to be executed ") +.required(true) +.description("The value that should be used to populate the query") +.expressionLanguageSupported(true) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final AllowableValue SPLIT= new AllowableValue("Split", "Split", +"String.splitUse a delimiter character or RegEx to split the results into attributes"); +public static final AllowableValue REGEX = new AllowableValue("RegEx", "RegEx", +"Use a regular expression to split the results into attributes "); +public static final AllowableValue NONE = new AllowableValue("None", "None", +"Do not split results"); + +public static final PropertyDescriptor QUERY_PARSER = new PropertyDescriptor.Builder() +.name("Results Parser") +.description("The method used to slice the results into attribute groups") +.allowableValues(SPLIT, REGEX, NONE) +.required(true) +.defaultValue(NONE.getValue()) +.build(); + +public static final PropertyDescriptor QUERY_PARSER_INPUT = new PropertyDescriptor.Builder() +.name("Parser RegEx") +.description("Choice between a splitter and regex matcher used to parse the results of the query into attribute groups") +.expressionLanguageSupported(false) +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + + +public static final Relationship REL_FOUND = new Relationship.Builder() +.name("found") +.description("Where to route flow files after successfully enriching attributes with data") +.build(); + +public static final Relationship REL_NOT_FOUND = new Relationship.Builder() +.name("not found") +.description("Where to route flow if data enrichment query rendered no results") --- End diff -- addressed --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72592085 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/AbstractEnrichProcessor.java --- @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractEnrichProcessor extends AbstractProcessor { +public static final PropertyDescriptor QUERY_INPUT = new PropertyDescriptor.Builder() +.name("Format the query should be to be executed ") --- End diff -- addressed --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72530345 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/test/java/org/apache/nifi/processors/enrich/TestQueryDNS.java --- @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; + +import java.util.HashMap; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + + +import javax.naming.Context; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttribute; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; + +import static org.junit.Assert.assertTrue; + +public class TestQueryDNS { +private QueryDNS queryDNS; +private TestRunner queryDNSTestRunner; + +@Before +public void setupTest() throws Exception { +this.queryDNS = new QueryDNS(); +this.queryDNSTestRunner = TestRunners.newTestRunner(queryDNS); + +Hashtable env = new Hashtable(); +env.put(Context.INITIAL_CONTEXT_FACTORY, FakeDNSInitialDirContextFactory.class.getName()); + +this.queryDNS.initializeContext(env); + +final DirContext mockContext = FakeDNSInitialDirContextFactory.getLatestMockContext(); + +// Capture JNDI's getAttibutes method containing the (String) queryValue and (String[]) queryType +Mockito.when( mockContext.getAttributes(Mockito.anyString(), Mockito.any(String[].class))) +.thenAnswer(new Answer() { +public Object answer(InvocationOnMock invocation) throws Throwable { +// Craft a false DNS response +// Note the DNS response will not make use of any of the mocked +// query contents (all input is discarded and replies synthetically +// generated +return craftResponse(invocation); +} +}); +} + +@Test +public void testVanillaQueryWithoutSplit() { +queryDNSTestRunner.setProperty(QueryDNS.DNS_QUERY_TYPE, "PTR"); +queryDNSTestRunner.setProperty(QueryDNS.DNS_RETRIES, "1"); +queryDNSTestRunner.setProperty(QueryDNS.DNS_TIMEOUT, "1000"); +queryDNSTestRunner.setProperty(QueryDNS.QUERY_INPUT, "${ip_address:getDelimitedField(4, '.'):trim()}" + +".${ip_address:getDelimitedField(3, '.'):trim()}" + +".${ip_address:getDelimitedField(2, '.'):trim()}" + +".${ip_address:getDelimitedField(1, '.'):trim()}" + +".in-addr.arpa"); +queryDNSTestRunner.setProperty(QueryDNS.QUERY_PARSER, QueryDNS.NONE.getValue()); + +final Map attributeMap = new HashMap<>(); +attributeMap.put("ip_address", "123.123.123.123"); + +queryDNSTestRunner.enqueue(new byte[0], attributeMap); +queryDNSTestRunner.enqueue("teste teste teste chocolate".getBytes()); + +queryDNSTestRunner.run(1,true, false); + +List results = queryDNSTestRunner.getFlowFilesForRelationship(QueryDNS.REL_FOUND); +assertTrue(results.size() == 1); +String result = results.get(0).getAttribute("enrich.dns.record0.group0"); + +assertTrue(result.contains("apache.nifi.org")); + + +} + +@Test +public void testValidDataWithSplit() { +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72530250 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72530181 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72529865 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72528893 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72527914 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/QueryDNS.java --- @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.naming.Context; +import javax.naming.NameNotFoundException; +import javax.naming.NamingEnumeration; +import javax.naming.NamingException; +import javax.naming.directory.Attributes; +import javax.naming.directory.BasicAttributes; +import javax.naming.directory.DirContext; +import javax.naming.directory.InitialDirContext; + +import org.apache.commons.lang3.StringUtils; + +import org.apache.nifi.annotation.behavior.EventDriven; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; + + +@EventDriven +@SideEffectFree +@SupportsBatching +@Tags({"dns", "enrich", "ip"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("A powerful DNS query processor primary designed to enrich DataFlows with DNS based APIs " + +"(e.g. RBLs, ShadowServer's ASN lookup) but that can be also used to perform regular DNS lookups.") +@WritesAttributes({ +@WritesAttribute(attribute = "enrich.dns.record*.group*", description = "The captured fields of the DNS query response for each of the records received"), +}) +public class QueryDNS extends AbstractEnrichProcessor { + +public static final PropertyDescriptor DNS_QUERY_TYPE = new PropertyDescriptor.Builder() +.name("DNS Query Type") +.description("The DNS query type to be used by the processor (e.g. TXT, A)") +.required(true) +.defaultValue("TXT") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_SERVER = new PropertyDescriptor.Builder() +.name("DNS Servers") +.description("A comma separated list of DNS servers to be used. (Defaults to system wide if none is used)") +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_TIMEOUT = new PropertyDescriptor.Builder() +.name("DNS Query Timeout") +.description("The amount of milliseconds to wait until considering a query as failed") +.required(true) +.defaultValue("1500") +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final PropertyDescriptor DNS_RETRIES = new PropertyDescriptor.Builder() +.name("DNS Query Retries") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72527360 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/AbstractEnrichProcessor.java --- @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractEnrichProcessor extends AbstractProcessor { +public static final PropertyDescriptor QUERY_INPUT = new PropertyDescriptor.Builder() +.name("Format the query should be to be executed ") +.required(true) +.description("The value that should be used to populate the query") +.expressionLanguageSupported(true) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final AllowableValue SPLIT= new AllowableValue("Split", "Split", +"String.splitUse a delimiter character or RegEx to split the results into attributes"); +public static final AllowableValue REGEX = new AllowableValue("RegEx", "RegEx", +"Use a regular expression to split the results into attributes "); +public static final AllowableValue NONE = new AllowableValue("None", "None", +"Do not split results"); + +public static final PropertyDescriptor QUERY_PARSER = new PropertyDescriptor.Builder() +.name("Results Parser") +.description("The method used to slice the results into attribute groups") +.allowableValues(SPLIT, REGEX, NONE) +.required(true) +.defaultValue(NONE.getValue()) +.build(); + +public static final PropertyDescriptor QUERY_PARSER_INPUT = new PropertyDescriptor.Builder() +.name("Parser RegEx") +.description("Choice between a splitter and regex matcher used to parse the results of the query into attribute groups") +.expressionLanguageSupported(false) +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + + +public static final Relationship REL_FOUND = new Relationship.Builder() +.name("found") +.description("Where to route flow files after successfully enriching attributes with data") +.build(); + +public static final Relationship REL_NOT_FOUND = new Relationship.Builder() +.name("not found") +.description("Where to route flow if data enrichment query rendered no results") +.build(); + + +@Override +public List customValidate(ValidationContext validationContext) { +final List results = new ArrayList<>(super.customValidate(validationContext)); + +final String chosenQUERY_PARSER = validationContext.getProperty(QUERY_PARSER).getValue(); +final boolean QUERY_PARSER_INPUT_isSet = validationContext.getProperty(QUERY_PARSER_INPUT).isSet(); + +if ((!chosenQUERY_PARSER.equals(NONE.getValue()) ) && ( !QUERY_PARSER_INPUT_isSet )) { +results.add(new ValidationResult.Builder().input("Parser RegEx") +.explanation("Split and Regex parsers require a valid Regular Expression") +
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72526887 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/AbstractEnrichProcessor.java --- @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractEnrichProcessor extends AbstractProcessor { +public static final PropertyDescriptor QUERY_INPUT = new PropertyDescriptor.Builder() +.name("Format the query should be to be executed ") +.required(true) +.description("The value that should be used to populate the query") +.expressionLanguageSupported(true) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final AllowableValue SPLIT= new AllowableValue("Split", "Split", +"String.splitUse a delimiter character or RegEx to split the results into attributes"); +public static final AllowableValue REGEX = new AllowableValue("RegEx", "RegEx", +"Use a regular expression to split the results into attributes "); +public static final AllowableValue NONE = new AllowableValue("None", "None", +"Do not split results"); + +public static final PropertyDescriptor QUERY_PARSER = new PropertyDescriptor.Builder() +.name("Results Parser") +.description("The method used to slice the results into attribute groups") +.allowableValues(SPLIT, REGEX, NONE) +.required(true) +.defaultValue(NONE.getValue()) +.build(); + +public static final PropertyDescriptor QUERY_PARSER_INPUT = new PropertyDescriptor.Builder() +.name("Parser RegEx") +.description("Choice between a splitter and regex matcher used to parse the results of the query into attribute groups") +.expressionLanguageSupported(false) +.required(false) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + + +public static final Relationship REL_FOUND = new Relationship.Builder() +.name("found") +.description("Where to route flow files after successfully enriching attributes with data") +.build(); + +public static final Relationship REL_NOT_FOUND = new Relationship.Builder() +.name("not found") +.description("Where to route flow if data enrichment query rendered no results") --- End diff -- 'flow files' (to be consistent with 'found' relationship --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72526705 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/AbstractEnrichProcessor.java --- @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractEnrichProcessor extends AbstractProcessor { +public static final PropertyDescriptor QUERY_INPUT = new PropertyDescriptor.Builder() +.name("Format the query should be to be executed ") +.required(true) +.description("The value that should be used to populate the query") +.expressionLanguageSupported(true) +.addValidator(StandardValidators.NON_EMPTY_VALIDATOR) +.build(); + +public static final AllowableValue SPLIT= new AllowableValue("Split", "Split", +"String.splitUse a delimiter character or RegEx to split the results into attributes"); --- End diff -- wording? --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72526574 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/AbstractEnrichProcessor.java --- @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractEnrichProcessor extends AbstractProcessor { +public static final PropertyDescriptor QUERY_INPUT = new PropertyDescriptor.Builder() +.name("Format the query should be to be executed ") --- End diff -- please use name() and displayName() when defining properties [1] https://mail-archives.apache.org/mod_mbox/nifi-dev/201605.mbox/%3c7e801f27-09cb-4c02-983d-f34ac4420...@hortonworks.com%3E --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user pvillard31 commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r72526422 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/main/java/org/apache/nifi/processors/enrich/AbstractEnrichProcessor.java --- @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.enrich; + + +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public abstract class AbstractEnrichProcessor extends AbstractProcessor { +public static final PropertyDescriptor QUERY_INPUT = new PropertyDescriptor.Builder() +.name("Format the query should be to be executed ") --- End diff -- wording? --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---
[GitHub] nifi pull request #496: NIFI-1965 - Implement QueryDNS Processor
Github user trixpan commented on a diff in the pull request: https://github.com/apache/nifi/pull/496#discussion_r71655778 --- Diff: nifi-nar-bundles/nifi-enrich-bundle/nifi-enrich-processors/src/test/java/org/apache/nifi/processors/TestQueryDNS.java --- @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors; + + +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Test; + + + +import static org.junit.Assert.assertTrue; + +public class TestQueryDNS { --- End diff -- addressed --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---