This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 056d0386a6db133085b7cab2150b8b3034f2bc74 Author: Gabor Gyimesi <[email protected]> AuthorDate: Tue Jun 7 13:15:55 2022 +0200 MINFICPP-1856 Add kubernetes log collection config examples Closes #1345 Signed-off-by: Marton Szasz <[email protected]> --- examples/kubernetes/README.md | 30 +++++ examples/kubernetes/daemon-set-log-collection.yml | 124 ++++++++++++++++++ examples/kubernetes/sidecar-log-collection.yml | 147 ++++++++++++++++++++++ 3 files changed, 301 insertions(+) diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md new file mode 100644 index 000000000..1ff7a47b2 --- /dev/null +++ b/examples/kubernetes/README.md @@ -0,0 +1,30 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +# Kubernetes Examples + +The following examples show different configurations that can be applied in Kubernetes for log collection use cases. + +## Cluster level log collection with MiNiFi C++ + +The [daemon-set-log-collection.yml](daemon-set-log-collection.yml) file has an example for cluster level log collection, which is done on every node by creating a daemon set. +The config includes a KubernetesControllerService that provides the namespace, pod, uid, container variables for the TailFile processor for getting the logs for the filtered Kubernetes objects. +In this specific example all container logs from the default namespace are collected and forwarded to Kafka. +The controller service can be modified to have additional filters for namespaces, pods, containers, for which more information can be found in the [CONTROLLERS.md](/CONTROLLERS.md#kubernetesControllerService) documentation. +This setup complies with the ["node logging agent"](https://kubernetes.io/docs/concepts/cluster-administration/logging/#using-a-node-logging-agent) architecture described in the Kubernetes documentation. + +## Pod level log collection with sidecar container using MiNiFi C++ + +The [sidecar-log-collection.yml](sidecar-log-collection.yml) file has an example for pod level log collection, which is done by creating a sidecar container in the same pod where the container we want to collect the logs from is present. In this specific example a pod with a NiFi container is instantiated with a MiNiFi sidecar container which collects, compresses and uploads the NiFi logs to an AWS S3 bucket. +This setup complies with the ["sidecar container with logging agent"](https://kubernetes.io/docs/concepts/cluster-administration/logging/#sidecar-container-with-logging-agent) architecture described in the Kubernetes documentation. diff --git a/examples/kubernetes/daemon-set-log-collection.yml b/examples/kubernetes/daemon-set-log-collection.yml new file mode 100644 index 000000000..ce2d63ee1 --- /dev/null +++ b/examples/kubernetes/daemon-set-log-collection.yml @@ -0,0 +1,124 @@ + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +data: + minifi-log.properties: | + spdlog.pattern=[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v + appender.stderr=stderr + logger.root=INFO,stderr + logger.org::apache::nifi::minifi=INFO,stderr + minifi.properties: | + nifi.flow.configuration.file=./conf/config.yml + nifi.administrative.yield.duration=30 sec + nifi.bored.yield.duration=100 millis + nifi.extension.path=../extensions/* + + nifi.provenance.repository.directory.default=${MINIFI_HOME}/provenance_repository + nifi.provenance.repository.max.storage.time=1 MIN + nifi.provenance.repository.max.storage.size=1 MB + nifi.flowfile.repository.directory.default=${MINIFI_HOME}/flowfile_repository + nifi.database.content.repository.directory.default=${MINIFI_HOME}/content_repository + nifi.provenance.repository.class.name=NoOpRepository + nifi.content.repository.class.name=DatabaseContentRepository + config.yml: | + MiNiFi Config Version: 3 + Flow Controller: + name: Read Kubernetes pod logs and publish them to Kafka + Processors: + - name: Tail Kubernetes log files + id: 891efd7f-2814-4068-9efd-5f258f4990d5 + class: org.apache.nifi.minifi.processors.TailFile + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + Properties: + tail-mode: Multiple file + tail-base-directory: /var/log/pods/${namespace}_${pod}_${uid}/${container} + File to Tail: '.*\.log' + Attribute Provider Service: KubernetesControllerService + Lookup frequency: 10 min + Recursive lookup: 'false' + Initial Start Position: Beginning of File + Input Delimiter: \n + Rolling Filename Pattern: '${filename}.log.*' + - name: Publish messages to Kafka topic test + id: fb880b73-bff7-4775-a854-e048ae09e07e + class: org.apache.nifi.processors.standard.PublishKafka + scheduling strategy: EVENT_DRIVEN + auto-terminated relationships list: + - success + - failure + Properties: + Batch Size: '10' + Client Name: test-client + Compress Codec: none + Delivery Guarantee: '1' + Known Brokers: kafka-broker:9092 + Message Timeout: 12 sec + Request Timeout: 10 sec + Topic Name: ${kubernetes.namespace}_${kubernetes.pod}_${kubernetes.container}_logs + Connections: + - name: TailFile/success/PublishKafka + id: d6675f90-62a6-4f98-b67f-00efeab78e5e + source id: 891efd7f-2814-4068-9efd-5f258f4990d5 + source relationship name: success + destination id: fb880b73-bff7-4775-a854-e048ae09e07e + Controller Services: + - name: KubernetesControllerService + id: 568559dc-3c81-4b01-bc08-760fadb953b0 + type: org.apache.nifi.minifi.controllers.KubernetesControllerService + Properties: + Namespace Filter: default + Remote Process Groups: [] +kind: ConfigMap +metadata: + labels: + k8s-app: minifi-log-collection + name: minifi-log-collection-config + namespace: default +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: log-collection-minifi + namespace: default + labels: + k8s-app: minifi-log-collection +spec: + selector: + matchLabels: + name: log-collection-minifi + template: + metadata: + labels: + name: log-collection-minifi + spec: + containers: + - name: minifi + image: apache/nifi-minifi-cpp:latest + volumeMounts: + - name: minificonfig + mountPath: /opt/minifi/minifi-current/conf/config.yml + subPath: config.yml + - name: minificonfig + mountPath: /opt/minifi/minifi-current/conf/minifi-log.properties + subPath: minifi-log.properties + volumes: + - name: nifi-logs + emptyDir: {} + - configMap: + defaultMode: 420 + name: minifi-log-collection-config + name: minificonfig diff --git a/examples/kubernetes/sidecar-log-collection.yml b/examples/kubernetes/sidecar-log-collection.yml new file mode 100644 index 000000000..540a55170 --- /dev/null +++ b/examples/kubernetes/sidecar-log-collection.yml @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +apiVersion: v1 +data: + minifi-log.properties: | + spdlog.pattern=[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v + appender.stderr=stderr + logger.root=INFO,stderr + logger.org::apache::nifi::minifi=INFO,stderr + minifi.properties: | + nifi.flow.configuration.file=./conf/config.yml + nifi.administrative.yield.duration=30 sec + nifi.bored.yield.duration=100 millis + nifi.extension.path=../extensions/* + + nifi.provenance.repository.directory.default=${MINIFI_HOME}/provenance_repository + nifi.provenance.repository.max.storage.time=1 MIN + nifi.provenance.repository.max.storage.size=1 MB + nifi.flowfile.repository.directory.default=${MINIFI_HOME}/flowfile_repository + nifi.database.content.repository.directory.default=${MINIFI_HOME}/content_repository + nifi.provenance.repository.class.name=NoOpRepository + nifi.content.repository.class.name=DatabaseContentRepository + config.yml: | + Flow Controller: + name: minifi-logging + Processors: + - id: 94b8e610-b4ed-1ec9-b26f-c839931bf3e2 + name: TailFile + class: org.apache.nifi.processors.standard.TailFile + scheduling strategy: TIMER_DRIVEN + scheduling period: 5 sec + auto-terminated relationships list: [] + Properties: + File to Tail: /nifi-logs/nifi-app.log + Lookup frequency: 1 min + - id: 261e8cf1-71ba-af86-fb2b-bc95764f91f8 + name: MergeContent + class: org.apache.nifi.processors.standard.MergeContent + scheduling strategy: EVENT_DRIVEN + auto-terminated relationships list: + - original + Properties: + Attribute Strategy: Keep Only Common Attributes + Maximum number of Bins: 100 + Minimum Group Size: 0 + Max Bin Age: 1 min + Minimum Number of Entries: 1000000 + Maximum Group Size: 1000000 + Maximum Number of Entries: 1000000 + Merge Strategy: Bin-Packing Algorithm + - id: 69335770-ee29-11eb-9a03-0242ac130003 + name: CompressContent + class: org.apache.nifi.processors.standard.CompressContent + scheduling strategy: EVENT_DRIVEN + auto-terminated relationships list: + - failure + Properties: + Compression Level: 6 + Compression Format: gzip + UpdateFileName: false + - id: fe198bd9-2a1c-316e-0000-000000000000 + name: PutS3Object + class: org.apache.nifi.minifi.azure.processors.PutS3Object + scheduling strategy: EVENT_DRIVEN + auto-terminated relationships list: + - success + Properties: + Bucket: test-bucket + AWS Credentials Provider service: AWSCredentialsService + Controller Services: + - name: AWSCredentialsService + id: 2094d776-2006-4d02-9bb9-28eac9d0fc95 + class: org.apache.nifi.minifi.aws.controllers.AWSCredentialsService + Properties: + Use Default Credentials: 'true' # Can be used in Amazon EKS to retrieve credentials from metadata otherwise use your AWS Access Key and Secret Key + Connections: + - id: 99f617e7-49a1-6078-8534-26af7d56ca08 + name: TailFile/success/MergeContent + source name: TailFile + source relationship names: + - success + destination name: MergeContent + - id: 24d6be1e-ee29-11eb-9a03-0242ac130003 + name: MergeContent/merged/CompressContent + source name: MergeContent + source relationship names: + - merged + destination name: CompressContent + - id: 67ea5c91-446a-393b-6274-b6fae2f475a2 + name: CompressContent/success/PutS3Object + source name: CompressContent + source relationship names: + - success + destination name: PutS3Object + Remote Process Groups: [] +kind: ConfigMap +metadata: + labels: + k8s-app: minifi-log-collection + name: minifi-log-collection-config + namespace: default +--- +apiVersion: v1 +kind: Pod +metadata: + name: log-collection-minifi-pod + namespace: default +spec: + containers: + - name: nifi + image: apache/nifi:latest + volumeMounts: + - name: nifi-logs + mountPath: /opt/nifi/nifi-current/logs + - name: sidecar-minifi + image: apache/nifi-minifi-cpp:latest + volumeMounts: + - name: nifi-logs + mountPath: /nifi-logs + - name: minificonfig + mountPath: /opt/minifi/minifi-current/conf/config.yml + subPath: config.yml + - name: minificonfig + mountPath: /opt/minifi/minifi-current/conf/minifi-log.properties + subPath: minifi-log.properties + volumes: + - name: nifi-logs + emptyDir: {} + - configMap: + defaultMode: 420 + name: minifi-log-collection-config + name: minificonfig
