lordgamez commented on code in PR #2072: URL: https://github.com/apache/nifi-minifi-cpp/pull/2072#discussion_r2668526046
########## extensions/grafana-loki/tests/features/resources/reverse-proxy/nginx-basic-auth.conf: ########## Review Comment: You are right, I was not aware of the copyright rules for github gists in case no license is defined, now I see that in this case the gist is copyrighted to the author and cannot be reused without permission. I deleted the files are rewritten them without the original source in https://github.com/apache/nifi-minifi-cpp/pull/2072/commits/8cf1a6c925cdb93ba6e1be3b8bd1f175ef7c4480 ########## extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py: ########## @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import requests +import time + + +def wait_for(action, timeout_seconds, *args, **kwargs) -> bool: + start_time = time.perf_counter() + while True: + result = action(*args, **kwargs) + if result: + return result + time.sleep(1) + if timeout_seconds < (time.perf_counter() - start_time): + break Review Comment: Updated in https://github.com/apache/nifi-minifi-cpp/pull/2072/commits/8cf1a6c925cdb93ba6e1be3b8bd1f175ef7c4480 ########## extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py: ########## @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import requests +import time + + +def wait_for(action, timeout_seconds, *args, **kwargs) -> bool: + start_time = time.perf_counter() + while True: + result = action(*args, **kwargs) + if result: + return result + time.sleep(1) + if timeout_seconds < (time.perf_counter() - start_time): + break + return False + + +def veify_log_lines_on_grafana_loki(host: str, lines: list[str], ssl: bool, tenant_id: str) -> bool: Review Comment: Fixed in https://github.com/apache/nifi-minifi-cpp/pull/2072/commits/8cf1a6c925cdb93ba6e1be3b8bd1f175ef7c4480 ########## extensions/grafana-loki/tests/features/steps/grafana_loki_container.py: ########## @@ -0,0 +1,145 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from OpenSSL import crypto + +from minifi_test_framework.core.helpers import wait_for_condition, retry_check +from minifi_test_framework.containers.container import Container +from minifi_test_framework.containers.file import File +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.ssl_utils import make_server_cert + + +class GrafanaLokiOptions: + def __init__(self, enable_ssl: bool = False, enable_multi_tenancy: bool = False): + self.enable_ssl = enable_ssl + self.enable_multi_tenancy = enable_multi_tenancy + + +class GrafanaLokiContainer(Container): + def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions): + super().__init__("grafana/loki:3.2.1", f"grafana-loki-server-{test_context.scenario_id}", test_context.network) + extra_ssl_settings = "" + if options.enable_ssl: + grafana_loki_cert, grafana_loki_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + + root_ca_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert) + self.files.append(File("/etc/loki/root_ca.crt", root_ca_content, permissions=0o644)) + + grafana_loki_cert_content = crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=grafana_loki_cert) + self.files.append(File("/etc/loki/cert.pem", grafana_loki_cert_content, permissions=0o644)) + + grafana_loki_key_content = crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=grafana_loki_key) + self.files.append(File("/etc/loki/key.pem", grafana_loki_key_content, permissions=0o644)) + + extra_ssl_settings = """ + http_tls_config: + cert_file: /etc/loki/cert.pem + key_file: /etc/loki/key.pem + client_ca_file: /etc/loki/root_ca.crt + client_auth_type: VerifyClientCertIfGiven + + grpc_tls_config: + cert_file: /etc/loki/cert.pem + key_file: /etc/loki/key.pem + client_ca_file: /etc/loki/root_ca.crt + client_auth_type: VerifyClientCertIfGiven + +query_scheduler: + grpc_client_config: + grpc_compression: snappy + tls_enabled: true + tls_ca_path: /etc/loki/root_ca.crt + tls_insecure_skip_verify: true + +ingester_client: + grpc_client_config: + grpc_compression: snappy + tls_enabled: true + tls_ca_path: /etc/loki/root_ca.crt + tls_insecure_skip_verify: true + +frontend: + grpc_client_config: + grpc_compression: snappy + tls_enabled: true + tls_ca_path: /etc/loki/root_ca.crt + tls_insecure_skip_verify: true + +frontend_worker: + grpc_client_config: + grpc_compression: snappy + tls_enabled: true + tls_ca_path: /etc/loki/root_ca.crt + tls_insecure_skip_verify: true +""" + + grafana_loki_yml_content = """ +auth_enabled: {enable_multi_tenancy} + +server: + http_listen_port: 3100 + grpc_listen_port: 9095 +{extra_ssl_settings} + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-05-15 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +ruler: + alertmanager_url: http://localhost:9093 + +analytics: + reporting_enabled: false +""".format(extra_ssl_settings=extra_ssl_settings, enable_multi_tenancy=options.enable_multi_tenancy) + + self.files.append(File("/etc/loki/local-config.yaml", grafana_loki_yml_content.encode(), permissions=0o644)) + + def deploy(self): + super().deploy() + finished_str = "Loki started" + return wait_for_condition( + condition=lambda: finished_str in self.get_logs(), + timeout_seconds=120, + bail_condition=lambda: self.exited, + context=None) + + @retry_check() + def are_lines_present(self, lines: str, timeout: int, ssl: bool, tenant_id: str = "") -> bool: + try: + self.client.containers.run("minifi-grafana-loki-helper:latest", ["python", "/scripts/check_log_lines_on_grafana.py", self.container_name, lines, str(timeout), str(ssl), tenant_id], + remove=True, stdout=True, stderr=True, network=self.network.name) + return True + except Exception as e: + logging.error(f"Failed to run python command in grafana loki helper docker: {e}") Review Comment: Added log in https://github.com/apache/nifi-minifi-cpp/pull/2072/commits/8cf1a6c925cdb93ba6e1be3b8bd1f175ef7c4480 ########## extensions/grafana-loki/tests/features/steps/steps.py: ########## @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from behave import step, then + +from minifi_test_framework.steps import checking_steps # noqa: F401 +from minifi_test_framework.steps import configuration_steps # noqa: F401 +from minifi_test_framework.steps import core_steps # noqa: F401 +from minifi_test_framework.steps import flow_building_steps # noqa: F401 +from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.helpers import log_due_to_failure +from grafana_loki_container import GrafanaLokiContainer, GrafanaLokiOptions +from reverse_proxy_container import ReverseProxyContainer + + +@step("a Grafana Loki server is set up") +def step_impl(context: MinifiTestContext): + context.containers["grafana-loki-server"] = GrafanaLokiContainer(context, GrafanaLokiOptions()) + + +@step("a Grafana Loki server is set up with multi-tenancy enabled") +def step_impl(context: MinifiTestContext): + context.containers["grafana-loki-server"] = GrafanaLokiContainer(context, GrafanaLokiOptions(enable_multi_tenancy=True)) + + +@step("a Grafana Loki server with SSL is set up") +def step_impl(context: MinifiTestContext): + context.containers["grafana-loki-server"] = GrafanaLokiContainer(context, GrafanaLokiOptions(enable_ssl=True)) + + +@then("\"{lines}\" lines are published to the Grafana Loki server in less than {timeout_seconds:d} seconds") +@then("\"{lines}\" line is published to the Grafana Loki server in less than {timeout_seconds:d} seconds") +def step_impl(context, lines: str, timeout_seconds: int): + assert context.containers["grafana-loki-server"].are_lines_present(lines, timeout_seconds, False) or log_due_to_failure(context) Review Comment: Updated in https://github.com/apache/nifi-minifi-cpp/pull/2072/commits/8cf1a6c925cdb93ba6e1be3b8bd1f175ef7c4480 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
