This is an automated email from the ASF dual-hosted git repository. harishgokul01 pushed a commit to branch demo in repository https://gitbox.apache.org/repos/asf/incubator-resilientdb.git
commit 811489294c0367af57ee6813b833f6b33ac90917 Author: harish876 <[email protected]> AuthorDate: Sat Dec 20 04:19:52 2025 +0000 Add .env to .gitignore, enable global stats increment in PBFT commitment, update response handling in Stats, and introduce ResLens Flamegraph Analysis Service with Gunicorn configuration and startup scripts. --- .gitignore | 1 + platform/consensus/ordering/pbft/commitment.cpp | 4 +- platform/statistic/stats.cpp | 5 +- scripts/deploy/script/restart_reslens.sh | 14 ++ service/tools/config/server/server.config | 5 +- .../tools/kv/api_tools/monitoring/gunicorn.conf.py | 38 ++++ .../tools/kv/api_tools/monitoring/requirements.txt | 10 + .../api_tools/monitoring/reslens_tools_service.py | 243 +++++++++++++++++++++ .../monitoring/start_reslens_tools_service.sh | 75 +++++++ 9 files changed, 389 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index acd52c56..d5db0cc9 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ resdb/ gmon.out .history/ service/tools/data/cert/ +*.env # Generated LLM cache and Pocketflow outputs llm_cache.json ecosystem/pocketflow/.pf-output/ diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 4d048331..4a1aacc0 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -245,7 +245,7 @@ int Commitment::ProcessPrepareMsg(std::unique_ptr<Context> context, return message_manager_->AddConsensusMsg(context->signature, std::move(request)); } - //global_stats_->IncPrepare(); + global_stats_->IncPrepare(); std::unique_ptr<Request> commit_request = resdb::NewRequest( Request::TYPE_COMMIT, *request, config_.GetSelfInfo().id()); commit_request->mutable_data_signature()->Clear(); @@ -288,7 +288,7 @@ int Commitment::ProcessCommitMsg(std::unique_ptr<Context> context, return message_manager_->AddConsensusMsg(context->signature, std::move(request)); } - //global_stats_->IncCommit(); + global_stats_->IncCommit(); // Add request to message_manager. // If it has received enough same requests(2f+1), message manager will // commit the request. diff --git a/platform/statistic/stats.cpp b/platform/statistic/stats.cpp index 21f1524f..dfc1957d 100644 --- a/platform/statistic/stats.cpp +++ b/platform/statistic/stats.cpp @@ -167,12 +167,13 @@ void Stats::CrowRoute() { res.set_header( "Access-Control-Allow-Headers", "Content-Type, Authorization"); // Specify allowed headers - + + res.body = "Not Enabled"; // Send your response if (enable_faulty_switch_) { make_faulty_.store(!make_faulty_.load()); + res.body = "Success"; } - res.body = "Success"; res.end(); }); CROW_ROUTE(app, "/transaction_data") diff --git a/scripts/deploy/script/restart_reslens.sh b/scripts/deploy/script/restart_reslens.sh new file mode 100755 index 00000000..229fe830 --- /dev/null +++ b/scripts/deploy/script/restart_reslens.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd /home/ubuntu/resilient-monitoring/ResLens-Middleware + +# Stop existing containers (if any) +sudo docker-compose down || true + +# Start containers in detached mode +sudo docker-compose up -d + + + diff --git a/service/tools/config/server/server.config b/service/tools/config/server/server.config index 88fd8c50..8b435651 100644 --- a/service/tools/config/server/server.config +++ b/service/tools/config/server/server.config @@ -47,8 +47,9 @@ block_cache_capacity: 100 }, require_txn_validation:true, - enable_viewchange:false, + enable_viewchange:true, enable_resview:true, - enable_faulty_switch:false + enable_faulty_switch:false, + recovery_enabled:false, } diff --git a/service/tools/kv/api_tools/monitoring/gunicorn.conf.py b/service/tools/kv/api_tools/monitoring/gunicorn.conf.py new file mode 100644 index 00000000..d763514e --- /dev/null +++ b/service/tools/kv/api_tools/monitoring/gunicorn.conf.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +""" +Gunicorn configuration for ResLens Flamegraph Analysis Service +""" + +# Server socket +bind = "0.0.0.0:8080" +backlog = 2048 + +# Worker processes +workers = 1 # Single worker for this service +worker_class = "sync" +worker_connections = 1000 +max_requests = 1000 +max_requests_jitter = 50 + +# Timeout +timeout = 30 +keepalive = 2 + +# Logging +accesslog = "-" +errorlog = "-" +loglevel = "info" +access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + +# Process naming +proc_name = "reslens-flamegraph-service" + +# Preload app for better performance +preload_app = True + +# Worker timeout +graceful_timeout = 30 + +# Restart workers after this many requests +max_requests = 1000 +max_requests_jitter = 50 \ No newline at end of file diff --git a/service/tools/kv/api_tools/monitoring/requirements.txt b/service/tools/kv/api_tools/monitoring/requirements.txt new file mode 100644 index 00000000..81e680cb --- /dev/null +++ b/service/tools/kv/api_tools/monitoring/requirements.txt @@ -0,0 +1,10 @@ +blinker==1.9.0 +click==8.2.1 +Flask==3.1.1 +flask-cors==6.0.1 +gunicorn==23.0.0 +itsdangerous==2.2.0 +Jinja2==3.1.6 +MarkupSafe==3.0.2 +packaging==25.0 +Werkzeug==3.1.3 diff --git a/service/tools/kv/api_tools/monitoring/reslens_tools_service.py b/service/tools/kv/api_tools/monitoring/reslens_tools_service.py new file mode 100644 index 00000000..964e7311 --- /dev/null +++ b/service/tools/kv/api_tools/monitoring/reslens_tools_service.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +""" +ResLens Flamegraph Analysis Service + +A simple HTTP service for executing ResilientDB random data operations +as part of the ResLens monitoring and analysis toolkit. + +This utility is used mainly to seed data to create a long running data seeding job for flamegroah analysis as these flamegraph processes need to run for more than 30s. +""" + +import os +import sys +import json +import subprocess +import random +import threading +import time +from flask import Flask, request, jsonify +from flask_cors import CORS + +app = Flask(__name__) +CORS(app) + +class ResLensToolsService: + def __init__(self): + self.project_root = self._find_project_root() + self.seeding_running = False + self.seeding_thread = None + self.seeding_lock = threading.Lock() + print(f"ResLens Tools Service - Using project root: {self.project_root}") + + def _find_project_root(self): + """Find the ResilientDB project root directory.""" + resilientdb_root = os.getenv("RESILIENTDB_ROOT") + if resilientdb_root: + return resilientdb_root + + # Since we're now in service/tools/kv/api_tools/monitoring, go up to project root + current_dir = os.path.dirname(os.path.abspath(__file__)) + project_root = os.path.join(current_dir, "../../../../../../") + + # Verify this is the project root by checking for bazel-bin + if os.path.exists(os.path.join(project_root, "bazel-bin")): + return project_root + + # Fallback to common paths + possible_paths = [ + "/opt/resilientdb", + "/home/ubuntu/incubator-resilientdb", + os.path.join(os.path.expanduser("~"), "resilientdb") + ] + + for path in possible_paths: + tool_path = os.path.join(path, "bazel-bin/service/tools/kv/api_tools/kv_service_tools") + if os.path.exists(tool_path): + return path + + return "/opt/resilientdb" + + def _get_tool_path(self): + """Get the path to the kv_service_tools binary.""" + return f"{self.project_root}/bazel-bin/service/tools/kv/api_tools/kv_service_tools" + + def _check_tool_exists(self): + """Check if the CLI tool exists.""" + tool_path = self._get_tool_path() + return os.path.exists(tool_path) and os.access(tool_path, os.X_OK) + + def start_seeding(self, count): + """Start data seeding job in background thread.""" + if self.seeding_running: + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "Seeding job already running" + } + + self.seeding_running = True + + def seeding_worker(): + for i in range(count): + if not self.seeding_running: + break + + key = f"key{random.randint(0, 499)}" + value = f"value{random.randint(0, 499)}" + + cmd = [ + self._get_tool_path(), + "--config", f"{self.project_root}/service/tools/config/interface/service.config", + "--cmd", "set", + "--key", key, + "--value", value + ] + + try: + subprocess.run(cmd, capture_output=True, text=True, timeout=30) + except (subprocess.TimeoutExpired, Exception): + # Silently continue on errors - no logging needed + pass + + time.sleep(0.1) + + with self.seeding_lock: + self.seeding_running = False + + self.seeding_thread = threading.Thread(target=seeding_worker, daemon=True) + self.seeding_thread.start() + + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "success", + "message": f"Started seeding job with {count} operations" + } + + def stop_seeding(self): + """Stop the data seeding job.""" + if not self.seeding_running: + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "No seeding job running" + } + + self.seeding_running = False + if self.seeding_thread and self.seeding_thread.is_alive(): + self.seeding_thread.join(timeout=5) + + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "success", + "message": "Seeding job stopped" + } + + def get_seeding_status(self): + """Get current seeding job status.""" + with self.seeding_lock: + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "running" if self.seeding_running else "stopped" + } + +# Global service instance +service = ResLensToolsService() + [email protected]('/health', methods=['GET']) +def health(): + """Health check endpoint.""" + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "ok" + }) + [email protected]('/seed', methods=['POST']) +def start_seeding(): + """Start data seeding job.""" + try: + data = request.get_json() + if not data: + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "No JSON data provided" + }), 400 + + count = data.get('count') + if count is None: + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "Missing 'count' parameter" + }), 400 + + if not isinstance(count, int) or count <= 0: + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "Count must be a positive integer" + }), 400 + + # Check if CLI tool exists before starting + if not service._check_tool_exists(): + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": f"CLI tool not found at {service._get_tool_path()}" + }), 503 + + return jsonify(service.start_seeding(count)) + + except Exception as e: + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": str(e) + }), 500 + [email protected]('/stop', methods=['POST']) +def stop_seeding(): + """Stop data seeding job.""" + return jsonify(service.stop_seeding()) + [email protected]('/status', methods=['GET']) +def get_status(): + """Get seeding job status.""" + return jsonify(service.get_seeding_status()) + [email protected]('/', methods=['GET']) +def root(): + """Root endpoint with service information.""" + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "description": "HTTP service for executing ResilientDB random data operations", + "endpoints": { + "GET /health": "Health check", + "POST /seed": "Start data seeding job (JSON body: {\"count\": 5})", + "POST /stop": "Stop data seeding job", + "GET /status": "Get seeding job status" + }, + "example": { + "POST /seed": { + "body": {"count": 10}, + "response": "Starts background job to execute 10 random set operations" + }, + "POST /stop": { + "body": "{}", + "response": "Stops the running seeding job" + } + } + }) + +if __name__ == '__main__': + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8080 + + print(f"Starting ResLens Flamegraph Analysis Service on port {port}") + print("Available endpoints:") + print(" GET /health - Health check") + print(" POST /seed - Start data seeding job") + print(" POST /stop - Stop data seeding job") + print(" GET /status - Get seeding job status") + print(" GET / - Service information") + + app.run(host='0.0.0.0', port=port, debug=False, threaded=True) \ No newline at end of file diff --git a/service/tools/kv/api_tools/monitoring/start_reslens_tools_service.sh b/service/tools/kv/api_tools/monitoring/start_reslens_tools_service.sh new file mode 100755 index 00000000..f4ac21f6 --- /dev/null +++ b/service/tools/kv/api_tools/monitoring/start_reslens_tools_service.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Startup script for ResLens Flamegraph Analysis Service +# + +# Set the directory to the script location +cd "$(dirname "$0")" + +# Function to find and activate virtual environment +activate_venv() { + # Check for common virtual environment locations + local venv_paths=( + "venv" + "env" + ".venv" + ".env" + "../venv" + "../env" + "../../venv" + "../../env" + "/opt/resilientdb/venv" + "/opt/resilientdb/env" + ) + + for venv_path in "${venv_paths[@]}"; do + if [[ -d "$venv_path" && -f "$venv_path/bin/activate" ]]; then + echo "Found virtual environment at: $venv_path" + source "$venv_path/bin/activate" + return 0 + fi + done + + # Check if we're already in a virtual environment + if [[ -n "$VIRTUAL_ENV" ]]; then + echo "Already in virtual environment: $VIRTUAL_ENV" + return 0 + fi + + echo "No virtual environment found. Using system Python." + return 1 +} + +# Try to activate virtual environment +if activate_venv; then + echo "Using virtual environment: $VIRTUAL_ENV" +else + echo "Using system Python" +fi + +# Check Python version +python_version=$(python3 --version 2>&1) +echo "Python version: $python_version" + +# Check if gunicorn is installed +if ! python3 -c "import gunicorn" &> /dev/null; then + echo "Gunicorn not found. Installing..." + pip install gunicorn +fi + +# Check if Flask is installed +if ! python3 -c "import flask" &> /dev/null; then + echo "Flask not found. Installing..." + pip install flask flask-cors +fi + +echo "Starting ResLens Flamegraph Analysis Service with Gunicorn..." + +# Run with gunicorn +gunicorn \ + --config gunicorn.conf.py \ + --bind 0.0.0.0:8080 \ + --workers 1 \ + --timeout 30 \ + --log-level info \ + reslens_tools_service:app \ No newline at end of file
