It sure would be nice if Radiator used IPC::Sharable or other similar
method to have shared stats structures across the forks so none of this
was necessary but here's my workaround. Thankfully it is possible, this
seemed like a better option that trying to do similar tricks with other
stats exporters. (And now I can get back to triggering alerts based on
per-client counters.)
TLDR; close/reactive management server port incrementing per fork, use
standalone http server to combine stats for each fork naively
incrementing from starting port until it encounters an error.
YMMV, but at least we can get aggregate stats via a single instance.
****
FarmSize 10
FarmChildHook file:"/etc/radiator/farmchild.pl"
<ServerHTTP>
Identifier MetricsHack
Port 8000
DefaultPrivilegeLevel 1
PageNotFoundHook file:"/etc/radiator/metrics-handler.pl"
</ServerHTTP>
*****
*****
/etc/radiator/farmchild.pl
sub {
my $server =
Radius::Configurable::find('ServerHTTP','MetricsHack');
my $port = $server->{Port} + $main::farmInstance;
&main::log($main::LOG_INFO, "FarmInstance $main::farmInstance
assigning ServerHTTP port to $port");
$server->{Port} = $port;
$server->activate();
}
*****
*****
/etc/radiator/metrics-handler.pl
sub {
my $uri = $_[0];
if($uri !~ /^\/metrics$/) {
return;
}
my $stats;
foreach ( keys %Radius::StatsLogGeneric::statistic_names) {
my $name = "radiator_$_";
my $value = $main::config->{Statistics}{$_};
$stats .= "$name $value\n";
}
for my $client (@{$main::config->{Client}}) {
my $name = $client->{Name};
for my $stat (keys $client->{Statistics} ) {
next if $stat eq 'Name';
$stats .= "radiatorClient_${stat}{client=\"$name\"} " .
$client->{Statistics}{$stat} . "\n";
}
}
return (200, $stats);
}
*****
*****
#!/usr/bin/perl
use strict;
use LWP::Simple;
use List::Util qw(sum);
use HTTP::Daemon;
use HTTP::Status;
use Sys::Syslog;
my $start_port = 8000;
my $server_port = 7999;
my $server = HTTP::Daemon->new( LocalPort => $server_port, Reuse => 1 )
or die;
while (my $client = $server->accept) {
while ( my $request = $client->get_request) {
if ( $request->method eq 'GET' and $request->uri->path
eq '/metrics' ) {
my $metrics = aggregate_metrics();
$client->send_response(HTTP::Response->new(200,undef,undef,$metrics));
} else {
$client->send_error(RC_FORBIDDEN);
}
}
$client->close;
}
sub aggregate_metrics {
my %counters;
my %gauges;
my $metrics;
my $count = 0;
my $port = $start_port;
while (1) {
$port++;
my $farmstat = get("http://127.0.0.1:$port/metrics";);
if ( $farmstat ) {
$count++;
for my $line (split /^/, $farmstat ) {
my ($label,$value) = split(/ /,$line);
if ( $label =~ /responseTime/ ) {
push(@{$gauges{$label}},$value);
next;
}
$counters{$label} += $value;
}
} else {
last;
}
}
for my $label (keys %gauges) {
$metrics .= $label . " " . (
sum(@{$gauges{$label}})/@{$gauges{$label}} ) . "\n";
}
for my $label (keys %counters) {
$metrics .= "$label $counters{$label}\n";
}
$metrics .="radiator_FarmActive $count\n";
eval {
openlog($0,'cons,pid','user');
syslog('info',"aggregated stats from $count farm workers");
closelog();
};
return $metrics;
}
*****
*****
# cat aggreagte-metrics.service
[Unit]
Description=Radiator Prometheus Metrics Aggregator Hack
[Service]
ExecStart=/etc/radiator/aggreagte-metrics.pl
[Install]
WantedBy=multi-user.target
*****
--
[email protected] sonic.net, inc.
System Architect 2260 Apollo Way
707.522.1000 Santa Rosa, CA
_______________________________________________
radiator mailing list
[email protected]
https://lists.open.com.au/mailman/listinfo/radiator
