From 3a17b22edf48b0ea5bf028bb1a354d1a5dc6f4f3 Mon Sep 17 00:00:00 2001 From: Neraud Date: Sat, 8 Jul 2017 09:16:07 +0200 Subject: [PATCH 1/5] Add configuration option for logtail path --- plugins/nginx/nginx_byprojects/README.md | 5 +++-- plugins/nginx/nginx_byprojects/byprojects_access | 6 +++++- plugins/nginx/nginx_byprojects/byprojects_bandwidth | 6 +++++- plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth | 6 +++++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/plugins/nginx/nginx_byprojects/README.md b/plugins/nginx/nginx_byprojects/README.md index d2b59505..f095a9b7 100644 --- a/plugins/nginx/nginx_byprojects/README.md +++ b/plugins/nginx/nginx_byprojects/README.md @@ -16,9 +16,10 @@ Counts the in/out bandwidth used by each projects/vhost. [Logtail] (https://www. ## Installation The setup is pretty straight forward. First you need to configure the plugin: -Define the path to logtail: +In your munin plugin configuration file (for example, a new dedicated /etc/munin/plugin-conf.d/nginx_byprojects), configure the plugins : - $logtail = '/usr/local/bin/logtail'; + [byprojects_*] + env.logtail /usr/local/bin/logtail Multiple logs can be used for the same project/vhost and a regular expression (regex) can be used as a filter: diff --git a/plugins/nginx/nginx_byprojects/byprojects_access b/plugins/nginx/nginx_byprojects/byprojects_access index 81d755dd..496a0137 100644 --- a/plugins/nginx/nginx_byprojects/byprojects_access +++ b/plugins/nginx/nginx_byprojects/byprojects_access @@ -23,11 +23,15 @@ use strict; # Test graph will be using everything file matching /home/test/log/access*.log # and stuff that match the expression '"[A-Z] /test/' in /var/log/access.log # such as '"GET /test/' +# +# Configuration +# [byprojects_*] +# env.logtail_path /usr/local/bin/logtail my $server = 'Nginx'; my $statepath = $ENV{MUNIN_PLUGSTATE}; -my $logtail = '/usr/local/bin/logtail'; +my $logtail = $ENV{logtail_path} || '/usr/local/bin/logtail'; my %logs = ( 'prod' => [ diff --git a/plugins/nginx/nginx_byprojects/byprojects_bandwidth b/plugins/nginx/nginx_byprojects/byprojects_bandwidth index 1ccce21a..7e6d796b 100644 --- a/plugins/nginx/nginx_byprojects/byprojects_bandwidth +++ b/plugins/nginx/nginx_byprojects/byprojects_bandwidth @@ -29,11 +29,15 @@ use strict; # Test graph will be using everything file matching /home/test/log/access*.log # and stuff that match the expression '"[A-Z] /test/' in /var/log/access.log # such as '"GET /test/' +# +# Configuration +# [byprojects_*] +# env.logtail_path /usr/local/bin/logtail my $server = 'Nginx'; my $statepath = $ENV{MUNIN_PLUGSTATE}; -my $logtail = '/usr/local/bin/logtail'; +my $logtail = $ENV{logtail_path} || '/usr/local/bin/logtail'; my %logs = ( 'prod' => [ diff --git a/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth b/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth index 8f02826b..55358b47 100644 --- a/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth +++ b/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth @@ -29,11 +29,15 @@ use strict; # Test graph will be using everything file matching /home/test/log/access*.log # and stuff that match the expression '"[A-Z] /test/' in /var/log/access.log # such as '"GET /test/' +# +# Configuration +# [byprojects_*] +# env.logtail_path /usr/local/bin/logtail my $server = 'Nginx'; my $statepath = $ENV{MUNIN_PLUGSTATE}; -my $logtail = '/usr/local/bin/logtail'; +my $logtail = $ENV{logtail_path} || '/usr/local/bin/logtail'; my %logs = ( 'prod' => [ From a1f7808b9e613dd1f25ed85c621140dbdaef16f4 Mon Sep 17 00:00:00 2001 From: Neraud Date: Sat, 8 Jul 2017 10:36:46 +0200 Subject: [PATCH 2/5] Add configuration option for the logs setup --- plugins/nginx/nginx_byprojects/README.md | 19 +++++----------- .../nginx/nginx_byprojects/byprojects_access | 21 +++++++----------- .../nginx_byprojects/byprojects_bandwidth | 22 +++++++------------ .../byprojects_inout_bandwidth | 22 +++++++------------ 4 files changed, 29 insertions(+), 55 deletions(-) diff --git a/plugins/nginx/nginx_byprojects/README.md b/plugins/nginx/nginx_byprojects/README.md index f095a9b7..9791d8ca 100644 --- a/plugins/nginx/nginx_byprojects/README.md +++ b/plugins/nginx/nginx_byprojects/README.md @@ -21,21 +21,12 @@ In your munin plugin configuration file (for example, a new dedicated /etc/munin [byprojects_*] env.logtail /usr/local/bin/logtail -Multiple logs can be used for the same project/vhost and a regular expression (regex) can be used as a filter: +Multiple logs can be used for the same project/vhost and a regular expression (regex) can be used as a filter. +Each log is defined in a dedicated environment variable, named env.site.[siteName]. The value is JSON formatted. - my %logs = ( - 'prod' => [ - {'path' => '/home/prod/log/access.log'} - ], - 'dev' => [ - {'path' => '/var/log/httpd/ssl-dev-access.log'}, - {'path' => '/home/dev/log/access*.log'} # glob is supported - ], - 'test' => [ - {'path' => '/var/log/access.log', 'regex' => '"[A-Z]+ /test/'}, - {'path' => '/home/test/log/access.log'} - ], - ); + env.site.prod [{"path":"/home/prod/log/access.log"}] + env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] + env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] In the previous example the prod project graph will be using everything in /home/prod/log/access.log. The test project will be using eveything in /home/test/log/access.log and stuff that match '"[A-Z] /test/' in /var/log/httpd/access.log (e.g. "GET /test/). diff --git a/plugins/nginx/nginx_byprojects/byprojects_access b/plugins/nginx/nginx_byprojects/byprojects_access index 496a0137..85a8f986 100644 --- a/plugins/nginx/nginx_byprojects/byprojects_access +++ b/plugins/nginx/nginx_byprojects/byprojects_access @@ -1,5 +1,6 @@ #!/usr/bin/perl -w use strict; +use JSON qw(decode_json); # # byprojects_access # @@ -27,25 +28,19 @@ use strict; # Configuration # [byprojects_*] # env.logtail_path /usr/local/bin/logtail +# env.site.prod [{"path":"/home/prod/log/access.log"}] +# env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] +# env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] my $server = 'Nginx'; my $statepath = $ENV{MUNIN_PLUGSTATE}; my $logtail = $ENV{logtail_path} || '/usr/local/bin/logtail'; -my %logs = ( - 'prod' => [ - {'path' => '/home/prod/log/access.log'} - ], - 'dev' => [ - {'path' => '/var/log/httpd/ssl-dev-access.log'}, - {'path' => '/home/dev/log/access*.log'} # glob is supported - ], - 'test' => [ - {'path' => '/var/log/access.log', 'regex' => '"[A-Z]+ /test/'}, - {'path' => '/home/test/log/access.log'} - ], -); +my @loglist = grep {$_ =~ /site\./} keys(%ENV); +my %envLogs = %ENV{@loglist}; +my %logs; +while(my($k, $v) = each %envLogs) { @logs{substr($k, 5)} = decode_json($v); } ########### diff --git a/plugins/nginx/nginx_byprojects/byprojects_bandwidth b/plugins/nginx/nginx_byprojects/byprojects_bandwidth index 7e6d796b..409da6aa 100644 --- a/plugins/nginx/nginx_byprojects/byprojects_bandwidth +++ b/plugins/nginx/nginx_byprojects/byprojects_bandwidth @@ -1,5 +1,6 @@ #!/usr/bin/perl -w use strict; +use JSON qw(decode_json); # # byprojects_bandwidth # @@ -33,26 +34,19 @@ use strict; # Configuration # [byprojects_*] # env.logtail_path /usr/local/bin/logtail +# env.site.prod [{"path":"/home/prod/log/access.log"}] +# env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] +# env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] my $server = 'Nginx'; my $statepath = $ENV{MUNIN_PLUGSTATE}; my $logtail = $ENV{logtail_path} || '/usr/local/bin/logtail'; -my %logs = ( - 'prod' => [ - {'path' => '/home/prod/log/access.log'} - ], - 'dev' => [ - {'path' => '/var/log/httpd/ssl-dev-access.log'}, - {'path' => '/home/dev/log/access.log'} - ], - 'test' => [ - {'path' => '/var/log/access.log', 'regex' => '"[A-Z]+ /test/'}, - {'path' => '/home/test/log/access.log'} - ], -); - +my @loglist = grep {$_ =~ /site\./} keys(%ENV); +my %envLogs = %ENV{@loglist}; +my %logs; +while(my($k, $v) = each %envLogs) { @logs{substr($k, 5)} = decode_json($v); } ########### diff --git a/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth b/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth index 55358b47..7093aaf1 100644 --- a/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth +++ b/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth @@ -1,5 +1,6 @@ #!/usr/bin/perl -w use strict; +use JSON qw(decode_json); # # byprojects_inout_bandwidth # @@ -33,26 +34,19 @@ use strict; # Configuration # [byprojects_*] # env.logtail_path /usr/local/bin/logtail +# env.site.prod [{"path":"/home/prod/log/access.log"}] +# env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] +# env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] my $server = 'Nginx'; my $statepath = $ENV{MUNIN_PLUGSTATE}; my $logtail = $ENV{logtail_path} || '/usr/local/bin/logtail'; -my %logs = ( - 'prod' => [ - {'path' => '/home/prod/log/access.log'} - ], - 'dev' => [ - {'path' => '/var/log/httpd/ssl-dev-access.log'}, - {'path' => '/home/dev/log/access.log'} - ], - 'test' => [ - {'path' => '/var/log/access.log', 'regex' => '"[A-Z]+ /test/'}, - {'path' => '/home/test/log/access.log'} - ], -); - +my @loglist = grep {$_ =~ /site\./} keys(%ENV); +my %envLogs = %ENV{@loglist}; +my %logs; +while(my($k, $v) = each %envLogs) { @logs{substr($k, 5)} = decode_json($v); } ########### From 9ce70486fcc533fc0cb77da7e6d0b9ad931d1bf1 Mon Sep 17 00:00:00 2001 From: Neraud Date: Sat, 8 Jul 2017 10:59:33 +0200 Subject: [PATCH 3/5] Minor documentation update Fix the markdown format for Logtail links. Add the restart munin-node step. --- plugins/nginx/nginx_byprojects/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/nginx/nginx_byprojects/README.md b/plugins/nginx/nginx_byprojects/README.md index 9791d8ca..7f93095f 100644 --- a/plugins/nginx/nginx_byprojects/README.md +++ b/plugins/nginx/nginx_byprojects/README.md @@ -6,11 +6,11 @@ Count the number of hits per projects/vhost. ![byproject_access](https://www.mantor.org/~northox/misc/munin-plugins/nginx_byprojects_access1-month.png "byproject_access") ## munin_byprojects_bandwidth -Count the total bandwidth used by each projects/vhost. [Logtail] (https://www.fourmilab.ch/webtools/logtail/) is required. +Count the total bandwidth used by each projects/vhost. [Logtail](https://www.fourmilab.ch/webtools/logtail/) is required. ![byproject_bandwidth](https://www.mantor.org/~northox/misc/munin-plugins/apache_byprojects_bandwidth-month.png "byproject_bandwidth") ## munin_byprojects_inout_bandwidth -Counts the in/out bandwidth used by each projects/vhost. [Logtail] (https://www.fourmilab.ch/webtools/logtail/) is required. +Counts the in/out bandwidth used by each projects/vhost. [Logtail](https://www.fourmilab.ch/webtools/logtail/) is required. ![byproject_inout_bandwidth](https://www.mantor.org/~northox/misc/munin-plugins/apache_byprojects_inout_bandwidth-month.png "byproject_inout_bandwidth") ## Installation @@ -34,5 +34,7 @@ Then link the file just as any other plugins. ln -s /usr/local/sbin/ /usr/local/etc/munin/plugins/ +And restart the munin-node service. + ## License MIT From 331e3da9be74b410ebae9fb5d3aff66a5cb640ba Mon Sep 17 00:00:00 2001 From: Neraud Date: Sat, 8 Jul 2017 11:14:15 +0200 Subject: [PATCH 4/5] Add executable bit --- plugins/nginx/nginx_byprojects/byprojects_access | 0 plugins/nginx/nginx_byprojects/byprojects_bandwidth | 0 plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 plugins/nginx/nginx_byprojects/byprojects_access mode change 100644 => 100755 plugins/nginx/nginx_byprojects/byprojects_bandwidth mode change 100644 => 100755 plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth diff --git a/plugins/nginx/nginx_byprojects/byprojects_access b/plugins/nginx/nginx_byprojects/byprojects_access old mode 100644 new mode 100755 diff --git a/plugins/nginx/nginx_byprojects/byprojects_bandwidth b/plugins/nginx/nginx_byprojects/byprojects_bandwidth old mode 100644 new mode 100755 diff --git a/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth b/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth old mode 100644 new mode 100755 From 69896717af0febbfb03ddfe80637c382682dd229 Mon Sep 17 00:00:00 2001 From: Neraud Date: Thu, 13 Jul 2017 12:59:05 +0200 Subject: [PATCH 5/5] Fix comment format : use spaces instead of tabs --- plugins/nginx/nginx_byprojects/byprojects_access | 6 +++--- plugins/nginx/nginx_byprojects/byprojects_bandwidth | 6 +++--- plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/nginx/nginx_byprojects/byprojects_access b/plugins/nginx/nginx_byprojects/byprojects_access index 85a8f986..bc400504 100755 --- a/plugins/nginx/nginx_byprojects/byprojects_access +++ b/plugins/nginx/nginx_byprojects/byprojects_access @@ -28,9 +28,9 @@ use JSON qw(decode_json); # Configuration # [byprojects_*] # env.logtail_path /usr/local/bin/logtail -# env.site.prod [{"path":"/home/prod/log/access.log"}] -# env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] -# env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] +# env.site.prod [{"path":"/home/prod/log/access.log"}] +# env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] +# env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] my $server = 'Nginx'; diff --git a/plugins/nginx/nginx_byprojects/byprojects_bandwidth b/plugins/nginx/nginx_byprojects/byprojects_bandwidth index 409da6aa..af245e96 100755 --- a/plugins/nginx/nginx_byprojects/byprojects_bandwidth +++ b/plugins/nginx/nginx_byprojects/byprojects_bandwidth @@ -34,9 +34,9 @@ use JSON qw(decode_json); # Configuration # [byprojects_*] # env.logtail_path /usr/local/bin/logtail -# env.site.prod [{"path":"/home/prod/log/access.log"}] -# env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] -# env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] +# env.site.prod [{"path":"/home/prod/log/access.log"}] +# env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] +# env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] my $server = 'Nginx'; diff --git a/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth b/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth index 7093aaf1..21b36902 100755 --- a/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth +++ b/plugins/nginx/nginx_byprojects/byprojects_inout_bandwidth @@ -34,9 +34,9 @@ use JSON qw(decode_json); # Configuration # [byprojects_*] # env.logtail_path /usr/local/bin/logtail -# env.site.prod [{"path":"/home/prod/log/access.log"}] -# env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] -# env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] +# env.site.prod [{"path":"/home/prod/log/access.log"}] +# env.site.dev [{"path":"/var/log/httpd/ssl-dev-access.log"}, {"path":"/home/dev/log/access*.log"}] +# env.site.test [{"path":"/var/log/access.log","regex":"\"[A-Z]+ /test/"}, {"path":"/home/test/log/access.log"}] my $server = 'Nginx';