From 2590132015e9e5ab066b60a9687b68a250e6833d Mon Sep 17 00:00:00 2001 From: "K.Cima" Date: Fri, 24 Mar 2017 23:14:47 +0900 Subject: [PATCH 1/5] Add Solaris io_disk plugin - Merge io_* plugins into one multigraph plugin - Change graph appearance like linux diskstats plugin RRD files generated by io_busy_, io_ios_, io_bytes_ can be taken over - Add queue and latency functions - Remove perl code and replace old Solaris::MapDev feature by using iostat - Add POD style document - Refactoring, using functions, replace variable names, check with shellcheck --- plugins/solaris/io_disk | 301 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100755 plugins/solaris/io_disk diff --git a/plugins/solaris/io_disk b/plugins/solaris/io_disk new file mode 100755 index 00000000..ee2c11d8 --- /dev/null +++ b/plugins/solaris/io_disk @@ -0,0 +1,301 @@ +#!/bin/bash + +: << =cut + +=head1 NAME + + io_disk - Multigraph plugin to monitor disks for Solaris. + + These functions are implemented: + ops : similar to iostat r/s, w/s + bytes : similar to iostat kr/s, kw/s + busy : similar to iostat %b, %w + queue : similar to iostat actv, wait + latency : similar to iostat asvc_t, wsvc_t + + Any device found in /usr/bin/kstat can be monitored. + + Tested with Solaris 10 and 11, and it should work with Solaris 8, 9. + +=head1 CONFIGURATION + + Make symlink: + cd /path/to/munin/etc/plugins + ln -s /path/to/munin/lib/plugins/io_disk . + + The RRD files generated by io_busy_, io_ops_, io_bytes_ can be taken over + by this plugin. + Thus, please remove symlinks of those plugins before using this plugin. + + By default, this plugin monitors disk devices. And also it can monitor + NFS and Tape devices as much as io_* plugins with setting environments. + +=head1 ENVIRONMENT VARIABLES + + env.class - kstat class. See man kstat -c option. + example: env.class /disk|nfs|tape/ + default: disk + + env.module - Module name. Only used in internal graph name. If you would not + to like to take over RRDs generated by io_*_sd plugins, please change it + to another name. + + example: env.module something + default: sd + + env.module_regex - kstat module. See man kstat -m option. + example: env.module_regex /^(s?sd|vdc|zvblk|dad|md|nfs|tape)$/ + default: /^(s?sd|vdc|zvblk|dad|md)$/ + + env.title_type - Device type shown in graph title. + example: env.title_type Disk Device, NFS, Tape + default: Disk Device + + env.exclude - Device instance name(s) to exclude seperated by white-space. + example: env.exclude sd0 ssd1 + default: none + + env.graph_width - Graph width in pixel + example: env.graph_width 450 + default: none (usually 400) + +=head1 AUTHOR + + Unknown Author + Improved by K.Cima https://github.com/shakemid + +=head1 LICENSE + + GPLv2 + +=head1 MAGIC MARKERS + + #%# family=auto + #%# capabilities=autoconf + +=cut + +# Include plugin.sh +. "${MUNIN_LIBDIR:-}/plugins/plugin.sh" +is_multigraph "$@" + +# Shell options +set -o nounset +set -o pipefail + +# Set environment variables +plugin_name=io +functions='ops bytes busy queue latency' +: "${class:=disk}" +: "${module:=sd}" +: "${module_regex:=/^(s?sd|vdc|zvblk|dad|md)\$/}" +: "${title_type:=Disk Device}" +: "${exclude:=}" +: "${graph_width:=}" + +# Create map of instance name (e.g. sd0) and logical device name (e.g. c0t0d0) +# Example: +# name_sd1=c0t0d0 +# name_ssd2=c0tAB_1234d0 (shorten long target) +# ... +declare $( paste -d= \ + <( iostat -x | awk '{print $1}' | sed -e '1,2d' -e 's/^/name_/' ) \ + <( iostat -xn | awk '{print $NF}' | sed -e '1,2d' -e 's/^\(c[0-9]*\)\(t.\{2\}\).*\(.\{4\}\)\(d[0-9]*\)$/\1\2_\3\4/' ) \ + ) + +# Functions + +preconfig() { + local func + func=$1 + + case "$func" in + ops) + conf_title='I/O Operations' + conf_in=reads + conf_out=writes + conf_cdef= + conf_gargs='--base 1000' + conf_vlabel='Ops per second write (-) / read (+)' + ;; + bytes) + conf_title='I/O Throughput' + conf_in=nread + conf_out=nwritten + conf_cdef= + conf_gargs='--base 1024' + conf_vlabel='Bytes per second write (-) / read (+)' + ;; + busy) + conf_title='Busy & Wait' + conf_in=rtime + conf_out=wtime + conf_cdef=',100,*' + conf_gargs='--base 1000 --lower-limit 0 --upper-limit 100' + conf_vlabel='% wait (-) / busy (+)' + ;; + queue) + conf_title='Queue Length' + conf_in=rlentime + conf_out=wlentime + conf_cdef= + conf_gargs='--base 1000' + conf_vlabel='Queue length wait (-) / actv (+)' + ;; + latency) + conf_title='Latency' + conf_in=rlentime + conf_out=wlentime + conf_cdef= + conf_gargs='--base 1000' + conf_vlabel='Seconds wsvc_t (-) / asvc_t (+)' + ;; + *) + echo "Unknown function: $func" + exit 1 + ;; + esac +} + +is_excluded() { + local dev i + dev=$1 + + for i in ${exclude} + do + if [ "$dev" = "$i" ]; then + return 0 + fi + done + + return 1 +} + +do_config() { + local func dev ref devname stat + + func=$1 + preconfig "$func" + + echo "multigraph ${plugin_name}_${func}_${module}" + + echo "graph_title $title_type $conf_title" + echo 'graph_category disk' + echo "graph_args $conf_gargs" + echo "graph_vlabel $conf_vlabel" + if [ -n "$graph_width" ]; then + echo "graph_width $graph_width" + fi + + kstat -p -c "$class" -m "$module_regex" -s "/^${conf_in}\$/" | sed 's/:/ /g' | awk '{ print $3 }' | \ + while read -r dev + do + if is_excluded "$dev" ; then + continue + fi + + # replace instance name to logical device name if exists + ref="name_$dev" + devname=${!ref:-} + + # reads and writes are necessary to calculate latency + if [ "$func" = 'latency' ]; then + for stat in reads writes + do + echo "${dev}_$stat.label dummy" + echo "${dev}_$stat.graph no" + echo "${dev}_$stat.type DERIVE" + echo "${dev}_$stat.min 0" + done + # rlentime / ( reads + writes ) + conf_cdef=,${dev}_reads,${dev}_writes,+,/ + fi + + echo "${dev}_${conf_out}.label dummy" + echo "${dev}_${conf_out}.graph no" + echo "${dev}_${conf_out}.type DERIVE" + echo "${dev}_${conf_out}.min 0" + + echo "${dev}_${conf_in}.label ${devname:-${dev}}" + echo "${dev}_${conf_in}.negative ${dev}_${conf_out}" + echo "${dev}_${conf_in}.type DERIVE" + echo "${dev}_${conf_in}.min 0" + + if [ -n "$conf_cdef" ]; then + echo "${dev}_${conf_out}.cdef ${dev}_${conf_out}${conf_cdef}" + echo "${dev}_${conf_in}.cdef ${dev}_${conf_in}${conf_cdef}" + fi + done + + echo +} + +do_getvalue() { + local func stat_regex dev stat value + + func=$1 + preconfig "$func" + + if [ "$func" = 'latency' ]; then + stat_regex="/^($conf_in|$conf_out|reads|writes)\$/" + else + stat_regex="/^($conf_in|$conf_out)\$/" + fi + + echo "multigraph ${plugin_name}_${func}_${module}" + + kstat -p -c "$class" -m "$module_regex" -s "$stat_regex" | sed 's/:/ /g' | awk '{ print $3,$4,$5 }' | \ + while read -r dev stat value + do + if is_excluded "$dev" ; then + continue + fi + + echo "${dev}_${stat}.value ${value%.*}" + done + + echo +} + +autoconf() { + if [ -x /usr/bin/kstat ]; then + echo yes + exit 0 + else + echo "no (/usr/bin/kstat not found)" + exit 0 + fi +} + +config() { + local func + + for func in $functions + do + do_config "$func" + done +} + +getvalue() { + local func + + for func in $functions + do + do_getvalue "$func" + done +} + +# main +case ${1:-} in +autoconf) + autoconf + ;; +config) + config + ;; +*) + getvalue + ;; +esac + +exit 0 From fe097689f945f6b2d2d5da1f9fe95e6668ecd3e6 Mon Sep 17 00:00:00 2001 From: "K.Cima" Date: Mon, 27 Mar 2017 16:23:33 +0900 Subject: [PATCH 2/5] Fix for https://github.com/munin-monitoring/contrib/pull/826 --- plugins/solaris/io_disk | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/plugins/solaris/io_disk b/plugins/solaris/io_disk index ee2c11d8..3e222d48 100755 --- a/plugins/solaris/io_disk +++ b/plugins/solaris/io_disk @@ -9,10 +9,11 @@ These functions are implemented: ops : similar to iostat r/s, w/s bytes : similar to iostat kr/s, kw/s - busy : similar to iostat %b, %w + busy : similar to iostat %b, %w (%w usually indicates 0 in Sol10,11) queue : similar to iostat actv, wait latency : similar to iostat asvc_t, wsvc_t + This plugin is improved from Solaris io_busy_, io_ops_, io_bytes_ plugins. Any device found in /usr/bin/kstat can be monitored. Tested with Solaris 10 and 11, and it should work with Solaris 8, 9. @@ -30,19 +31,26 @@ By default, this plugin monitors disk devices. And also it can monitor NFS and Tape devices as much as io_* plugins with setting environments. + Note that instance names of nfs (e.g. nfs1) can be changed after reboot or + remount, this plugin may not monitor nfs properly. (same as original ones) + =head1 ENVIRONMENT VARIABLES env.class - kstat class. See man kstat -c option. example: env.class /disk|nfs|tape/ default: disk - env.module - Module name. Only used in internal graph name. If you would not - to like to take over RRDs generated by io_*_sd plugins, please change it - to another name. - + env.module - Module name. Only used in internal graph name. example: env.module something default: sd + If you would not like to take over RRDs generated by io_*_sd plugins, + please change it to another name. + + And also, if you would like to take over RRDs of nfs, tape devices, + please set it 'nfs' or 'tape' (and set env.class, env.module_regex). + It may be good to link like ln -s /path/to/io_disk io_nfs if necessary. + env.module_regex - kstat module. See man kstat -m option. example: env.module_regex /^(s?sd|vdc|zvblk|dad|md|nfs|tape)$/ default: /^(s?sd|vdc|zvblk|dad|md)$/ @@ -187,16 +195,15 @@ do_config() { echo "graph_width $graph_width" fi + # Get device instance names by kstat kstat -p -c "$class" -m "$module_regex" -s "/^${conf_in}\$/" | sed 's/:/ /g' | awk '{ print $3 }' | \ while read -r dev do - if is_excluded "$dev" ; then - continue - fi + is_excluded "$dev" && continue - # replace instance name to logical device name if exists + # Replace instance name to logical device name if exists ref="name_$dev" - devname=${!ref:-} + devname=${!ref:-} # ${!varname} means indirect evaluation (similar to eval) # reads and writes are necessary to calculate latency if [ "$func" = 'latency' ]; then @@ -244,12 +251,18 @@ do_getvalue() { echo "multigraph ${plugin_name}_${func}_${module}" + # Get device instance names, stat names and values by kstat + + # kstat output example: + # $ kstat -p -c disk -m '/^sd$/' -s '/^reads$/' + # sd:0:sd0:reads 51432610 + # sd:1:sd1:reads 52671435 + # ... + kstat -p -c "$class" -m "$module_regex" -s "$stat_regex" | sed 's/:/ /g' | awk '{ print $3,$4,$5 }' | \ while read -r dev stat value do - if is_excluded "$dev" ; then - continue - fi + is_excluded "$dev" && continue echo "${dev}_${stat}.value ${value%.*}" done @@ -285,7 +298,7 @@ getvalue() { done } -# main +# Main case ${1:-} in autoconf) autoconf From 13cb4192d9be63f64b421c95eeffd2c7be0f6148 Mon Sep 17 00:00:00 2001 From: "K.Cima" Date: Tue, 28 Mar 2017 12:10:22 +0900 Subject: [PATCH 3/5] Remove useless set -o pipefail --- plugins/solaris/io_disk | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/solaris/io_disk b/plugins/solaris/io_disk index 3e222d48..f50d58b0 100755 --- a/plugins/solaris/io_disk +++ b/plugins/solaris/io_disk @@ -88,8 +88,7 @@ is_multigraph "$@" # Shell options -set -o nounset -set -o pipefail +set -o nounset # Like perl use strict; # Set environment variables plugin_name=io From 544b2cfb4a721c5a472d78f042b23eec7a20d684 Mon Sep 17 00:00:00 2001 From: "K.Cima" Date: Wed, 29 Mar 2017 12:09:21 +0900 Subject: [PATCH 4/5] Fix method to create device maps because sometimes iostat may hang in process substitution and for readability --- plugins/solaris/io_disk | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/solaris/io_disk b/plugins/solaris/io_disk index f50d58b0..8ea87e0d 100755 --- a/plugins/solaris/io_disk +++ b/plugins/solaris/io_disk @@ -105,10 +105,11 @@ functions='ops bytes busy queue latency' # name_sd1=c0t0d0 # name_ssd2=c0tAB_1234d0 (shorten long target) # ... -declare $( paste -d= \ - <( iostat -x | awk '{print $1}' | sed -e '1,2d' -e 's/^/name_/' ) \ - <( iostat -xn | awk '{print $NF}' | sed -e '1,2d' -e 's/^\(c[0-9]*\)\(t.\{2\}\).*\(.\{4\}\)\(d[0-9]*\)$/\1\2_\3\4/' ) \ - ) +instance_names=$( iostat -x | sed -e '1,2d' | awk '{print $1}' | \ + sed -e 's/^/name_/' ) +logical_device_names=$( iostat -xn | sed -e '1,2d' | awk '{print $NF}' | \ + sed -e 's/^\(c[0-9]*\)\(t.\{2\}\).*\(.\{4\}\)\(d[0-9]*\)$/\1\2_\3\4/' ) +declare $( paste -d= <( echo "$instance_names" ) <( echo "$logical_device_names" ) ) # Functions From c7a7f086ec54e3d18ab56171974dc8fb1cf1d80c Mon Sep 17 00:00:00 2001 From: "K.Cima" Date: Wed, 29 Mar 2017 23:13:34 +0900 Subject: [PATCH 5/5] Add size module --- plugins/solaris/io_disk | 91 ++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 29 deletions(-) diff --git a/plugins/solaris/io_disk b/plugins/solaris/io_disk index 8ea87e0d..509cf1ca 100755 --- a/plugins/solaris/io_disk +++ b/plugins/solaris/io_disk @@ -12,6 +12,7 @@ busy : similar to iostat %b, %w (%w usually indicates 0 in Sol10,11) queue : similar to iostat actv, wait latency : similar to iostat asvc_t, wsvc_t + size : average io size This plugin is improved from Solaris io_busy_, io_ops_, io_bytes_ plugins. Any device found in /usr/bin/kstat can be monitored. @@ -52,7 +53,7 @@ It may be good to link like ln -s /path/to/io_disk io_nfs if necessary. env.module_regex - kstat module. See man kstat -m option. - example: env.module_regex /^(s?sd|vdc|zvblk|dad|md|nfs|tape)$/ + example: env.module_regex /^(s?sd|vdc|zvblk|dad|md|nfs|st)$/ default: /^(s?sd|vdc|zvblk|dad|md)$/ env.title_type - Device type shown in graph title. @@ -92,7 +93,7 @@ set -o nounset # Like perl use strict; # Set environment variables plugin_name=io -functions='ops bytes busy queue latency' +functions='ops bytes busy queue latency size' : "${class:=disk}" : "${module:=sd}" : "${module_regex:=/^(s?sd|vdc|zvblk|dad|md)\$/}" @@ -120,43 +121,57 @@ preconfig() { case "$func" in ops) conf_title='I/O Operations' - conf_in=reads - conf_out=writes - conf_cdef= conf_gargs='--base 1000' conf_vlabel='Ops per second write (-) / read (+)' + conf_in=reads + conf_out=writes + conf_in_cdef= + conf_out_cdef= ;; bytes) conf_title='I/O Throughput' - conf_in=nread - conf_out=nwritten - conf_cdef= conf_gargs='--base 1024' conf_vlabel='Bytes per second write (-) / read (+)' + conf_in=nread + conf_out=nwritten + conf_in_cdef= + conf_out_cdef= ;; busy) conf_title='Busy & Wait' - conf_in=rtime - conf_out=wtime - conf_cdef=',100,*' conf_gargs='--base 1000 --lower-limit 0 --upper-limit 100' conf_vlabel='% wait (-) / busy (+)' + conf_in=rtime + conf_out=wtime + conf_in_cdef=',100,*' + conf_out_cdef=',100,*' ;; queue) conf_title='Queue Length' - conf_in=rlentime - conf_out=wlentime - conf_cdef= conf_gargs='--base 1000' conf_vlabel='Queue length wait (-) / actv (+)' + conf_in=rlentime + conf_out=wlentime + conf_in_cdef= + conf_out_cdef= ;; latency) conf_title='Latency' - conf_in=rlentime - conf_out=wlentime - conf_cdef= conf_gargs='--base 1000' conf_vlabel='Seconds wsvc_t (-) / asvc_t (+)' + conf_in=rlentime + conf_out=wlentime + conf_in_cdef= + conf_out_cdef= + ;; + size) + conf_title='I/O Size' + conf_gargs='--base 1024' + conf_vlabel='Average size write (-) / read (+)' + conf_in=nread + conf_out=nwritten + conf_in_cdef= + conf_out_cdef= ;; *) echo "Unknown function: $func" @@ -205,8 +220,9 @@ do_config() { ref="name_$dev" devname=${!ref:-} # ${!varname} means indirect evaluation (similar to eval) - # reads and writes are necessary to calculate latency - if [ "$func" = 'latency' ]; then + # reads and writes are necessary to calculate latency, size + case "$func" in + latency|size) for stat in reads writes do echo "${dev}_$stat.label dummy" @@ -214,23 +230,37 @@ do_config() { echo "${dev}_$stat.type DERIVE" echo "${dev}_$stat.min 0" done - # rlentime / ( reads + writes ) - conf_cdef=,${dev}_reads,${dev}_writes,+,/ - fi + ;; + esac + # Set cdef for latency, size + case "$func" in + latency) + # rlentime / ( reads + writes ) + conf_in_cdef=,${dev}_reads,${dev}_writes,+,/ + conf_out_cdef=,${dev}_reads,${dev}_writes,+,/ + ;; + size) + conf_in_cdef=,${dev}_reads,/ + conf_out_cdef=,${dev}_writes,/ + ;; + esac + + # Print data attributes echo "${dev}_${conf_out}.label dummy" echo "${dev}_${conf_out}.graph no" echo "${dev}_${conf_out}.type DERIVE" echo "${dev}_${conf_out}.min 0" + if [ -n "${conf_out_cdef:-}" ]; then + echo "${dev}_${conf_out}.cdef ${dev}_${conf_out}${conf_out_cdef}" + fi echo "${dev}_${conf_in}.label ${devname:-${dev}}" echo "${dev}_${conf_in}.negative ${dev}_${conf_out}" echo "${dev}_${conf_in}.type DERIVE" echo "${dev}_${conf_in}.min 0" - - if [ -n "$conf_cdef" ]; then - echo "${dev}_${conf_out}.cdef ${dev}_${conf_out}${conf_cdef}" - echo "${dev}_${conf_in}.cdef ${dev}_${conf_in}${conf_cdef}" + if [ -n "${conf_in_cdef:-}" ]; then + echo "${dev}_${conf_in}.cdef ${dev}_${conf_in}${conf_in_cdef}" fi done @@ -243,11 +273,14 @@ do_getvalue() { func=$1 preconfig "$func" - if [ "$func" = 'latency' ]; then + case "$func" in + latency|size) stat_regex="/^($conf_in|$conf_out|reads|writes)\$/" - else + ;; + *) stat_regex="/^($conf_in|$conf_out)\$/" - fi + ;; + esac echo "multigraph ${plugin_name}_${func}_${module}"