From ef960abcf9efcd8a35d3fd0c27c84e4de95f15cf Mon Sep 17 00:00:00 2001 From: antonio Date: Tue, 27 Aug 2013 21:48:51 +0200 Subject: [PATCH 1/3] Commit includes: fix slash in url params, examples update, fix issue #1 and #2, qps graph uses request instead of avgRequestPerSecond, new 'memory' graph, deprecation of availableram parameter --- plugins/solr/solr4_ | 208 ++++++++++++++++++++++++++------------------ 1 file changed, 125 insertions(+), 83 deletions(-) diff --git a/plugins/solr/solr4_ b/plugins/solr/solr4_ index 80d38317..10d48c19 100755 --- a/plugins/solr/solr4_ +++ b/plugins/solr/solr4_ @@ -20,40 +20,27 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. # -# Munin plugin for monitoring a multicore solr 4.* installation via mbean. -# It calls: -# > http://localhost:8080/solr/admin/cores?action=STATUS&wt=json -# and -# > http://localhost:8080/solr/corename/admin/mbeans?stats=true&wt=json -# for each core to retrieve cores and data. Verify those urls on your instance. +# Solr 4.* munin graph plugin +# Plugins configuration parameters: # -# Configuration parameters: # [solr_*] -# host_port -# qpshandler_ -# availableram +# env.host_port +# env.url +# env.qpshandler_ # -# Example: -# host_port solrhost:8080 -# qpshandler_select /select -# availableram 3221225472 +# ex: +# env.host_port solrhost:8080 +# env.url /solr +# env.qpshandler_select /select # -# Defined checks: -# numdocs -# qps -# indexsize -# requesttimes -# documentcache -# fieldvaluecache -# filtercache -# queryresultcache -# -# Installation example: +# Install plugins: # ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_numdocs_core_1 # ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_requesttimes_select -# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_qps_core_1_select +# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_core_1_select +# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_indexsize +# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_memory # -# Source repo: https://github.com/averni/munin-solr + import sys import os @@ -72,7 +59,7 @@ def parse_params(): data = params['core'].rsplit('_', 1) handler = data.pop() params['params'] = { - 'handler': os.environ.get('qpshandler_%s' % handler, '/select') + 'handler': os.environ.get('qpshandler_%s' % handler, 'standard') } if not data: params['core'] = '' @@ -90,7 +77,7 @@ class CheckException(Exception): class JSONReader: @classmethod - def readValue(cls, struct, path): + def readValue(cls, struct, path, convert = None): if not path[0] in struct: return -1 obj = struct[path[0]] @@ -98,15 +85,18 @@ class JSONReader: return -1 for k in path[1:]: obj = obj[k] + if convert: + return convert(obj) return obj class SolrCoresAdmin: - def __init__(self, host): + def __init__(self, host, solrurl): self.host = host + self.solrurl = solrurl self.data = None def fetchcores(self): - uri = "/solr/admin/cores?action=STATUS&wt=json" + uri = os.path.join(self.solrurl, "admin/cores?action=STATUS&wt=json") conn = httplib.HTTPConnection(self.host) conn.request("GET", uri) res = conn.getresponse() @@ -135,13 +125,14 @@ class SolrCoresAdmin: return ret class SolrCoreMBean: - def __init__(self, host, core): + def __init__(self, host, solrurl, core): self.host = host self.data = None self.core = core + self.solrurl = solrurl def _fetch(self): - uri = "/solr/%s/admin/mbeans?stats=true&wt=json" % self.core + uri = os.path.join(self.solrurl, "%s/admin/mbeans?stats=true&wt=json" % self.core) conn = httplib.HTTPConnection(self.host) conn.request("GET", uri) res = conn.getresponse() @@ -159,37 +150,59 @@ class SolrCoreMBean: data[key] = el else: key = el + self._fetchSystem() - def _read(self, path): + def _fetchSystem(self): + uri = os.path.join(self.solrurl, "%s/admin/system?stats=true&wt=json" % self.core) + conn = httplib.HTTPConnection(self.host) + conn.request("GET", uri) + res = conn.getresponse() + data = res.read() + if res.status != 200: + raise CheckException("System fetch failed: %s\n%s" %( str(res.status), res.read())) + self.data['system'] = json.loads(data) + + + def _readInt(self, path): + return self._read(path, int) + + def _readFloat(self, path): + return self._read(path, float) + + def _read(self, path, convert = None): if self.data is None: self._fetch() - return JSONReader.readValue(self.data, path) + return JSONReader.readValue(self.data, path, convert) def _readCache(self, cache): result = {} - for key in ['lookups', 'hits', 'inserts', 'evictions', 'hitratio']: + for key, ftype in [('lookups', int), ('hits', int), ('inserts', int), ('evictions', int), ('hitratio', float)]: path = ['solr-mbeans', 'CACHE', cache, 'stats', 'cumulative_%s' % key] - result[key] = self._read(path) - result['size'] = self._read(['solr-mbeans', 'CACHE', cache, 'stats', 'size']) + result[key] = self._read(path, ftype) + result['size'] = self._readInt(['solr-mbeans', 'CACHE', cache, 'stats', 'size']) return result def getCore(self): return self.core + def requestcount(self, handler): + path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats', 'requests'] + return self._readInt(path) + def qps(self, handler): path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats', 'avgRequestsPerSecond'] - return self._read(path) + return self._readFloat(path) def requesttimes(self, handler): times = {} path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats'] for perc in ['avgTimePerRequest', '75thPcRequestTime', '99thPcRequestTime']: - times[perc] = self._read(path + [perc]) + times[perc] = self._read(path + [perc], float) return times def numdocs(self): path = ['solr-mbeans', 'CORE', 'searcher', 'stats', 'numDocs'] - return self._read(path) + return self._readInt(path) def documentcache(self): return self._readCache('documentCache') @@ -203,6 +216,13 @@ class SolrCoreMBean: def queryresultcache(self): return self._readCache('queryResultCache') + def memory(self): + data = self._read(['system', 'jvm', 'memory', 'raw']) + del data['used%'] + for k in data.keys(): + data[k] = int(data[k]) + return data + ############################################################################# # Graph Templates @@ -240,14 +260,19 @@ evictions.draw LINE2 """ -QPSMAIN_GRAPH_TPL = """graph_title Solr {core} {handler} Request per second" -graph_args -l 0 +QPSMAIN_GRAPH_TPL = """graph_title Solr {core} {handler} Request per second +graph_args --base 1000 -r --lower-limit 0 +graph_scale no graph_vlabel request / second graph_category solr +graph_period second +graph_order {gorder} {cores_qps_graphs}""" QPSCORE_GRAPH_TPL = """qps_{core}.label {core} Request per second -qps_{core}.type LINESTACK1 +qps_{core}.draw {gtype} +qps_{core}.type DERIVE +qps_{core}.min 0 qps_{core}.graph yes""" REQUESTTIMES_GRAPH_TPL = """multigraph {core}_requesttimes @@ -256,15 +281,14 @@ graph_args -l 0 graph_vlabel millis graph_category solr savgtimeperrequest_{core}.label {core} Avg time per request -savgtimeperrequest_{core}.type gauge +savgtimeperrequest_{core}.type GAUGE savgtimeperrequest_{core}.graph yes s75thpcrequesttime_{core}.label {core} 75th perc -s75thpcrequesttime_{core}.type gauge +s75thpcrequesttime_{core}.type GAUGE s75thpcrequesttime_{core}.graph yes s99thpcrequesttime_{core}.label {core} 99th perc -s99thpcrequesttime_{core}.type gauge +s99thpcrequesttime_{core}.type GAUGE s99thpcrequesttime_{core}.graph yes - """ NUMDOCS_GRAPH_TPL = """graph_title Solr Docs %s @@ -272,39 +296,42 @@ graph_vlabel docs docs.label Docs graph_category solr""" -INDEXSIZE_GRAPH_TPL = """graph_args --base 1024 -l 0 --upper-limit {availableram} +INDEXSIZE_GRAPH_TPL = """graph_args --base 1024 -l 0 graph_vlabel Bytes graph_title Index Size graph_category solr -graph_info Solr Index Memory Usage. +graph_info Solr Index Size. graph_order {cores} {cores_config} +xmx.label Xmx +xmx.colour ff0000 """ INDEXSIZECORE_GRAPH_TPL = """{core}.label {core} {core}.draw STACK""" +MEMORYUSAGE_GRAPH_TPL = """graph_args --base 1024 -l 0 --upper-limit {availableram} +graph_vlabel Bytes +graph_title Solr memory usage +graph_category solr +graph_info Solr Memory Usage. +used.label Used +max.label Max +max.colour ff0000 +""" + ############################################################################# # Graph managment -CHECKS_DEFINED = [ - 'numdocs', - 'qps', - 'indexsize', - 'requesttimes', - 'documentcache', - 'fieldvaluecache', - 'filtercache', - 'queryresultcache' -] class SolrMuninGraph: - def __init__(self, hostport, solrmbean): - self.solrcoresadmin = SolrCoresAdmin(hostport) + def __init__(self, hostport, solrurl, params): + self.solrcoresadmin = SolrCoresAdmin(hostport, solrurl) self.hostport = hostport + self.solrurl = solrurl self.params = params def _getMBean(self, core): - return SolrCoreMBean(self.hostport, core) + return SolrCoreMBean(self.hostport, self.solrurl, core) def _cacheConfig(self, cacheType, cacheName): return CACHE_GRAPH_TPL.format(core=self.params['core'], cacheType=cacheType, cacheName=cacheName) @@ -318,16 +345,14 @@ class SolrMuninGraph: data = getattr(solrmbean, cacheType)() results.append('multigraph solr_{core}_{cacheType}_hit_rates'.format(core=self.params['core'], cacheType=cacheType)) for label in hits_fields: - results.append("%s.value %s" % (label, data[label])) + results.append("%s.value %.8f" % (label, data[label])) results.append('multigraph solr_{core}_{cacheType}_size'.format(core=self.params['core'], cacheType=cacheType)) for label in size_fields: - results.append("%s.value %s" % (label, data[label])) + results.append("%s.value %d" % (label, data[label])) return "\n".join(results) def config(self, mtype): - if not mtype: - raise CheckException("""Check missing. Available checks: \n\t%s""" % '\n\t'.join(CHECKS_DEFINED)) - if not hasattr(self, '%sConfig' % mtype): + if not mtype or not hasattr(self, '%sConfig' % mtype): raise CheckException("Unknown check %s" % mtype) return getattr(self, '%sConfig' % mtype)() @@ -345,12 +370,13 @@ class SolrMuninGraph: def qpsConfig(self): cores = self._getCores() - graph = [QPSCORE_GRAPH_TPL.format(core=c) for c in cores ] + graph = [QPSCORE_GRAPH_TPL.format(core=c, gtype='LINESTACK1') for pos,c in enumerate(cores) ] return QPSMAIN_GRAPH_TPL.format( cores_qps_graphs='\n'.join(graph), handler=self.params['params']['handler'], core=self.params['core'], - cores_qps_cdefs='%s,%s' % (','.join(map(lambda x: 'qps_%s' % x, cores)),','.join(['+']*(len(cores)-1))) + cores_qps_cdefs='%s,%s' % (','.join(map(lambda x: 'qps_%s' % x, cores)),','.join(['+']*(len(cores)-1))), + gorder=','.join(cores) ) def qps(self): @@ -358,7 +384,7 @@ class SolrMuninGraph: cores = self._getCores() for c in cores: mbean = self._getMBean(c) - results.append('qps_%s.value %s' % (c, mbean.qps(self.params['params']['handler']))) + results.append('qps_%s.value %d' % (c, mbean.requestcount(self.params['params']['handler']))) return '\n'.join(results) def requesttimesConfig(self): @@ -373,7 +399,7 @@ class SolrMuninGraph: mbean = self._getMBean(c) results.append('multigraph {core}_requesttimes'.format(core=c)) for k, time in mbean.requesttimes(self.params['params']['handler']).items(): - results.append('s%s_%s.value %s' % (k.lower(), c, time)) + results.append('s%s_%s.value %.5f' % (k.lower(), c, time)) return '\n'.join(results) def numdocsConfig(self): @@ -381,20 +407,36 @@ class SolrMuninGraph: def numdocs(self): mbean = self._getMBean(self.params['core']) - return 'docs.value %s' % mbean.numdocs(**self.params['params']) + return 'docs.value %d' % mbean.numdocs(**self.params['params']) def indexsizeConfig(self): cores = self._getCores() - availableram = os.environ.get('availableram', 16868532224) graph = [ INDEXSIZECORE_GRAPH_TPL.format(core=c) for c in cores] - return INDEXSIZE_GRAPH_TPL.format(cores=" ".join(cores), cores_config="\n".join(graph), availableram=availableram) + return INDEXSIZE_GRAPH_TPL.format(cores=" ".join(cores), cores_config="\n".join(graph)) def indexsize(self): results = [] for c, size in self.solrcoresadmin.indexsize(**self.params['params']).items(): - results.append("%s.value %s" % (c, size)) + results.append("%s.value %d" % (c, size)) + cores = self._getCores() + mbean = self._getMBean(cores[0]) + memory = mbean.memory() + results.append('xmx.value %d' % memory['max']) return "\n".join(results) + def memoryConfig(self): + cores = self._getCores() + mbean = self._getMBean(cores[0]) + memory = mbean.memory() + return MEMORYUSAGE_GRAPH_TPL.format(availableram=memory['max'] * 1.05) + + def memory(self): + results = [] + cores = self._getCores() + mbean = self._getMBean(cores[0]) + memory = mbean.memory() + return '\n'.join(['used.value %d' % memory['used'], 'max.value %d' % memory['max']]) + def documentcacheConfig(self): return self._cacheConfig('documentcache', 'Document Cache') @@ -422,10 +464,10 @@ class SolrMuninGraph: if __name__ == '__main__': params = parse_params() SOLR_HOST_PORT = os.environ.get('host_port', 'localhost:8080').replace('http://', '') - mb = SolrMuninGraph(SOLR_HOST_PORT, params) - try: - if hasattr(mb, params['op']): - print getattr(mb, params['op'])(params['type']) - except Exception, ex: - print "ERROR: %s" % ex - exit(1) + SOLR_URL = os.environ.get('url', '/solr') + if SOLR_URL[0] != '/': + SOLR_URL = '/' + SOLR_URL + mb = SolrMuninGraph(SOLR_HOST_PORT, SOLR_URL, params) + if hasattr(mb, params['op']): + print getattr(mb, params['op'])(params['type']) + From b9781d98a916aa1b04b6be77921bb392ccf3f56d Mon Sep 17 00:00:00 2001 From: antonio Date: Thu, 12 Sep 2013 23:57:59 +0200 Subject: [PATCH 2/3] fix issue #3 --- plugins/solr/solr4_ | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/plugins/solr/solr4_ b/plugins/solr/solr4_ index 10d48c19..ca468b0b 100755 --- a/plugins/solr/solr4_ +++ b/plugins/solr/solr4_ @@ -336,6 +336,15 @@ class SolrMuninGraph: def _cacheConfig(self, cacheType, cacheName): return CACHE_GRAPH_TPL.format(core=self.params['core'], cacheType=cacheType, cacheName=cacheName) + def _format4Value(self, value): + if isinstance(value, basestring): + return "%s" + if isinstance(value, int): + return "%d" + if isinstance(value, float): + return "%.6f" + return "%s" + def _cacheFetch(self, cacheType, fields = None): fields = fields or ['size', 'lookups', 'hits', 'inserts', 'evictions'] hits_fields = ['lookups', 'hits', 'inserts'] @@ -345,7 +354,8 @@ class SolrMuninGraph: data = getattr(solrmbean, cacheType)() results.append('multigraph solr_{core}_{cacheType}_hit_rates'.format(core=self.params['core'], cacheType=cacheType)) for label in hits_fields: - results.append("%s.value %.8f" % (label, data[label])) + vformat = self._format4Value(data[label]) + results.append(("%s.value " + vformat) % (label, data[label])) results.append('multigraph solr_{core}_{cacheType}_size'.format(core=self.params['core'], cacheType=cacheType)) for label in size_fields: results.append("%s.value %d" % (label, data[label])) From 5cc126eebccb71b57b802dc9a739d42415f2eb58 Mon Sep 17 00:00:00 2001 From: antonio Date: Wed, 18 Sep 2013 13:29:26 +0200 Subject: [PATCH 3/3] Comment cleanup --- plugins/solr/solr4_ | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/plugins/solr/solr4_ b/plugins/solr/solr4_ index ca468b0b..690aed66 100755 --- a/plugins/solr/solr4_ +++ b/plugins/solr/solr4_ @@ -21,25 +21,30 @@ # DEALINGS IN THE SOFTWARE. # # Solr 4.* munin graph plugin -# Plugins configuration parameters: +# Project repo: https://github.com/averni/munin-solr +# +# Plugin configuration parameters: # # [solr_*] # env.host_port # env.url # env.qpshandler_ # -# ex: -# env.host_port solrhost:8080 -# env.url /solr -# env.qpshandler_select /select +# Example: +# [solr_*] +# env.host_port solrhost:8080 +# env.url /solr +# env.qpshandler_select /select # # Install plugins: # ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_numdocs_core_1 # ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_requesttimes_select -# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_core_1_select +# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_qps +# ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_qps_core_1_select # ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_indexsize # ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_memory # +# import sys