From 276169a6c937ecc322859e24f8e48ca366691ee3 Mon Sep 17 00:00:00 2001 From: Alban Date: Thu, 13 Feb 2020 16:11:16 +0100 Subject: [PATCH] Update mongodb plugins * Update mongodb plugins * fix category * chmod +x * flake8-CI * flake8 adjustments for mongo_collection_ and env python3 --- plugins/mongodb/mongo_btree | 22 ++++- plugins/mongodb/mongo_collection_ | 135 +++++++++++++++--------------- plugins/mongodb/mongo_conn | 22 ++++- plugins/mongodb/mongo_lag | 42 ++++++---- plugins/mongodb/mongo_lock | 22 ++++- plugins/mongodb/mongo_mem | 84 +++++++++++++------ plugins/mongodb/mongo_ops | 90 ++++++++++++++------ plugins/mongodb/mongodb_conn | 4 +- plugins/mongodb/mongodb_docs | 85 +++++++++++++++++++ 9 files changed, 360 insertions(+), 146 deletions(-) create mode 100755 plugins/mongodb/mongodb_docs diff --git a/plugins/mongodb/mongo_btree b/plugins/mongodb/mongo_btree index bddd972a..ea27faaa 100755 --- a/plugins/mongodb/mongo_btree +++ b/plugins/mongodb/mongo_btree @@ -1,6 +1,22 @@ -#!/usr/bin/python +#!/usr/bin/env python +""" +=head1 NAME + MongoDB btree Plugin -## GENERATED FILE - DO NOT EDIT +=head1 APPLICABLE SYSTEMS + + Works until MongoDB 2.7. The "indexCounters" field was removed in 2.8 version. + +=head1 CONFIGURATION + + [mongo_btree] + env.MONGO_DB_URI mongodb://user:password@host:port/dbname + +=head1 AUTHOR + + Original script there : https://github.com/comerford/mongo-munin + Doc added by Alban Espie-Guillon +""" import urllib2 import sys @@ -50,5 +66,3 @@ if __name__ == "__main__": doConfig() else: doData() - - diff --git a/plugins/mongodb/mongo_collection_ b/plugins/mongodb/mongo_collection_ index 229435fe..37d939e7 100755 --- a/plugins/mongodb/mongo_collection_ +++ b/plugins/mongodb/mongo_collection_ @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # vim: set sts=4 sw=4 encoding=utf-8 @@ -29,8 +29,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#%# family=auto -#%# capabilities=suggest autoconf +# #%# family=auto +# #%# capabilities=suggest autoconf import pymongo @@ -40,7 +40,7 @@ settings_host = '127.0.0.1' settings_port = 27017 # mongodb_uri will override host and port settings_mongodb_uri = '' -settings_db = 'mydb' +settings_db = 'admin' settings_user = '' settings_password = '' settings_ignoredb = {} @@ -88,98 +88,97 @@ typeIndex['indexsize']['category'] = 'db' def getCollstats(graphtype): - if settings_mongodb_uri: - con = pymongo.MongoClient(settings_mongodb_uri) - else: - con = pymongo.MongoClient(settings_host, int(settings_port)) + if settings_mongodb_uri: + con = pymongo.MongoClient(settings_mongodb_uri) + else: + con = pymongo.MongoClient(settings_host, int(settings_port)) - if settings_user: - db = con['admin'] - db.authenticate(settings_user, settings_password) + if settings_user: + db = con[settings_db] + db.authenticate(settings_user, settings_password) - stats_tmp = {} - for dbname in con.database_names(): - if dbname in settings_ignoredb: - continue - db = con[dbname] - for coll in db.collection_names(): - if coll.startswith('system.'): - continue - stats = db.command("collstats", coll) - collname = dbname + "_" + coll.replace('.', '_') - if not stats_tmp.has_key(collname): - stats_tmp[collname] = {} - stats_tmp[collname]['value'] = 0 - stats_tmp[collname]['dbname'] = dbname - if typeIndex[graphtype]['index'] in stats: - stats_tmp[collname]['value'] += long(stats[typeIndex[graphtype]['index']]) + stats_tmp = {} + for dbname in con.list_database_names(): + if dbname in settings_ignoredb: + continue + db = con[dbname] + for coll in db.list_collection_names(): + if coll.startswith('system.'): + continue + stats = db.command("collstats", coll) + collname = dbname + "_" + coll.replace('.', '_') + if collname not in stats_tmp: + stats_tmp[collname] = {} + stats_tmp[collname]['value'] = 0 + stats_tmp[collname]['dbname'] = dbname + if typeIndex[graphtype]['index'] in stats: + stats_tmp[collname]['value'] += int(stats[typeIndex[graphtype]['index']]) + + con.close() + + for collname, item in sorted(stats_tmp.items()): + yield ("%s" % collname, item['value'], item['dbname']) - con.close() - - for collname, item in sorted(stats_tmp.items()): - yield ("%s" % collname, item['value'], item['dbname']) - - -def doData(base,graphtype): +def doData(base, graphtype): lastdb = "" for coll, stats, db in sorted(getCollstats(graphtype), key=itemgetter(2)): if lastdb != db: - print "multigraph " + base + "_" + graphtype + "_" + db + print("multigraph " + base + "_" + graphtype + "_" + db) lastdb = db - print "%s_%s.value %s" % (graphtype, coll, stats) + print("%s_%s.value %s" % (graphtype, coll, stats)) -def doConfig(base,graphtype): +def doConfig(base, graphtype): lastdb = "" - for k,v,d in sorted(getCollstats(graphtype), key=itemgetter(2)): + for k, v, d in sorted(getCollstats(graphtype), key=itemgetter(2)): if lastdb != d: - print "multigraph " + base + "_" + graphtype + "_" + d + print("multigraph " + base + "_" + graphtype + "_" + d) lastdb = d -# print "graph_total total" - print "graph_title MongoDB " + typeIndex[graphtype]['title'] + " for database " + d - print "graph_args --base " + typeIndex[graphtype]['base'] + " " + typeIndex[graphtype]['scale'] - print "graph_vlabel " + typeIndex[graphtype]['yaxis'] - print "graph_category db" - print "%s_%s.label %s" % (graphtype, k, k) - print "%s_%s.min 0" % (graphtype, k) - print "%s_%s.draw LINE1" % (graphtype, k) + print("graph_title MongoDB " + typeIndex[graphtype]['title'] + " for database " + d) + print("graph_args --base " + typeIndex[graphtype]['base'] + " " + typeIndex[graphtype]['scale']) + print("graph_vlabel " + typeIndex[graphtype]['yaxis']) + print("graph_category db") + print("%s_%s.label %s" % (graphtype, k, k)) + print("%s_%s.min 0" % (graphtype, k)) + print("%s_%s.draw LINE1" % (graphtype, k)) + def doSuggest(): - print "keys" - for k in typeIndex.keys(): - print k + print("keys") + for k in typeIndex.keys(): + print(k) if __name__ == "__main__": - from sys import argv,exit - from os import environ,path + from sys import argv, exit + from os import environ, path import re # Could be done by a for loop # but i think if's are faster - if 'HOST' in environ: - settings_host = environ['HOST'] - if 'PORT' in environ: - settings_port = environ['PORT'] - if 'DB' in environ: - settings_db = environ['DB'] - if 'MONGO_USER' in environ: - settings_user = environ['MONGO_USER'] - if 'PASSWORD' in environ: - settings_password = environ['PASSWORD'] - if 'IGNOREDB' in environ: - settings_ignoredb = environ['IGNOREDB'].split(',') + if 'host' in environ: + settings_host = environ['host'] + if 'port' in environ: + settings_port = environ['port'] + if 'db' in environ: + settings_db = environ['db'] + if 'username' in environ: + settings_user = environ['username'] + if 'password' in environ: + settings_password = environ['password'] + if 'ignoredb' in environ: + settings_ignoredb = environ['ignoredb'].split(',') m = re.search('^(.*)_([a-zA-Z0-9]*)$', path.basename(argv[0])) if len(argv) < 2: - doData(m.group(1),m.group(2)) + doData(m.group(1), m.group(2)) elif argv[1] == "config": - doConfig(m.group(1),m.group(2)) + doConfig(m.group(1), m.group(2)) elif argv[1] == "autoconf": - print "yes" + print("yes") elif argv[1] == "suggest": doSuggest() else: - print "invalid argument" + print("invalid argument") exit(1) diff --git a/plugins/mongodb/mongo_conn b/plugins/mongodb/mongo_conn index 62324c65..727c2cda 100755 --- a/plugins/mongodb/mongo_conn +++ b/plugins/mongodb/mongo_conn @@ -1,6 +1,22 @@ -#!/usr/bin/python +#!/usr/bin/env python +""" +=head1 NAME + MongoDB connections Plugin -## GENERATED FILE - DO NOT EDIT +=head1 APPLICABLE SYSTEMS + + Works until MongoDB 3.6. The httpinterface was later removed. + +=head1 CONFIGURATION + + [mongo_lock] + env.MONGO_DB_URI mongodb://user:password@host:port/dbname + +=head1 AUTHOR + + Original script there : https://github.com/comerford/mongo-munin + Doc added by Alban Espie-Guillon +""" import urllib2 import sys @@ -40,5 +56,3 @@ if __name__ == "__main__": doConfig() else: doData() - - diff --git a/plugins/mongodb/mongo_lag b/plugins/mongodb/mongo_lag index 681798ff..8a44541c 100755 --- a/plugins/mongodb/mongo_lag +++ b/plugins/mongodb/mongo_lag @@ -1,24 +1,38 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ +=head1 NAME MongoDB Replication Lag - ~~~~~~~~~~~~~~~~~~~~~~~ Connects to a single mongo instance and retrieve replication lag for all connected members. - munin-node.conf: +=head1 APPLICABLE SYSTEMS + + MongoDB 3.X and 4.X with pymongo installed. + +=head1 CONFIGURATION + + munin-node.conf + defauts for host is 127.0.0.1 and port 27017 + and will work without being defined : + [mongo_lag] env.host 127.0.0.1 env.port 27017 env.username user env.password P@55w0rd - :author: Stefan Andersen - :license: The Beer Ware License (Revision 42) - wrote this file. As long - as you retain this notice you can do whatever you want - with this stuff. If we meet some day, and you think - this stuff is worth it, you can buy me a beer in return. +=head1 AUTHOR + + Stefan Andersen + Updated by Alban Espie-Guillon + +=head1 LICENSE + The Beer Ware License (Revision 42) + wrote this file. As long + as you retain this notice you can do whatever you want + with this stuff. If we meet some day, and you think + this stuff is worth it, you can buy me a beer in return. """ import os import sys @@ -33,7 +47,7 @@ def _get_members(): if username: connAuth = conn['admin'] connAuth.authenticate(username, password) - + repl_status = conn.admin.command("replSetGetStatus") members = {} @@ -51,17 +65,17 @@ def run(): for member in members: lag = (primary_optime - members[member]['optimeDate']).seconds - print "{0}.value {1}".format(member, lag) + print("{0}.value {1}".format(member, lag)) def config(): - print """graph_title MongoDB replication lag + print("""graph_title MongoDB replication lag graph_args --base 1000 graph_vlabel Replication lag (seconds) graph_category db -""" +""") for member in _get_members(): - print "{0}.label {0}".format(member) + print("{0}.label {0}".format(member)) if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == "config": diff --git a/plugins/mongodb/mongo_lock b/plugins/mongodb/mongo_lock index ef490d7d..9bd84791 100755 --- a/plugins/mongodb/mongo_lock +++ b/plugins/mongodb/mongo_lock @@ -1,6 +1,22 @@ -#!/usr/bin/python +#!/usr/bin/env python +""" +=head1 NAME + MongoDB lock Plugin -## GENERATED FILE - DO NOT EDIT +=head1 APPLICABLE SYSTEMS + + MongoDB 2.X. The "lockTime" field was removed in later versions. + +=head1 CONFIGURATION + + [mongo_lock] + env.MONGO_DB_URI mongodb://user:password@host:port/dbname + +=head1 AUTHOR + + Original script there : https://github.com/comerford/mongo-munin + Doc added by Alban Espie-Guillon +""" import urllib2 import sys @@ -47,5 +63,3 @@ if __name__ == "__main__": doConfig() else: doData() - - diff --git a/plugins/mongodb/mongo_mem b/plugins/mongodb/mongo_mem index 8f720602..6daeb7ad 100755 --- a/plugins/mongodb/mongo_mem +++ b/plugins/mongodb/mongo_mem @@ -1,49 +1,83 @@ -#!/usr/bin/python +#!/usr/bin/env python3 +""" +=head1 NAME + MongoDB mem Plugin -## GENERATED FILE - DO NOT EDIT +=head1 APPLICABLE SYSTEMS -import urllib2 + MongoDB 3.X and 4.X with pymongo installed. + +=head1 CONFIGURATION + + munin-node.conf + defauts for host is 127.0.0.1 and port 27017 + and will work without being defined : + + [mongo_mem] + env.host 127.0.0.1 + env.port 27017 + env.username user + env.password P@55w0rd + env.db dbname + + or + + [mongodb_mem] + env.MONGO_DB_URI mongodb://user:password@host:port/dbname + +=head1 AUTHOR + + Original script there : https://github.com/comerford/mongo-munin + Updated by Alban Espie-Guillon +""" import sys - -try: - import json -except ImportError: - import simplejson as json - +import os +import pymongo def getServerStatus(): - raw = urllib2.urlopen( "http://127.0.0.1:28017/_status" ).read() - return json.loads( raw )["serverStatus"] + if 'MONGO_DB_URI' in os.environ: + c = pymongo.MongoClient(os.environ['MONGO_DB_URI']) + + elif 'username' and 'password' in os.environ: + host = os.environ.get('host', '127.0.0.1') + port = os.environ.get('port', 27017) + username = os.environ.get('username', '') + password = os.environ.get('password', '') + c = pymongo.MongoClient(host, int(port)) + if username: + cAuth = c['admin'] + cAuth.authenticate(username, password) + + else: + c = pymongo.MongoClient() + + return c.admin.command('serverStatus', workingSet=True) def ok(s): return s == "resident" or s == "virtual" or s == "mapped" def doData(): - for k,v in getServerStatus()["mem"].iteritems(): + for k,v in getServerStatus()["mem"].items(): if ok(k): print( str(k) + ".value " + str(v * 1024 * 1024) ) -def doConfig(): - print "graph_title MongoDB memory usage" - print "graph_args --base 1024 -l 0 --vertical-label Bytes" - print "graph_category db" +def doConfig(): + print(""" +graph_title MongoDB memory usage +graph_args --base 1024 -l 0 --vertical-label Bytes +graph_category mongodb +""") for k in getServerStatus()["mem"]: if ok( k ): - print k + ".label " + k - print k + ".draw LINE1" - - - - - + print(k + ".label " + k) + print(k + ".draw LINE1") if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "config": doConfig() else: doData() - - diff --git a/plugins/mongodb/mongo_ops b/plugins/mongodb/mongo_ops index 15f6b8b9..216fe270 100755 --- a/plugins/mongodb/mongo_ops +++ b/plugins/mongodb/mongo_ops @@ -1,45 +1,85 @@ -#!/usr/bin/python +#!/usr/bin/env python3 +""" +=head1 NAME + MongoDB ops Plugin -## GENERATED FILE - DO NOT EDIT +=head1 APPLICABLE SYSTEMS -import urllib2 + MongoDB 3.X and 4.X with pymongo installed. + +=head1 CONFIGURATION + + munin-node.conf + defauts for host is 127.0.0.1 and port 27017 + and will work without being defined : + + [mongodb_ops] + env.host 127.0.0.1 + env.port 27017 + env.username user + env.password P@55w0rd + env.db dbname + + or + + [mongodb_ops] + env.MONGO_DB_URI mongodb://user:password@host:port/dbname + +=head1 AUTHOR + + Original script there : https://github.com/comerford/mongo-munin + Updated by Alban Espie-Guillon +""" import sys - -try: - import json -except ImportError: - import simplejson as json - +import os +import pymongo def getServerStatus(): - raw = urllib2.urlopen( "http://127.0.0.1:28017/_status" ).read() - return json.loads( raw )["serverStatus"] + if 'MONGO_DB_URI' in os.environ: + c = pymongo.MongoClient(os.environ['MONGO_DB_URI']) + + elif 'username' and 'password' in os.environ: + host = os.environ.get('host', '127.0.0.1') + port = os.environ.get('port', 27017) + username = os.environ.get('username', '') + password = os.environ.get('password', '') + c = pymongo.MongoClient(host, int(port)) + if username: + cAuth = c['admin'] + cAuth.authenticate(username, password) + + else: + c = pymongo.MongoClient() + + return c.admin.command('serverStatus', workingSet=True) def doData(): ss = getServerStatus() - for k,v in ss["opcounters"].iteritems(): + for k,v in ss["opcounters"].items(): print( str(k) + ".value " + str(v) ) -def doConfig(): - print "graph_title MongoDB ops" - print "graph_args --base 1000 -l 0" - print "graph_vlabel ops / ${graph_period}" - print "graph_category db" - print "graph_total total" +def doConfig(): + print(""" +graph_title MongoDB ops +graph_args --base 1000 -l 0 +graph_vlabel ops / ${graph_period} +graph_category db +graph_total total +""") for k in getServerStatus()["opcounters"]: - print k + ".label " + k - print k + ".min 0" - print k + ".type COUNTER" - print k + ".max 500000" - print k + ".draw LINE1" + print(k + ".label " + k) + print(k + ".min 0") + print(k + ".type COUNTER") + print(k + ".max 500000") + print(k + ".draw LINE1") + if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "config": doConfig() else: doData() - - diff --git a/plugins/mongodb/mongodb_conn b/plugins/mongodb/mongodb_conn index 5772ea5c..831b7cda 100755 --- a/plugins/mongodb/mongodb_conn +++ b/plugins/mongodb/mongodb_conn @@ -53,8 +53,8 @@ def _get_connections(): def run(): connections = _get_connections() - for connection, value in connections.items(): - print(connection + ".value", value) + for c, v in connections.items(): + print(str(c) + ".value " + str(v)) def config(): diff --git a/plugins/mongodb/mongodb_docs b/plugins/mongodb/mongodb_docs new file mode 100755 index 00000000..c1eb201a --- /dev/null +++ b/plugins/mongodb/mongodb_docs @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +=head1 NAME + MongoDB docs Plugin + +=head1 APPLICABLE SYSTEMS + + MongoDB 3.X and 4.X with pymongo installed. + +=head1 CONFIGURATION + + munin-node.conf + defauts for host is 127.0.0.1 and port 27017 + and will work without being defined : + + [mongodb_docs] + env.host 127.0.0.1 + env.port 27017 + env.username user + env.password P@55w0rd + env.db dbname + + or + + [mongodb_docs] + env.MONGO_DB_URI mongodb://user:password@host:port/dbname + +=head1 AUTHOR + + Original script there : https://github.com/comerford/mongo-munin + Updated by Alban Espie-Guillon +""" +import sys +import os +import pymongo + + +def getServerStatus(): + if 'MONGO_DB_URI' in os.environ: + c = pymongo.MongoClient(os.environ['MONGO_DB_URI']) + + elif 'username' and 'password' in os.environ: + host = os.environ.get('host', '127.0.0.1') + port = os.environ.get('port', 27017) + username = os.environ.get('username', '') + password = os.environ.get('password', '') + c = pymongo.MongoClient(host, int(port)) + if username: + cAuth = c['admin'] + cAuth.authenticate(username, password) + + else: + c = pymongo.MongoClient() + + return c.admin.command('serverStatus', workingSet=True) + + +def doData(): + ss = getServerStatus() + for k, v in ss["metrics"]["document"].items(): + print(str(k) + ".value " + str(v)) + + +def doConfig(): + print(""" +graph_title MongoDB documents +graph_args --base 1000 -l 0 +graph_vlabel documents +graph_category db +""") + + for k in getServerStatus()["metrics"]["document"]: + print(k + ".label " + k) + print(k + ".min 0") + print(k + ".type COUNTER") + print(k + ".max 500000") + print(k + ".draw LINE1") + + +if __name__ == "__main__": + + if len(sys.argv) > 1 and sys.argv[1] == "config": + doConfig() + else: + doData()