From 36192adcef9a1c2c796437fee2d1881ef1437e84 Mon Sep 17 00:00:00 2001
From: Marco Bardelli <bardelli.marco@gmail.com>
Date: Mon, 27 Jul 2015 10:48:29 +0200
Subject: [PATCH 1/4] allow SSL with pymongo >= 3.0
---
mongodbtools/collection_stats.py | 33 ++++++++++++++++++++++++----
mongodbtools/index_stats.py | 36 ++++++++++++++++++++++++++-----
mongodbtools/redundant_indexes.py | 32 +++++++++++++++++++++++----
requirements.txt | 2 +-
4 files changed, 89 insertions(+), 14 deletions(-)
diff --git a/mongodbtools/collection_stats.py b/mongodbtools/collection_stats.py
index 4e03464..bf85651 100755
--- a/mongodbtools/collection_stats.py
+++ b/mongodbtools/collection_stats.py
@@ -7,9 +7,17 @@
from prettytable import PrettyTable
import psutil
-from pymongo import Connection
from pymongo import ReadPreference
from optparse import OptionParser
+from distutils.version import StrictVersion
+import pymongo
+
+HAS_PYMONGO3 = bool(StrictVersion(pymongo.version) >= StrictVersion('3.0'))
+
+if HAS_PYMONGO3:
+ from pymongo import MongoClient
+else:
+ from pymongo import Connection as MongoClient
def compute_signature(index):
signature = index["ns"]
@@ -50,18 +58,34 @@ def get_cli_options():
default="",
metavar="PASSWORD",
help="Admin password if authentication is enabled")
+ parser.add_option("--ssl-cert",
+ dest="ssl_certfile",
+ default=None,
+ metavar="CERTIFICATE",
+ help="SSL Certificate to use is SSL is enabled (only with pymongo >= 3)")
+ parser.add_option("--ssl-ca-certs",
+ dest="ssl_ca_certs",
+ default=None,
+ metavar="CA",
+ help="SSL Certificate of CA for certificate validation if SSL is enabled (only with pymongo >= 3)")
(options, args) = parser.parse_args()
return options
-def get_connection(host, port, username, password):
+def get_connection(host, port, username, password, ssl_certfile=None, ssl_ca_certs=None):
userPass = ""
if username and password:
userPass = username + ":" + password + "@"
mongoURI = "mongodb://" + userPass + host + ":" + str(port)
- return Connection(host=mongoURI, read_preference=ReadPreference.SECONDARY)
+
+ conn_kwargs = dict(host=mongoURI, read_preference=ReadPreference.SECONDARY)
+
+ if HAS_PYMONGO3:
+ conn_kwargs.update(dict(ssl_certfile=ssl_certfile, ssl_ca_certs=ssl_ca_certs))
+
+ return MongoClient(**conn_kwargs)
# From http://www.5dollarwhitebox.org/drupal/node/84
def convert_bytes(bytes):
@@ -92,7 +116,8 @@ def main(options):
}
all_stats = []
- connection = get_connection(options.host, options.port, options.user, options.password)
+ connection = get_connection(options.host, options.port, options.user, options.password,
+ options.ssl_certfile, options.ssl_ca_certs)
all_db_stats = {}
diff --git a/mongodbtools/index_stats.py b/mongodbtools/index_stats.py
index fd1313d..6a73a2e 100755
--- a/mongodbtools/index_stats.py
+++ b/mongodbtools/index_stats.py
@@ -7,9 +7,18 @@
from prettytable import PrettyTable
import psutil
-from pymongo import Connection
+from socket import getfqdn
from pymongo import ReadPreference
from optparse import OptionParser
+from distutils.version import StrictVersion
+import pymongo
+
+HAS_PYMONGO3 = bool(StrictVersion(pymongo.version) >= StrictVersion('3.0'))
+
+if HAS_PYMONGO3:
+ from pymongo import MongoClient
+else:
+ from pymongo import Connection as MongoClient
def compute_signature(index):
signature = index["ns"]
@@ -70,18 +79,34 @@ def get_cli_options():
default="",
metavar="PASSWORD",
help="Admin password if authentication is enabled")
+ parser.add_option("--ssl-cert",
+ dest="ssl_certfile",
+ default=None,
+ metavar="CERTIFICATE",
+ help="SSL Certificate to use is SSL is enabled")
+ parser.add_option("--ssl-ca-certs",
+ dest="ssl_ca_certs",
+ default=None,
+ metavar="CA",
+ help="SSL Certificate of CA for certificate validation if SSL is enabled")
(options, args) = parser.parse_args()
return options
-def get_connection(host, port, username, password):
+def get_connection(host, port, username, password, ssl_certfile=None, ssl_ca_certs=None):
userPass = ""
if username and password:
userPass = username + ":" + password + "@"
mongoURI = "mongodb://" + userPass + host + ":" + str(port)
- return Connection(host=mongoURI, read_preference=ReadPreference.SECONDARY)
+
+ conn_kwargs = dict(host=mongoURI, read_preference=ReadPreference.SECONDARY)
+
+ if HAS_PYMONGO3:
+ conn_kwargs.update(dict(ssl_certfile=ssl_certfile, ssl_ca_certs=ssl_ca_certs))
+
+ return MongoClient(**conn_kwargs)
def main(options):
summary_stats = {
@@ -91,7 +116,8 @@ def main(options):
}
all_stats = []
- connection = get_connection(options.host, options.port, options.user, options.password)
+ connection = get_connection(options.host, options.port, options.user, options.password,
+ options.ssl_certfile, options.ssl_ca_certs)
all_db_stats = {}
@@ -164,7 +190,7 @@ def main(options):
print "Total Index Size:", convert_bytes(summary_stats["indexSize"])
# this is only meaningful if we're running the script on localhost
- if options.host == "localhost":
+ if options.host == "localhost" or options.host == getfqdn():
ram_headroom = psutil.phymem_usage()[0] - summary_stats["indexSize"]
print "RAM Headroom:", convert_bytes(ram_headroom)
print "RAM Used: %s (%s%%)" % (convert_bytes(psutil.phymem_usage()[1]), psutil.phymem_usage()[3])
diff --git a/mongodbtools/redundant_indexes.py b/mongodbtools/redundant_indexes.py
index 12b202e..6cd8807 100755
--- a/mongodbtools/redundant_indexes.py
+++ b/mongodbtools/redundant_indexes.py
@@ -6,10 +6,17 @@
with just fields {field1:1}, the latter index is not needed since the first index already
indexes the necessary fields.
"""
-from pymongo import Connection
from pymongo import ReadPreference
from optparse import OptionParser
+from distutils.version import StrictVersion
+import pymongo
+HAS_PYMONGO3 = bool(StrictVersion(pymongo.version) >= StrictVersion('3.0'))
+
+if HAS_PYMONGO3:
+ from pymongo import MongoClient
+else:
+ from pymongo import Connection as MongoClient
def get_cli_options():
parser = OptionParser(usage="usage: python %prog [options]",
@@ -40,21 +47,38 @@ def get_cli_options():
default="",
metavar="PASSWORD",
help="Admin password if authentication is enabled")
+ parser.add_option("--ssl-cert",
+ dest="ssl_certfile",
+ default=None,
+ metavar="CERTIFICATE",
+ help="SSL Certificate to use is SSL is enabled (only with pymongo >= 3)")
+ parser.add_option("--ssl-ca-certs",
+ dest="ssl_ca_certs",
+ default=None,
+ metavar="CA",
+ help="SSL Certificate of CA for certificate validation if SSL is enabled (only with pymongo >= 3)")
(options, args) = parser.parse_args()
return options
-def get_connection(host, port, username, password):
+def get_connection(host, port, username, password, ssl_certfile=None, ssl_ca_certs=None):
userPass = ""
if username and password:
userPass = username + ":" + password + "@"
mongoURI = "mongodb://" + userPass + host + ":" + str(port)
- return Connection(host=mongoURI, read_preference=ReadPreference.SECONDARY)
+
+ conn_kwargs = dict(host=mongoURI, read_preference=ReadPreference.SECONDARY)
+
+ if HAS_PYMONGO3:
+ conn_kwargs.update(dict(ssl_certfile=ssl_certfile, ssl_ca_certs=ssl_ca_certs))
+
+ return MongoClient(**conn_kwargs)
def main(options):
- connection = get_connection(options.host, options.port, options.user, options.password)
+ connection = get_connection(options.host, options.port, options.user, options.password,
+ options.ssl_certfile, options.ssl_ca_certs)
def compute_signature(index):
signature = index["ns"]
diff --git a/requirements.txt b/requirements.txt
index c880503..237758a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-pymongo==2.1
+pymongo>=2.1
PrettyTable==0.7.1
psutil==0.3.0
mongoengine==0.5.0
From f885307d50d308b3a1a91ace5d7e9e3e91eed67d Mon Sep 17 00:00:00 2001
From: Marco Bardelli <bardelli.marco@gmail.com>
Date: Mon, 27 Jul 2015 11:09:56 +0200
Subject: [PATCH 2/4] use print as function looking forward to py3
---
mongodbtools/collection_stats.py | 20 ++++++++++----------
mongodbtools/index_stats.py | 24 ++++++++++++------------
mongodbtools/redundant_indexes.py | 8 ++++----
3 files changed, 26 insertions(+), 26 deletions(-)
diff --git a/mongodbtools/collection_stats.py b/mongodbtools/collection_stats.py
index bf85651..de41de7 100755
--- a/mongodbtools/collection_stats.py
+++ b/mongodbtools/collection_stats.py
@@ -17,7 +17,7 @@
if HAS_PYMONGO3:
from pymongo import MongoClient
else:
- from pymongo import Connection as MongoClient
+ from pymongo import Connection as MongoClient # pylint: disable=E0611
def compute_signature(index):
signature = index["ns"]
@@ -26,7 +26,7 @@ def compute_signature(index):
return signature
def get_collection_stats(database, collection):
- print "Checking DB: %s" % collection.full_name
+ print("Checking DB: %s" % collection.full_name)
return database.command("collstats", collection.name)
def get_cli_options():
@@ -170,18 +170,18 @@ def main(options):
])
print
- print x.get_string(sortby="% Size")
- print "Total Documents:", summary_stats["count"]
- print "Total Data Size:", convert_bytes(summary_stats["size"])
- print "Total Index Size:", convert_bytes(summary_stats["indexSize"])
- print "Total Storage Size:", convert_bytes(summary_stats["storageSize"])
+ print(x.get_string(sortby="% Size"))
+ print("Total Documents:", summary_stats["count"])
+ print("Total Data Size:", convert_bytes(summary_stats["size"]))
+ print("Total Index Size:", convert_bytes(summary_stats["indexSize"]))
+ print("Total Storage Size:", convert_bytes(summary_stats["storageSize"]))
# this is only meaningful if we're running the script on localhost
if options.host == "localhost":
ram_headroom = psutil.phymem_usage()[0] - summary_stats["indexSize"]
- print "RAM Headroom:", convert_bytes(ram_headroom)
- print "RAM Used: %s (%s%%)" % (convert_bytes(psutil.phymem_usage()[1]), psutil.phymem_usage()[3])
- print "Available RAM Headroom:", convert_bytes((100 - psutil.phymem_usage()[3]) / 100 * ram_headroom)
+ print("RAM Headroom:", convert_bytes(ram_headroom))
+ print("RAM Used: %s (%s%%)" % (convert_bytes(psutil.phymem_usage()[1]), psutil.phymem_usage()[3]))
+ print("Available RAM Headroom:", convert_bytes((100 - psutil.phymem_usage()[3]) / 100 * ram_headroom))
if __name__ == "__main__":
options = get_cli_options()
diff --git a/mongodbtools/index_stats.py b/mongodbtools/index_stats.py
index 6a73a2e..c3f9daa 100755
--- a/mongodbtools/index_stats.py
+++ b/mongodbtools/index_stats.py
@@ -18,7 +18,7 @@
if HAS_PYMONGO3:
from pymongo import MongoClient
else:
- from pymongo import Connection as MongoClient
+ from pymongo import Connection as MongoClient # pylint: disable=E0611
def compute_signature(index):
signature = index["ns"]
@@ -27,7 +27,7 @@ def compute_signature(index):
return signature
def get_collection_stats(database, collection):
- print "Checking DB: %s" % collection.full_name
+ print("Checking DB: %s" % collection.full_name)
return database.command("collstats", collection.name)
# From http://www.5dollarwhitebox.org/drupal/node/84
@@ -167,11 +167,11 @@ def main(options):
x.add_row(row)
- print "Index Overview"
- print x.get_string(sortby="Collection")
+ print("Index Overview")
+ print(x.get_string(sortby="Collection"))
print
- print "Top 5 Largest Indexes"
+ print("Top 5 Largest Indexes")
x = PrettyTable(["Collection", "Index","% Size", "Index Size"])
x.align["Collection"] = "l"
x.align["Index"] = "l"
@@ -182,19 +182,19 @@ def main(options):
top_five_indexes = sorted(index_size_mapping.keys(), reverse=True)[0:5]
for size in top_five_indexes:
x.add_row(index_size_mapping.get(size))
- print x
+ print(x)
print
- print "Total Documents:", summary_stats["count"]
- print "Total Data Size:", convert_bytes(summary_stats["size"])
- print "Total Index Size:", convert_bytes(summary_stats["indexSize"])
+ print("Total Documents:", summary_stats["count"])
+ print("Total Data Size:", convert_bytes(summary_stats["size"]))
+ print("Total Index Size:", convert_bytes(summary_stats["indexSize"]))
# this is only meaningful if we're running the script on localhost
if options.host == "localhost" or options.host == getfqdn():
ram_headroom = psutil.phymem_usage()[0] - summary_stats["indexSize"]
- print "RAM Headroom:", convert_bytes(ram_headroom)
- print "RAM Used: %s (%s%%)" % (convert_bytes(psutil.phymem_usage()[1]), psutil.phymem_usage()[3])
- print "Available RAM Headroom:", convert_bytes((100 - psutil.phymem_usage()[3]) / 100 * ram_headroom)
+ print("RAM Headroom:", convert_bytes(ram_headroom))
+ print("RAM Used: %s (%s%%)" % (convert_bytes(psutil.phymem_usage()[1]), psutil.phymem_usage()[3]))
+ print("Available RAM Headroom:", convert_bytes((100 - psutil.phymem_usage()[3]) / 100 * ram_headroom))
if __name__ == "__main__":
options = get_cli_options()
diff --git a/mongodbtools/redundant_indexes.py b/mongodbtools/redundant_indexes.py
index 6cd8807..64c843d 100755
--- a/mongodbtools/redundant_indexes.py
+++ b/mongodbtools/redundant_indexes.py
@@ -16,7 +16,7 @@
if HAS_PYMONGO3:
from pymongo import MongoClient
else:
- from pymongo import Connection as MongoClient
+ from pymongo import Connection as MongoClient # pylint: disable=E0611
def get_cli_options():
parser = OptionParser(usage="usage: python %prog [options]",
@@ -90,7 +90,7 @@ def compute_signature(index):
return signature
def report_redundant_indexes(current_db):
- print "Checking DB: %s" % current_db.name
+ print("Checking DB: %s" % current_db.name)
indexes = current_db.system.indexes.find()
index_map = {}
for index in indexes:
@@ -102,11 +102,11 @@ def report_redundant_indexes(current_db):
if signature == other_sig:
continue
if other_sig.startswith(signature):
- print "Index %s[%s] may be redundant with %s[%s]" % (
+ print ("Index %s[%s] may be redundant with %s[%s]" % (
index_map[signature]["ns"],
index_map[signature]["name"],
index_map[other_sig]["ns"],
- index_map[other_sig]["name"])
+ index_map[other_sig]["name"]))
databases= []
if options.database:
From 0fdb38f81264db054fcd1122dd0d2a7b055cbfbd Mon Sep 17 00:00:00 2001
From: Marco Bardelli <bardelli.marco@gmail.com>
Date: Mon, 27 Jul 2015 13:25:03 +0200
Subject: [PATCH 3/4] better format for print
---
mongodbtools/collection_stats.py | 15 ++++++++-------
mongodbtools/index_stats.py | 10 +++++-----
2 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/mongodbtools/collection_stats.py b/mongodbtools/collection_stats.py
index de41de7..943a08e 100755
--- a/mongodbtools/collection_stats.py
+++ b/mongodbtools/collection_stats.py
@@ -7,6 +7,7 @@
from prettytable import PrettyTable
import psutil
+from socket import getfqdn
from pymongo import ReadPreference
from optparse import OptionParser
from distutils.version import StrictVersion
@@ -171,17 +172,17 @@ def main(options):
print
print(x.get_string(sortby="% Size"))
- print("Total Documents:", summary_stats["count"])
- print("Total Data Size:", convert_bytes(summary_stats["size"]))
- print("Total Index Size:", convert_bytes(summary_stats["indexSize"]))
- print("Total Storage Size:", convert_bytes(summary_stats["storageSize"]))
+ print("Total Documents: %s" % summary_stats["count"])
+ print("Total Data Size: %s" % convert_bytes(summary_stats["size"]))
+ print("Total Index Size: %s" % convert_bytes(summary_stats["indexSize"]))
+ print("Total Storage Size: %s" % convert_bytes(summary_stats["storageSize"]))
# this is only meaningful if we're running the script on localhost
- if options.host == "localhost":
+ if options.host == "localhost" or options.host == getfqdn():
ram_headroom = psutil.phymem_usage()[0] - summary_stats["indexSize"]
- print("RAM Headroom:", convert_bytes(ram_headroom))
+ print("RAM Headroom: %s" % convert_bytes(ram_headroom))
print("RAM Used: %s (%s%%)" % (convert_bytes(psutil.phymem_usage()[1]), psutil.phymem_usage()[3]))
- print("Available RAM Headroom:", convert_bytes((100 - psutil.phymem_usage()[3]) / 100 * ram_headroom))
+ print("Available RAM Headroom: %s" % convert_bytes((100 - psutil.phymem_usage()[3]) / 100 * ram_headroom))
if __name__ == "__main__":
options = get_cli_options()
diff --git a/mongodbtools/index_stats.py b/mongodbtools/index_stats.py
index c3f9daa..0bccb21 100755
--- a/mongodbtools/index_stats.py
+++ b/mongodbtools/index_stats.py
@@ -185,16 +185,16 @@ def main(options):
print(x)
print
- print("Total Documents:", summary_stats["count"])
- print("Total Data Size:", convert_bytes(summary_stats["size"]))
- print("Total Index Size:", convert_bytes(summary_stats["indexSize"]))
+ print("Total Documents: %s" % summary_stats["count"])
+ print("Total Data Size: %s" % convert_bytes(summary_stats["size"]))
+ print("Total Index Size: %s" % convert_bytes(summary_stats["indexSize"]))
# this is only meaningful if we're running the script on localhost
if options.host == "localhost" or options.host == getfqdn():
ram_headroom = psutil.phymem_usage()[0] - summary_stats["indexSize"]
- print("RAM Headroom:", convert_bytes(ram_headroom))
+ print("RAM Headroom: %s" % convert_bytes(ram_headroom))
print("RAM Used: %s (%s%%)" % (convert_bytes(psutil.phymem_usage()[1]), psutil.phymem_usage()[3]))
- print("Available RAM Headroom:", convert_bytes((100 - psutil.phymem_usage()[3]) / 100 * ram_headroom))
+ print("Available RAM Headroom: %s" % convert_bytes((100 - psutil.phymem_usage()[3]) / 100 * ram_headroom))
if __name__ == "__main__":
options = get_cli_options()
From 16a3afe90050e88c34b91f32ca52bf622492f672 Mon Sep 17 00:00:00 2001
From: Marco Bardelli <bardelli.marco@gmail.com>
Date: Mon, 27 Jul 2015 13:25:38 +0200
Subject: [PATCH 4/4] allow console scripts main invocation w/o param
---
mongodbtools/collection_stats.py | 5 ++++-
mongodbtools/index_stats.py | 5 ++++-
mongodbtools/redundant_indexes.py | 5 ++++-
3 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/mongodbtools/collection_stats.py b/mongodbtools/collection_stats.py
index 943a08e..8bade4b 100755
--- a/mongodbtools/collection_stats.py
+++ b/mongodbtools/collection_stats.py
@@ -108,7 +108,10 @@ def convert_bytes(bytes):
size = '%.2fb' % bytes
return size
-def main(options):
+def main(options=None):
+ if options is None:
+ options = get_cli_options()
+
summary_stats = {
"count" : 0,
"size" : 0,
diff --git a/mongodbtools/index_stats.py b/mongodbtools/index_stats.py
index 0bccb21..11ccfdc 100755
--- a/mongodbtools/index_stats.py
+++ b/mongodbtools/index_stats.py
@@ -108,7 +108,10 @@ def get_connection(host, port, username, password, ssl_certfile=None, ssl_ca_cer
return MongoClient(**conn_kwargs)
-def main(options):
+def main(options=None):
+ if options is None:
+ options = get_cli_options()
+
summary_stats = {
"count" : 0,
"size" : 0,
diff --git a/mongodbtools/redundant_indexes.py b/mongodbtools/redundant_indexes.py
index 64c843d..fa5e843 100755
--- a/mongodbtools/redundant_indexes.py
+++ b/mongodbtools/redundant_indexes.py
@@ -76,7 +76,10 @@ def get_connection(host, port, username, password, ssl_certfile=None, ssl_ca_cer
return MongoClient(**conn_kwargs)
-def main(options):
+def main(options=None):
+ if options is None:
+ options = get_cli_options()
+
connection = get_connection(options.host, options.port, options.user, options.password,
options.ssl_certfile, options.ssl_ca_certs)