Changes needed to support pytest + refactor to make installable with pip

Author: KarlLevik

README.md

@@ -24,7 +24,7 @@ You only need the database connector(s) for the database system type(s) that you
 
 ## Configuration
 
-Copy the files `*.example.cfg` files to just `*.cfg`:
+Copy the files `conf/*.example.cfg` files to just `conf/*.cfg`:
 
  * `config.example.cfg` to `config.cfg`
  * `jobs.example.cfg` to `jobs.cfg`
@@ -52,3 +52,9 @@ You also need to add database credentials to the `datasources.cfg` file and emai
 If your database system is not yet supported, then you need to amend the `create_report` in the `DBReport` class.
 
 See also the *.example.cfg files for examples.
+
+To install the source code in editable mode for development:
+```bash
+git clone [email protected]:DiamondLightSource/db-jobs.git
+pip install --user -e db-jobs
+```

bin/runjob.py

@@ -0,0 +1,16 @@
+#
+# Copyright 2022 Karl Levik
+#
+
+import argparse
+import logging
+import dbjobs
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-j", "--job", help = "Specify job")
+parser.add_argument("-d", "--dir", help = "Specify config dir")
+
+args = parser.parse_args()
+
+j = dbjobs.create("job", job_section=args.job, conf_dir=args.dir, log_level=logging.DEBUG)
+j.run_job()

bin/runreport.py

@@ -0,0 +1,20 @@
+#
+# Copyright 2021 Karl Levik
+#
+import argparse
+import logging
+import dbjobs
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-j", "--job", help = "Specify job")
+parser.add_argument("-d", "--dir", help = "Specify config dir")
+parser.add_argument("-i", "--interval", help = "Specify interval")
+parser.add_argument("-m", "--start_month", help = "Specify start month")
+parser.add_argument("-y", "--start_year", help = "Specify start year")
+
+args = parser.parse_args()
+#r = DBReport(job_section=args.job, interval=args.interval, start_year=args.start_year, start_month=self.start_month, log_level=logging.DEBUG)
+r = dbjobs.create("report", job_section=args.job, conf_dir=args.dir, log_level=logging.DEBUG)
+r.make_sql(args.interval, args.start_year, args.start_month)
+r.run_job()
+r.send_email(f'{args.job} for {args.interval} starting {r.start_date}')

config.example.cfg conf/config.example.cfg

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"

datasources.example.cfg conf/datasources.example.cfg

@@ -0,0 +1,20 @@
+[metadata]
+name = dbjobs
+version = 0.5.0
+description = Python package to execute SQL statements and optionally generate reports from the result sets.
+description-file = README.md
+long_description = This package allows the execution of SQL statements. Several different database systems are supported. Optionally, the result sets (if any) can be used to create reports in either xlsx or csv format, and the reports can be emailed as attachments. SQL statements, database connecttion details and email preferences are stored in configuration files.
+author = Diamond Light Source
+license = Apache License, Version 2.0
+
+[options]
+packages = find:
+package_dir =
+	=src
+python_requires = >=3.7
+scripts =
+	bin/runjob.py
+	bin/runreport.py
+
+[options.packages.find]
+where = src

jobs.example.cfg conf/jobs.example.cfg

@@ -0,0 +1,4 @@
+import setuptools
+
+if __name__ == "__main__":
+    setuptools.setup()

pyproject.toml

@@ -0,0 +1,13 @@
+__version__ = "0.5.0"
+
+def create(type, job_section, conf_dir, log_level):
+    if type == "job":
+        from dbjobs.dbjob import DBJob
+        return DBJob(job_section=job_section, conf_dir=conf_dir, log_level=log_level)
+    elif type == "report":
+        from dbjobs.dbreport import DBReport
+        return DBReport(job_section=job_section, conf_dir=conf_dir, log_level=log_level)
+    else:
+        raise AttributeError(
+            f"{type} is not a supported dbjob type. Supported types are 'job' and 'report'"
+        )

runjob.py

@@ -0,0 +1,14 @@
+import sys
+
+class Base(object):
+    """Base class for project"""
+
+    def error(self, msg):
+        sys.exit(f"ERROR: {msg}")
+
+    def warning(self, msg):
+        print(f"WARNING: {msg}")
+
+    def debug(self, msg):
+        if self.debug_opt:
+            print(f"DEBUG: {msg}")

runreport.py

@@ -5,7 +5,8 @@
 import atexit
 from logging.handlers import RotatingFileHandler
 from datetime import datetime, timedelta, date
-import sys, os, copy
+import sys, os, copy, os.path
+from dbjobs.base import Base
 try:
     import pytds
 except ImportError:
@@ -19,27 +20,20 @@
 except ImportError:
     psycopg2 = None
 
+import configparser
 
-# Trick to make it work with both Python 2 and 3:
-try:
-  import configparser
-except ImportError:
-  import ConfigParser as configparser
-
-class DBJob():
+class DBJob(Base):
     """Utility methods to execute a database query with logging based on 
     configuration and command-line parameters"""
 
-    def __init__(self, log_level=logging.DEBUG):
-        if len(sys.argv) <= 1:
-            msg = "No parameters"
-            logging.getLogger().error(msg)
-            raise AttributeError(msg)
+    def __init__(self, job_section=None, conf_dir=None, log_level=logging.DEBUG):
+        if job_section is None:
+            self.error("Job section is required.")
 
-        self.read_config(sys.argv[1])
+        self.read_config(job_section, conf_dir)
         self.working_dir = self.config['directory']
         self.fileprefix = self.job['file_prefix']
-        self.set_logging(level = log_level, filepath = os.path.join(self.working_dir, '%s.log' % self.fileprefix))
+        self.set_logging(level = log_level, filepath = os.path.join(self.working_dir, f"{self.fileprefix}.log"))
         logging.getLogger().info("DBJob %s started" % self.job['fullname'])
         atexit.register(self.clean_up)
         self.sql = self.job['sql']
@@ -157,13 +151,22 @@ def get_section_items(self, conf_file, conf_section):
 
         return dict(config.items(conf_section))
 
-    def read_config(self, job_section):
+    def read_config(self, job_section, conf_dir=None):
         """Read the email settings, job configuration and DB credentials from
         the config.cfg, reports.cfg and datasources.cfg config files"""
 
-        self.config = self.get_section_items("config.cfg", job_section)
-        self.job = self.get_section_items("jobs.cfg", job_section)
+        conf_file = "config.cfg"
+        jobs_file = "jobs.cfg"
+        ds_file = "datasources.cfg"
+
+        if conf_dir:
+            conf_file = os.path.join(conf_dir, conf_file)
+            jobs_file = os.path.join(conf_dir, jobs_file)
+            ds_file = os.path.join(conf_dir, ds_file)
+
+        self.config = self.get_section_items(conf_file, job_section)
+        self.job = self.get_section_items(jobs_file, job_section)
         ds_section = self.job['datasource']
-        self.datasource = self.get_section_items("datasources.cfg", ds_section)
+        self.datasource = self.get_section_items(ds_file, ds_section)
 
         return True

setup.cfg

@@ -13,67 +13,61 @@
 import pytds
 import mysql.connector
 import psycopg2
-from dbjob import DBJob
+from dbjobs.dbjob import DBJob
 
-# Trick to make it work with both Python 2 and 3:
-try:
-  import configparser
-except ImportError:
-  import ConfigParser as configparser
+import configparser
 
 class DBReport(DBJob):
     """Utility methods to create a report and send it as en email attachment"""
 
-    def __init__(self, log_level=logging.DEBUG):
-        super().__init__(log_level=log_level)
-        self.get_parameters()
-
-        nowstr = str(datetime.now().strftime('%Y%m%d-%H%M%S'))
+    def __init__(self, job_section=None, conf_dir=None, log_level=logging.DEBUG):
+        super().__init__(job_section=job_section, conf_dir=conf_dir, log_level=log_level)
         self.filesuffix = self.job['file_suffix']
-        self.filename = '%s%s_%s-%s_%s%s' % (self.fileprefix, self.interval, self.start_year, self.start_month, nowstr, self.filesuffix)
-        self.set_logging(level = log_level, filepath = os.path.join(self.working_dir, '%s%s_%s-%s.log' % (self.fileprefix, self.interval, self.start_year, self.start_month)))
 
-    def get_parameters(self):
+    def get_start_date(self, interval=None, start_year=None, start_month=None):
         # Get input parameters, otherwise use default values
-        self.interval = 'month'
-
         today = date.today()
         first = today.replace(day=1)
         prev_date = first - timedelta(days=1)
 
-        if len(sys.argv) > 2:
-            self.interval = sys.argv[2]
+        if interval:
+            self.interval = interval
+        else:
+            self.interval = "month"
 
-        if len(sys.argv) >= 2:
-            if self.interval == 'month':
-                self.start_year = prev_date.year
-                self.start_month = prev_date.month
-            elif self.interval == 'year':
-                self.start_year = prev_date.year - 1
-                self.start_month = prev_date.month
-            else:
-                err_msg = 'interval must be "month" or "year"'
-                logging.getLogger().error(err_msg)
-                raise AttributeError(err_msg)
+        if self.interval == "month": 
+            self.start_year = prev_date.year
+            self.start_month = prev_date.month
 
-            if len(sys.argv) > 3:
-                self.start_year = sys.argv[3]  # e.g. 2018
-                if len(sys.argv) > 4:
-                    self.start_month = sys.argv[4] # e.g. 02
+        elif self.interval == "year":
+            self.start_year = prev_date.year - 1
+            self.start_month = prev_date.month
 
-        self.start_date = '%s/%s/01' % (self.start_year, self.start_month)
+        else:
+            self.error('interval must be "month" or "year"')
+        
+        if start_year:
+            self.start_year = start_year
+        if start_month:
+            self.start_month = start_month
+
+        return f'{self.start_year}/{self.start_month}/01'
 
-    def read_config(self, job_section):
-        super().read_config(job_section)
+    def read_config(self, job_section, conf_dir):
+        super().read_config(job_section, conf_dir=conf_dir)
         self.sender = self.config['sender']
         self.recipients = self.config['recipients']
 
-    def make_sql(self):
+    def make_sql(self, interval, start_year, start_month):
         """Create proper SQL from the template - merge the headers in as aliases"""
         self.headers = self.job['sql_headers'].split(',')
         fmt = copy.deepcopy(self.headers)
+
+        self.start_date = self.get_start_date(interval, start_year, start_month)
         fmt.append(self.start_date)
-        fmt.append(self.interval)
+        fmt.append(interval)
+        nowstr = str(datetime.now().strftime('%Y%m%d-%H%M%S'))
+        self.filename = '%s%s_%s-%s_%s%s' % (self.fileprefix, interval, start_year, start_month, nowstr, self.filesuffix)
         self.sql = self.job['sql'].format(*fmt)
 
     def run_job(self):
@@ -171,7 +165,7 @@ def create_csv(self, result_set):
         logging.getLogger().debug(msg)
 
 
-    def send_email(self):
+    def send_email(self, subject):
         report_name = self.job["fullname"]
         attach_report = True
         if self.config["attach"].lower() == "no":
@@ -180,7 +174,7 @@ def send_email(self):
             filepath = os.path.join(self.working_dir, self.filename)
 
             message = MIMEMultipart()
-            message['Subject'] = '%s for %s starting %s' % (report_name, self.interval, self.start_date)
+            message['Subject'] = subject
             message['From'] = self.sender
             message['To'] = self.recipients
 

setup.py

@@ -0,0 +1,24 @@
+# pytest configuration file
+
+import os
+import pytest
+import dbjobs
+import logging
+
+
+@pytest.fixture(scope="session")
+def testconfdir():
+    """Return the directory path to the configuration files."""
+    config_dir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "..", "conf")
+    )
+    if not os.path.exists(config_dir):
+        pytest.skip(
+            "No configuration dir found. Skipping end-to-end tests."
+        )
+    return config_dir
+
+@pytest.fixture
+def testdbreport(testconfdir):
+    """Return a DBReport object for the test configuration."""
+    yield dbjobs.create("report", job_section="FaultReport", conf_dir=testconfdir, log_level=logging.DEBUG)

src/dbjobs/__init__.py

@@ -0,0 +1,10 @@
+import time
+
+
+def test_report(testdbreport):
+    testdbreport.make_sql("month", "2022", "01")
+    testdbreport.run_job()
+    # testdbreport.send_email()
+
+    # assert that output file exists in expected dir and so on ...
+