Top

geoDSS.base module

The base module provides the rules_set class. This class is all you need to work with to run your testing.

Example

import geoDSS                                                        # be sure this is possible by installing geoDSS 
                                                                     # or invoking this from the right working dir

r = geoDSS.rules_set('geoDSS/examples/rule_sets/unit_test.yaml')     # and then check this path as well
r.execute( subject = {  'result': True,
                        'geometry': 'SRID=28992;POINT((125000 360000))'
                     })
print(r.report())
# -*- coding: utf-8 -*-
'''
The base module provides the rules_set class. This class is all you need to work with to run your testing.

Example
-------

    import geoDSS                                                        # be sure this is possible by installing geoDSS 
                                                                         # or invoking this from the right working dir

    r = geoDSS.rules_set('geoDSS/examples/rule_sets/unit_test.yaml')     # and then check this path as well
    r.execute( subject = {  'result': True,
                            'geometry': 'SRID=28992;POINT((125000 360000))'
                         })
    print(r.report())
'''

import copy
import datetime
import locale
import os
import sys
from abc import ABCMeta, abstractmethod
try:
    import exceptions
except:
    pass

import logging
import logging.handlers

import loaders
import processors
import tests
import reporters

class rules_set(object):
    '''
    Container object for a set of rules.
    '''

    class Rules(object):
        '''
        A convenience object to stick rules on.

        Especially handy for the evaluate test so that we can write nice things in the rule_set file like 'rules.test1'
        '''

        pass

        # todo: add an iterator or __getitem__ and use this class instead of the list (keep order!)


    def __init__(self, rules_set_file, loader_module = loaders.yaml_loader, **kwargs):
        '''
        Reads the rules_set with the load_rule_set method of the loader_moduler.

        If not specified the default yaml loader is used.

        `rules_set_file` is expected to be a file containing the rules set.
        '''

        dss = sys.modules[__name__]

        self.definition = None
        self.definition = loader_module.load_rule_set(rules_set_file)
        if not 'title' in self.definition.keys():
            self.definition['title'] = self.definition['name'] 
        if not 'description' in self.definition.keys():
            self.definition['description'] = '' 
        if not 'settings' in self.definition.keys():
            self.definition['settings'] = None
        if not 'logging' in self.definition.keys():
            self.definition['logging'] = {}
        if not 'break_on_true' in self.definition.keys():
            self.definition['break_on_true'] = False
        if not 'break_on_false' in self.definition.keys():
            self.definition['break_on_false'] = False

        self.logger = self._setup_logging()

        if not 'reporter' in self.definition.keys():
            self.reporter_module = reporters.md
        else:
            _group, _class = self.definition['reporter'].split('.',2)
            if hasattr(dss, _group):                                        # this allows for reporters outside the reporters module
                _group = getattr(dss, _group)
            else:
                self.logger.error('The module %s is not defined yet.' % _group)                         
                raise exceptions.NotImplementedError('The module %s is not defined yet. Did you mean to use the reporters module?' % _group)
            if hasattr(_group, _class):
                self.reporter_module = getattr(_group, _class)
                self.logger.debug("Selecting reporter %s" % self.reporter_module.__name__)
            else:
                self.logger.error('The reporter %s is not defined yet.' % self.definition['reporter'])
                raise exceptions.NotImplementedError('The reporter %s is not defined yet or has unmet dependencies.' % self.definition['reporter'])

        self.reporter_args = None
        if 'reporter_args' in self.definition.keys():
            self.reporter_args = self.definition['reporter_args']

        self._rules = self.Rules()
        self.rules = []
        for rule in self.definition['rules']:
            name = rule.keys()[0]
            definition = rule[name]
            _group, _class = definition['type'].split('.',2)
            if hasattr(dss, _group):
                _group = getattr(dss, _group)
            else:
                self.logger.error('The module %s is not defined yet.' % _group)
                raise exceptions.NotImplementedError('The module %s is not defined yet.Did you mean to use the tests or processors module?' % _group)
            if hasattr(_group, _class):
                # the class is in a module eg unit_test.unit_test
                _group = getattr(_group, _class)
                test_to_add = getattr(_group, _class)
                self.logger.debug("Adding rule with name: %s" % name)
                self.rules.append(test_to_add(name, definition, logger = self.logger, rules = self._rules, settings = self.definition['settings']))
            else:
                self.logger.error('The type %s is not defined yet.' % definition['type'])
                raise exceptions.NotImplementedError('The type %s is not defined yet or has unmet dependencies.' % definition['type'])


    def _setup_logging(self):
        '''
        Sets up the logger.

        It uses settings from the rule_set file:

            - logging:
              - level                       Python log level. Usually one of: DEBUG, INFO, ERROR (defaults to INFO)
              - format                      Python logging format string. Default: '%(asctime)s %(name)-12s %(levelname)-7s  %(message)s'
              - file                        A writeable file to write the log. This file will be log-rotated.
        '''

        logger = logging.getLogger()
        logger.name = self.definition['name']
        log_level = logging.INFO
        if 'level' in self.definition['logging'].keys():
            log_level = logging.getLevelName(self.definition['logging']['level'])
        logger.setLevel(log_level)
        log_format = '%(asctime)s %(name)-12s %(levelname)-7s  %(message)s'
        if 'format' in self.definition['logging'].keys():
            log_format = self.definition['logging']['format']
        formatter = logging.Formatter( log_format )
        if 'file' in self.definition['logging'].keys():
            hdlr = logging.handlers.RotatingFileHandler(self.definition['logging']['file'], maxBytes= 1 * 1000000, backupCount=5)
        else:
            hdlr = logging.StreamHandler()
        hdlr.setFormatter(formatter)
        logger.addHandler(hdlr)

        logger.info("Logger is activated on level: %s" % logging.getLevelName(log_level) )
        return logger


    def execute(self, subject):
        '''
        Executes processors and tests in the rules_set.

        this method should be called BEFORE the report method.

        `subject`       is expected to be a dict which is passed to each rule.
        '''

        self.result = []                                                 # the rule_set has it's own result we can report on; we fill it with the subjects
        self.result.append(copy.deepcopy(subject))                       # add the first subject to the rule_set result

        self.logger.info("Start execution of rules")
        self.logger.debug('First subject: ' + str(subject))

        for rule in self.rules:
            self.logger.debug('Executing rule "%s" with subject "%s"' % (str(rule.name), str(subject)))
            result = rule.execute(subject)
            setattr(self._rules, rule.name, rule)                        # we add a reference to each executed rule here
            if result:
                self.result.append(copy.deepcopy(subject))               # and add each subject to the rule_set result as well
                subject = result
                if self.definition['break_on_true'] and result.decision:
                    self.logger.info('Rule "%s" evaluated to True and ended execution due to break_on_true being True.' % str(rule.name))
                    break
                if self.definition['break_on_false'] and not result.decision:
                    self.logger.info('Rule "%s" evaluated to False and ended execution due to break_on_false being True.' % str(rule.name))
                    break
            if not result:
                self.logger.info('Rule "%s" returned False to end execution.' % str(rule.name))
                break
        self.logger.info("Finished execution of rules")


    def report(self, **kwargs):
        '''
        Report the result of the rules.

        this method should be called AFTER the execute method.

        The default reporter module is reporters.md.

        if `reporter` is part of the definition of the rule set,
        that reporter will be used.

        if `reporter_args` is part of the definition of the rule set,
         the arguments defined there will be passed to the reporter.
        
        if `output_format` is passed as a parameter to the report function,
        this will overrule the `output_format` defined in `reporter_args`.

        A yaml snippet to illustrate the selection of the markdown reporter with html output:

            name: test_bag_geocoder
            title: Test the BAG geocoder
            description: Test the BAG geocoder by geocoding an address and report the result
            reporter: reporters.md
            reporter_args:
              output_format: html

        If no arguments are defined in the rule set, 
        arguments passed in this report method will be passed to the reporter.

        The 'rule_set_reporter' method in the default reporter reporters.md accepts 
        an optional `output_format` parameter which is expected to be one of:

        - 'markdown'    (default)
        - 'html'
        - 'pdf'
        '''

        if self.reporter_args:
            if 'output_format' in kwargs:
                return self.reporter_module.rule_set_reporter(self, output_format = kwargs['output_format'], **self.reporter_args)
            else:
                return self.reporter_module.rule_set_reporter(self, **self.reporter_args)
        else:
            return self.reporter_module.rule_set_reporter(self, **kwargs)

Classes

class rules_set

Container object for a set of rules.

class rules_set(object):
    '''
    Container object for a set of rules.
    '''

    class Rules(object):
        '''
        A convenience object to stick rules on.

        Especially handy for the evaluate test so that we can write nice things in the rule_set file like 'rules.test1'
        '''

        pass

        # todo: add an iterator or __getitem__ and use this class instead of the list (keep order!)


    def __init__(self, rules_set_file, loader_module = loaders.yaml_loader, **kwargs):
        '''
        Reads the rules_set with the load_rule_set method of the loader_moduler.

        If not specified the default yaml loader is used.

        `rules_set_file` is expected to be a file containing the rules set.
        '''

        dss = sys.modules[__name__]

        self.definition = None
        self.definition = loader_module.load_rule_set(rules_set_file)
        if not 'title' in self.definition.keys():
            self.definition['title'] = self.definition['name'] 
        if not 'description' in self.definition.keys():
            self.definition['description'] = '' 
        if not 'settings' in self.definition.keys():
            self.definition['settings'] = None
        if not 'logging' in self.definition.keys():
            self.definition['logging'] = {}
        if not 'break_on_true' in self.definition.keys():
            self.definition['break_on_true'] = False
        if not 'break_on_false' in self.definition.keys():
            self.definition['break_on_false'] = False

        self.logger = self._setup_logging()

        if not 'reporter' in self.definition.keys():
            self.reporter_module = reporters.md
        else:
            _group, _class = self.definition['reporter'].split('.',2)
            if hasattr(dss, _group):                                        # this allows for reporters outside the reporters module
                _group = getattr(dss, _group)
            else:
                self.logger.error('The module %s is not defined yet.' % _group)                         
                raise exceptions.NotImplementedError('The module %s is not defined yet. Did you mean to use the reporters module?' % _group)
            if hasattr(_group, _class):
                self.reporter_module = getattr(_group, _class)
                self.logger.debug("Selecting reporter %s" % self.reporter_module.__name__)
            else:
                self.logger.error('The reporter %s is not defined yet.' % self.definition['reporter'])
                raise exceptions.NotImplementedError('The reporter %s is not defined yet or has unmet dependencies.' % self.definition['reporter'])

        self.reporter_args = None
        if 'reporter_args' in self.definition.keys():
            self.reporter_args = self.definition['reporter_args']

        self._rules = self.Rules()
        self.rules = []
        for rule in self.definition['rules']:
            name = rule.keys()[0]
            definition = rule[name]
            _group, _class = definition['type'].split('.',2)
            if hasattr(dss, _group):
                _group = getattr(dss, _group)
            else:
                self.logger.error('The module %s is not defined yet.' % _group)
                raise exceptions.NotImplementedError('The module %s is not defined yet.Did you mean to use the tests or processors module?' % _group)
            if hasattr(_group, _class):
                # the class is in a module eg unit_test.unit_test
                _group = getattr(_group, _class)
                test_to_add = getattr(_group, _class)
                self.logger.debug("Adding rule with name: %s" % name)
                self.rules.append(test_to_add(name, definition, logger = self.logger, rules = self._rules, settings = self.definition['settings']))
            else:
                self.logger.error('The type %s is not defined yet.' % definition['type'])
                raise exceptions.NotImplementedError('The type %s is not defined yet or has unmet dependencies.' % definition['type'])


    def _setup_logging(self):
        '''
        Sets up the logger.

        It uses settings from the rule_set file:

            - logging:
              - level                       Python log level. Usually one of: DEBUG, INFO, ERROR (defaults to INFO)
              - format                      Python logging format string. Default: '%(asctime)s %(name)-12s %(levelname)-7s  %(message)s'
              - file                        A writeable file to write the log. This file will be log-rotated.
        '''

        logger = logging.getLogger()
        logger.name = self.definition['name']
        log_level = logging.INFO
        if 'level' in self.definition['logging'].keys():
            log_level = logging.getLevelName(self.definition['logging']['level'])
        logger.setLevel(log_level)
        log_format = '%(asctime)s %(name)-12s %(levelname)-7s  %(message)s'
        if 'format' in self.definition['logging'].keys():
            log_format = self.definition['logging']['format']
        formatter = logging.Formatter( log_format )
        if 'file' in self.definition['logging'].keys():
            hdlr = logging.handlers.RotatingFileHandler(self.definition['logging']['file'], maxBytes= 1 * 1000000, backupCount=5)
        else:
            hdlr = logging.StreamHandler()
        hdlr.setFormatter(formatter)
        logger.addHandler(hdlr)

        logger.info("Logger is activated on level: %s" % logging.getLevelName(log_level) )
        return logger


    def execute(self, subject):
        '''
        Executes processors and tests in the rules_set.

        this method should be called BEFORE the report method.

        `subject`       is expected to be a dict which is passed to each rule.
        '''

        self.result = []                                                 # the rule_set has it's own result we can report on; we fill it with the subjects
        self.result.append(copy.deepcopy(subject))                       # add the first subject to the rule_set result

        self.logger.info("Start execution of rules")
        self.logger.debug('First subject: ' + str(subject))

        for rule in self.rules:
            self.logger.debug('Executing rule "%s" with subject "%s"' % (str(rule.name), str(subject)))
            result = rule.execute(subject)
            setattr(self._rules, rule.name, rule)                        # we add a reference to each executed rule here
            if result:
                self.result.append(copy.deepcopy(subject))               # and add each subject to the rule_set result as well
                subject = result
                if self.definition['break_on_true'] and result.decision:
                    self.logger.info('Rule "%s" evaluated to True and ended execution due to break_on_true being True.' % str(rule.name))
                    break
                if self.definition['break_on_false'] and not result.decision:
                    self.logger.info('Rule "%s" evaluated to False and ended execution due to break_on_false being True.' % str(rule.name))
                    break
            if not result:
                self.logger.info('Rule "%s" returned False to end execution.' % str(rule.name))
                break
        self.logger.info("Finished execution of rules")


    def report(self, **kwargs):
        '''
        Report the result of the rules.

        this method should be called AFTER the execute method.

        The default reporter module is reporters.md.

        if `reporter` is part of the definition of the rule set,
        that reporter will be used.

        if `reporter_args` is part of the definition of the rule set,
         the arguments defined there will be passed to the reporter.
        
        if `output_format` is passed as a parameter to the report function,
        this will overrule the `output_format` defined in `reporter_args`.

        A yaml snippet to illustrate the selection of the markdown reporter with html output:

            name: test_bag_geocoder
            title: Test the BAG geocoder
            description: Test the BAG geocoder by geocoding an address and report the result
            reporter: reporters.md
            reporter_args:
              output_format: html

        If no arguments are defined in the rule set, 
        arguments passed in this report method will be passed to the reporter.

        The 'rule_set_reporter' method in the default reporter reporters.md accepts 
        an optional `output_format` parameter which is expected to be one of:

        - 'markdown'    (default)
        - 'html'
        - 'pdf'
        '''

        if self.reporter_args:
            if 'output_format' in kwargs:
                return self.reporter_module.rule_set_reporter(self, output_format = kwargs['output_format'], **self.reporter_args)
            else:
                return self.reporter_module.rule_set_reporter(self, **self.reporter_args)
        else:
            return self.reporter_module.rule_set_reporter(self, **kwargs)

Ancestors (in MRO)

Class variables

var Rules

Instance variables

var definition

var logger

var reporter_args

var rules

Methods

def __init__(

self, rules_set_file, loader_module=<module 'geoDSS.loaders.yaml_loader' from '/home/marco/MD2/Projecten/OGG_RUD/code/versioned/geoDSS/geoDSS/loaders/yaml_loader.pyc'>, **kwargs)

Reads the rules_set with the load_rule_set method of the loader_moduler.

If not specified the default yaml loader is used.

rules_set_file is expected to be a file containing the rules set.

def __init__(self, rules_set_file, loader_module = loaders.yaml_loader, **kwargs):
    '''
    Reads the rules_set with the load_rule_set method of the loader_moduler.
    If not specified the default yaml loader is used.
    `rules_set_file` is expected to be a file containing the rules set.
    '''
    dss = sys.modules[__name__]
    self.definition = None
    self.definition = loader_module.load_rule_set(rules_set_file)
    if not 'title' in self.definition.keys():
        self.definition['title'] = self.definition['name'] 
    if not 'description' in self.definition.keys():
        self.definition['description'] = '' 
    if not 'settings' in self.definition.keys():
        self.definition['settings'] = None
    if not 'logging' in self.definition.keys():
        self.definition['logging'] = {}
    if not 'break_on_true' in self.definition.keys():
        self.definition['break_on_true'] = False
    if not 'break_on_false' in self.definition.keys():
        self.definition['break_on_false'] = False
    self.logger = self._setup_logging()
    if not 'reporter' in self.definition.keys():
        self.reporter_module = reporters.md
    else:
        _group, _class = self.definition['reporter'].split('.',2)
        if hasattr(dss, _group):                                        # this allows for reporters outside the reporters module
            _group = getattr(dss, _group)
        else:
            self.logger.error('The module %s is not defined yet.' % _group)                         
            raise exceptions.NotImplementedError('The module %s is not defined yet. Did you mean to use the reporters module?' % _group)
        if hasattr(_group, _class):
            self.reporter_module = getattr(_group, _class)
            self.logger.debug("Selecting reporter %s" % self.reporter_module.__name__)
        else:
            self.logger.error('The reporter %s is not defined yet.' % self.definition['reporter'])
            raise exceptions.NotImplementedError('The reporter %s is not defined yet or has unmet dependencies.' % self.definition['reporter'])
    self.reporter_args = None
    if 'reporter_args' in self.definition.keys():
        self.reporter_args = self.definition['reporter_args']
    self._rules = self.Rules()
    self.rules = []
    for rule in self.definition['rules']:
        name = rule.keys()[0]
        definition = rule[name]
        _group, _class = definition['type'].split('.',2)
        if hasattr(dss, _group):
            _group = getattr(dss, _group)
        else:
            self.logger.error('The module %s is not defined yet.' % _group)
            raise exceptions.NotImplementedError('The module %s is not defined yet.Did you mean to use the tests or processors module?' % _group)
        if hasattr(_group, _class):
            # the class is in a module eg unit_test.unit_test
            _group = getattr(_group, _class)
            test_to_add = getattr(_group, _class)
            self.logger.debug("Adding rule with name: %s" % name)
            self.rules.append(test_to_add(name, definition, logger = self.logger, rules = self._rules, settings = self.definition['settings']))
        else:
            self.logger.error('The type %s is not defined yet.' % definition['type'])
            raise exceptions.NotImplementedError('The type %s is not defined yet or has unmet dependencies.' % definition['type'])

def execute(

self, subject)

Executes processors and tests in the rules_set.

this method should be called BEFORE the report method.

subject is expected to be a dict which is passed to each rule.

def execute(self, subject):
    '''
    Executes processors and tests in the rules_set.
    this method should be called BEFORE the report method.
    `subject`       is expected to be a dict which is passed to each rule.
    '''
    self.result = []                                                 # the rule_set has it's own result we can report on; we fill it with the subjects
    self.result.append(copy.deepcopy(subject))                       # add the first subject to the rule_set result
    self.logger.info("Start execution of rules")
    self.logger.debug('First subject: ' + str(subject))
    for rule in self.rules:
        self.logger.debug('Executing rule "%s" with subject "%s"' % (str(rule.name), str(subject)))
        result = rule.execute(subject)
        setattr(self._rules, rule.name, rule)                        # we add a reference to each executed rule here
        if result:
            self.result.append(copy.deepcopy(subject))               # and add each subject to the rule_set result as well
            subject = result
            if self.definition['break_on_true'] and result.decision:
                self.logger.info('Rule "%s" evaluated to True and ended execution due to break_on_true being True.' % str(rule.name))
                break
            if self.definition['break_on_false'] and not result.decision:
                self.logger.info('Rule "%s" evaluated to False and ended execution due to break_on_false being True.' % str(rule.name))
                break
        if not result:
            self.logger.info('Rule "%s" returned False to end execution.' % str(rule.name))
            break
    self.logger.info("Finished execution of rules")

def report(

self, **kwargs)

Report the result of the rules.

this method should be called AFTER the execute method.

The default reporter module is reporters.md.

if reporter is part of the definition of the rule set, that reporter will be used.

if reporter_args is part of the definition of the rule set, the arguments defined there will be passed to the reporter.

if output_format is passed as a parameter to the report function, this will overrule the output_format defined in reporter_args.

A yaml snippet to illustrate the selection of the markdown reporter with html output:

name: test_bag_geocoder
title: Test the BAG geocoder
description: Test the BAG geocoder by geocoding an address and report the result
reporter: reporters.md
reporter_args:
  output_format: html

If no arguments are defined in the rule set, arguments passed in this report method will be passed to the reporter.

The 'rule_set_reporter' method in the default reporter reporters.md accepts an optional output_format parameter which is expected to be one of:

  • 'markdown' (default)
  • 'html'
  • 'pdf'
def report(self, **kwargs):
    '''
    Report the result of the rules.
    this method should be called AFTER the execute method.
    The default reporter module is reporters.md.
    if `reporter` is part of the definition of the rule set,
    that reporter will be used.
    if `reporter_args` is part of the definition of the rule set,
     the arguments defined there will be passed to the reporter.
    
    if `output_format` is passed as a parameter to the report function,
    this will overrule the `output_format` defined in `reporter_args`.
    A yaml snippet to illustrate the selection of the markdown reporter with html output:
        name: test_bag_geocoder
        title: Test the BAG geocoder
        description: Test the BAG geocoder by geocoding an address and report the result
        reporter: reporters.md
        reporter_args:
          output_format: html
    If no arguments are defined in the rule set, 
    arguments passed in this report method will be passed to the reporter.
    The 'rule_set_reporter' method in the default reporter reporters.md accepts 
    an optional `output_format` parameter which is expected to be one of:
    - 'markdown'    (default)
    - 'html'
    - 'pdf'
    '''
    if self.reporter_args:
        if 'output_format' in kwargs:
            return self.reporter_module.rule_set_reporter(self, output_format = kwargs['output_format'], **self.reporter_args)
        else:
            return self.reporter_module.rule_set_reporter(self, **self.reporter_args)
    else:
        return self.reporter_module.rule_set_reporter(self, **kwargs)