geoDSS.interfaces module
This example code provides several interfaces to geoDSS:
- command line
- cgi
- wsgi
Of course this example is functional, but might be improved on with eg. better cgi handling, more error checking, logging and communication with the user.
Enable citg (cgi)
Adapt the global variable ENABLE_CGTIB to enable cgitb.
NOTICE
cgitb doesn't always play nice with apache: (https://bugs.python.org/issue8704)
WARNING
don't enable cgitb in a production environment!
and remember: don't enable cgitb in a production environment!
Fetch remote rule_set files
Adapt the global variable ENABLE_NON_LOCAL_RULE_SET_FILES to enable fetching of rule_set files from a remote host.
WARNING
Do not enable this in a production environment with the standard geoDSS processors and tests.
You allow the execution of arbitrary SQL commands in your database.
And probably some more ugly stuff as allowing the user to create an open proxy.
Logging
Adapt the global variable DEBUG_LEVEL to control what's written to stderr.
This setting only affects interfaces.py. geoDSS utilyzes a logger which is configured via the rule_set_file. When using cgi, most of the time you'll find the things written to stderr in your webservers logs. You can set
DEBUG_LEVEL = {False | "DEBUG"}
#!/usr/bin/env python #!C:/Python27/python.exe -u # -*- coding: utf-8 -*- ''' This example code provides several interfaces to geoDSS: - command line - cgi - wsgi Of course this example is functional, but might be improved on with eg. better cgi handling, more error checking, logging and communication with the user. Enable citg (cgi) ----------------- Adapt the global variable ENABLE_CGTIB to enable cgitb. **NOTICE** >cgitb doesn't always play nice with apache: (https://bugs.python.org/issue8704) **WARNING** >don't enable cgitb in a production environment! >and remember: don't enable cgitb in a production environment! Fetch remote rule_set files --------------------------- Adapt the global variable ENABLE_NON_LOCAL_RULE_SET_FILES to enable fetching of rule_set files from a remote host. **WARNING** >Do not enable this in a production environment with the standard geoDSS processors and tests. >You allow the execution of arbitrary SQL commands in your database. >And probably some more ugly stuff as allowing the user to create an open proxy. Logging ------- Adapt the global variable DEBUG_LEVEL to control what's written to stderr. This setting only affects interfaces.py. geoDSS utilyzes a logger which is configured via the rule_set_file. When using cgi, most of the time you'll find the things written to stderr in your webservers logs. You can set >DEBUG_LEVEL = {False | "DEBUG"} ''' ENABLE_CGTIB = False ENABLE_NON_LOCAL_RULE_SET_FILES = False DEBUG_LEVEL = "DEBUG" import argparse import cgi import json import logging import os import sys import traceback try: import exceptions except: pass if ENABLE_CGTIB: import cgitb # make sure we can import the package when run as script if __name__ == '__main__' and __package__ is None: from os import sys, path sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) import geoDSS from geoDSS import loaders from geoDSS import ui_generators def sanitize_headers(headers): ''' removes hop-by-hop headers as these are not supported by wsgi. ''' hop_by_hop_headers = ('connection','keep-alive','public','proxy-authenticate','transfer-encoding','upgrade') keys = [x.lower() for x in headers.keys()] for h in hop_by_hop_headers: if h in keys: headers.pop(h) return headers def load_execute_report(params): ''' main entrypoint command line, cgi and wsgi interfaces end up here. `params` a CGI field storage object. ''' mime = 'text/plain' try: output_format = params['output_format'][0] if output_format in ['html']: mime = 'text/html; charset=utf-8' if output_format in ['pdf']: mime = 'application/pdf' #mime = 'text/plain' elif output_format in ['markdown']: mime = 'text/plain; charset=utf-8' except: output_format = 'markdown' response_headers = {'Content-Type': mime} status = '400 Bad Request' if 'form' in params: form_definition = params['form'][0] if form_definition[-4:] == '.htm' or form_definition[-5:] =='.html': # just serve this html file if os.path.exists(form_definition): with open(form_definition,'rb') as f: data = f.read() else: return status, response_headers, "Could not find requested form" else: template = None if 'template' in params: template = params['template'][0] try: data = geoDSS.ui_generators.form.generate(form_yaml = form_definition, template = template) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not generate form with error: %s \n" % str(e)) sys.stderr.write(traceback.format_exc()) return status, response_headers, "Could not generate form with error: " + str(e) elif 'rule_set_file' in params: try: subject = json.loads(params['subject'][0]) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not parse subject with error: %s \n" % str(e)) #sys.stderr.write("geoDSS: " + params['subject'][0]) return status, response_headers, "Could not parse subject with error: " + str(e) rule_set_file = params['rule_set_file'][0] base_name,extension = os.path.splitext(rule_set_file) loader_module = loaders.yaml_loader if extension == '.json': loader_module = loaders.json_loader if ENABLE_NON_LOCAL_RULE_SET_FILES: try: rule_set_file = params['protocol'][0] + '://' + rule_set_file except: pass try: r = geoDSS.rules_set(rule_set_file, loader_module) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not load rule_set with error: %s \n" % str(e)) sys.stderr.write(traceback.format_exc()) return status, response_headers, "Could not load rule_set with error: " + str(e) try: r.execute(subject) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not evaluate rule_set with error: %s \n" % str(e)) sys.stderr.write(traceback.format_exc()) return status, response_headers, "Could not evaluate rule_set with error: " + str(e) try: sys.stderr.write("trying to generate a report with output format: " + output_format) data = r.report(output_format = output_format) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not report on rule_set with error: %s \n" % str(e)) sys.stderr.write(traceback.format_exc()) return status, response_headers, "Could not report on rule_set with error: " + str(e) else: data = "You are almost there... Please specify a form to load or a rule_set to proces." status = '200 OK' response_headers["Content-Length"] = str(len(data)) return status, response_headers, data def application(environ, start_response): ''' WSGI entry method: called by WSGI framework. ''' if ENABLE_CGTIB: cgitb.enable() if DEBUG_LEVEL: sys.stderr.write("geoDSS: enabling cgitb. \n") if DEBUG_LEVEL: sys.stderr.write('geoDSS: WSGI environ=%s \n' % str(environ)) if "REQUEST_METHOD" in environ and environ['REQUEST_METHOD'] == 'POST': query_string = environ['wsgi.input'] elif "REQUEST_METHOD" in environ and environ['REQUEST_METHOD'] == 'GET': query_string = environ['QUERY_STRING'] else: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Only HTTP POST and GET are supported. \n") raise NotImplemented('Only HTTP POST and GET are supported') params = cgi.parse_qs(query_string, keep_blank_values=True) status, response_headers, data = load_execute_report(params) headers = sanitize_headers(response_headers) start_response(status, headers.items()) #return [data] return [data.encode('utf-8')] # needed for apache to encode unicode to bytes string def cgi_application(params): ''' CGI entry method: called locally. example: `http://localhost:8000/cgi-bin/geodss.cgi?output_format=html&rule_set_file=geoDSS/geoDSS/examples/rule_sets/bag_geocoder_test.yaml&subject={%22postcode%22:%20%224171KG%22,%20%22huisnummer%22:%20%2274%22}` ''' status, response_headers, data = load_execute_report(params) response_headers = sanitize_headers(response_headers) # this one not necessary, but it keeps in line with the wsgi interface headers = '\r\n'.join( ["%s: %s;" % (key, value) for key, value in response_headers.iteritems()] ) + '\r\n' + '\r\n' sys.stdout.write(headers) sys.stdout.write(data.encode('utf-8')) # Get form/query params (CGI) parameters = cgi.FieldStorage(keep_blank_values = True) param_count = len(parameters.keys()) if param_count > 0 and 'wsgi.version' not in parameters and 'REQUEST_METHOD' in os.environ: # We are called as CGI: handle with CGI function if ENABLE_CGTIB: if DEBUG_LEVEL: sys.stderr.write("geoDSS: enabling cgitb. \n") cgitb.enable() if os.environ['REQUEST_METHOD'] == "GET": query_string = os.environ['QUERY_STRING'] params = cgi.parse_qs(query_string, keep_blank_values=True) elif os.environ['REQUEST_METHOD'] == "POST": params = {} for param in parameters.keys(): try: params[param] = parameters.getList(param) except: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Received POST data in a way not supported yet. Doing a guess. \n") params[param] = [parameters.value] else: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Only HTTP POST and GET are supported. \n") raise NotImplemented('Only HTTP POST and GET are supported') cgi_application(params) elif __name__ == '__main__': ''' this script is invoked from command line >example: `./geoDSS examples/rule_sets/unit_test.yaml '{"result": true, "geometry": "SRID=28992;POINT(125000 360000)" }'` > `./geoDSS examples/rule_sets/various.yaml '{"result": true, "geometry": "SRID=28992;POINT(125000 360000)", "brzo": "true" }'` ''' parser = argparse.ArgumentParser(description = 'Invoke geoDSS from command line using the default markdown reporter with output_format markdown.', epilog = '''Example: ./geoDSS examples/rule_sets/unit_test.yaml '{"result": true, "geometry": "SRID=28992;POINT(125000 360000)" }' ''') parser.add_argument("rule_set_file", help = 'The file containing the rule set.') parser.add_argument("subject", help = 'Either a JSON-string defining a python dict containing a subject or a path to a file containing such a JSON-string.') parser.add_argument("--output_file", help = 'An output file to write the results to. If not given, output is to stdout.') parser.add_argument("--file_mode", default = "wb" , choices=('wb', 'wt', 'ab', 'at'), help = 'Python file mode. mode starting with "a" will append, starting with "w" will overwrite.' ) args = parser.parse_args() if os.path.exists(args.subject): with open(args.subject, 'r') as f: subject_string = f.read() else: subject_string = args.subject params = { 'rule_set_file': [args.rule_set_file], 'subject': [subject_string], 'output_format': ['markdown']} status, response_headers, data = load_execute_report(params) if args.output_file: with open(args.output_file,args.file_mode) as f: f.write(data) else: print(data) else: # Standard WSGI: do nothing as WSGI entry-function 'application' will be invoked pass
Module variables
var DEBUG_LEVEL
var ENABLE_CGTIB
var ENABLE_NON_LOCAL_RULE_SET_FILES
var param_count
var parameters
Functions
def application(
environ, start_response)
WSGI entry method: called by WSGI framework.
def application(environ, start_response): ''' WSGI entry method: called by WSGI framework. ''' if ENABLE_CGTIB: cgitb.enable() if DEBUG_LEVEL: sys.stderr.write("geoDSS: enabling cgitb. \n") if DEBUG_LEVEL: sys.stderr.write('geoDSS: WSGI environ=%s \n' % str(environ)) if "REQUEST_METHOD" in environ and environ['REQUEST_METHOD'] == 'POST': query_string = environ['wsgi.input'] elif "REQUEST_METHOD" in environ and environ['REQUEST_METHOD'] == 'GET': query_string = environ['QUERY_STRING'] else: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Only HTTP POST and GET are supported. \n") raise NotImplemented('Only HTTP POST and GET are supported') params = cgi.parse_qs(query_string, keep_blank_values=True) status, response_headers, data = load_execute_report(params) headers = sanitize_headers(response_headers) start_response(status, headers.items()) #return [data] return [data.encode('utf-8')] # needed for apache to encode unicode to bytes string
def cgi_application(
params)
CGI entry method: called locally.
example: http://localhost:8000/cgi-bin/geodss.cgi?output_format=html&rule_set_file=geoDSS/geoDSS/examples/rule_sets/bag_geocoder_test.yaml&subject={%22postcode%22:%20%224171KG%22,%20%22huisnummer%22:%20%2274%22}
def cgi_application(params): ''' CGI entry method: called locally. example: `http://localhost:8000/cgi-bin/geodss.cgi?output_format=html&rule_set_file=geoDSS/geoDSS/examples/rule_sets/bag_geocoder_test.yaml&subject={%22postcode%22:%20%224171KG%22,%20%22huisnummer%22:%20%2274%22}` ''' status, response_headers, data = load_execute_report(params) response_headers = sanitize_headers(response_headers) # this one not necessary, but it keeps in line with the wsgi interface headers = '\r\n'.join( ["%s: %s;" % (key, value) for key, value in response_headers.iteritems()] ) + '\r\n' + '\r\n' sys.stdout.write(headers) sys.stdout.write(data.encode('utf-8'))
def load_execute_report(
params)
main entrypoint
command line, cgi and wsgi interfaces end up here.
params
a CGI field storage object.
def load_execute_report(params): ''' main entrypoint command line, cgi and wsgi interfaces end up here. `params` a CGI field storage object. ''' mime = 'text/plain' try: output_format = params['output_format'][0] if output_format in ['html']: mime = 'text/html; charset=utf-8' if output_format in ['pdf']: mime = 'application/pdf' #mime = 'text/plain' elif output_format in ['markdown']: mime = 'text/plain; charset=utf-8' except: output_format = 'markdown' response_headers = {'Content-Type': mime} status = '400 Bad Request' if 'form' in params: form_definition = params['form'][0] if form_definition[-4:] == '.htm' or form_definition[-5:] =='.html': # just serve this html file if os.path.exists(form_definition): with open(form_definition,'rb') as f: data = f.read() else: return status, response_headers, "Could not find requested form" else: template = None if 'template' in params: template = params['template'][0] try: data = geoDSS.ui_generators.form.generate(form_yaml = form_definition, template = template) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not generate form with error: %s \n" % str(e)) sys.stderr.write(traceback.format_exc()) return status, response_headers, "Could not generate form with error: " + str(e) elif 'rule_set_file' in params: try: subject = json.loads(params['subject'][0]) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not parse subject with error: %s \n" % str(e)) #sys.stderr.write("geoDSS: " + params['subject'][0]) return status, response_headers, "Could not parse subject with error: " + str(e) rule_set_file = params['rule_set_file'][0] base_name,extension = os.path.splitext(rule_set_file) loader_module = loaders.yaml_loader if extension == '.json': loader_module = loaders.json_loader if ENABLE_NON_LOCAL_RULE_SET_FILES: try: rule_set_file = params['protocol'][0] + '://' + rule_set_file except: pass try: r = geoDSS.rules_set(rule_set_file, loader_module) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not load rule_set with error: %s \n" % str(e)) sys.stderr.write(traceback.format_exc()) return status, response_headers, "Could not load rule_set with error: " + str(e) try: r.execute(subject) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not evaluate rule_set with error: %s \n" % str(e)) sys.stderr.write(traceback.format_exc()) return status, response_headers, "Could not evaluate rule_set with error: " + str(e) try: sys.stderr.write("trying to generate a report with output format: " + output_format) data = r.report(output_format = output_format) except Exception as e: if DEBUG_LEVEL: sys.stderr.write("geoDSS: Could not report on rule_set with error: %s \n" % str(e)) sys.stderr.write(traceback.format_exc()) return status, response_headers, "Could not report on rule_set with error: " + str(e) else: data = "You are almost there... Please specify a form to load or a rule_set to proces." status = '200 OK' response_headers["Content-Length"] = str(len(data)) return status, response_headers, data
def sanitize_headers(
headers)
removes hop-by-hop headers as these are not supported by wsgi.
def sanitize_headers(headers): ''' removes hop-by-hop headers as these are not supported by wsgi. ''' hop_by_hop_headers = ('connection','keep-alive','public','proxy-authenticate','transfer-encoding','upgrade') keys = [x.lower() for x in headers.keys()] for h in hop_by_hop_headers: if h in keys: headers.pop(h) return headers