#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
'''Simple python server to display an index page and static objects.
.. note:: External module included: webpy (http://webpy.org/)
.. moduleauthor:: Ana Oprea
Usage:
>>> # call the class from top level
>>> start_server.py
'''
from __future__ import with_statement, absolute_import, print_function
from optparse import OptionParser
import sys
import os
import logging
import time
from itertools import chain
# webpy
try:
from . import web
except (ValueError, ImportError):
import web
# config file
try:
from . import config_pytomo
except (ValueError):
import config_pytomo
# rrd library to interact with
try:
from .lib_rrdtools import PytomoRRD, create_options
except (ValueError):
from lib_rrdtools import PytomoRRD, create_options
# database connection
try:
from .lib_database import PytomoDatabase
except (ValueError):
from lib_database import PytomoDatabase
from sqlite3 import Error
# file operations
try:
from .lib_io import ALL_PLOTS, ALL_KEY, LINKS_KEY, MAIN_KEY, DB_KEY, \
DB_EXTENSION, get_latest_specific_file, write_index, \
check_templates_exist, index_filename
except (ValueError):
from lib_io import ALL_PLOTS, ALL_KEY, LINKS_KEY, MAIN_KEY, DB_KEY, \
DB_EXTENSION, get_latest_specific_file, write_index, \
check_templates_exist, index_filename
# to check if files/dirs exist and set name
try:
from .start_pytomo import check_out_files
except ValueError:
from start_pytomo import check_out_files
# port range to run the server
MIN_PORT = 1024
MAX_PORT = 65535
# refresh time in seconds to reload the main page
REFRESH_TIME = 90
# first line in the html to be displayed in the browser
START_RENDERING = 1
# global directory where the databases are stored (can be changed if the user
# requests a specific one in cli)
DB_DIR = config_pytomo.DATABASE_DIR
#DATABASE = config_pytomo.DATABASE
DATABASE = get_latest_specific_file(DB_DIR, DB_EXTENSION)
# rrd needs minimum 3 points to plot graphs
RRD_MIN_NR_POINTS = 3
[docs]class Index:
''' Class that serves the main page.
Will search for a .html file under the folder set in render below.
'''
def __init__(self):
# search for the main page that will be displayed in the templates dir
self.render = web.template.render(config_pytomo.TEMPLATES_DIR,
cache=False)
[docs] def GET(self, parameter):
'''Retrieves the main page from the parameter_timestamp_index.html
template, based on the selected database as parameter.
'''
# verify if the user has already specified a certain database on which
# to display the plots, otherwise choose the latest database
# database is a parameter in the link
# example: http://127.0.0.1:5555/Database_archive?db=databases/large.db
# has as 'db' parameter the database 'databases/large.db'
try:
#global DATABASE
#DATABASE = web.input().db
database = web.input().db
except AttributeError:
config_pytomo.LOG.debug('Could not get a database specified by the'
' user')
database = DATABASE
config_pytomo.LOG.debug('Database: %s' % database)
try:
db_nr_rows = PytomoDatabase(database).count_rows()
except Error, mes:
db_nr_rows = RRD_MIN_NR_POINTS - 1
config_pytomo.LOG.error('Unable to extract data with error: %s'
% mes)
# if database is has less than 3 points no graphs will be created
# (rrd limitation)
if db_nr_rows < RRD_MIN_NR_POINTS:
config_pytomo.LOG.warning('Database %s does not have enough points'
' to create graphs' % database)
timestamp = None
write_index(timestamp, database, DB_DIR)
else:
try:
timestamp = PytomoDatabase(database).fetch_start_time()
except Error, mes:
timestamp = None
config_pytomo.LOG.error('Unable to extract data with error:'
'%s' % mes)
latest_db = get_latest_specific_file(DB_DIR, DB_EXTENSION)
# check if:
# the modification time of the db has changed in the REFRESH_TIME s
# or if template files don't exist
# or if there is a new database that is not in the db archive list
if (((time.time() - os.path.getmtime(database)) < REFRESH_TIME)
or not check_templates_exist(timestamp)
or (os.path.getmtime(index_filename(DB_KEY, timestamp))
< os.path.getmtime(latest_db))):
config_pytomo.LOG.debug('RRD operations start...')
pytomo_rrd = PytomoRRD(database)
pytomo_rrd.update_pytomo_rrd()
#pytomo_rrd.fetch_pytomo_rrd()
pytomo_rrd.plot_pytomo_rrd()
config_pytomo.LOG.debug('RRD operations end...')
write_index(timestamp, database, DB_DIR)
config_pytomo.LOG.debug('Start rendering page...')
# these headers allow browsers to display the page
web.header('Content-type','text/html')
web.header('Transfer-Encoding','chunked')
# if the user provides a legitimate link, it will be displayed
# otherwise the main graphs will be displayed
# parameter is taken from the link, for example:
# http://127.0.0.1:5555/PlaybackDuration
# has as parameter PlaybackDuration
# on the main page (no parameter) the main plots are displayed
if (parameter in chain(ALL_PLOTS[ALL_KEY], ALL_PLOTS.keys(),
[LINKS_KEY, DB_KEY])):
template_file = index_filename(parameter, timestamp)
else:
template_file = index_filename(MAIN_KEY, timestamp)
try:
f_index = open(template_file, 'r')
except IOError:
config_pytomo.LOG.error('Template file %s does not exist!' %
template_file)
# skip the first line that allows to process the parameter given by the
# link
for line in f_index.readlines()[START_RENDERING:]:
yield line
f_index.close()
config_pytomo.LOG.debug('End rendering page...')
[docs]class Static:
''' Class that serves the static objects of the main page.
Will search for elements under the directories mentioned in urls related to
this class.
'''
def __init__(self):
self.object_path = None
[docs] def GET(self, media, filename):
'''Retrieves the static objects located in the main page.'''
try:
self.object_path = open(media + '/' + filename, 'rb')
except IOError, mes:
config_pytomo.LOG.exception(mes)
config_pytomo.LOG.error('File, %s, does not exist.' % filename)
yield self.object_path.read()
[docs]class Doc:
''' Class that serves the documentation pages.
'''
def __init__(self):
self.object_path = None
[docs] def GET(self, filename):
'''Retrieves the documentation files.'''
try:
self.object_path = open(config_pytomo.DOC_DIR + filename, 'rb')
except IOError, mes:
config_pytomo.LOG.exception(mes)
config_pytomo.LOG.error('File, %s, does not exist.' % filename)
return
yield self.object_path.read()
[docs]class Pdf:
''' Class that serves the PDF reports.
Will search for elements under the directories mentioned in urls related to
this class.
'''
def __init__(self):
self.object_path = None
[docs] def GET(self, media, filename):
'''Retrieves the static objects located in the main page.'''
# these headers allow browsers to display the page
web.header('Content-type','application/octet-stream')
try:
self.object_path = open(media + '/' + filename, 'rb')
except IOError, mes:
config_pytomo.LOG.exception(mes)
config_pytomo.LOG.error('File, %s, does not exist.' % filename)
yield self.object_path.read()
[docs]def main(argv=None):
'Program wrapper'
if argv is None:
argv = sys.argv[1:]
usage = '%prog [-v] [-f database] [-d database_directory] [port]'
parser = OptionParser(usage=usage)
create_options(parser)
(options, args) = parser.parse_args(argv)
# Intialize the logger for standalone testing Logging
if not config_pytomo.LOG:
configure_logger_web()
if options.verbose:
config_pytomo.LOG.setLevel(logging.DEBUG)
# set port for web server
port = None
if len(args) == 0:
# add default port
port = int(config_pytomo.WEB_DEFAULT_PORT)
elif len(args) == 1:
# set port
try:
port = int(args[0])
except ValueError, mes:
config_pytomo.LOG.exception(mes)
parser.error('You must specify an integer in the range [%d,%d]'
% (MIN_PORT, MAX_PORT))
if (port < MIN_PORT or port > MAX_PORT):
parser.error('Port %d is not in the range [%d,%d])' %
(port, MIN_PORT, MAX_PORT))
else:
parser.error('Only the port on which to run the application should be '
'specified, in the range [%d,%d]' % (MIN_PORT, MAX_PORT))
# if a directory is specified, the latest database there is selected
if options.db_dir_name:
global DB_DIR
DB_DIR = options.db_dir_name
global DATABASE
# if database is specified
if options.db_name:
DATABASE = options.db_name
else:
# last database in directory is selected
DATABASE = get_latest_specific_file(DB_DIR, DB_EXTENSION)
if not DATABASE:
print('Please start a crawl first, in order to have a database to '
'extract data from!\n Use:\n For Linux: ./start_crawl.py\n '
'For Windows: pytomo_windows_x86.exe\n')
return 1
config_pytomo.LOG.debug('Initial database: %s' % DATABASE)
# the main page (index)
# static objects found under folders below in the current dir
# each element is treated by its respective class
urls = (
'/(js|css|%s|%s)/(.*)' % (config_pytomo.IMAGE_DIR,
config_pytomo.RRD_PLOT_DIR), 'Static',
'/(%s)/(.*)' % (config_pytomo.PDF_DIR), 'Pdf',
'/%s(.*)' % config_pytomo.DOC_DIR, 'Doc',
'/(.*)', 'Index'
)
# hack due to the fact that web.application.__init__() expects the
# globals() dictionary as second parameter, so application would only
# work with port number as option (no -v or anything else)
app = web.application(urls, {'Static': Static, 'Pdf': Pdf, 'Doc': Doc,
'Index': Index}, autoreload=True)
# parameter is taken from the link, for example:
# http://127.0.0.1:5555/PlaybackDuration
# has as parameter PlaybackDuration
# on the main page (no parameter) the main plots are displayed
config_pytomo.LOG.debug('Launching server on port %d' % port)
# hack due to line 54 in /usr/lib/pymodules/python2.6/web/wsgi.py
# return httpserver.runsimple(func, validip(listget(sys.argv, 1, '')))
if len(sys.argv) == 0 or len(sys.argv) == 1:
sys.argv.append(str(port))
else:
sys.argv[1] = str(port)
try:
print('Launching server on port %d' % port)
print('Type Ctrl-C to stop server.')
app.run()
except IOError:
parser.error('Address already in use, start the application with '
'another port number in the range [%d,%d])'
% (MIN_PORT,MAX_PORT))
except KeyboardInterrupt:
print('\nServer stopped by user')
config_pytomo.LOG.critical('Server stopped by user')
return 1
return 0
if __name__ == '__main__':
import doctest
doctest.testmod()
sys.exit(main())