[open-ils-commits] r860 - in constrictor/trunk: . constrictor (erickson)
svn at svn.open-ils.org
svn at svn.open-ils.org
Sun Apr 11 10:51:38 EDT 2010
Author: erickson
Date: 2010-04-11 10:51:35 -0400 (Sun, 11 Apr 2010)
New Revision: 860
Removed:
constrictor/trunk/constrictor/db.py
Modified:
constrictor/trunk/constrictor.py
constrictor/trunk/constrictor/data.py
constrictor/trunk/constrictor/utils.py
Log:
added option to bypass inserting timing data into the db. removed old db.py module, which was replaced by data.py
Modified: constrictor/trunk/constrictor/data.py
===================================================================
--- constrictor/trunk/constrictor/data.py 2010-04-11 14:51:35 UTC (rev 859)
+++ constrictor/trunk/constrictor/data.py 2010-04-11 14:51:35 UTC (rev 860)
@@ -29,6 +29,7 @@
self.meta.bind = self.engine
self.runtime_data = []
self.task_set_id = None
+ self.insert_data = True
Data.singleton = self
def disconnect(self):
@@ -67,28 +68,34 @@
def create_task_set(self):
+
self.task_set_start_time = time.time()
- res = self.engine.execute(
- self.task_set_table.insert().values(start_time = self.task_set_start_time)
- )
- #self.task_set_id = res.inserted_primary_key
- self.task_set_id = res.last_inserted_ids()[0]
- res.close()
+
+ if self.insert_data:
+
+ res = self.engine.execute(
+ self.task_set_table.insert().values(start_time = self.task_set_start_time)
+ )
+ #self.task_set_id = res.inserted_primary_key
+ self.task_set_id = res.last_inserted_ids()[0]
+ res.close()
def store_data(self):
end_time = time.time()
# close out the task set
- self.engine.execute(
- self.task_set_table.update(
- ).where(
- self.task_set_table.c.id == self.task_set_id
- ).values(
- end_time = end_time
- )
- ).close()
+ if self.insert_data:
+ self.engine.execute(
+ self.task_set_table.update(
+ ).where(
+ self.task_set_table.c.id == self.task_set_id
+ ).values(
+ end_time = end_time
+ )
+ ).close()
+ task_counts = {}
task_times = 0
total_tasks = len(self.runtime_data)
counter = 0
@@ -99,29 +106,38 @@
# TODO: Create a batch insert SQL string to insert, since
# one row at a time is pretty slow
for task in self.runtime_data:
-
+
task_times += task['duration']
log.log_debug("Storing " + task['name'])
- self.engine.execute(
- self.task_table.insert().values(
- task_name = task['name'],
- task_set = self.task_set_id,
- run_time = task['run_time'],
- thread_id = task['thread_id'],
- duration = '%0.3f' % task['duration'],
- success = task['success']
- )
- ).close()
+ if task['name'] not in task_counts:
+ task_counts[task['name']] = 0;
+ task_counts[task['name']] += 1
+
+ if self.insert_data:
+ self.engine.execute(
+ self.task_table.insert().values(
+ task_name = task['name'],
+ task_set = self.task_set_id,
+ run_time = task['run_time'],
+ thread_id = task['thread_id'],
+ duration = '%0.3f' % task['duration'],
+ success = task['success']
+ )
+ ).close()
+
counter += 1
if counter %100 == 0:
log.log_info("Inserted %d rows" % counter)
# log some basic stats
task_set_time = end_time - self.task_set_start_time
+ log.log_info("-"*70)
+ log.log_info("Task Counts -> %s" % str(task_counts))
log.log_info("Total Tasks -> %d" % len(self.runtime_data))
log.log_info("Total time -> %0.3f seconds" % task_set_time)
log.log_info("Average task time -> %0.3f seconds" % (task_times / total_tasks))
log.log_info("Real Time / Total Tasks -> %0.3f seconds/task" % (task_set_time / total_tasks))
+ log.log_info("-"*70)
Deleted: constrictor/trunk/constrictor/db.py
===================================================================
--- constrictor/trunk/constrictor/db.py 2010-04-11 14:51:35 UTC (rev 859)
+++ constrictor/trunk/constrictor/db.py 2010-04-11 14:51:35 UTC (rev 860)
@@ -1,188 +0,0 @@
-# -----------------------------------------------------------------------
-# Copyright (C) 2007-2008 King County Library System
-# Bill Erickson <erickson at esilibrary.com>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-# -----------------------------------------------------------------------
-
-from pysqlite2 import dbapi2 as sqlite
-import sys, threading, thread, time
-from log import *
-from properties import Properties
-
-# wrap DB access in a semaphore
-dbSema = threading.BoundedSemaphore(value=1)
-
-
-class DBConnection(object):
-
- # the ID of the currently executing task set
- taskSetID = 0
-
- def __init__(self, dbFile):
- self.conn = sqlite.connect(dbFile)
-
- def createTaskSet(self):
- SQL = '''
- insert into task_set (start_time) values (%f)
- ''' % time.time()
- cur = self.execute(SQL, True)
- DBConnection.taskSetID = cur.lastrowid
-
- def finishTaskSet(self):
- SQL = '''
- update task_set set end_time = %f where id = %d
- ''' % (time.time(), DBConnection.taskSetID)
- self.execute(SQL, True)
-
-
- def createTables(self):
- taskSQL = '''
- CREATE TABLE IF NOT EXISTS task (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- task_name TEXT,
- task_set INTEGER,
- runtime REAL,
- runner INTEGER,
- thread_id INTEGER,
- duration INTEGER,
- success INTEGER
- );
- '''
-
- taskSetSQL = '''
- CREATE TABLE IF NOT EXISTS task_set (
- id INTEGER PRIMARY KEY AUTOINCREMENT,
- start_time REAL,
- end_time REAL
- );
- '''
-
- self.execute(taskSQL)
- self.execute(taskSetSQL, True)
-
-
- def createTaskSummary(self):
- summary = {}
-
- cur = self.execute(
- 'select end_time - start_time from task_set where id = %d' % DBConnection.taskSetID)
- row = cur.fetchone()
- summary['task_set_duration'] = float(row[0]) or ''
-
- cur = self.execute(
- 'select avg(duration) from task where success = 1 and task_set = %d' % DBConnection.taskSetID)
- row = cur.fetchone()
- summary['avg_task_duration'] = row[0] or ''
-
- cur = self.execute(
- 'select count(*) from task where success = 1 and task_set = %d' % DBConnection.taskSetID)
- row = cur.fetchone()
- summary['num_task_success'] = row[0]
-
- cur = self.execute(
- 'select count(*) from task where success = 0 and task_set = %d' % DBConnection.taskSetID)
- row = cur.fetchone()
- summary['num_task_failed'] = row[0]
-
- summary['amortized_task_duration'] = ''
- if summary['num_task_success'] > 0:
- summary['amortized_task_duration'] = \
- float(summary['task_set_duration']) / float(summary['num_task_success'])
-
- summary['amortized_tasks_per_second'] = ''
- if summary['num_task_success'] > 0:
- summary['amortized_tasks_per_second'] = 1.0 / float(summary['amortized_task_duration'])
-
- props = Properties.get_properties()
- summary['thread_count'] = props.get_property('constrictor.numThreads')
- summary['iteration_count'] = props.get_property('constrictor.numIterations')
-
- self.makeTaskTypeSummary(summary)
-
- log_info('created summary %s' % summary)
- return summary
-
- def makeTaskTypeSummary(self, summary):
- ''' build a summary string for each task type of <task_name>:<num_runs>:<avg_duration>; '''
-
- summary['task_type_summary'] = ''
- tasks = self.execute(
- 'select distinct(task_name) from task where task_set = %d' % DBConnection.taskSetID)
-
- for t in tasks:
- t = t[0]
-
- # grab the average duration for this type of task
- cur = self.execute(
- 'select avg(duration) from task where task_set = %d and task_name = "%s"' % (
- DBConnection.taskSetID, t))
- avg = cur.fetchone()[0]
-
- # grab the number of times this task was run
- cur = self.execute(
- 'select count(*) from task where task_set = %d and task_name = "%s"' % (
- DBConnection.taskSetID, t))
- row = cur.fetchone()
- count = row[0]
-
- summary['task_type_summary'] += '%s:%d:%f;' % (t, count, float(avg))
-
-
-
-
- def dropTables(self):
- SQL = 'drop table task;'
- self.execute(SQL, True)
-
-
- def disconnect(self):
- if self.conn is not None:
- self.conn.close()
-
-
- def execute(self, sql, commit=False):
- log_debug('SQL ' + sql)
- from script import ScriptThread
- cur = None
- try:
- dbSema.acquire()
- cur = self.conn.cursor()
- cur.execute(sql)
- if commit:
- self.conn.commit()
- except Exception, e:
- sys.stderr.write('DB error: thread = %d : %s\n' % (ScriptThread.get_thread_id(), str(e)))
- sys.stderr.flush()
- dbSema.release()
- sys.exit(1)
-
- dbSema.release()
- return cur
-
- def insertTask(self, task):
- from script import ScriptThread
-
- SQL = """
- insert into task (task_name, runtime, runner, thread_id, task_set, duration, success)
- values ('%s', %f, %d, %d, %d, %f, %d) """ % (
- task.name,
- time.time(),
- 1, # XXX get me from the task runner ?
- ScriptThread.get_thread_id(),
- DBConnection.taskSetID,
- task.duration,
- task.success
- )
- self.execute(SQL, True)
-
-
-
Modified: constrictor/trunk/constrictor/utils.py
===================================================================
--- constrictor/trunk/constrictor/utils.py 2010-04-11 14:51:35 UTC (rev 859)
+++ constrictor/trunk/constrictor/utils.py 2010-04-11 14:51:35 UTC (rev 860)
@@ -48,11 +48,12 @@
print "WARNING: Unable to store properties to file\n%s" % str(e)
-def init_db():
+def init_db(store_data = True):
''' connect to the db and make sure the tables exist '''
data = constrictor.data.Data(
os.path.join(os.getcwd(), props.get_property('constrictor.dbFile'))
)
+ data.insert_data = store_data
data.create_schema()
data.create_task_set()
data.disconnect()
Modified: constrictor/trunk/constrictor.py
===================================================================
--- constrictor/trunk/constrictor.py 2010-04-11 14:51:35 UTC (rev 859)
+++ constrictor/trunk/constrictor.py 2010-04-11 14:51:35 UTC (rev 860)
@@ -25,6 +25,7 @@
props = None
props_filename = PROPS_FILENAME
drone_controller = None
+store_data = True
def usage():
print '''
@@ -43,6 +44,7 @@
-p port to listen for controller connections on
-l listen address for incoming controller connections
-x clear the local cache file cache
+ -n do not store the results in the database
''' % sys.argv[0]
sys.exit(0)
@@ -51,9 +53,10 @@
def read_args_and_props():
global props
global props_filename
+ global store_data
# see if we have any command-line args that override the properties file
- ops, args = getopt.getopt(sys.argv[1:], 's:t:i:d:p:l:f:hx')
+ ops, args = getopt.getopt(sys.argv[1:], 's:t:i:d:p:l:f:hxn')
options = dict( (k,v) for k,v in ops )
if options.has_key('-f'):
@@ -76,6 +79,8 @@
props.set_property('constrictor.port', options['-p'])
if options.has_key('-l'):
props.set_property('constrictor.listenAddress', options['-l'])
+ if options.has_key('-n'):
+ store_data = False
if options.has_key('-x'):
# delete the cache directory
@@ -130,7 +135,7 @@
log_info('running ' + script)
f = open_script(scriptDirs, script)
if f:
- init_db()
+ init_db(store_data)
try:
exec(f)
except Exception, e:
@@ -143,7 +148,7 @@
drone_controller.shutdown()
else:
- init_db()
+ init_db(store_data)
script = props.get_property('constrictor.script') # execute the requested script
ScriptThread.reset_thread_seed()
f = open_script(scriptDirs, script)
More information about the open-ils-commits
mailing list