[open-ils-commits] r855 - constrictor/trunk/constrictor (erickson)

[svn at svn.open-ils.org]

Author: erickson
Date: 2010-04-08 14:04:09 -0400 (Thu, 08 Apr 2010)
New Revision: 855

Modified:
   constrictor/trunk/constrictor/data.py
Log:
some more expressive logging during data inserts

Modified: constrictor/trunk/constrictor/data.py
===================================================================
--- constrictor/trunk/constrictor/data.py	2010-04-07 18:37:42 UTC (rev 854)
+++ constrictor/trunk/constrictor/data.py	2010-04-08 18:04:09 UTC (rev 855)
@@ -78,7 +78,6 @@
     def store_data(self):
 
         end_time = time.time()
-        log.log_info("Inserting data into data store...")
 
         # close out the task set
         self.engine.execute(
@@ -91,11 +90,19 @@
         ).close()
 
         task_times = 0
+        total_tasks = len(self.runtime_data)
+        counter = 0
 
+        log.log_info("Inserting %d rows of data into data store..." % total_tasks)
+
         # insert all of the task data
+        # TODO: Create a batch insert SQL string to insert, since
+        # one row at a time is pretty slow
         for task in self.runtime_data:
+
             task_times += task['duration']
             log.log_debug("Storing " + task['name'])
+
             self.engine.execute(
                 self.task_table.insert().values(
                     task_name = task['name'],
@@ -107,10 +114,14 @@
                 )
             ).close()
 
+            counter += 1
+            if counter %100 == 0:
+                log.log_info("Inserted %d rows" % counter)
+
         # log some basic stats
         task_set_time = end_time - self.task_set_start_time
         log.log_info("Total Tasks        -> %d" % len(self.runtime_data))
         log.log_info("Total time         -> %0.3f seconds" % task_set_time)
-        log.log_info("Average task time  -> %0.3f seconds" % (task_times / len(self.runtime_data)))
-        log.log_info("Time / Total Tasks -> %0.3f seconds/task" % (task_set_time / len(self.runtime_data)))
+        log.log_info("Average task time  -> %0.3f seconds" % (task_times / total_tasks))
+        log.log_info("Real Time / Total Tasks -> %0.3f seconds/task" % (task_set_time / total_tasks))