diff -r fadb27984bec -r 8a3d4a250bc9 src/python/SConscript --- a/src/python/SConscript Wed Apr 24 11:28:17 2013 +0100 +++ b/src/python/SConscript Wed Apr 24 11:28:21 2013 +0100 @@ -53,6 +53,7 @@ PySource('m5.stats', 'm5/stats/context.py') PySource('m5.stats', 'm5/stats/display.py') PySource('m5.stats', 'm5/stats/info.py') +PySource('m5.stats', 'm5/stats/sql.py') PySource('m5.util', 'm5/util/__init__.py') PySource('m5.util', 'm5/util/attrdict.py') PySource('m5.util', 'm5/util/code_formatter.py') diff -r fadb27984bec -r 8a3d4a250bc9 src/python/m5/main.py --- a/src/python/m5/main.py Wed Apr 24 11:28:17 2013 +0100 +++ b/src/python/m5/main.py Wed Apr 24 11:28:21 2013 +0100 @@ -82,6 +82,9 @@ group("Statistics Options") option("--stats-file", metavar="FILE", default="stats.txt", help="Sets the output file for statistics [Default: %default]") + option("--stats-db-file", metavar="FILE", default="", + help = "Sets the output database file for statistics [Default: \ + %default]") # Configuration Options group("Configuration Options") @@ -176,7 +179,7 @@ import stats import trace - from util import fatal + from util import fatal, warn if len(args) == 0: options, arguments = parse_options() @@ -311,7 +314,15 @@ sys.path[0:0] = options.path # set stats options - stats.initText(options.stats_file) + if options.stats_db_file: + stats.init_SQL(options.outdir, options.stats_db_file) + + if options.stats_file: + stats.init_text(options.stats_file) + + # Check that at least one stats output format is enabled + if not stats.stats_output_enabled(): + warn("Unable to output statistics.") # set debugging options debug.setRemoteGDBPort(options.remote_gdb_port) diff -r fadb27984bec -r 8a3d4a250bc9 src/python/m5/stats/__init__.py --- a/src/python/m5/stats/__init__.py Wed Apr 24 11:28:17 2013 +0100 +++ b/src/python/m5/stats/__init__.py Wed Apr 24 11:28:21 2013 +0100 @@ -48,6 +48,19 @@ from m5.stats.context import stat_context from m5.util import attrdict, fatal, panic, SortedDict +# Try and include SQLAlchemy. If this is successful we allow it to be used, +# otherwise it is disabled. +try: + from sqlalchemy import * + from sqlalchemy.orm import sessionmaker + from m5.stats.sql import * + SQL_ENABLED = True +except: + SQL_ENABLED = False + +# Global variable to determine if statistics output is enabled +STATS_OUTPUT_ENABLED = False + def initSimStats(): internal.stats.initSimStats() @@ -256,7 +269,7 @@ ## @brief Keeps track of the number of stats dumps which have occured. dumpCount = 0 -class display_text(object): +class OutputText(object): def __init__(self, filename, desc): self.filename = filename self.desc = desc @@ -293,19 +306,84 @@ display.descriptions = save_desc +# Keep track of the number of stats dumps performed. Lets us keep +# track of the data we write to the SQL database. +dump_count = 0 + +# @brief Class which outputs the stats to a database +class OutputSQL(object): + # @brief Create the database and add the tables used to store the stats. + def __init__(self, filename): + self.filename = filename + self.db = create_database(self.filename) + Session = sessionmaker(bind = self.db) + create_tables(self.db) + self.session = Session() + + # @brief Write the stats to the database. On the first dump we also write + # the information about the stats to the database. This is only done once. + def __call__(self, context): + global dump_count + + # On the first dump write the information about the stats to the + # database. + if dump_count == 0: + for name,stat in context.iterate(): + add_stat_info(stat, self.session) + + # Write the values to the database. + for name,stat in context.iterate(): + store_stat_value(stat, self.session, dump_count) + + # Commit our changes to the database. All changes are commited once + # to allow SQLAlchemy to optimize the database accesses, resulting in + # faster stats dumps. + self.session.commit() + + outputList = [] # @brief Add the stats file as an output and add it to outputList # @param filename The filename to which the stats are written # @param desc Bool which sets whether the descriptions are written to the # output file or not. -def initText(filename, desc=True): - output = display_text(filename, desc) +def init_text(filename, desc=True): + output = OutputText(filename, desc) outputList.append(output) + global STATS_OUTPUT_ENABLED + STATS_OUTPUT_ENABLED = True +# @brief Add the stats database as an output and add it to outputList +# @param filename The filename to which the stats are written +# @return True if SQL stats are enabled, False otherwise. +def init_SQL(outputDirectory, filename): + global SQL_ENABLED + + # Take the supplied filename and prepend the output directory. + import os + filename = os.path.join(outputDirectory, filename) + + if SQL_ENABLED: + output = OutputSQL(filename) + outputList.append(output) + global STATS_OUTPUT_ENABLED + STATS_OUTPUT_ENABLED = True + return True + else: + return False + +# @brief Check that at least one statistics output format is enabled +# @return True if at least one output format is enabled, False otherwise +def stats_output_enabled(): + return STATS_OUTPUT_ENABLED + lastDump = 0 # @brief Dump all statistics values to the registered outputs. def dump(): + # If there is no registered output, just return + if not STATS_OUTPUT_ENABLED: + return + curTick = m5.curTick() global lastDump @@ -326,6 +404,9 @@ global dumpCount dumpCount += 1 + global dump_count + dump_count = dump_count + 1 + ## @brief Reset all statistics to the base state. def reset(): # call reset stats on all SimObjects diff -r fadb27984bec -r 8a3d4a250bc9 src/python/m5/stats/sql.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/python/m5/stats/sql.py Wed Apr 24 11:28:21 2013 +0100 @@ -0,0 +1,312 @@ +# +# Copyright (c) 2012 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Sascha Bischoff +# + +try: + from sqlalchemy import * + from sqlalchemy.orm import sessionmaker +except: + print "Unable to import sqlalchemy!" + raise + +from m5.util import fatal, panic + + +# @brief Create the database used to store the stats. If it exists, we delete +# it. +# @param filename The filename used to store the stats. +# @return A handle to the database. +def create_database(filename): + import os + + if os.path.exists(filename): + os.remove(filename) + + try: + db = create_engine('sqlite:///' + filename) + except: + panic("Failed to open database %s!", filename) + db.echo = False + return db + + +# @brief Create the tables used to store the stats and information about them. +# @param db The database in which to create the tables. +def create_tables(db): + metadata = MetaData(db) + + # Stores the information about the stats + stats_table = Table('stats', metadata, + Column('id', Integer, primary_key = True), + Column('name', String), + Column('desc', String), + Column('subnames', String), + Column('y_subnames', String), + Column('subdescs', String), + Column('precision', Integer), + Column('prereq', Integer), + Column('flags', Integer), + Column('x', Integer), + Column('y', Integer), + Column('type', String), + Column('formula', String), + ) + + # Stores scalar values + scalar_value_table = Table('scalarValue', metadata, + Column('id', Integer), + Column('dump', Integer), + Column('value', Float), + ) + + # Stores vectors, 2d vectors and is also used to store formulas as they can + # be scalars or vectors based on the stats used in the calculation. + vector_value_table = Table('vectorValue', metadata, + Column('id', Integer), + Column('dump', Integer), + Column('value', Binary), + ) + + # Stores distributions. + dist_value_table = Table('distValue', metadata, + Column('id', Integer), + Column('dump', Integer), + Column('sum', Float), + Column('squares', Float), + Column('samples', Float), + Column('min', Float), + Column('max', Float), + Column('bucket', Float), + Column('vector', Binary), + Column('min_val', Float), + Column('max_val', Float), + Column('underflow', Float), + Column('overflow', Float), + ) + + metadata.create_all() + + +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + +# @brief Class used to insert the information about stats into the database. +class StatsInfoClass(Base): + __tablename__ = 'stats' + + id = Column(Integer, primary_key = True) + name = Column(String) + desc = Column(String) + subnames = Column(String) + y_subnames = Column(String) + subdescs = Column(String) + precision = Column(Integer) + prereq = Column(Integer) + flags = Column(Integer) + x = Column(Integer) + y = Column(Integer) + type = Column(String) + formula = Column(String) + + def __init__(self, stat): + self.id = stat.id + self.name = stat.name + self.desc = stat.desc + self.flags = stat.flags + self.precision = stat.precision + if stat.prereq: + self.prereq = stat.prereq.id + self.type = stat.type + + if stat.type == "VectorInfo": + self.subnames = ','.join(stat.subnames) + self.subdescs = ','.join(stat.subdescs) + elif stat.type == "FormulaInfo": + self.subnames = ','.join(stat.subnames) + self.subdescs = ','.join(stat.subdescs) + self.formula = stat.formula + elif stat.type == "Vector2dInfo": + self.subnames = ','.join(stat.subnames) + self.y_subnames = ','.join(stat.y_subnames) + self.subdescs = ','.join(stat.subdescs) + self.x = stat.x + self.y = stat.y + elif stat.type == "DistInfo": + from m5.stats.info import Deviation, Distribution, Histogram + + data = stat.value + + if type(data) == Deviation: + the_type = "Deviation" + elif type(data) == Distribution: + the_type = "Distribution" + elif type(data) == Histogram: + the_type = "Histogram" + + self.type = the_type + + def __repr__(self): + return "" % (self.id, self.name, self.desc, self.subnames, + self.y_subnames, self.subdescs, self.precision, self.prereq, + self.flags, self.x, self.y, self.type, self.formula) + + +# @brief Class used to insert scalar stats into the database. +class ScalarValueClass(Base): + __tablename__ = 'scalarValue' + + id = Column(Integer, primary_key = True) + dump = Column(Integer) + value = Column(Integer) + + def __init__(self, id, dump, value): + self.id = id + self.dump = dump + self.value = value + + + def __repr__(self): + return "" % (self.id, self.dump, self.value) + + +# @brief Class used to insert vector stats into the database. +class VectorValueClass(Base): + __tablename__ = 'vectorValue' + + id = Column(Integer, primary_key = True) + dump = Column(Integer) + value = Column(Binary) + + def __init__(self, id, dump, value): + import array + self.id = id + self.dump = dump + a = array.array('f', value) + self.value = a.tostring() + + def __repr__(self): + return "" % (self.id, self.dump, self.value) + + +# @brief Class used to insert dictribution stats into the database. +class DistValueClass(Base): + __tablename__ = 'distValue' + + id = Column(Integer, primary_key = True) + dump = Column(Integer) + sum = Column(Float) + squares = Column(Float) + samples = Column(Float) + min = Column(Float) + max = Column(Float) + bucket = Column(Float) + vector = Column(Binary) + min_val = Column(Float) + max_val = Column(Float) + underflow = Column(Float) + overflow = Column(Float) + + def __init__(self, id, dump): + self.id = id + self.dump = dump + + + def __repr__(self): + return "" % (self.id, self.dump, self.sum, self.squares, + self.samples, self.min, self.max, self.bucket, self.vector, + self.min_val, self.max_val, self.underflow, self.overflow) + + +# @brief Add the information about a stat. +# @param name The name of the stat. +# @param stat The stat itself. +# @param session The session associated with the database. +def add_stat_info(stat, session): + temp = StatsInfoClass(stat) + session.add(temp) + + +# @brief Stores the value of a stat. +# @param stat The stat itself. +# @param session The session associated with the database. +# @param dumpCount The number of dumps that have occured. Used to store multiple +# stats dumps in one database. +def store_stat_value(stat, session, dumpCount): + if stat.type == "ScalarInfo": + temp = ScalarValueClass(id = stat.id, dump = dumpCount, + value = stat.value) + session.add(temp) + + elif stat.type == "VectorInfo" or stat.type == "Vector2dInfo" \ + or stat.type == "FormulaInfo": + temp = VectorValueClass(id = stat.id, dump = dumpCount, + value = stat.value) + session.add(temp) + + elif stat.type == "DistInfo": + from m5.stats.info import Deviation, Distribution, Histogram + import array + + data = stat.value + + temp = DistValueClass(id = stat.id, dump = dumpCount) + + temp.sum = data.sum + temp.squares = data.squares + temp.samples = data.samples + + if type(data) == Distribution or type(data) == Histogram: + temp.min = data.min + temp.max = data.max + temp.bucket = data.bucket_size + a = array.array('f', data.vector) + temp.vector = a.tostring() + + if type(data) == Distribution: + temp.min_val = data.min_val + temp.max_val = data.max_val + temp.underflow = data.underflow + temp.overflow = data.overflow + + session.add(temp) + + else: + panic("Unable to output stat %s. Unsupported stat type!", name)