diff -r a129a7aecb97 -r 35ce7e2339f3 util/stats/extractSQLStats.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/util/stats/extractSQLStats.py Tue Jan 15 18:31:32 2013 +0000 @@ -0,0 +1,220 @@ +#!/usr/bin/python +# +# Copyright (c) 2012 ARM Limited +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Sascha Bischoff +# + +# This script extracts the data stored in the stats database and is able to +# either write it to the screen, or to a simple csv file. It is designed as a +# rough guide for how to extract the data from the SQLite database used by +# the gem5 stats system. +# +# In order to inspect the raw data contained in the gem5 stats database, Ubuntu +# and RedHat users are able to use a module called sqlitebrowser. This allows +# the tables to be inspected and queries to be designed and tested. + +import optparse +import re +import sys + +try: + from sqlalchemy import * + from sqlalchemy.orm import sessionmaker +except: + print "Unable to import SQLAlchemy! Exiting..." + sys.exit(1) + +def main(): + parser = optparse.OptionParser() + + parser.add_option("-i", "--input", type="string", default="", + help="Name (and path to) the input stats DB. Default=%default") + + parser.add_option("-o", "--output", type="string", default="", + help="Name (and path to) the output file. Default=%default") + + parser.add_option("-n", "--name", type="string", default="", + help="Name of the stat to extract. Default=%default") + + parser.add_option("-d", "--dump", type="int", default=0, + help="Index of dump to extract. Default=%default") + + (options, args) = parser.parse_args() + + # Check that we have been passed an input file, and connect to the DB + if options.input: + # Connect to the database + try: + db = create_engine('sqlite:///' + options.input) + # Set db.echo to True to debug the generated queries + db.echo = False + except: + print "Unable to open the database! Exiting..." + sys.exit(1) + else: + print "Please supply an input file! Exiting..." + sys.exit(1) + + if not options.output: + print "Warning: no output file supplied. Writing to stdout." + else: + try: + outputFile = open(options.output, "w") + except IOError, e: + print e + sys.exit(1) + + if not options.name: + print "Please provide the name of a stat to extract (-n)." + sys.exit(1) + + # Create the session + Session = sessionmaker(bind = db) + session = Session() + + metadata = MetaData(db) + + #Define the tables we use in the database: + + try: + # Stores the information about the stats + statsTable = Table('stats', metadata, autoload=True) + + # Stores scalar values + scalarValueTable = Table('scalarValue', metadata, autoload=True) + + # Stores vectors, 2d vectors and is also used to store formulas as they can + # be scalars or vectors based on the stats used in the calculation. + vectorValueTable = Table('vectorValue', metadata, autoload=True) + + # Stores distributions. + distValueTable = Table('distValue', metadata, autoload=True) + except: + print "The input DB file is not a valid gem5 stats dump! Exiting..." + sys.exit(1) + + # Get all of the information about the stats we care about + statInfo = (session.query(statsTable) + .filter(statsTable.c.name.like('%' + options.name + '%')) + .all()) + + for item in statInfo: + # As the different stats are stored in different tables, we need to + # tailor the querys to the type of the stat. + if item.type == "ScalarInfo": + # For stat called item.name, get value for dump options.dump + # Need to join the statsTable and the scalarValueTable based on the + # id of the stat, then filter by name and index of the dump. + for result in (session.query(statsTable.join(scalarValueTable, + statsTable.c.id==scalarValueTable.c.id) + ) + .filter(statsTable.c.id == scalarValueTable.c.id) + .filter(statsTable.c.name == item.name) + .filter(scalarValueTable.c.dump == options.dump) + .all()): + + # If the user has specificed an output file, write to that. + # Otherwise, write to the screen. + if options.output: + outputFile.write(result.name + "," + str(result.value) + + "\n") + else: + print result.name + ":", str(result.value) + + elif item.type == "VectorInfo" or \ + item.type == "FormulaInfo" or \ + item.type == "Vector2dInfo": + # For stat called item.name, get value for dump options.dump + # Need to join the statsTable and the vectorValueTable based on the + # id of the stat, then filter by name and index of the dump. + for result in (session.query(statsTable.join(vectorValueTable, + statsTable.c.id==vectorValueTable.c.id) + ) + .filter(statsTable.c.id == vectorValueTable.c.id) + .filter(statsTable.c.name == item.name) + .filter(vectorValueTable.c.dump == options.dump) + .all()): + + # The vector values are stored in binary, and therefore need to + # be extracted. + import array + temp = array.array('f') + temp.fromstring(result.value) + + # If the user has specificed an output file, write to that. + # Otherwise, write to the screen. + if options.output: + outputFile.write(result.name + "," + + ",".join(str(v) for v in temp) + "\n") + else: + print result.name + ":", ",".join(str(v) for v in temp) + + elif item.type =="Histogram" or \ + item.type =="Distribution" or \ + item.type =="Deviation": + # For stat called item.name, get value for dump options.dump + # Need to join the statsTable and the distValueTable based on the + # id of the stat, then filter by name and index of the dump. + for result in (session.query(statsTable.join(distValueTable, + statsTable.c.id==distValueTable.c.id) + ) + .filter(statsTable.c.id == distValueTable.c.id) + .filter(distValueTable.c.dump == options.dump) + .filter(statsTable.c.name == item.name) + .all()): + + # The vector is stored in binary, and therefore need to be + # extracted. + import array + temp = array.array('f') + temp.fromstring(result.vector) + + # If the user has specificed an output file, write to that. + # Otherwise, write to the screen. + if options.output: + outputFile.write(result.name + "," + + ",".join(str(v) for v in temp) + "\n") + else: + print result.name + ":", ",".join(str(v) for v in temp) + + else: + print "Unhandled type: " + item.type + sys.exit(1) + +if __name__ == "__main__": + main() +