diff -r 5761c79f8bcf -r 8df9230c6de8 new_tests/conftest.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/new_tests/conftest.py Tue Aug 30 12:38:35 2011 -0500 @@ -0,0 +1,235 @@ +# Copyright (c) 2011 ARM Limited +# All rights reserved. +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Ali Saidi + + +import pytest +import sys, os, subprocess + +# A class to just store names of various objects +class T(object): + alpha = 'alpha' + arm = 'arm' + mips = 'mips' + sparc = 'sparc' + x86 = 'x86' + + atomic = 'simple-atomic' + timing = 'simple-timing' + inorder = 'inorder' + o3 = 'o3' + + classic = 'classic' + ruby = 'ruby' + + quick = 'quick' + medium = 'medium' + fast = 'fast' + +# This is the base class of all the tests we currently use +# Beacuse of the way pytest calls test objects we can't have an __init__ +# method on this case, thus we store per-test state as items in the class +# name, sname, and variants must be overridden in the base class +class BaseTest(object): + name = 'ERR: Must set name' + sname = 'ERR_must_set_name' + variants = () + + # This function is used create a directory name for a specific test + # The base case is just joining arch/cpu/mem/length however + # It can be overrride to do more creative things if needed + def canonical_name(self, variant): + return self.sname + ".%(arch)s.%(cpu)s.%(mem)s.%(length)s" % variant + + # Return a command line to run a test + # This too can be overridden as required, but as a default we run the test + # python file and pass everything in the variants dict as a=foo parameters + # The target script can then decide how it wants to handle that + def run_script(self, variant): + items = [__file__] + for x in variant.items(): + items.append("%s=%s" % (x[0], x[1])) + return items + +class BaseSETest(BaseTest): + mode = 'SE' + +class BaseFSTest(BaseTest): + mode = 'FS' + +# Class that does the heavy lifting for actually running a test and processing +# the resulting output. The class is meant to be very generic and call methods +# on the input test case (test_info) to get anything that isn't generic +class gem5Test(object): + def __init__(self, cls, _options, v): + # Test info contains an instance of the test class we're going to run + self.test_info = cls() + # The pytest options object so we can look at command line flags + self.options = _options + # A list of test variants to run + self.variant = v + + # Directory that all the tests live in + self.test_root = os.path.dirname(__file__) + # Root of gem5 repository + self.gem5_root = os.path.dirname(self.test_root) + # Regression testing configs directory + self.test_configs = os.path.join(self.test_root, 'configs') + + # We probably need a better way to locate the build directory + self.binary_dir = './build/%s_%s' % (self.variant['arch'].upper(), + self.test_info.mode) + + # Binary to run + self.binary = os.path.join(self.binary_dir, + 'gem5.%s' % self.options.variant) + # Directory to create test output in build/ARCH_MD/tests/ + self.test_dir = os.path.join(self.binary_dir, 'tests', + self.test_info.sname, + self.test_info.canonical_name(self.variant)) + # Directory for references tests// + self.ref_dir = os.path.join(self.test_root, self.test_info.sname, + self.test_info.canonical_name(self.variant)) + + # Should a test be run? Compare options to cpu, mode, and length + # This could be arbitrarily complex + def run_test(self): + if len(self.options.length) and \ + self.variant['length'] not in self.options.length: + return False + if len(self.options.cpus) and \ + self.variant['cpu'] not in self.options.cpus: + return False + if len(self.options.modes) and \ + self.test_info.mode not in self.options.modes: + return False + return True + + # Check if two files are the same and if not raise an error + def compare_file(self, fname): + ref_file = os.path.join(self.ref_dir, fname) + test_file = os.path.join(self.test_dir, fname) + + ref_lines = open(ref_file, "r").readlines() + test_lines = open(test_file, "r").readlines() + + if fname == 'simout': + # Remove gem5 (started|executing|compiled) + ref_lines = map(lambda x: re.sub('(gem5 (started|executing|compiled)).*', + '\\1', x), ref_lines) + test_lines = map(lambda x: re.sub('(gem5 (started|executing|compiled)).*', + '\\1', x), test_lines) + + diff = unified_diff(ref_lines, test_lines, ref_file, test_file) + if len(diff): + print diff + pytest.fail("%s differs", fname) + + # Check that the simulator and reference directory have the same output + def check_output(self): + ref_files = set(os.listdir(self.ref_dir)) + gen_files = set(os.listdir(self.test_dir)) + + diffs = ref_files ^ gen_files + + if len(diffs): + pytest.fail("Test did not generate all output files: %s", + ','.join(diffs)) + + normal_files = gen_files - 'stats.txt' + + for f in normal_files: + self.compare_file(f) + + # Check the stats are the same + def check_stats(self): + # code to diff stats (python or txt) goes here + if False: + pytest.fail("Stats were different") + + # Actually run a test + def test(self): + # Check if we actually want to run this test + if not self.run_test(): + pytest.skip("Test disabled with command line parameters") + + # build a command line + cmd_line = [self.binary, '-d', self.test_dir, '-re'] + cmd_line.extend(self.test_info.run_script(self.variant)) + + # run gem5 + retcode = subprocess.call(cmd_line) + + # check output + if retcode: + pytest.fail('gem5 returned status code: %d' % retcode) + + self.check_output() + self.check_stats() + + +# Factory to generate instances of gem5Test for each possible test we might run +# Every class that has Test in it's name is run through this generator function +# currently there isn't much error checking, but for every variant in the +# BaseTest.variants variable a new instance of a gem5Test() is created and +# initialized with the test, pytest options, and a particular variant in the list +def pytest_generate_tests(metafunc): + # For every test specified in the test class + for v in metafunc.cls.variants: + # Create a gem5Test object to support running that test + test_instance = gem5Test(metafunc.cls, metafunc.config.option, v) + # Add it to the list of tests to call + metafunc.addcall(funcargs=dict(test_instance=test_instance)) + +# Add a set of options to pytest. The options can be supplied multipile times, +# so if you wanted to run alpha and arm tests you could run: +# py.test --arch=arm --arch=alpha +def pytest_addoption(parser): + parser.addoption("--length", action="append", default=[], + choices=["quick", "medium", "long"], + help="Run only tests of a certain length (can be specified multiple times)") + parser.addoption("--cpus", action="append", default=[], + choices=["o3", "simple-atomic", "inorder", "simple-timing"], + help="Run only tests with certain cpu models") + parser.addoption("--modes", action="append", default=[], + choices=["SE", "FS"], + help="Run only tests with certain gem5 modes (can be specified multiple times)") + parser.addoption("--archs", action="append", default=[], + choices=['alpha','arm','mips','power','sparc','x86'], + help="Run only tests with certain architectures") + parser.addoption("--variant", action="store", default="opt", + choices=['debug','opt','fast','prof'], + help="Which variant of gem5 would you like to test?") diff -r 5761c79f8bcf -r 8df9230c6de8 new_tests/hello_world/test_hello.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/new_tests/hello_world/test_hello.py Tue Aug 30 12:38:35 2011 -0500 @@ -0,0 +1,85 @@ +# Copyright (c) 2011 ARM Limited +# All rights reserved. +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Ali Saidi + +# Simple test hello world test +import sys, os + +# Find the root of the test directory and add it to the +# system paths so we can find things like conftest +test_root = os.path.dirname(__file__) +tests_root = os.path.dirname(test_root) +sys.path.append(tests_root) + +# If we're running within gem5 cal the regression test method +if __name__ == '__m5_main__': + regression_test() + + +# Describe a hello world test +from conftest import BaseSETest, T +class TestHelloWorld(BaseSETest): + # Pretty name + name = 'Hello World' + # short name + sname = 'hello_world' + # Variants to run (just two at the moment) + variants = (dict(arch=T.arm, cpu=T.atomic, mem=T.classic, length=T.quick), + dict(arch=T.arm, cpu=T.timing, mem=T.classic, length=T.quick)) + + + # Test must be named test_* + def test_hello(self, test_instance): + # Call the gem5Test class test function + test_instance.test() + +def regression_test(): + # gem5 is running at this point and we have the contents + import m5, sys + + # Reconstruct the various parameters from the command line + params = {} + for x in xrange(1,len(sys.argv)): + key,item = sys.argv[x].split('=') + params[key] = "%s" % item + + # Could execfile scripts here or have majority of code inline + # Another option would be to break down a config into multiple + # steps. Something like, where each script should have access + # to the params dict, or we could pass it in as a global + # execfile(params['mem'] + '.py') + # execfile(params['cpu'] + '.py') + # execfile('simple_binary.py')