diff -r 5f265b5de1c5 -r 0e0cf51add93 SConstruct --- a/SConstruct Wed Apr 23 13:08:20 2014 +0100 +++ b/SConstruct Wed Apr 23 13:08:28 2014 +0100 @@ -109,6 +109,7 @@ raise # Global Python includes +import itertools import os import re import subprocess @@ -1160,16 +1161,21 @@ ################################################### main['ALL_ISA_LIST'] = all_isa_list +all_isa_deps = {} def make_switching_dir(dname, switch_headers, env): # Generate the header. target[0] is the full path of the output # header to generate. 'source' is a dummy variable, since we get the # list of ISAs from env['ALL_ISA_LIST']. def gen_switch_hdr(target, source, env): fname = str(target[0]) - f = open(fname, 'w') isa = env['TARGET_ISA'].lower() - print >>f, '#include "%s/%s/%s"' % (dname, isa, basename(fname)) - f.close() + try: + f = open(fname, 'w') + print >>f, '#include "%s/%s/%s"' % (dname, isa, basename(fname)) + f.close() + except IOError: + print "Failed to create %s" % fname + raise # Build SCons Action object. 'varlist' specifies env vars that this # action depends on; when env['ALL_ISA_LIST'] changes these actions @@ -1180,8 +1186,37 @@ # Instantiate actions for each header for hdr in switch_headers: env.Command(hdr, [], switch_hdr_action) + + isa_target = Dir('.').up().name.lower().replace('_', '-') + env['PHONY_BASE'] = '#'+isa_target + all_isa_deps[isa_target] = None + Export('make_switching_dir') +# all-isas -> all-deps -> all-environs -> all_targets +main.Alias('#all-isas', []) +main.Alias('#all-deps', '#all-isas') + +# Dummy target to ensure all environments are created before telling +# SCons what to actually make (the command line arguments). We attach +# them to the dependence graph after the environments are complete. +ORIG_BUILD_TARGETS = list(BUILD_TARGETS) # force a copy; gets closure to work. +def environsComplete(target, source, env): + for t in ORIG_BUILD_TARGETS: + main.Depends('#all-targets', t) + +# Each build/* switching_dir attaches its *-environs target to #all-environs. +main.Append(BUILDERS = {'CompleteEnvirons' : + Builder(action=MakeAction(environsComplete, None))}) +main.CompleteEnvirons('#all-environs', []) + +def doNothing(**ignored): pass +main.Append(BUILDERS = {'Dummy': Builder(action=MakeAction(doNothing, None))}) + +# The final target to which all the original targets ultimately get attached. +main.Dummy('#all-targets', '#all-environs') +BUILD_TARGETS[:] = ['#all-targets'] + ################################################### # # Define build environments for selected configurations. @@ -1290,14 +1325,25 @@ # The src/SConscript file sets up the build rules in 'env' according # to the configured variables. It returns a list of environments, # one for each variant build (debug, opt, etc.) - envList = SConscript('src/SConscript', variant_dir = variant_path, - exports = 'env') + SConscript('src/SConscript', variant_dir = variant_path, exports = 'env') - # Set up the regression tests for each build. - for e in envList: - SConscript('tests/SConscript', - variant_dir = joinpath(variant_path, 'tests', e.Label), - exports = { 'env' : e }, duplicate = False) +def pairwise(iterable): + "s -> (s0,s1), (s1,s2), (s2, s3), ..." + a, b = itertools.tee(iterable) + b.next() + return itertools.izip(a, b) + +# Create false dependencies so SCons will parse ISAs, establish +# dependencies, and setup the build Environments serially. Either +# SCons (likely) and/or our SConscripts (possibly) cannot cope with -j +# greater than 1. It appears to be standard race condition stuff; it +# doesn't always fail, but usually, and the behaviors are different. +# Every time I tried to remove this, builds would fail in some +# creative new way. So, don't do that. You'll want to, though, because +# tests/SConscript takes a long time to make its Environments. +for t1, t2 in pairwise(sorted(all_isa_deps.iterkeys())): + main.Depends('#%s-deps' % t2, '#%s-deps' % t1) + main.Depends('#%s-environs' % t2, '#%s-environs' % t1) # base help text Help(''' diff -r 5f265b5de1c5 -r 0e0cf51add93 src/SConscript --- a/src/SConscript Wed Apr 23 13:08:20 2014 +0100 +++ b/src/SConscript Wed Apr 23 13:08:28 2014 +0100 @@ -149,6 +149,14 @@ def __eq__(self, other): return self.filename == other.filename def __ne__(self, other): return self.filename != other.filename + @staticmethod + def done(): + def disabled(cls, name, *ignored): + raise RuntimeError("Additional SourceFile '%s'" % name,\ + "declared, but targets deps are already fixed.") + SourceFile.__init__ = disabled + + class Source(SourceFile): '''Add a c/c++ source file to the build''' def __init__(self, source, Werror=True, swig=False, **guards): @@ -877,20 +885,26 @@ # # List of constructed environments to pass back to SConstruct -envList = [] +date_source = Source('base/date.cc', skip_lib=True) -date_source = Source('base/date.cc', skip_lib=True) +# Capture this directory for the closure makeEnv, otherwise when it is +# called, it won't know what directory it should use. +variant_dir = Dir('.').path +def variant(*path): + return os.path.join(variant_dir, *path) +def variantd(*path): + return variant(*path)+'/' # Function to create a new build environment as clone of current # environment 'env' with modified object suffix and optional stripped # binary. Additional keyword arguments are appended to corresponding # build environment vars. -def makeEnv(label, objsfx, strip = False, **kwargs): +def makeEnv(env, label, objsfx, strip = False, **kwargs): # SCons doesn't know to append a library suffix when there is a '.' in the # name. Use '_' instead. - libname = 'gem5_' + label - exename = 'gem5.' + label - secondary_exename = 'm5.' + label + libname = variant('gem5_' + label) + exename = variant('gem5.' + label) + secondary_exename = variant('m5.' + label) new_env = env.Clone(OBJSUFFIX=objsfx, SHOBJSUFFIX=objsfx + 's') new_env.Label = label @@ -978,8 +992,8 @@ test_objs = [ make_obj(s, static=True) for s in test_sources ] if test.main: test_objs += main_objs - testname = "unittest/%s.%s" % (test.target, label) - new_env.Program(testname, test_objs + static_objs) + path = variant('unittest/%s.%s' % (test.target, label)) + new_env.Program(path, test_objs + static_objs) progname = exename if strip: @@ -999,7 +1013,7 @@ MakeAction('ln $SOURCE $TARGET', Transform("HARDLINK"))) new_env.M5Binary = targets[0] - envList.append(new_env) + return new_env # Start out with the compiler flags common to all compilers, # i.e. they all use -g for opt and -g -pg for prof @@ -1065,39 +1079,77 @@ if 'all' in needed_envs: needed_envs += target_types -# Debug binary -if 'debug' in needed_envs: - makeEnv('debug', '.do', - CCFLAGS = Split(ccflags['debug']), - CPPDEFINES = ['DEBUG', 'TRACING_ON=1'], - LINKFLAGS = Split(ldflags['debug'])) +gem5_root = Dir('.').up().up().abspath +def makeEnvirons(target, source, env): + # cause any later Source() calls to be fatal, as a diagnostic. + Source.done() -# Optimized binary -if 'opt' in needed_envs: - makeEnv('opt', '.o', - CCFLAGS = Split(ccflags['opt']), - CPPDEFINES = ['TRACING_ON=1'], - LINKFLAGS = Split(ldflags['opt'])) + envList = [] -# "Fast" binary -if 'fast' in needed_envs: - makeEnv('fast', '.fo', strip = True, - CCFLAGS = Split(ccflags['fast']), - CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'], - LINKFLAGS = Split(ldflags['fast'])) + # Debug binary + if 'debug' in needed_envs: + envList.append( + makeEnv(env, 'debug', '.do', + CCFLAGS = Split(ccflags['debug']), + CPPDEFINES = ['DEBUG', 'TRACING_ON=1'], + LINKFLAGS = Split(ldflags['debug']))) -# Profiled binary using gprof -if 'prof' in needed_envs: - makeEnv('prof', '.po', - CCFLAGS = Split(ccflags['prof']), - CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'], - LINKFLAGS = Split(ldflags['prof'])) + # Optimized binary + if 'opt' in needed_envs: + envList.append( + makeEnv(env, 'opt', '.o', + CCFLAGS = Split(ccflags['opt']), + CPPDEFINES = ['TRACING_ON=1'], + LINKFLAGS = Split(ldflags['opt']))) -# Profiled binary using google-pprof -if 'perf' in needed_envs: - makeEnv('perf', '.gpo', - CCFLAGS = Split(ccflags['perf']), - CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'], - LINKFLAGS = Split(ldflags['perf'])) + # "Fast" binary + if 'fast' in needed_envs: + envList.append( + makeEnv(env, 'fast', '.fo', strip = True, + CCFLAGS = Split(ccflags['fast']), + CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'], + LINKFLAGS = Split(ldflags['fast']))) -Return('envList') + # Profiled binary using gprof + if 'prof' in needed_envs: + envList.append( + makeEnv(env, 'prof', '.po', + CCFLAGS = Split(ccflags['prof']), + CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'], + LINKFLAGS = Split(ldflags['prof']))) + + # Profiled binary using google-pprof + if 'perf' in needed_envs: + envList.append( + makeEnv(env, 'perf', '.gpo', + CCFLAGS = Split(ccflags['perf']), + CPPDEFINES = ['NDEBUG', 'TRACING_ON=0'], + LINKFLAGS = Split(ldflags['perf']))) + + # Set up the regression tests for each build. + for e in envList: + SConscript(os.path.join(gem5_root, 'tests', 'SConscript'), + variant_dir = variantd('tests', e.Label), + exports = { 'env' : e }, duplicate = False) + +# The MakeEnvirons Builder defers the full dependency collection until +# after processing the ISA definition (due to dynamically generated +# source files). Add this dependency to all targets so they will wait +# until the environments are completely set up. Otherwise, a second +# process (e.g. -j2 or higher) will try to compile the requested target, +# not know how, and fail. +env.Append(BUILDERS = {'MakeEnvirons' : + Builder(action=MakeAction(makeEnvirons, + Transform("ENVIRONS", 1)))}) + +isa_target = env['PHONY_BASE'] + '-deps' +environs = env['PHONY_BASE'] + '-environs' +env.Depends('#all-deps', isa_target) +env.Depends('#all-environs', environs) +env.ScanISA(isa_target, File('arch/%s/generated/inc.d' % env['TARGET_ISA'])) +envSetup = env.MakeEnvirons(environs, isa_target) + +# make sure no -deps targets occur before all ISAs are complete +env.Depends(isa_target, '#all-isas') +# likewise for -environs targets and all the -deps targets +env.Depends(environs, '#all-deps') diff -r 5f265b5de1c5 -r 0e0cf51add93 src/arch/SConscript --- a/src/arch/SConscript Wed Apr 23 13:08:20 2014 +0100 +++ b/src/arch/SConscript Wed Apr 23 13:08:28 2014 +0100 @@ -30,6 +30,7 @@ import sys import os +import re Import('*') @@ -97,18 +98,32 @@ cpu_models = list(env['CPU_MODELS']) cpu_models.append('CheckerCPU') - # Several files are generated from the ISA description. - # We always get the basic decoder and header file. - target = [ 'decoder.cc', 'decoder.hh', 'max_inst_regs.hh' ] - # We also get an execute file for each selected CPU model. - target += [CpuModel.dict[cpu].filename for cpu in cpu_models] - # List the isa parser as a source. source += [ isa_parser ] # Add in the CPU models. source += [ Value(m) for m in cpu_models ] - return [os.path.join("generated", t) for t in target], source + # Specify different targets depending on if we're running the ISA + # parser for its dependency information, or for the generated files. + # (As an optimization, the ISA parser detects the useless second run + # and skips doing any work, if the first run was performed, since it + # always generates all its files). The way we track this in SCons is the + # _isa_outputs value in the environment (env). If it's unset, we + # don't know what the dependencies are so we ask for generated/inc.d to + # be generated so they can be acquired. If we know what they are, then + # it's because we've already processed inc.d and then claim that our + # outputs (targets) will be thus. + isa = env['TARGET_ISA'] + key = '%s_isa_outputs' % isa + if key in env: + targets = [ os.path.join('generated', f) for f in env[key] ] + else: + targets = [ os.path.join('generated','inc.d') ] + + def prefix(s): + return os.path.join(target[0].dir.up().abspath, s) + + return [ prefix(t) for t in targets ], source ARCH_DIR = Dir('.') @@ -133,6 +148,53 @@ env.Append(BUILDERS = { 'ISADesc' : isa_desc_builder }) +# The ISA is generated twice: the first time to find out what it generates, +# and the second time to make scons happy by telling the ISADesc builder +# what it will make before it builds it. +def scan_isa_deps(target, source, env): + # Process dependency file generated by the ISA parser -- + # add the listed files to the dependency tree of the build. + source = source[0] + archbase = source.dir.up().path + + try: + depfile = open(source.abspath, 'r') + except: + print "scan_isa_deps: Can't open ISA deps file '%s' in %s" % \ + (source.path,os.getcwd()) + raise + + # Scan through the lines + targets = {} + for line in depfile: + # Read the dependency line with the format + # : [ * ] + m = re.match(r'^\s*([^:]+\.([^\.:]+))\s*:\s*(.*)', line) + assert(m) + targ, extn = m.group(1,2) + deps = m.group(3).split() + + files = [ targ ] + deps + for f in files: + targets[f] = True + # Eliminate unnecessary re-generation if we already generated it + env.Precious(os.path.join(archbase, 'generated', f)) + + files = [ os.path.join(archbase, 'generated', f) for f in files ] + + if extn == 'cc': + Source(os.path.join(archbase,'generated', targ)) + depfile.close() + env[env['TARGET_ISA'] + '_isa_outputs'] = targets.keys() + + isa = env.ISADesc(os.path.join(archbase,'isa','main.isa')) + for t in targets: + env.Depends('#all-isas', isa) + +env.Append(BUILDERS = {'ScanISA' : + Builder(action=MakeAction(scan_isa_deps, + Transform("NEW DEPS", 1)))}) + DebugFlag('IntRegs') DebugFlag('FloatRegs') DebugFlag('CCRegs') diff -r 5f265b5de1c5 -r 0e0cf51add93 tests/SConscript --- a/tests/SConscript Wed Apr 23 13:08:20 2014 +0100 +++ b/tests/SConscript Wed Apr 23 13:08:28 2014 +0100 @@ -267,14 +267,14 @@ def test_builder(env, ref_dir): """Define a test.""" - (category, mode, name, _ref, isa, opsys, config) = ref_dir.split('/') - assert(_ref == 'ref') + path = list(ref_dir.split('/')) - # target path (where test output goes) is the same except without - # the 'ref' component - tgt_dir = os.path.join(category, mode, name, isa, opsys, config) + # target path (where test output goes) consists of category, mode, + # name, isa, opsys, and config (skips the 'ref' component) + assert(path.pop(-4) == 'ref') + tgt_dir = os.path.join(*path[-6:]) - # prepend file name with tgt_dir + # local closure for prepending target path to filename def tgt(f): return os.path.join(tgt_dir, f) @@ -342,11 +342,10 @@ else: configs = [c + "-ruby-" + env['PROTOCOL'] for c in configs] -cwd = os.getcwd() -os.chdir(str(Dir('.').srcdir)) +src = Dir('.').srcdir for config in configs: - dirs = glob.glob('*/*/*/ref/%s/*/%s' % (env['TARGET_ISA'], config)) + dirs = src.glob('*/*/*/ref/%s/*/%s' % (env['TARGET_ISA'], config)) for d in dirs: + d = str(d) if not os.path.exists(os.path.join(d, 'skip')): test_builder(env, d) -os.chdir(cwd)