diff -r 79181d97f42d -r 666aa039ec0d configs/common/FSConfig.py --- a/configs/common/FSConfig.py Mon Aug 10 22:41:08 2015 -0500 +++ b/configs/common/FSConfig.py Mon Aug 10 22:41:39 2015 -0500 @@ -204,7 +204,7 @@ def makeArmSystem(mem_mode, machine_type, num_cpus=1, mdesc=None, dtb_filename=None, bare_metal=False, cmdline=None, - external_memory=""): + external_memory="", options=None): assert machine_type if bare_metal: @@ -250,6 +250,28 @@ # Attach any PCI devices this platform supports self.realview.attachPciDevices() + if options.pd_gem5 and options.dual == None: + # instantiate a client EtheTap object and pass along the port and ip + # information of server (Swtich) + self.ethertap0 = EtherTap(server = False, + port = options.switch_tap_port, + server_ip = options.switch_tap_ip, + sync_quantum = options.sync_quantum) + # instantiate a "pd-gem5 connector" Etherlink (mode 2) and pass along + # speed and latency information of NIC and Link + self.etherlink = EtherLink(mode = 1, + delay = options.etherlink_delay, + speed = options.etherlink_speed, + ni_delay = options.nic_delay, + ni_speed = options.nic_speed) + # connect int1 of Etherlink to Ethertap + self.etherlink.int1 = self.ethertap0.tap + # connect int0 of Etherlink to NIC + self.etherlink.int0 = self.realview.ethernet.interface + if options.etherdump: + self.etherdump = EtherDump(file=dumpfile) + self.etherlink.dump = Parent.etherdump + # default to an IDE controller rather than a CF one try: self.realview.ide.disks = [self.cf0] diff -r 79181d97f42d -r 666aa039ec0d configs/common/Options.py --- a/configs/common/Options.py Mon Aug 10 22:41:08 2015 -0500 +++ b/configs/common/Options.py Mon Aug 10 22:41:39 2015 -0500 @@ -57,6 +57,21 @@ sys.exit(0) def addCommonOptions(parser): + # pd-gem5 options + parser.add_option("--sync-quantum",type="string",default='10us') + parser.add_option("--pd-gem5", action="store_true") + parser.add_option("--num-nodes",type="int",default=1) + parser.add_option("--switch-tap-ip",type="string",default="127.0.0.1", + help = """ + ip address of the machine which is simulating switch""") + parser.add_option("--switch-tap-port",type="int",default=3500) + parser.add_option("--etherlink-delay", type="string", default='10us') + parser.add_option("--nic-delay", type="string", default='0us') + parser.add_option("--sw-delay", type="string", default='0us') + parser.add_option("--etherlink-speed", type="string", default='1Gbps') + parser.add_option("--nic-speed", type="string", default='1Gbps') + parser.add_option("--sw-speed", type="string", default='1Gbps') + parser.add_option("--switch-queue-size",type="int",default=100) # system options parser.add_option("--list-cpu-types", action="callback", callback=_listCpuTypes, diff -r 79181d97f42d -r 666aa039ec0d configs/example/fs.py --- a/configs/example/fs.py Mon Aug 10 22:41:08 2015 -0500 +++ b/configs/example/fs.py Mon Aug 10 22:41:39 2015 -0500 @@ -99,7 +99,8 @@ options.num_cpus, bm[0], options.dtb_filename, bare_metal=options.bare_metal, cmdline=cmdline, - external_memory=options.external_memory_system) + external_memory=options.external_memory_system, + options=options) if options.enable_context_switch_stats_dump: test_sys.enable_context_switch_stats_dump = True else: diff -r 79181d97f42d -r 666aa039ec0d configs/example/sw.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/configs/example/sw.py Mon Aug 10 22:41:39 2015 -0500 @@ -0,0 +1,84 @@ +# Copyright (c) 2015 The University of Wisconsin Madison +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Mohammad Alian + +# This is an example of an n port network switch (star topology) to work in +# pd-gem5. Users can extend this to have different different topologies + +import optparse +import sys + +import m5 +from m5.defines import buildEnv +from m5.objects import * +from m5.util import addToPath, fatal + +addToPath('../common') + +import Simulation +import Options + +def build_switch(options): + # instantiate an EtherSwitch with "num_node" ports. Also pass along + # timing parameters + switch = EtherSwitch(port_count = options.num_nodes, + delay = options.sw_delay) + # instantiate etherlinks to connect switch box ports to ethertap objects + switch.portlink = [EtherLink(mode = 1, + delay = options.etherlink_delay, + speed = options.etherlink_speed, + ni_speed = options.sw_speed) + for i in xrange(options.num_nodes)] + + # instantiate ethertap objects in server mode + switch.porttap = [EtherTap(server = True, + sync_quantum = options.sync_quantum) + for i in xrange(options.num_nodes)] + for (i, link) in enumerate(switch.portlink): + link.int1 = switch.porttap[i].tap + link.int0 = switch.interface[i] + + return switch +# Add options +parser = optparse.OptionParser() +Options.addCommonOptions(parser) +Options.addFSOptions(parser) +(options, args) = parser.parse_args() + +system = build_switch(options) +root = Root(full_system = True, system = system) +Simulation.run(options, root, None, None) + diff -r 79181d97f42d -r 666aa039ec0d util/pd-gem5/pd-gem5.conf --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/util/pd-gem5/pd-gem5.conf Mon Aug 10 22:41:39 2015 -0500 @@ -0,0 +1,143 @@ +# Copyright (c) 2015 The University of Wisconsin Madison +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Mohammad Alian +# +# pd-gem5 configuration file +# +############################################################################### +# pd-gem5 run name, pd-gem5 creates a direcotry with this name and run cluster +# simulation there +run_name= TEST +# time between each syncrhonization point (in usec) +sync_period= 10us +# Map each simulated node to a physical host +# Simulated node names are tux0, ..., tux($(num_nodes) -1), sw +# syntax: : +# +# set to "HTCondor" to run that node using HTCondor, a +# simulation pool management software: http://research.cs.wisc.edu/htcondor/ +# if you want to simulate (4 nodes + 1 local switch) with simple cpu under +# one gem5 process, then map 4 simulated nodes to one physical machine: +# e.g, the following machine assignemt use 5 physical machines (all localhost) +# to simulate 7 nodes: +# machine_names= localhost:tux0,tux1,tux2,tux3 localhost:tux4 localhost:tux5 localhost:tux6 localhost:sw +machine_names= localhost:tux0 localhost:tux1 localhost:tux2 localhost:tux3 localhost:sw +############################################################################### +pd-gem5_dir= /user/home/gem5 +############################################################################### +# this is the layout of directories in pd-gem5 +# rundir and ckptdir will be created automatically after launching pd-gem5, +# however, if you wish to restore from a checkpoint, that checkpoint should +# already be presented under ckptdir +# User is responsible for scriptdir content +# +# $(pd-gem5_dir)/ +# configs/ +# examples/ +# ... +# rundir/ +# $(run_name)/ +# sw/ +# tux0/ +# ... +# tux($(num_nodes) - 1)/ +# ckptdir/ +# $(ckpt_dir)/ +# sw/ +# tux0/ +# ... +# tux($(num_nodes) - 1)/ +# scriptdir/ +# $(script_dir)/ +# tux0.sh +# ... +# tux($(num_nodes) - 1).sh +# $(run_dir)/ +# $(run_name)/ +# sw/ +# tux0/ +# ... +# tux($(num_nodes) - 1)/ +############################################################################### +run_dir= /user/home/gem5/rundir +# gem5 full-system script name, located under $pd-gem5_dir/configs/examples +fs_script= fs.py +# pd-gem5 will pick scripts for gem5 processes (excluding tux0) from this path: +# $(pd-gem5_dir)/scriptdir/$(script_dir) +# "script_dir" is the relative path (from $(pd-gem5_dir)) for the directory that +# contains scripts for each node. The naming convetion for script files is: +# tux0.sh, tux1.sh ... tux($(num_nodes) - 1).sh +# set script_dir to 0 if you don't want to pass any script to gem5 processes +script_dir= TEST_SCRIPT +# tux0 script +script_tux0= /user/home/gem5/scriptdir/TEST_SCRIPT/tux0.sh +# relative path to checkpoint files: +# $(pd-gem5_dir)/ckptdir/$(ckpt_dir) +ckpt_dir= TEST_CKPT +# relative path to gem5 binrary +# $(pd-gem5_dir)/$(gem5_binary) +gem5_binary= gem5.opt +############################################################################### +# Network Parameters +# WE ASSUME THAT LINK BANDWIDTH IS NOT THE NETWORK BANDWIDTH BOTTLENECK +# Then make sure that: link_speed >= Max(nic_speed, sw_speed) +############################################################################### +link_speed= 1Gbps +link_delay= 10us +nic_speed= 1Gbps +nic_delay= 1us +sw_speed= 1Gbps +sw_delay= 1us +delay_var= 0ns +queue_size= 200 +############################################################################### +# Other command line options +# Other than "--script", "--checkpoint-dir" and network options +# ATTENTION: DO NOT INSERT NEWLINE IN THE STRING +############################################################################### +other_command_line_options= --disk-image=/user/home/gem5/disks/aarch32-ubuntu-natty-headless.img +############################################################################### +# Debug Parametes +############################################################################### +# enable trace base debuging on a subset of nodes, set to 0 to disable debuging +# e.g: "trace_on= tux0,tux3,sw" +# enables traces on gem5 processes simulating tux0, tux3 and switch box +trace_on= 0 +#trace_on= tux0,tux1,tux2,tux3,sw +debug_flags= Ethernet +debug_file= NIC.trace +# debug start tick +debug_start= 0 diff -r 79181d97f42d -r 666aa039ec0d util/pd-gem5/pd-gem5.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/util/pd-gem5/pd-gem5.py Mon Aug 10 22:41:39 2015 -0500 @@ -0,0 +1,362 @@ +#!/usr/bin/python + +# Copyright (c) 2015 The University of Wisconsin Madison +# All rights reserved +# +# The license below extends only to copyright in the software and shall +# not be construed as granting a license to any other intellectual +# property including but not limited to intellectual property relating +# to a hardware implementation of the functionality of the software +# licensed hereunder. You may use the software subject to the license +# terms below provided that you ensure that this notice is replicated +# unmodified and in its entirety in all distributions of the software, +# modified or unmodified, in source code or in binary form. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer; +# redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution; +# neither the name of the copyright holders nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Authors: Mohammad Alian + +# This script launch pd-gem5 +# usage: python pd-gem5.py + +import sys, os, thread, commands +import socket +import time +import subprocess +import threading +import signal +import fcntl +import struct +import re +import collections +configFile = sys.argv[1] +# A dictionary of parameters +params = {} +machines = {} +# dictionary that maps each simulated node to a physical node +sim_to_phy = {} +# dictionary that maps each subprocess to a simulated node +gem5_processes = {} +this = socket.gethostname() +run_dir = "" +ckpt_dir = "" +script_dir = "" +submit_script = "" +sync_ip = '' +sync_port = 5000 +sw_ip = '' +sw_port = '' +running = True +monitoring = True +num_nodes = 0 + +# kill all threads and gem5 processes +def cleanup(): + global running + # kill barrier, monitor and main threads + running = False + time.sleep(1) + # kill gem5 processes if they are still running + for p in gem5_processes.keys(): + if type(p) == str: + submission_line = subprocess.Popen("grep submitted %s/%s/pdgem5sim.log"\ + %(run_dir, gem5_processes[p]), stdout=subprocess.PIPE, stderr=subprocess.PIPE, \ + shell = True).communicate()[0].rstrip() + if submission_line != '': + job_id = re.split('\.|\(', submission_line)[1] + os.system("condor_rm %s > /dev/null" %(job_id)) + else: + try: + os.killpg(p.pid, signal.SIGTERM) + except Exception as ex: + pass + sys.exit(0) + +# submit a job to HTCondor +# MODIFY THIS FUNCTION IF YOU ARE USING OTHER SIMULATION POOL MANAGEMENT SOFTWARE +def submit_to_HTCondor(job_path): + submit_script =("executable = /bin/sh\n" + "arguments = %s\n" + "initialdir = %s\n" + "output = %s\n" + "error = %s\n" + "log = %s\n" + "Rank = TARGET.Mips\n" + "Requirements =\n" + "universe = vanilla\n" + "getenv = true\n" + "queue\n")\ + % (job_path + "/job.sh", + job_path, + job_path + "/pdgem5sim.out", + job_path + "/pdgem5sim.err", + job_path + "/pdgem5sim.log") + f0 = open(job_path + '/condor.rcS', 'w') + f0.write(submit_script) + f0.close() + command_line = 'condor_submit ' + job_path + '/condor.rcS' + os.system(command_line) + +# returns the ip address of a given network interface +def get_ip_address(ifname): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + return socket.inet_ntoa( fcntl.ioctl(s.fileno(), 0x8915,\ + struct.pack('256s', ifname[:15]))[20:24]) + +# get the ip address of this machines, we run barrier on this machine +sync_ip = get_ip_address('eth0') + +def launch_gem5_process(cmd, sim): + global gem5_processes + job_path = run_dir + '/' + sim + if sim_to_phy[sim] == 'HTCondor': + f0 = open(job_path + '/job.sh', 'w') + f0.write('#!/bin/sh\n') + f0.write(cmd) + f0.close() + key = 'HTCondor_' + sim + gem5_processes[key] = sim + submit_to_HTCondor(job_path) + else: + host_ip = socket.gethostbyname(sim_to_phy[sim]) + command_line = '' + # if the physical host is localhost then we don't need to ssh + if (host_ip == '127.0.0.1') or (host_ip == '127.0.1.1') or\ + host_ip == sync_ip: + command_line = 'cd ' + job_path + ';' + cmd +\ + ' >pdgem5sim.out 2>pdgem5sim.err' + # we should ssh to machines and launch cmd on them + else: + command_line = 'ssh ' + sim_to_phy[sim] + ' \'cd ' + job_path + ';'\ + + cmd + ' >pdgem5sim.out 2>pdgem5sim.err\'' + p = subprocess.Popen(command_line, shell = True, preexec_fn=os.setsid) + gem5_processes[p] = sim + tap_connected(sim) + +def perpare_dir(): + # clean up the old files + os.system("rm -rf %s > /dev/null" % (run_dir)) + + # make new directories + os.system("mkdir -p %s > /dev/null 2>&1" %(run_dir)) + # make sure that checkpoint directory exists + os.system("mkdir -p %s > /dev/null 2>&1" %(params['pd-gem5_dir'] +'/ckptdir/')) + # make run directory for each node, make sure that checkpoint dir exists for + # each node + for machine in machines: + (phy, sim) = machine.split(':') + os.system("mkdir -p %s/%s > /dev/null 2>&1" %(run_dir, sim)) + os.system("mkdir -p %s/%s > /dev/null 2>&1" %((params['pd-gem5_dir'] +\ + '/ckptdir/' + params['ckpt_dir'] + '/', sim))) + +# this function prepare gem5 commandline for each simualted node +def prepare_cmd(sims): + cmd = '' + cmd_debug = params['pd-gem5_dir'] + '/' + params['gem5_binary'] + ' ' + \ + '--debug-flags=' + params['debug_flags'] + ' ' + if params['debug_start'] != '0': + cmd_debug += '--debug-start=' + params['debug_start'] + ' ' + cmd_nodebug = params['pd-gem5_dir'] + '/' + params['gem5_binary'] + ' ' + + debug_flag = False + for sim in sims.split(','): + if sim in params['trace_on']: + debug_flag = True + break + if debug_flag: + cmd = cmd_debug + params['pd-gem5_dir'] + '/configs/example/' + else: + cmd = cmd_nodebug + params['pd-gem5_dir'] + '/configs/example/' + + # add switch specific options + if sims.split(',')[0] == 'sw': + cmd += 'sw.py ' + # add node specific options + else: + cmd += params['fs_script'] + ' '\ + + '--switch-tap-ip=' + sw_ip + ' '\ + + '--switch-tap-port=' + sw_port + ' ' + if len(sims.split(',')) == 1: + cmd += '--pd-gem5 ' + # right now we just support quad or mono + else: + cmd += '--quad ' + if params['script_dir'] != '0': + if len(sims.split(',')) == 4: + if sims.split(',')[0] != 'tux0': + cmd += '--script=' + script_dir + '/' + sims.split(',')[0] + '.sh '\ + + '--script1=' + script_dir + '/' + sims.split(',')[1] + '.sh '\ + + '--script2=' + script_dir + '/' + sims.split(',')[2] + '.sh '\ + + '--script3=' + script_dir + '/' + sims.split(',')[3] + '.sh ' + else: + cmd += '--script=' + params['script_tux0'] + ' '\ + + '--script1=' + script_dir + '/' + sims.split(',')[1] + '.sh '\ + + '--script2=' + script_dir + '/' + sims.split(',')[2] + '.sh '\ + + '--script3=' + script_dir + '/' + sims.split(',')[3] + '.sh ' + else: + if sims.split(',')[0] != 'tux0': + cmd += '--script=' + script_dir + '/' + sims.split(',')[0] + '.sh ' + else: + cmd += '--script=' + params['script_tux0'] + ' ' + # add common options for both switch and nodes + cmd += '--checkpoint-dir=' + ckpt_dir + '/' + sims + ' '\ + + params['other_command_line_options'] + ' '\ + '--sync-quantum=' + params['sync_period'] + ' '\ + + '--num-nodes=' + str(num_nodes - 1) + ' '\ + + '--nic-delay=' + params['nic_delay'] + ' '\ + + '--nic-speed=' + params['nic_speed'] + ' '\ + + '--etherlink-delay=' + params['link_delay'] + ' '\ + + '--etherlink-speed=' + params['link_speed'] + ' '\ + + '--sw-delay=' + params['sw_delay'] + ' '\ + + '--sw-speed=' + params['sw_speed'] + ' ' + if 'etherdump' in params.keys(): + cmd += '--etherdump=' + run_dir + '/' + sims + '/etherdump.pcap ' + return cmd + +def get_sw_ip_port(): + tap_line = '' + # wait till sw gem5 process updates the content of it's log file + time.sleep(1) + if (os.path.isfile("%s/sw/pdgem5sim.err" %(run_dir)) == False): + cleanup() + sys.exit(0) + while tap_line == '': + tap_line = subprocess.Popen("grep tap %s/sw/pdgem5sim.err"\ + %(run_dir), stdout=subprocess.PIPE,\ + shell = True).communicate()[0].rstrip() + sw_ip = tap_line.split(' ')[6] + sw_port = tap_line.split(' ')[7].rstrip() + return sw_ip, sw_port + +def tap_connected(sim): + line = '' + while line == '' and running: + line = subprocess.Popen("grep Listening %s/%s/pdgem5sim.err"\ + %(run_dir, sim), stdout=subprocess.PIPE,\ + shell = True).communicate()[0].rstrip() + +def check_nodes_status(): + flag = False + for p in gem5_processes.keys(): + # type of p is "strting" if gem5 process is launched using HTCondor + if type(p) == str: + submission_line = '' + # wait till gem5 process updates the content of it's log file + while submission_line == '': + submission_line = subprocess.Popen("grep submitted %s/%s/pdgem5sim.log"\ + %(run_dir, gem5_processes[p]), stdout=subprocess.PIPE,\ + stderr=subprocess.PIPE, shell = True).communicate()[0].rstrip() + time.sleep(1) + + job_id = re.split('\.|\(', submission_line)[1] + job_status = subprocess.Popen("condor_q %s | grep %s" %(job_id, job_id), + stdout=subprocess.PIPE, shell = True).communicate()[0].rstrip() + if job_status == '': + print ("gem5 process simulating %s is finished/killed. Please" + " check %s/%s/pdgem5sim.err(out) for more information")\ + %(gem5_processes[p], run_dir, gem5_processes[p]) + flag = True + else: + # poll output is None if process is still running + if p.poll() != None: + print ("gem5 process simulating %s is finished/killed. Please" + " check %s/%s/pdgem5sim.err(out) for more information")\ + %(gem5_processes[p], run_dir, gem5_processes[p]) + flag = True + return flag + +def monitor_cluster(): + while monitoring: + time.sleep(1) + if check_nodes_status(): + cleanup() + return + +# read confing file and extract the parmeters +configCont = open(configFile,'r').readlines() + +for i in configCont: + if len(i) > 1 and not i.strip()[0] == '#': + iSplit = map(lambda x:x.strip(), i.split('=')[1:]) + params[i.split('=')[0]] = '='.join(iSplit) + +print "preparing to start pd-gem5" +print "configuration file = " + params['run_name'] + +run_dir = params['run_dir'] + '/' + params['run_name'] +ckpt_dir = params['pd-gem5_dir'] + '/ckptdir/' + params['ckpt_dir'] +script_dir = params['pd-gem5_dir'] + '/scriptdir/' + params['script_dir'] +machines = params['machine_names'].split(' ') +num_nodes = len(machines) + +for machine in machines: + (phy, sim) = machine.split(':') + #num_nodes += len(sim.split(',')) + # right now we just support quad or mono + if (len(sim.split(',')) != 1) and (len(sim.split(',')) != 4): + print ("right now we just support quad or mono simulation!"\ + "please modify machine_names config\n") + running = False + monitoring = False + sys.exit(0) + sim_to_phy[sim] = phy + +# clean up previous directories, create new ones +perpare_dir() +# take a copy of "config file" to run_dir +os.system("cp %s %s"%(configFile,run_dir)) + +# start monitoring the progress of launched gem5 processes and kill simulation +# if any of them encountered any problem + +t_monitor = threading.Thread(name='monitor_thread', target = monitor_cluster) +t_monitor.daemon = True +#threads.append(t_monitor) +t_monitor.start() + +cmds = {} +# prepare cmd for switch +cmds['sw'] = prepare_cmd('sw') + +try: + launch_gem5_process(cmds['sw'], 'sw') + print "switch started" + # get ip address of the physical machine that is simulating switch + + # the port number of the switch tap device + sw_ip, sw_port = get_sw_ip_port() + sim_to_phy_ordered = collections.OrderedDict(sorted(sim_to_phy.items())) + # prepare cmd for nodes and start them + for sim in sim_to_phy_ordered.keys(): + if sim != 'sw' and running: + cmds[sim] = prepare_cmd(sim) + launch_gem5_process(cmds[sim], sim) + print "%s started" %(sim) + + # busy loop till simulation finish OR keyboard interrupt receive + print "pd-gem5 started" + while running: + time.sleep(1) +except KeyboardInterrupt: + print "CTRL+C pressed.\nLet's cleanup and exit pd-gem5 ..." + cleanup()