diff -r a9f8d3c4c995 -r 8b6602b580e5 configs/common/FSConfig.py
--- a/configs/common/FSConfig.py	Mon Jan 10 15:50:50 2011 -0800
+++ b/configs/common/FSConfig.py	Mon Jan 10 16:21:43 2011 -0800
@@ -408,7 +408,8 @@
     for i in range(3, 15):
         assignISAInt(i, i)
 
-def makeLinuxX86System(mem_mode, numCPUs = 1, mdesc = None, Ruby = False):
+def makeLinuxX86System(mem_mode, options, mdesc = None, Ruby = False):
+    numCPUs = options.num_cpus
     self = LinuxX86System()
 
     # Build up the x86 system and then specialize it for Linux
@@ -418,6 +419,22 @@
     # just to avoid corner cases.
     assert(self.physmem.range.second.getValue() >= 0x200000)
 
+    # set work count options
+    if options.work_item_id != None:
+        self.work_item_id = options.work_item_id
+    if options.work_begin_cpu_id_exit != None:
+        self.work_begin_cpu_id_exit = options.work_begin_cpu_id_exit
+    if options.work_end_exit_count != None:
+        self.work_end_exit_count = options.work_end_exit_count
+    if options.work_end_checkpoint_count != None:
+        self.work_end_ckpt_count = options.work_end_checkpoint_count
+    if options.work_begin_exit_count != None:
+        self.work_begin_exit_count = options.work_begin_exit_count
+    if options.work_begin_checkpoint_count != None:
+        self.work_begin_ckpt_count = options.work_begin_checkpoint_count
+    if options.work_cpus_checkpoint_count != None:
+        self.work_cpus_ckpt_count = options.work_cpus_checkpoint_count
+
     # Mark the first megabyte of memory as reserved
     self.e820_table.entries.append(X86E820Entry(
                 addr = 0,
diff -r a9f8d3c4c995 -r 8b6602b580e5 configs/common/Options.py
--- a/configs/common/Options.py	Mon Jan 10 15:50:50 2011 -0800
+++ b/configs/common/Options.py	Mon Jan 10 16:21:43 2011 -0800
@@ -54,6 +54,14 @@
 parser.add_option("--maxtime", type="float")
 parser.add_option("--maxinsts", type="int")
 parser.add_option("--prog_intvl", type="int")
+parser.add_option("--work-item-id", action="store", type="int",
+                  help="the specific work id for exit & checkpointing")
+parser.add_option("--work-begin-cpu-id-exit", action="store", type="int",
+                  help="exit when work starts on the specified cpu")
+parser.add_option("--work-end-exit-count", action="store", type="int",
+                  help="exit at specified work end count")
+parser.add_option("--work-begin-exit-count", action="store", type="int",
+                  help="exit at specified work begin count")
 
 
 # Checkpointing options
@@ -69,6 +77,12 @@
     help="restore from checkpoint <N>")
 parser.add_option("--checkpoint-at-end", action="store_true",
                   help="take a checkpoint at end of run")
+parser.add_option("--work-begin-checkpoint-count", action="store", type="int",
+                  help="checkpoint at specified work begin count")
+parser.add_option("--work-end-checkpoint-count", action="store", type="int",
+                  help="checkpoint at specified work end count")
+parser.add_option("--work-cpus-checkpoint-count", action="store", type="int",
+                  help="checkpoint and exit when active cpu count is reached")
 
 
 # CPU Switching - default switch model goes from a checkpoint
diff -r a9f8d3c4c995 -r 8b6602b580e5 configs/example/fs.py
--- a/configs/example/fs.py	Mon Jan 10 15:50:50 2011 -0800
+++ b/configs/example/fs.py	Mon Jan 10 16:21:43 2011 -0800
@@ -127,7 +127,7 @@
 elif buildEnv['TARGET_ISA'] == "sparc":
     test_sys = makeSparcSystem(test_mem_mode, bm[0])
 elif buildEnv['TARGET_ISA'] == "x86":
-    test_sys = makeLinuxX86System(test_mem_mode, np, bm[0])
+    test_sys = makeLinuxX86System(test_mem_mode, options, bm[0])
 elif buildEnv['TARGET_ISA'] == "arm":
     test_sys = makeLinuxArmSystem(test_mem_mode, bm[0],
             bare_metal=options.bare_metal, machine_type=options.machine_type)
diff -r a9f8d3c4c995 -r 8b6602b580e5 configs/example/ruby_fs.py
--- a/configs/example/ruby_fs.py	Mon Jan 10 15:50:50 2011 -0800
+++ b/configs/example/ruby_fs.py	Mon Jan 10 16:21:43 2011 -0800
@@ -117,7 +117,7 @@
                                      system.piobus,
                                      system.dma_devices)
 elif buildEnv['TARGET_ISA'] == "x86":
-    system = makeLinuxX86System(test_mem_mode, options.num_cpus, bm[0], True)
+    system = makeLinuxX86System(test_mem_mode, options, bm[0], True)
     system.ruby = Ruby.create_system(options,
                                      system,
                                      system.piobus)
diff -r a9f8d3c4c995 -r 8b6602b580e5 src/arch/x86/isa/decoder/two_byte_opcodes.isa
--- a/src/arch/x86/isa/decoder/two_byte_opcodes.isa	Mon Jan 10 15:50:50 2011 -0800
+++ b/src/arch/x86/isa/decoder/two_byte_opcodes.isa	Mon Jan 10 16:21:43 2011 -0800
@@ -216,6 +216,12 @@
                         0x59: m5reserved5({{
                             warn("M5 reserved opcode 5 ignored.\n");
                         }}, IsNonSpeculative);
+                        0x5a: m5_work_begin({{
+                            PseudoInst::workbegin(xc->tcBase(), Rdi, Rsi);
+                        }}, IsNonSpeculative);
+                        0x5b: m5_work_end({{
+                            PseudoInst::workend(xc->tcBase(), Rdi, Rsi);
+                        }}, IsNonSpeculative);
                         default: Inst::UD2();
                     }
                 }
diff -r a9f8d3c4c995 -r 8b6602b580e5 src/cpu/base.hh
--- a/src/cpu/base.hh	Mon Jan 10 15:50:50 2011 -0800
+++ b/src/cpu/base.hh	Mon Jan 10 16:21:43 2011 -0800
@@ -103,6 +103,8 @@
     inline Tick ticks(int numCycles) const { return clock * numCycles; }
     inline Tick curCycle() const { return curTick / clock; }
     inline Tick tickToCycles(Tick val) const { return val / clock; }
+    inline void workItemBegin() { numWorkItemsStarted++; }
+    inline void workItemEnd() { numWorkItemsCompleted++; }
     // @todo remove me after debugging with legion done
     Tick instCount() { return instCnt; }
 
@@ -316,6 +318,8 @@
   public:
     // Number of CPU cycles simulated
     Stats::Scalar numCycles;
+    Stats::Scalar numWorkItemsStarted;
+    Stats::Scalar numWorkItemsCompleted;
 };
 
 #endif // __CPU_BASE_HH__
diff -r a9f8d3c4c995 -r 8b6602b580e5 src/cpu/base.cc
--- a/src/cpu/base.cc	Mon Jan 10 15:50:50 2011 -0800
+++ b/src/cpu/base.cc	Mon Jan 10 16:21:43 2011 -0800
@@ -255,6 +255,16 @@
         .desc("number of cpu cycles simulated")
         ;
 
+    numWorkItemsStarted
+        .name(name() + ".numWorkItemsStarted")
+        .desc("number of work items this cpu started")
+        ;
+
+    numWorkItemsCompleted
+        .name(name() + ".numWorkItemsCompleted")
+        .desc("number of work items this cpu completed")
+        ;
+
     int size = threadContexts.size();
     if (size > 1) {
         for (int i = 0; i < size; ++i) {
diff -r a9f8d3c4c995 -r 8b6602b580e5 src/sim/SConscript
--- a/src/sim/SConscript	Mon Jan 10 15:50:50 2011 -0800
+++ b/src/sim/SConscript	Mon Jan 10 16:21:43 2011 -0800
@@ -77,3 +77,4 @@
 TraceFlag('Thread')
 TraceFlag('Timer')
 TraceFlag('VtoPhys')
+TraceFlag('WorkItems')
diff -r a9f8d3c4c995 -r 8b6602b580e5 src/sim/System.py
--- a/src/sim/System.py	Mon Jan 10 15:50:50 2011 -0800
+++ b/src/sim/System.py	Mon Jan 10 16:21:43 2011 -0800
@@ -46,6 +46,20 @@
     physmem = Param.PhysicalMemory(Parent.any, "physical memory")
     mem_mode = Param.MemoryMode('atomic', "The mode the memory system is in")
 
+    work_item_id = Param.Int(-1, "specific work item id")
+    work_begin_cpu_id_exit = Param.Int(-1,
+        "work started on specific id, now exit simulation")
+    work_begin_ckpt_count = Param.Counter(0,
+        "create checkpoint when work items begin count value is reached")
+    work_begin_exit_count = Param.Counter(0,
+        "exit simulation when work items begin count value is reached")
+    work_end_ckpt_count = Param.Counter(0,
+        "create checkpoint when work items end count value is reached")
+    work_end_exit_count = Param.Counter(0,
+        "exit simulation when work items end count value is reached")
+    work_cpus_ckpt_count = Param.Counter(0,
+        "create checkpoint when active cpu count value is reached")
+
     max_total_system_insts = Param.Counter(0,
         "terminate when the entire system has executed this instruction count")
 
diff -r a9f8d3c4c995 -r 8b6602b580e5 src/sim/pseudo_inst.hh
--- a/src/sim/pseudo_inst.hh	Mon Jan 10 15:50:50 2011 -0800
+++ b/src/sim/pseudo_inst.hh	Mon Jan 10 16:21:43 2011 -0800
@@ -63,5 +63,7 @@
 void m5checkpoint(ThreadContext *tc, Tick delay, Tick period);
 void debugbreak(ThreadContext *tc);
 void switchcpu(ThreadContext *tc);
+void workbegin(ThreadContext *tc, uint64_t workid, uint64_t threadid);
+void workend(ThreadContext *tc, uint64_t workid, uint64_t threadid);
 
 } // namespace PseudoInst
diff -r a9f8d3c4c995 -r 8b6602b580e5 src/sim/pseudo_inst.cc
--- a/src/sim/pseudo_inst.cc	Mon Jan 10 15:50:50 2011 -0800
+++ b/src/sim/pseudo_inst.cc	Mon Jan 10 16:21:43 2011 -0800
@@ -326,4 +326,121 @@
     exitSimLoop("switchcpu");
 }
 
+//
+// This function is executed when annotated work items begin.  Depending on 
+// what the user specified at the command line, the simulation may exit and/or
+// take a checkpoint when a certain when a certain work item begins.
+//
+void
+workbegin(ThreadContext *tc, uint64_t workid, uint64_t threadid)
+{
+    tc->getCpuPtr()->workItemBegin();
+    System *sys = tc->getSystemPtr();
+
+    DPRINTF(WorkItems, "Work Begin workid: %d, threadid %d\n", workid, 
+            threadid);
+
+    //
+    // If specified, determine if this is the specific work item the user
+    // identified
+    //
+    if (sys->params()->work_item_id == -1 || 
+        sys->params()->work_item_id == workid) {
+
+        uint64_t systemWorkBeginCount = sys->incWorkItemsBegin();
+        int cpuId = tc->getCpuPtr()->cpuId();
+
+        if (sys->params()->work_cpus_ckpt_count != 0 &&
+            sys->markWorkItem(cpuId) >= sys->params()->work_cpus_ckpt_count) {
+            //
+            // If active cpus equals checkpoint count, create checkpoint
+            //
+            Event *event = new SimLoopExitEvent("checkpoint", 0);
+            mainEventQueue.schedule(event, curTick);
+        }
+
+        if (systemWorkBeginCount == sys->params()->work_begin_ckpt_count) {
+            //
+            // Note: the string specified as the cause of the exit event must
+            // exactly equal "checkpoint" inorder to create a checkpoint
+            //
+            Event *event = new SimLoopExitEvent("checkpoint", 0);
+            mainEventQueue.schedule(event, curTick);
+        }
+
+        if (systemWorkBeginCount == sys->params()->work_begin_exit_count) {
+            //
+            // If a certain number of work items started, exit simulation
+            //
+            Event *event = new SimLoopExitEvent("work started count reach", 0);
+            mainEventQueue.schedule(event, curTick);
+        }
+
+        if (tc->getCpuPtr()->cpuId() == sys->params()->work_begin_cpu_id_exit) {
+            //
+            // If work started on the specific cpu id specified, exit simulation
+            //
+            Event *event = new SimLoopExitEvent("work started on specific cpu",
+                                                0);
+
+            mainEventQueue.schedule(event, curTick+1);
+        }
+    }
+}
+
+//
+// This function is executed when annotated work items end.  Depending on 
+// what the user specified at the command line, the simulation may exit and/or
+// take a checkpoint when a certain when a certain work item ends.
+//
+void
+workend(ThreadContext *tc, uint64_t workid, uint64_t threadid)
+{
+    tc->getCpuPtr()->workItemEnd();
+    System *sys = tc->getSystemPtr();
+
+    DPRINTF(WorkItems, "Work End workid: %d, threadid %d\n", workid, threadid);
+
+    //
+    // If specified, determine if this is the specific work item the user
+    // identified
+    //
+    if (sys->params()->work_item_id == -1 || 
+        sys->params()->work_item_id == workid) {
+
+        uint64_t systemWorkEndCount = sys->incWorkItemsEnd();
+        int cpuId = tc->getCpuPtr()->cpuId();
+
+        if (sys->params()->work_cpus_ckpt_count != 0 &&
+            sys->markWorkItem(cpuId) >= sys->params()->work_cpus_ckpt_count) {
+            //
+            // If active cpus equals checkpoint count, create checkpoint
+            //
+            Event *event = new SimLoopExitEvent("checkpoint", 0);
+            mainEventQueue.schedule(event, curTick);
+        }
+
+        if (sys->params()->work_end_ckpt_count != 0 &&
+            systemWorkEndCount == sys->params()->work_end_ckpt_count) {
+            //
+            // If total work items completed equals checkpoint count, create
+            // checkpoint
+            //
+            Event *event = new SimLoopExitEvent("checkpoint", 0);
+            mainEventQueue.schedule(event, curTick);
+        }
+
+        if (sys->params()->work_end_exit_count != 0 &&
+            systemWorkEndCount == sys->params()->work_end_exit_count) {
+            //
+            // If total work items completed equals exit count, exit simulation
+            //
+            Event *event = new SimLoopExitEvent("work items exit count reached",
+                                                0);
+
+            mainEventQueue.schedule(event, curTick);
+        }
+    }
+}
+
 } // namespace PseudoInst
diff -r a9f8d3c4c995 -r 8b6602b580e5 src/sim/system.hh
--- a/src/sim/system.hh	Mon Jan 10 15:50:50 2011 -0800
+++ b/src/sim/system.hh	Mon Jan 10 16:21:43 2011 -0800
@@ -160,6 +160,35 @@
 
   protected:
     Enums::MemoryMode memoryMode;
+    uint64_t workItemsBegin;
+    uint64_t workItemsEnd;
+    std::vector<bool> activeCpus;
+
+  public:
+    uint64_t 
+    incWorkItemsBegin()
+    {
+        return ++workItemsBegin;
+    }
+
+    uint64_t 
+    incWorkItemsEnd()
+    {
+        return ++workItemsEnd;
+    }
+
+    int 
+    markWorkItem(int index)
+    {
+        int count = 0;
+        assert(index < activeCpus.size());
+        activeCpus[index] = true;
+        for (std::vector<bool>::iterator i = activeCpus.begin(); 
+             i < activeCpus.end(); i++) {
+            if (*i) count++;
+        }
+        return count;
+    }
 
 #if FULL_SYSTEM
     /**
diff -r a9f8d3c4c995 -r 8b6602b580e5 src/sim/system.cc
--- a/src/sim/system.cc	Mon Jan 10 15:50:50 2011 -0800
+++ b/src/sim/system.cc	Mon Jan 10 16:21:43 2011 -0800
@@ -71,7 +71,10 @@
       pagePtr(0),
       nextPID(0),
 #endif
-      memoryMode(p->mem_mode), _params(p),
+      memoryMode(p->mem_mode),
+      workItemsBegin(0),
+      workItemsEnd(0),
+      _params(p),
       totalNumInsts(0), 
       instEventQueue("system instruction-based event queue")
 {
@@ -143,6 +146,8 @@
 
     // increment the number of running systms
     numSystemsRunning++;
+
+    activeCpus.clear();
 }
 
 System::~System()
@@ -217,6 +222,8 @@
         remoteGDB[id] = rgdb;
     }
 
+    activeCpus.push_back(false);
+
     return id;
 }
 
diff -r a9f8d3c4c995 -r 8b6602b580e5 util/m5/m5op_x86.S
--- a/util/m5/m5op_x86.S	Mon Jan 10 15:50:50 2011 -0800
+++ b/util/m5/m5op_x86.S	Mon Jan 10 16:21:43 2011 -0800
@@ -59,3 +59,5 @@
 TWO_BYTE_OP(m5_switchcpu, switchcpu_func)
 TWO_BYTE_OP(m5_addsymbol, addsymbol_func)
 TWO_BYTE_OP(m5_panic, panic_func)
+TWO_BYTE_OP(m5_work_begin, work_begin_func)
+TWO_BYTE_OP(m5_work_end, work_end_func)
diff -r a9f8d3c4c995 -r 8b6602b580e5 util/m5/m5ops.h
--- a/util/m5/m5ops.h	Mon Jan 10 15:50:50 2011 -0800
+++ b/util/m5/m5ops.h	Mon Jan 10 16:21:43 2011 -0800
@@ -57,6 +57,9 @@
 #define reserved4_func          0x58 // Reserved for user
 #define reserved5_func          0x59 // Reserved for user
 
+#define work_begin_func          0x5a
+#define work_end_func            0x5b
+
 // These operations are for critical path annotation
 #define annotate_func     0x55
 #define an_bsm            0x1