diff --git a/src/arch/arm/table_walker.hh b/src/arch/arm/table_walker.hh
--- a/src/arch/arm/table_walker.hh
+++ b/src/arch/arm/table_walker.hh
@@ -364,6 +364,9 @@
     /** Port to issue translation requests from */
     SnoopingDmaPort port;
 
+    /** If we're draining keep the drain event around until we're drained */
+    Event *drainEvent;
+
     /** TLB that is initiating these table walks */
     TLB *tlb;
 
@@ -389,6 +392,8 @@
         return dynamic_cast<const Params *>(_params);
     }
 
+    /** Checks if all state is cleared and if so, completes drain */
+    void completeDrain();
     virtual unsigned int drain(Event *de);
     virtual void resume();
     virtual MasterPort& getMasterPort(const std::string &if_name,
diff --git a/src/arch/arm/table_walker.cc b/src/arch/arm/table_walker.cc
--- a/src/arch/arm/table_walker.cc
+++ b/src/arch/arm/table_walker.cc
@@ -43,6 +43,7 @@
 #include "cpu/base.hh"
 #include "cpu/thread_context.hh"
 #include "debug/Checkpoint.hh"
+#include "debug/Drain.hh"
 #include "debug/TLB.hh"
 #include "debug/TLBVerbose.hh"
 #include "sim/system.hh"
@@ -51,7 +52,7 @@
 
 TableWalker::TableWalker(const Params *p)
     : MemObject(p), port(this, params()->sys, params()->min_backoff,
-                         params()->max_backoff),
+                         params()->max_backoff), drainEvent(NULL),
       tlb(NULL), currState(NULL), pending(false),
       masterId(p->sys->getMasterId(name())),
       doL1DescEvent(this), doL2DescEvent(this), doProcessEvent(this)
@@ -64,20 +65,32 @@
     ;
 }
 
+void
+TableWalker::completeDrain()
+{
+    if (drainEvent && stateQueueL2.empty() && stateQueueL2.empty() &&
+        pendingQueue.empty()) {
+        changeState(Drained);
+        DPRINTF(Drain, "TableWalker done draining, processing drain event\n");
+        drainEvent->process();
+        drainEvent = NULL;
+    }
+}
+
 unsigned int
 TableWalker::drain(Event *de)
 {
-    if (stateQueueL1.size() || stateQueueL2.size() || pendingQueue.size())
-    {
+    unsigned int count = port.drain(de);
+
+    if (stateQueueL1.size() || stateQueueL2.size() || pendingQueue.size()) {
+        drainEvent = de;
         changeState(Draining);
-        DPRINTF(Checkpoint, "TableWalker busy, wait to drain\n");
-        return 1;
-    }
-    else
-    {
+        DPRINTF(Drain, "TableWalker not drained\n");
+        return ++count;
+    } else {
         changeState(Drained);
-        DPRINTF(Checkpoint, "TableWalker free, no need to drain\n");
-        return 0;
+        DPRINTF(Drain, "TableWalker free, no need to drain\n");
+        return count;
     }
 }
 
@@ -86,8 +99,8 @@
 {
     MemObject::resume();
     if ((params()->sys->getMemoryMode() == Enums::timing) && currState) {
-            delete currState;
-            currState = NULL;
+        delete currState;
+        currState = NULL;
     }
 }
 
@@ -667,6 +680,7 @@
     doL1Descriptor();
 
     stateQueueL1.pop_front();
+    completeDrain();
     // Check if fault was generated
     if (currState->fault != NoFault) {
         currState->transState->finish(currState->fault, currState->req,
@@ -723,6 +737,7 @@
 
 
     stateQueueL2.pop_front();
+    completeDrain();
     pending = false;
     nextWalk(currState->tc);
 
diff --git a/src/cpu/base.cc b/src/cpu/base.cc
--- a/src/cpu/base.cc
+++ b/src/cpu/base.cc
@@ -386,8 +386,6 @@
 {
     assert(threadContexts.size() == oldCPU->threadContexts.size());
 
-    _cpuId = oldCPU->cpuId();
-
     ThreadID size = threadContexts.size();
     for (ThreadID i = 0; i < size; ++i) {
         ThreadContext *newTC = threadContexts[i];
@@ -418,11 +416,13 @@
             assert(old_itb_port);
             SlavePort &slavePort = old_itb_port->getSlavePort();
             new_itb_port->bind(slavePort);
+            old_itb_port->unBind();
         }
         if (new_dtb_port && !new_dtb_port->isConnected()) {
             assert(old_dtb_port);
             SlavePort &slavePort = old_dtb_port->getSlavePort();
             new_dtb_port->bind(slavePort);
+            old_dtb_port->unBind();
         }
 
         // Checker whether or not we have to transfer CheckerCPU
@@ -444,18 +444,26 @@
                 assert(old_checker_itb_port);
                 SlavePort &slavePort = old_checker_itb_port->getSlavePort();;
                 new_checker_itb_port->bind(slavePort);
+                old_checker_itb_port->unBind();
             }
             if (new_checker_dtb_port && !new_checker_dtb_port->isConnected()) {
                 assert(old_checker_dtb_port);
                 SlavePort &slavePort = old_checker_dtb_port->getSlavePort();;
                 new_checker_dtb_port->bind(slavePort);
+                old_checker_dtb_port->unBind();
             }
         }
     }
 
+    int tmpId = _cpuId;
+    _cpuId = oldCPU->cpuId();
+
     interrupts = oldCPU->interrupts;
     interrupts->setCPU(this);
 
+    oldCPU->interrupts = NULL;
+    oldCPU->_cpuId = tmpId;
+
     if (FullSystem) {
         for (ThreadID i = 0; i < size; ++i)
             threadContexts[i]->profileClear();
@@ -469,10 +477,12 @@
     // CPU.
     if (!getInstPort().isConnected()) {
         getInstPort().bind(oldCPU->getInstPort().getSlavePort());
+        oldCPU->getInstPort().unBind();
     }
 
     if (!getDataPort().isConnected()) {
         getDataPort().bind(oldCPU->getDataPort().getSlavePort());
+        oldCPU->getDataPort().unBind();
     }
 }
 
diff --git a/src/cpu/o3/commit_impl.hh b/src/cpu/o3/commit_impl.hh
--- a/src/cpu/o3/commit_impl.hh
+++ b/src/cpu/o3/commit_impl.hh
@@ -631,7 +631,7 @@
     wroteToTimeBuffer = false;
     _nextStatus = Inactive;
 
-    if (drainPending && rob->isEmpty() && !iewStage->hasStoresToWB()) {
+    if (drainPending && cpu->instList.empty() && !iewStage->hasStoresToWB()) {
         cpu->signalDrained();
         drainPending = false;
         return;
diff --git a/src/cpu/o3/cpu.cc b/src/cpu/o3/cpu.cc
--- a/src/cpu/o3/cpu.cc
+++ b/src/cpu/o3/cpu.cc
@@ -55,6 +55,7 @@
 #include "cpu/simple_thread.hh"
 #include "cpu/thread_context.hh"
 #include "debug/Activity.hh"
+#include "debug/Drain.hh"
 #include "debug/O3CPU.hh"
 #include "debug/Quiesce.hh"
 #include "enums/MemoryMode.hh"
@@ -260,7 +261,7 @@
     if (!deferRegistration) {
         _status = Running;
     } else {
-        _status = Idle;
+        _status = SwitchedOut;
     }
 
     if (params->checker) {
@@ -1119,9 +1120,8 @@
     DPRINTF(O3CPU, "Switching out\n");
 
     // If the CPU isn't doing anything, then return immediately.
-    if (_status == Idle || _status == SwitchedOut) {
+    if (_status == SwitchedOut)
         return 0;
-    }
 
     drainCount = 0;
     fetch.drain();
@@ -1142,6 +1142,8 @@
         wakeCPU();
         activityRec.activity();
 
+        DPRINTF(Drain, "CPU not drained\n");
+
         return 1;
     } else {
         return 0;
@@ -1160,7 +1162,7 @@
 
     changeState(SimObject::Running);
 
-    if (_status == SwitchedOut || _status == Idle)
+    if (_status == SwitchedOut)
         return;
 
     assert(system->getMemoryMode() == Enums::timing);
@@ -1183,6 +1185,7 @@
         BaseCPU::switchOut();
 
         if (drainEvent) {
+            DPRINTF(Drain, "CPU done draining, processing drain event\n");
             drainEvent->process();
             drainEvent = NULL;
         }
diff --git a/src/cpu/o3/fetch_impl.hh b/src/cpu/o3/fetch_impl.hh
--- a/src/cpu/o3/fetch_impl.hh
+++ b/src/cpu/o3/fetch_impl.hh
@@ -132,8 +132,10 @@
     // Get the size of an instruction.
     instSize = sizeof(TheISA::MachInst);
 
-    for (int i = 0; i < Impl::MaxThreads; i++)
+    for (int i = 0; i < Impl::MaxThreads; i++) {
+        cacheData[i] = NULL;
         decoder[i] = new TheISA::Decoder(NULL);
+    }
 }
 
 template <class Impl>
@@ -346,7 +348,8 @@
 
     for (ThreadID tid = 0; tid < numThreads; tid++) {
         // Create space to store a cache line.
-        cacheData[tid] = new uint8_t[cacheBlkSize];
+        if (!cacheData[tid])
+            cacheData[tid] = new uint8_t[cacheBlkSize];
         cacheDataPC[tid] = 0;
         cacheDataValid[tid] = false;
     }
diff --git a/src/cpu/simple/timing.cc b/src/cpu/simple/timing.cc
--- a/src/cpu/simple/timing.cc
+++ b/src/cpu/simple/timing.cc
@@ -48,6 +48,7 @@
 #include "cpu/simple/timing.hh"
 #include "cpu/exetrace.hh"
 #include "debug/Config.hh"
+#include "debug/Drain.hh"
 #include "debug/ExecFaulting.hh"
 #include "debug/SimpleCPU.hh"
 #include "mem/packet.hh"
@@ -129,6 +130,7 @@
     } else {
         changeState(SimObject::Draining);
         drainEvent = drain_event;
+        DPRINTF(Drain, "CPU not drained\n");
         return 1;
     }
 }
@@ -829,7 +831,7 @@
 void
 TimingSimpleCPU::completeDrain()
 {
-    DPRINTF(Config, "Done draining\n");
+    DPRINTF(Drain, "CPU done draining, processing drain event\n");
     changeState(SimObject::Drained);
     drainEvent->process();
 }
diff --git a/src/dev/copy_engine.cc b/src/dev/copy_engine.cc
--- a/src/dev/copy_engine.cc
+++ b/src/dev/copy_engine.cc
@@ -49,6 +49,7 @@
 #include "base/cp_annotate.hh"
 #include "base/trace.hh"
 #include "debug/DMACopyEngine.hh"
+#include "debug/Drain.hh"
 #include "dev/copy_engine.hh"
 #include "mem/packet.hh"
 #include "mem/packet_access.hh"
@@ -638,7 +639,7 @@
 CopyEngine::CopyEngineChannel::inDrain()
 {
     if (ce->getState() == SimObject::Draining) {
-        DPRINTF(DMACopyEngine, "processing drain\n");
+        DPRINTF(Drain, "CopyEngine done draining, processing drain event\n");
         assert(drainEvent);
         drainEvent->process();
         drainEvent = NULL;
@@ -655,7 +656,7 @@
     unsigned int count = 1;
     count += cePort.drain(de);
 
-    DPRINTF(DMACopyEngine, "unable to drain, returning %d\n", count);
+    DPRINTF(Drain, "CopyEngineChannel not drained\n");
     drainEvent = de;
     return count;
 }
@@ -673,7 +674,7 @@
     else
         changeState(Drained);
 
-    DPRINTF(DMACopyEngine, "call to CopyEngine::drain() returning %d\n", count);
+    DPRINTF(Drain, "CopyEngine not drained\n");
     return count;
 }
 
diff --git a/src/dev/dma_device.cc b/src/dev/dma_device.cc
--- a/src/dev/dma_device.cc
+++ b/src/dev/dma_device.cc
@@ -43,6 +43,7 @@
 
 #include "base/chunk_generator.hh"
 #include "debug/DMA.hh"
+#include "debug/Drain.hh"
 #include "dev/dma_device.hh"
 #include "sim/system.hh"
 
@@ -103,7 +104,7 @@
         delete pkt->req;
         delete pkt;
 
-        if (pendingCount == 0 && drainEvent) {
+        if (pendingCount == 0 && transmitList.empty() && drainEvent) {
             drainEvent->process();
             drainEvent = NULL;
         }
@@ -142,9 +143,10 @@
 unsigned int
 DmaPort::drain(Event *de)
 {
-    if (pendingCount == 0)
+    if (transmitList.empty() && pendingCount == 0)
         return 0;
     drainEvent = de;
+    DPRINTF(Drain, "DmaPort not drained\n");
     return 1;
 }
 
@@ -182,8 +184,6 @@
 DmaPort::dmaAction(Packet::Command cmd, Addr addr, int size, Event *event,
                    uint8_t *data, Tick delay, Request::Flags flag)
 {
-    assert(device->getState() == SimObject::Running);
-
     DmaReqState *reqState = new DmaReqState(event, size, delay);
 
 
@@ -287,7 +287,8 @@
         assert(pendingCount >= 0);
         delete pkt;
 
-        if (pendingCount == 0 && drainEvent) {
+        if (pendingCount == 0 && transmitList.empty() && drainEvent) {
+            DPRINTF(Drain, "DmaPort done draining, processing drain event\n");
             drainEvent->process();
             drainEvent = NULL;
         }
diff --git a/src/dev/i8254xGBe.cc b/src/dev/i8254xGBe.cc
--- a/src/dev/i8254xGBe.cc
+++ b/src/dev/i8254xGBe.cc
@@ -44,6 +44,7 @@
 
 #include "base/inet.hh"
 #include "base/trace.hh"
+#include "debug/Drain.hh"
 #include "debug/EthernetAll.hh"
 #include "dev/i8254xGBe.hh"
 #include "mem/packet.hh"
@@ -2072,12 +2073,12 @@
     if (tickEvent.scheduled())
         deschedule(tickEvent);
 
-    if (count)
+    if (count) {
+        DPRINTF(Drain, "IGbE not drained\n");
         changeState(Draining);
-    else
+    } else
         changeState(Drained);
 
-    DPRINTF(EthernetSM, "got drain() returning %d", count);
     return count;
 }
 
@@ -2100,12 +2101,12 @@
     if (!drainEvent)
         return;
 
-    DPRINTF(EthernetSM, "checkDrain() in drain\n");
     txFifoTick = false;
     txTick = false;
     rxTick = false;
     if (!rxDescCache.hasOutstandingEvents() &&
         !txDescCache.hasOutstandingEvents()) {
+        DPRINTF(Drain, "IGbE done draining, processing drain event\n");
         drainEvent->process();
         drainEvent = NULL;
     }
diff --git a/src/mem/bus.cc b/src/mem/bus.cc
--- a/src/mem/bus.cc
+++ b/src/mem/bus.cc
@@ -52,6 +52,7 @@
 #include "base/trace.hh"
 #include "debug/Bus.hh"
 #include "debug/BusAddrRanges.hh"
+#include "debug/Drain.hh"
 #include "mem/bus.hh"
 
 BaseBus::BaseBus(const BaseBusParams *p)
@@ -246,6 +247,7 @@
         // we see a retry from the destination
         retryWaiting();
     } else if (drainEvent) {
+        DPRINTF(Drain, "Bus done draining, processing drain event\n");
         //If we weren't able to drain before, do it now.
         drainEvent->process();
         // Clear the drain event once we're done with it.
@@ -498,6 +500,7 @@
     //waiting. We might be idle but have someone waiting if the device we
     //contacted for a retry didn't actually retry.
     if (!retryList.empty() || state != IDLE) {
+        DPRINTF(Drain, "Bus not drained\n");
         drainEvent = de;
         return 1;
     }
diff --git a/src/mem/cache/base.cc b/src/mem/cache/base.cc
--- a/src/mem/cache/base.cc
+++ b/src/mem/cache/base.cc
@@ -48,6 +48,7 @@
 #include "cpu/base.hh"
 #include "cpu/smt.hh"
 #include "debug/Cache.hh"
+#include "debug/Drain.hh"
 #include "mem/cache/base.hh"
 #include "mem/cache/mshr.hh"
 #include "sim/full_system.hh"
@@ -752,6 +753,7 @@
         drainEvent = de;
 
         changeState(SimObject::Draining);
+        DPRINTF(Drain, "Cache not drained\n");
         return count;
     }
 
diff --git a/src/mem/packet_queue.cc b/src/mem/packet_queue.cc
--- a/src/mem/packet_queue.cc
+++ b/src/mem/packet_queue.cc
@@ -41,6 +41,7 @@
  *          Andreas Hansson
  */
 
+#include "debug/Drain.hh"
 #include "debug/PacketQueue.hh"
 #include "mem/packet_queue.hh"
 
@@ -168,7 +169,9 @@
             em.schedule(&sendEvent, std::max(nextReady, curTick() + 1));
     } else {
         // no more to send, so if we're draining, we may be done
-        if (drainEvent && !sendEvent.scheduled()) {
+        if (drainEvent && transmitList.empty() && !sendEvent.scheduled()) {
+            DPRINTF(Drain, "PacketQueue done draining,"
+                    "processing drain event\n");
             drainEvent->process();
             drainEvent = NULL;
         }
@@ -201,6 +204,7 @@
 {
     if (transmitList.empty() && !sendEvent.scheduled())
         return 0;
+    DPRINTF(Drain, "PacketQueue not drained\n");
     drainEvent = de;
     return 1;
 }
diff --git a/src/mem/port.hh b/src/mem/port.hh
--- a/src/mem/port.hh
+++ b/src/mem/port.hh
@@ -140,6 +140,7 @@
                PortID id = InvalidPortID);
     virtual ~MasterPort();
 
+    void unBind();
     void bind(SlavePort& slave_port);
     SlavePort& getSlavePort() const;
     bool isConnected() const;
@@ -297,6 +298,7 @@
               PortID id = InvalidPortID);
     virtual ~SlavePort();
 
+    void unBind();
     void bind(MasterPort& master_port);
     MasterPort& getMasterPort() const;
     bool isConnected() const;
diff --git a/src/mem/port.cc b/src/mem/port.cc
--- a/src/mem/port.cc
+++ b/src/mem/port.cc
@@ -82,6 +82,12 @@
 }
 
 void
+MasterPort::unBind()
+{
+    _slavePort = NULL;
+}
+
+void
 MasterPort::bind(SlavePort& slave_port)
 {
     // master port keeps track of the slave port
@@ -167,6 +173,12 @@
 }
 
 void
+SlavePort::unBind()
+{
+    _masterPort = NULL;
+}
+
+void
 SlavePort::bind(MasterPort& master_port)
 {
     _masterPort = &master_port;
diff --git a/src/mem/ruby/system/RubyPort.cc b/src/mem/ruby/system/RubyPort.cc
--- a/src/mem/ruby/system/RubyPort.cc
+++ b/src/mem/ruby/system/RubyPort.cc
@@ -41,6 +41,7 @@
 
 #include "cpu/testers/rubytest/RubyTester.hh"
 #include "debug/Config.hh"
+#include "debug/Drain.hh"
 #include "debug/Ruby.hh"
 #include "mem/protocol/AccessPermission.hh"
 #include "mem/ruby/slicc_interface/AbstractController.hh"
@@ -524,8 +525,9 @@
     //If we weren't able to drain before, we might be able to now.
     if (drainEvent != NULL) {
         unsigned int drainCount = getDrainCount(drainEvent);
-        DPRINTF(Config, "Drain count: %u\n", drainCount);
+        DPRINTF(Drain, "Drain count: %u\n", drainCount);
         if (drainCount == 0) {
+            DPRINTF(Drain, "RubyPort done draining, processing drain event\n");
             drainEvent->process();
             // Clear the drain event once we're done with it.
             drainEvent = NULL;
@@ -584,6 +586,7 @@
     if (count != 0) {
         drainEvent = de;
 
+        DPRINTF(Drain, "RubyPort not drained\n");
         changeState(SimObject::Draining);
         return count;
     }
diff --git a/src/python/m5/simulate.py b/src/python/m5/simulate.py
--- a/src/python/m5/simulate.py
+++ b/src/python/m5/simulate.py
@@ -166,12 +166,22 @@
 def drain(root):
     all_drained = False
     drain_event = internal.event.createCountedDrain()
-    unready_objs = sum(obj.drain(drain_event) for obj in root.descendants())
+    unready_objs = 0
+    for obj in root.descendants():
+        tmp_objs = obj.drain(drain_event)
+        if tmp_objs > 0:
+            unready_objs += tmp_objs
+            print obj.get_name() + " not drained and contributes " + \
+                  str(tmp_objs) + " objects that need to be drained"
+            tmp_objs = 0
     # If we've got some objects that can't drain immediately, then simulate
     if unready_objs > 0:
         drain_event.setCount(unready_objs)
+        print "re-entering simulation inside drain() with " + \
+              str(unready_objs) + " unready objects"
         simulate()
     else:
+        print "everything is now drained"
         all_drained = True
     internal.event.cleanupCountedDrain(drain_event)
     return all_drained
diff --git a/src/sim/SConscript b/src/sim/SConscript
--- a/src/sim/SConscript
+++ b/src/sim/SConscript
@@ -62,6 +62,7 @@
 
 DebugFlag('Checkpoint')
 DebugFlag('Config')
+DebugFlag('Drain')
 DebugFlag('Event')
 DebugFlag('Fault')
 DebugFlag('Flow')