diff -r 4a46f0f39f34 -r 7445f17936f5 src/mem/coherent_xbar.cc --- a/src/mem/coherent_xbar.cc Wed Nov 04 17:52:32 2015 +0000 +++ b/src/mem/coherent_xbar.cc Wed Nov 04 17:52:33 2015 +0000 @@ -220,10 +220,11 @@ } // forwardTiming snooped into peer caches of the sender, and if - // this is a clean evict, but the packet is found in a cache, do - // not forward it - if (pkt->cmd == MemCmd::CleanEvict && pkt->isBlockCached()) { - DPRINTF(CoherentXBar, "recvTimingReq: Clean evict 0x%x still cached, " + // this is a clean evict or clean writeback, but the packet is + // found in a cache, do not forward it + if ((pkt->cmd == MemCmd::CleanEvict || + pkt->cmd == MemCmd::WritebackClean) && pkt->isBlockCached()) { + DPRINTF(CoherentXBar, "Clean evict/writeback %#llx still cached, " "not forwarding\n", pkt->getAddr()); // update the layer state and schedule an idle event @@ -634,8 +635,9 @@ // forwardAtomic snooped into peer caches of the sender, and if // this is a clean evict, but the packet is found in a cache, do // not forward it - if (pkt->cmd == MemCmd::CleanEvict && pkt->isBlockCached()) { - DPRINTF(CoherentXBar, "recvAtomic: Clean evict 0x%x still cached, " + if ((pkt->cmd == MemCmd::CleanEvict || + pkt->cmd == MemCmd::WritebackClean) && pkt->isBlockCached()) { + DPRINTF(CoherentXBar, "Clean evict/writeback %#llx still cached, " "not forwarding\n", pkt->getAddr()); return 0; } diff -r 4a46f0f39f34 -r 7445f17936f5 src/mem/packet.hh --- a/src/mem/packet.hh Wed Nov 04 17:52:32 2015 +0000 +++ b/src/mem/packet.hh Wed Nov 04 17:52:33 2015 +0000 @@ -86,7 +86,8 @@ ReadRespWithInvalidate, WriteReq, WriteResp, - Writeback, + WritebackDirty, + WritebackClean, CleanEvict, SoftPFReq, HardPFReq, @@ -144,6 +145,7 @@ IsRequest, //!< Issued by requester IsResponse, //!< Issue by responder NeedsResponse, //!< Requester needs response from target + IsWriteback, IsSWPrefetch, IsHWPrefetch, IsLlsc, //!< Alpha/MIPS LL or SC access @@ -192,6 +194,7 @@ bool needsExclusive() const { return testCmdAttrib(NeedsExclusive); } bool needsResponse() const { return testCmdAttrib(NeedsResponse); } bool isInvalidate() const { return testCmdAttrib(IsInvalidate); } + bool isWriteback() const { return testCmdAttrib(IsWriteback); } /** * Check if this particular packet type carries payload data. Note @@ -491,6 +494,7 @@ bool needsExclusive() const { return cmd.needsExclusive(); } bool needsResponse() const { return cmd.needsResponse(); } bool isInvalidate() const { return cmd.isInvalidate(); } + bool isWriteback() const { return cmd.isWriteback(); } bool hasData() const { return cmd.hasData(); } bool isLLSC() const { return cmd.isLLSC(); } bool isError() const { return cmd.isError(); } @@ -1013,8 +1017,7 @@ bool evictingBlock() const { - return (cmd == MemCmd::Writeback || - cmd == MemCmd::CleanEvict); + return (isWriteback() || cmd == MemCmd::CleanEvict); } /** diff -r 4a46f0f39f34 -r 7445f17936f5 src/mem/packet.cc --- a/src/mem/packet.cc Wed Nov 04 17:52:32 2015 +0000 +++ b/src/mem/packet.cc Wed Nov 04 17:52:33 2015 +0000 @@ -84,9 +84,14 @@ WriteResp, "WriteReq" }, /* WriteResp */ { SET3(IsWrite, NeedsExclusive, IsResponse), InvalidCmd, "WriteResp" }, - /* Writeback */ - { SET4(IsWrite, NeedsExclusive, IsRequest, HasData), - InvalidCmd, "Writeback" }, + /* WritebackDirty */ + { SET4(IsWrite, IsRequest, HasData, IsWriteback), + InvalidCmd, "WritebackDirty" }, + /* WritebackClean - This allows the upstream cache to writeback a + * line to the downstream cache without it being considered + * dirty. */ + { SET4(IsWrite, IsRequest, HasData, IsWriteback), + InvalidCmd, "WritebackClean" }, /* CleanEvict */ { SET1(IsRequest), InvalidCmd, "CleanEvict" }, /* SoftPFReq */ diff -r 4a46f0f39f34 -r 7445f17936f5 src/mem/snoop_filter.cc --- a/src/mem/snoop_filter.cc Wed Nov 04 17:52:32 2015 +0000 +++ b/src/mem/snoop_filter.cc Wed Nov 04 17:52:33 2015 +0000 @@ -207,7 +207,7 @@ // not the invalidation. Previously Writebacks did not generate upward // snoops so this was never an aissue. Now that Writebacks generate snoops // we need to special case for Writebacks. - assert(cpkt->cmd == MemCmd::Writeback || cpkt->req->isUncacheable() || + assert(cpkt->isWriteback() || cpkt->req->isUncacheable() || (cpkt->isInvalidate() == cpkt->needsExclusive())); if (cpkt->isInvalidate() && !sf_item.requested) { // Early clear of the holder, if no other request is currently going on @@ -270,7 +270,7 @@ //assert(sf_item.holder == 0); sf_item.holder = 0; } - assert(cpkt->cmd != MemCmd::Writeback); + assert(!cpkt->isWriteback()); sf_item.holder |= req_mask; sf_item.requested &= ~req_mask; assert(sf_item.requested | sf_item.holder); diff -r 4a46f0f39f34 -r 7445f17936f5 src/mem/cache/Cache.py --- a/src/mem/cache/Cache.py Wed Nov 04 17:52:32 2015 +0000 +++ b/src/mem/cache/Cache.py Wed Nov 04 17:52:33 2015 +0000 @@ -103,3 +103,11 @@ # cache a line is dropped for a mostly exclusive cache. clusivity = Param.Clusivity('mostly_incl', "Clusivity with upstream cache") + + # Determine if this cache sends out writebacks for clean lines, or + # simply clean evicts. In cases where a downstream cache is mostly + # exclusive with respect to this cache (acting as a victim cache), + # the clean writebacks are essential for performance. In general + # this should be set to True for anything but the last-level + # cache. + writeback_clean = Param.Bool(False, "Writeback clean lines") diff -r 4a46f0f39f34 -r 7445f17936f5 src/mem/cache/base.hh --- a/src/mem/cache/base.hh Wed Nov 04 17:52:32 2015 +0000 +++ b/src/mem/cache/base.hh Wed Nov 04 17:52:33 2015 +0000 @@ -521,9 +521,6 @@ // should only see writes or clean evicts here assert(pkt->isWrite() || pkt->cmd == MemCmd::CleanEvict); - // if this is a read-only cache we should never see any writes - assert(!(isReadOnly && pkt->isWrite())); - return allocateBufferInternal(&writeBuffer, blockAlign(pkt->getAddr()), blkSize, pkt, time, true); diff -r 4a46f0f39f34 -r 7445f17936f5 src/mem/cache/cache.hh --- a/src/mem/cache/cache.hh Wed Nov 04 17:52:32 2015 +0000 +++ b/src/mem/cache/cache.hh Wed Nov 04 17:52:33 2015 +0000 @@ -202,6 +202,15 @@ */ const Enums::Clusivity clusivity; + /** + * Determine if clean lines should be written back or not. In + * cases where a downstream cache is mostly inclusive we likely + * want it to act as a victim cache also for lines that have not + * been modified. Hence, we cannot simply drop the line (or send a + * clean evict), but rather need to send the actual data. + */ + const bool writebackClean; + /** * Upstream caches need this packet until true is returned, so * hold it for deletion until a subsequent call diff -r 4a46f0f39f34 -r 7445f17936f5 src/mem/cache/cache.cc --- a/src/mem/cache/cache.cc Wed Nov 04 17:52:32 2015 +0000 +++ b/src/mem/cache/cache.cc Wed Nov 04 17:52:33 2015 +0000 @@ -70,6 +70,7 @@ doFastWrites(true), prefetchOnAccess(p->prefetch_on_access), clusivity(p->clusivity), + writebackClean(p->writeback_clean), tempBlockWriteback(nullptr), writebackTempBlockAtomicEvent(this, false, EventBase::Delayed_Writeback_Pri) @@ -317,7 +318,7 @@ // flush and invalidate any existing block CacheBlk *old_blk(tags->findBlock(pkt->getAddr(), pkt->isSecure())); if (old_blk && old_blk->isValid()) { - if (old_blk->isDirty()) + if (old_blk->isDirty() || writebackClean) writebacks.push_back(writebackBlk(old_blk)); else writebacks.push_back(cleanEvictBlk(old_blk)); @@ -356,26 +357,50 @@ if (writeBuffer.findMatches(pkt->getAddr(), pkt->isSecure(), outgoing)) { assert(outgoing.size() == 1); - PacketPtr wbPkt = outgoing[0]->getTarget()->pkt; - assert(pkt->cmd == MemCmd::CleanEvict && - wbPkt->cmd == MemCmd::Writeback); - // As the CleanEvict is coming from above, it would have snooped - // into other peer caches of the same level while traversing the - // crossbar. If a copy of the block had been found, the CleanEvict - // would have been deleted in the crossbar. Now that the - // CleanEvict is here we can be sure none of the other upper level - // caches connected to this cache have the block, so we can clear - // the BLOCK_CACHED flag in the Writeback if set and discard the - // CleanEvict by returning true. - wbPkt->clearBlockCached(); - return true; + MSHR *wb_entry = outgoing[0]; + assert(wb_entry->getNumTargets() == 1); + PacketPtr wbPkt = wb_entry->getTarget()->pkt; + assert(wbPkt->isWriteback()); + + if (pkt->cmd == MemCmd::CleanEvict || + pkt->cmd == MemCmd::WritebackClean) { + // The CleanEvict and WritebackClean snoops into other + // peer caches of the same level while traversing the + // crossbar. If a copy of the block is found, the + // packet is deleted in the crossbar. Hence, none of + // the other upper level caches connected to this + // cache have the block, so we can clear the + // BLOCK_CACHED flag in the Writeback if set and + // discard the CleanEvict by returning true. + wbPkt->clearBlockCached(); + return true; + } else { + assert(pkt->cmd == MemCmd::WritebackDirty); + // Dirty writeback from above trumps our clean + // writeback... discard here + // Note: markInService will remove entry from writeback buffer. + markInService(wb_entry, false); + delete wbPkt; + } } } // Writeback handling is special case. We can write the block into // the cache without having a writeable copy (or any copy at all). - if (pkt->cmd == MemCmd::Writeback) { + if (pkt->isWriteback()) { assert(blkSize == pkt->getSize()); + + // we could get a clean writeback while we are having + // outstanding accesses to a block, do the simple thing for + // now and drop the clean writeback so that we do not upset + // any ordering/decisions about ownership already taken + if (pkt->cmd == MemCmd::WritebackClean && + mshrQueue.findMatch(pkt->getAddr(), pkt->isSecure())) { + DPRINTF(Cache, "Clean writeback %#llx to block with MSHR, " + "dropping\n", pkt->getAddr()); + return true; + } + if (blk == NULL) { // need to do a replacement blk = allocateBlock(pkt->getAddr(), pkt->isSecure(), writebacks); @@ -391,7 +416,11 @@ blk->status |= BlkSecure; } } - blk->status |= BlkDirty; + // only mark the block dirty if we got a writeback command, + // and leave it as is for a clean writeback + if (pkt->cmd == MemCmd::WritebackDirty) { + blk->status |= BlkDirty; + } // if shared is not asserted we got the writeback in modified // state, if it is asserted we are in the owned state if (!pkt->sharedAsserted()) { @@ -463,7 +492,13 @@ // this is a non-snoop request packet which does not require a // response. delete wbPkt; + } else if (wbPkt->cmd == MemCmd::WritebackClean) { + // clean writeback, do not send since the block is + // still cached above + assert(writebackClean); + delete wbPkt; } else { + assert(wbPkt->cmd == MemCmd::WritebackDirty); // Set BLOCK_CACHED flag in Writeback and send below, so that // the Writeback does not reset the bit corresponding to this // address in the snoop filter below. @@ -490,7 +525,7 @@ // isCachedAbove returns true we set BLOCK_CACHED flag in Writebacks // and discard CleanEvicts. if (isCachedAbove(wbPkt, false)) { - if (wbPkt->cmd == MemCmd::Writeback) { + if (wbPkt->cmd == MemCmd::WritebackDirty) { // Set BLOCK_CACHED flag in Writeback and send below, // so that the Writeback does not reset the bit // corresponding to this address in the snoop filter @@ -694,6 +729,10 @@ // by access(), that calls accessBlock() function. cpuSidePort->schedTimingResp(pkt, request_time, true); } else { + DPRINTF(Cache, "%s satisfied %s addr %#llx, no response needed\n", + __func__, pkt->cmdString(), pkt->getAddr(), + pkt->getSize()); + // queue the packet for deletion, as the sending cache is // still relying on it; if the block is found in access(), // CleanEvict and Writeback messages will be deleted @@ -765,10 +804,12 @@ // Coalesce unless it was a software prefetch (see above). if (pkt) { - assert(pkt->cmd != MemCmd::Writeback); - // CleanEvicts corresponding to blocks which have outstanding - // requests in MSHRs can be deleted here. - if (pkt->cmd == MemCmd::CleanEvict) { + assert(!pkt->isWriteback()); + // CleanEvicts or WritebackCleans corresponding to + // blocks which have outstanding requests in MSHRs are + // simply dropped + if (pkt->cmd == MemCmd::CleanEvict || + pkt->cmd == MemCmd::WritebackClean) { pendingDelete.reset(pkt); } else { DPRINTF(Cache, "%s coalescing MSHR for %s addr %#llx size %d\n", @@ -1106,8 +1147,8 @@ schedule(writebackTempBlockAtomicEvent, curTick()); } - tempBlockWriteback = blk->isDirty() ? writebackBlk(blk) : - cleanEvictBlk(blk); + tempBlockWriteback = (blk->isDirty() || writebackClean) ? + writebackBlk(blk) : cleanEvictBlk(blk); blk->invalidate(); } @@ -1456,7 +1497,7 @@ // Writebacks/CleanEvicts to write buffer. It specifies the latency to // allocate an internal buffer and to schedule an event to the // queued port. - if (blk->isDirty()) { + if (blk->isDirty() || writebackClean) { PacketPtr wbPkt = writebackBlk(blk); allocateWriteBuffer(wbPkt, forward_time); // Set BLOCK_CACHED flag if cached above. @@ -1482,8 +1523,9 @@ PacketPtr Cache::writebackBlk(CacheBlk *blk) { - chatty_assert(!isReadOnly, "Writeback from read-only cache"); - assert(blk && blk->isValid() && blk->isDirty()); + chatty_assert(!isReadOnly || writebackClean, + "Writeback from read-only cache"); + assert(blk && blk->isValid() && (blk->isDirty() || writebackClean)); writebacks[Request::wbMasterId]++; @@ -1497,7 +1539,13 @@ blk->task_id= ContextSwitchTaskId::Unknown; blk->tickInserted = curTick(); - PacketPtr writeback = new Packet(writebackReq, MemCmd::Writeback); + PacketPtr writeback = + new Packet(writebackReq, blk->isDirty() ? + MemCmd::WritebackDirty : MemCmd::WritebackClean); + + DPRINTF(Cache, "Create Writeback %#llx writable: %d, dirty: %d\n", + writeback->getAddr(), blk->isWritable(), blk->isDirty()); + if (blk->isWritable()) { // not asserting shared means we pass the block in modified // state, mark our own block non-writeable @@ -1507,16 +1555,19 @@ writeback->assertShared(); } + // make sure the block is not marked dirty + blk->status &= ~BlkDirty; + writeback->allocate(); std::memcpy(writeback->getPtr(), blk->data, blkSize); - blk->status &= ~BlkDirty; return writeback; } PacketPtr Cache::cleanEvictBlk(CacheBlk *blk) { + assert(!writebackClean); assert(blk && blk->isValid() && !blk->isDirty()); // Creating a zero sized write, a message to the snoop filter Request *req = @@ -1626,7 +1677,7 @@ // Will send up Writeback/CleanEvict snoops via isCachedAbove // when pushing this writeback list into the write buffer. - if (blk->isDirty()) { + if (blk->isDirty() || writebackClean) { // Save writeback packet for handling by caller writebacks.push_back(writebackBlk(blk)); } else { @@ -2062,7 +2113,7 @@ return; } - if (wb_pkt->cmd == MemCmd::Writeback) { + if (wb_pkt->cmd == MemCmd::WritebackDirty) { assert(!pkt->memInhibitAsserted()); pkt->assertMemInhibit(); if (!pkt->needsExclusive()) { @@ -2080,18 +2131,27 @@ doTimingSupplyResponse(pkt, wb_pkt->getConstPtr(), false, false); } else { - assert(wb_pkt->cmd == MemCmd::CleanEvict); + // on hitting a clean writeback we play it safe and do not + // provide a response, the block may be dirty somewhere + // else + assert(wb_pkt->cmd == MemCmd::CleanEvict || + wb_pkt->cmd == MemCmd::WritebackClean); // The cache technically holds the block until the - // corresponding CleanEvict message reaches the crossbar + // corresponding message reaches the crossbar // below. Therefore when a snoop encounters a CleanEvict - // message we must set assertShared (just like when it - // encounters a Writeback) to avoid the snoop filter - // prematurely clearing the holder bit in the crossbar - // below - if (!pkt->needsExclusive()) + // or WritebackClean message we must set assertShared + // (just like when it encounters a Writeback) to avoid the + // snoop filter prematurely clearing the holder bit in the + // crossbar below + if (!pkt->needsExclusive()) { pkt->assertShared(); - else + // the writeback is no longer passing exclusivity (the + // receiving cache should consider the block owned + // rather than modified) + wb_pkt->assertShared(); + } else { assert(pkt->isInvalidate()); + } } if (pkt->isInvalidate()) { @@ -2310,7 +2370,7 @@ mshr->blkAddr); // Deallocate the mshr target - if (tgt_pkt->cmd != MemCmd::Writeback) { + if (!tgt_pkt->isWriteback()) { if (mshr->queue->forceDeallocateTarget(mshr)) { // Clear block if this deallocation resulted freed an // mshr when all had previously been utilized diff -r 4a46f0f39f34 -r 7445f17936f5 configs/common/Caches.py --- a/configs/common/Caches.py Wed Nov 04 17:52:32 2015 +0000 +++ b/configs/common/Caches.py Wed Nov 04 17:52:33 2015 +0000 @@ -55,6 +55,8 @@ class L1_ICache(L1Cache): is_read_only = True + # Writeback clean lines as well + writeback_clean = True class L1_DCache(L1Cache): pass @@ -89,3 +91,5 @@ is_read_only = False else: is_read_only = True + # Writeback clean lines as well + writeback_clean = True diff -r 4a46f0f39f34 -r 7445f17936f5 configs/common/O3_ARM_v7a.py --- a/configs/common/O3_ARM_v7a.py Wed Nov 04 17:52:32 2015 +0000 +++ b/configs/common/O3_ARM_v7a.py Wed Nov 04 17:52:33 2015 +0000 @@ -151,6 +151,8 @@ assoc = 2 forward_snoops = False is_read_only = True + # Writeback clean lines as well + writeback_clean = True # Data Cache class O3_ARM_v7a_DCache(Cache): @@ -161,6 +163,8 @@ size = '32kB' assoc = 2 write_buffers = 16 + # Consider the L2 a victim cache also for clean lines + writeback_clean = True # TLB Cache # Use a cache as a L2 TLB @@ -174,6 +178,8 @@ write_buffers = 16 forward_snoops = False is_read_only = True + # Writeback clean lines as well + writeback_clean = True # L2 Cache class O3_ARM_v7aL2(Cache): diff -r 4a46f0f39f34 -r 7445f17936f5 src/mem/abstract_mem.cc --- a/src/mem/abstract_mem.cc Wed Nov 04 17:52:32 2015 +0000 +++ b/src/mem/abstract_mem.cc Wed Nov 04 17:52:33 2015 +0000 @@ -329,7 +329,7 @@ return; } - if (pkt->cmd == MemCmd::CleanEvict) { + if (pkt->cmd == MemCmd::CleanEvict || pkt->cmd == MemCmd::WritebackClean) { DPRINTF(MemoryAccess, "CleanEvict on 0x%x: not responding\n", pkt->getAddr()); return;