diff -r b2f995b2099c -r 1b834b01d375 src/mem/protocol/RubySlicc_MemControl.sm --- a/src/mem/protocol/RubySlicc_MemControl.sm Thu Feb 11 15:46:51 2016 -0600 +++ b/src/mem/protocol/RubySlicc_MemControl.sm Thu Feb 11 16:21:52 2016 -0600 @@ -57,6 +57,7 @@ MachineID OriginalRequestorMachId, desc="What component originally requested"; DataBlock DataBlk, desc="Data to writeback"; MessageSizeType MessageSize, desc="size category of the message"; + int Len, desc="size of the memory request"; // Not all fields used by all protocols: PrefetchBit Prefetch, desc="Is this a prefetch request"; bool ReadX, desc="Exclusive"; diff -r b2f995b2099c -r 1b834b01d375 src/mem/ruby/slicc_interface/AbstractController.hh --- a/src/mem/ruby/slicc_interface/AbstractController.hh Thu Feb 11 15:46:51 2016 -0600 +++ b/src/mem/ruby/slicc_interface/AbstractController.hh Thu Feb 11 16:21:52 2016 -0600 @@ -77,7 +77,8 @@ bool isBlocked(Addr); virtual MessageBuffer* getMandatoryQueue() const = 0; - virtual MessageBuffer* getMemoryQueue() const = 0; + virtual MessageBuffer* getMemReqQueue() const = 0; + virtual MessageBuffer* getMemRespQueue() const = 0; virtual AccessPermission getAccessPermission(const Addr &addr) = 0; virtual void print(std::ostream & out) const = 0; @@ -116,12 +117,13 @@ BaseMasterPort& getMasterPort(const std::string& if_name, PortID idx = InvalidPortID); - void queueMemoryRead(const MachineID &id, Addr addr, Cycles latency); - void queueMemoryWrite(const MachineID &id, Addr addr, Cycles latency, - const DataBlock &block); - void queueMemoryWritePartial(const MachineID &id, Addr addr, Cycles latency, - const DataBlock &block, int size); - void recvTimingResp(PacketPtr pkt); + /** + * Called by the MemoryPort when the memory controller issues a retry to + * indicate that it can accept further requests. + */ + void unblockPort(); + bool recvTimingResp(PacketPtr pkt); + void sendRetryRespIfBlocked(); public: MachineID getMachineID() const { return m_machineID; } @@ -140,6 +142,7 @@ void wakeUpBuffers(Addr addr); void wakeUpAllBuffers(Addr addr); void wakeUpAllBuffers(); + void serviceMemoryQueue(); protected: const NodeID m_version; @@ -189,30 +192,47 @@ /** * Port that forwards requests and receives responses from the - * memory controller. It has a queue of packets not yet sent. + * memory controller. */ - class MemoryPort : public QueuedMasterPort + class MemoryPort : public MasterPort { private: - // Packet queues used to store outgoing requests and snoop responses. - ReqPacketQueue reqQueue; - SnoopRespPacketQueue snoopRespQueue; - // Controller that operates this port. AbstractController *controller; + bool receiverBlocked; public: MemoryPort(const std::string &_name, AbstractController *_controller, const std::string &_label); - // Function for receiving a timing response from the peer port. - // Currently the pkt is handed to the coherence controller - // associated with this port. + /** + * Check if the port's slave is blocked from receiving packets + */ + bool isBlocked() { return receiverBlocked; } + /** + * Tries sending the packet, and if the send fails, sets the + * receiverBlocked variable. + */ + bool trySendTimingReq(PacketPtr pkt); + /** + * Indicates to the memory port that the memory controller has become + * unblocked, so the port can retry memory requests. + */ + void recvReqRetry(); + /** + * Receives a packet, which the controller queues in its memory input + * queue (getMemoryInQueue()). If the queue is blocked, recvTimingResp + * must return false to the memory controller indicating that the + * packet could not be accepted. + */ bool recvTimingResp(PacketPtr pkt); }; /* Master port to the memory controller. */ MemoryPort memoryPort; + bool m_needs_retry_resp; + bool m_retry_resp_scheduled; + Tick m_send_retry_resp_tick; // State that is stored in packets sent to the memory controller. struct SenderState : public Packet::SenderState diff -r b2f995b2099c -r 1b834b01d375 src/mem/ruby/slicc_interface/AbstractController.cc --- a/src/mem/ruby/slicc_interface/AbstractController.cc Thu Feb 11 15:46:51 2016 -0600 +++ b/src/mem/ruby/slicc_interface/AbstractController.cc Thu Feb 11 16:21:52 2016 -0600 @@ -29,6 +29,7 @@ #include "mem/ruby/slicc_interface/AbstractController.hh" #include "debug/RubyQueue.hh" +#include "debug/RubyMemory.hh" #include "mem/protocol/MemoryMsg.hh" #include "mem/ruby/system/RubySystem.hh" #include "mem/ruby/system/Sequencer.hh" @@ -42,7 +43,9 @@ m_number_of_TBEs(p->number_of_TBEs), m_transitions_per_cycle(p->transitions_per_cycle), m_buffer_size(p->buffer_size), m_recycle_latency(p->recycle_latency), - memoryPort(csprintf("%s.memory", name()), this, "") + memoryPort(csprintf("%s.memory", name()), this, ""), + m_needs_retry_resp(false), m_retry_resp_scheduled(false), + m_send_retry_resp_tick(0) { if (m_version == 0) { // Combine the statistics from all controllers @@ -61,6 +64,9 @@ m_delayVCHistogram.push_back(new Stats::Histogram()); m_delayVCHistogram[i]->init(10); } + if (getMemReqQueue()) { + getMemReqQueue()->setConsumer(this); + } } void @@ -215,74 +221,66 @@ } void -AbstractController::queueMemoryRead(const MachineID &id, Addr addr, - Cycles latency) +AbstractController::serviceMemoryQueue() { - RequestPtr req = new Request(addr, RubySystem::getBlockSizeBytes(), 0, - m_masterId); + assert(getMemReqQueue()); + assert(getMemReqQueue()->isReady(clockEdge())); - PacketPtr pkt = Packet::createRead(req); - uint8_t *newData = new uint8_t[RubySystem::getBlockSizeBytes()]; - pkt->dataDynamic(newData); + if (memoryPort.isBlocked()) { + return; + } - SenderState *s = new SenderState(id); + const MemoryMsg *mem_msg = (const MemoryMsg*)getMemReqQueue()->peek(); + unsigned int req_size = RubySystem::getBlockSizeBytes(); + if (mem_msg->m_Len > 0) { + req_size = mem_msg->m_Len; + } + + RequestPtr req = new Request(mem_msg->m_addr, req_size, 0, m_masterId); + PacketPtr pkt; + if (mem_msg->getType() == MemoryRequestType_MEMORY_WB) { + pkt = Packet::createWrite(req); + uint8_t *newData = new uint8_t[req_size]; + pkt->dataDynamic(newData); + memcpy(newData, mem_msg->m_DataBlk.getData(0, req_size), req_size); + } else if (mem_msg->getType() == MemoryRequestType_MEMORY_READ) { + pkt = Packet::createRead(req); + uint8_t *newData = new uint8_t[req_size]; + pkt->dataDynamic(newData); + } else { + panic("Unknown memory request type (%s) for addr %p", + MemoryRequestType_to_string(mem_msg->getType()), + mem_msg->m_addr); + } + + SenderState *s = new SenderState(mem_msg->m_Sender); pkt->pushSenderState(s); - // Use functional rather than timing accesses during warmup + DPRINTF(RubyMemory, "Trying to send mem %s to addr %p\n", + (pkt->isRead() ? "read" : "write"), pkt->getAddr()); + if (RubySystem::getWarmupEnabled()) { + // Use functional rather than timing accesses during warmup memoryPort.sendFunctional(pkt); + getMemReqQueue()->dequeue(clockEdge()); + // Since the queue was popped, the controller may be able + // to make more progress. Make sure it wakes up + scheduleEvent(Cycles(1)); recvTimingResp(pkt); return; } - memoryPort.schedTimingReq(pkt, clockEdge(latency)); -} - -void -AbstractController::queueMemoryWrite(const MachineID &id, Addr addr, - Cycles latency, const DataBlock &block) -{ - RequestPtr req = new Request(addr, RubySystem::getBlockSizeBytes(), 0, - m_masterId); - - PacketPtr pkt = Packet::createWrite(req); - uint8_t *newData = new uint8_t[RubySystem::getBlockSizeBytes()]; - pkt->dataDynamic(newData); - memcpy(newData, block.getData(0, RubySystem::getBlockSizeBytes()), - RubySystem::getBlockSizeBytes()); - - SenderState *s = new SenderState(id); - pkt->pushSenderState(s); - - // Use functional rather than timing accesses during warmup - if (RubySystem::getWarmupEnabled()) { - memoryPort.sendFunctional(pkt); - recvTimingResp(pkt); - return; + if (!memoryPort.trySendTimingReq(pkt)) { + DPRINTF(RubyMemory, " Port blocked %s to addr %p\n", + (pkt->isRead() ? "read" : "write"), pkt->getAddr()); + delete pkt->req; + delete pkt; + } else { + getMemReqQueue()->dequeue(clockEdge()); + // Since the queue was popped, the controller may be able + // to make more progress. Make sure it wakes up + scheduleEvent(Cycles(1)); } - - // Create a block and copy data from the block. - memoryPort.schedTimingReq(pkt, clockEdge(latency)); -} - -void -AbstractController::queueMemoryWritePartial(const MachineID &id, Addr addr, - Cycles latency, - const DataBlock &block, int size) -{ - RequestPtr req = new Request(addr, RubySystem::getBlockSizeBytes(), 0, - m_masterId); - - PacketPtr pkt = Packet::createWrite(req); - uint8_t *newData = new uint8_t[size]; - pkt->dataDynamic(newData); - memcpy(newData, block.getData(getOffset(addr), size), size); - - SenderState *s = new SenderState(id); - pkt->pushSenderState(s); - - // Create a block and copy data from the block. - memoryPort.schedTimingReq(pkt, clockEdge(latency)); } void @@ -294,24 +292,67 @@ int AbstractController::functionalMemoryWrite(PacketPtr pkt) { - int num_functional_writes = 0; - - // Check the buffer from the controller to the memory. - if (memoryPort.checkFunctional(pkt)) { - num_functional_writes++; - } - - // Update memory itself. memoryPort.sendFunctional(pkt); - return num_functional_writes + 1; + return 1; } void +AbstractController::sendRetryRespIfBlocked() +{ + // Only controllers with access to a memory input queue can send retries + // to the memories. Verify that this is such a controller. + assert(getMemRespQueue()); + + // TODO: HACK!: This assumes that the memory resp queue is popped in + // successive cycles in order to avoid races with wakeup. This is + // very hacky and should be fixed, but should work as long as the resp + // queue has more than one buffer slot. FIXME! + if (m_retry_resp_scheduled && (curTick() >= m_send_retry_resp_tick)) { + // If a retry is scheduled and the current cycle is later than + // when the retry was scheduled, send the retry + DPRINTF(RubyMemory, "Sending memory port unblock\n"); + memoryPort.sendRetryResp(); + m_send_retry_resp_tick = 0; + m_retry_resp_scheduled = false; + } + + if (m_needs_retry_resp) { + // Since this function is called when a directory dequeues a request + // from the memory queue, the queue will not say there is room until + // the next cycle. Set a signal to send the retry in the next cycle + // and make sure a wakeup is scheduled for the next cycle. + m_send_retry_resp_tick = nextCycle(); + m_retry_resp_scheduled = true; + scheduleEvent(Cycles(1)); + m_needs_retry_resp = false; + } +} + +void +AbstractController::unblockPort() +{ + // Since the out port is unblocked, the controller may be able + // to make more progress. Make sure it wakes up + scheduleEvent(Cycles(1)); +} + +bool AbstractController::recvTimingResp(PacketPtr pkt) { - assert(getMemoryQueue()); + assert(getMemRespQueue()); assert(pkt->isResponse()); + if (!getMemRespQueue()->areNSlotsAvailable(1, clockEdge())) { + DPRINTF(RubyMemory, "Memory queue blocked, cannot receive %p\n", + pkt->getAddr()); + m_needs_retry_resp = true; + // Make sure the controller wakes up to unblock the queue + scheduleEvent(Cycles(1)); + return false; + } + + DPRINTF(RubyMemory, "Receiving request from mem %p\n", pkt->getAddr()); + std::shared_ptr msg = std::make_shared(clockEdge()); (*msg).m_addr = pkt->getAddr(); (*msg).m_Sender = m_machineID; @@ -334,24 +375,40 @@ panic("Incorrect packet type received from memory controller!"); } - getMemoryQueue()->enqueue(msg, clockEdge(), cyclesToTicks(Cycles(1))); + getMemRespQueue()->enqueue(msg, clockEdge(), cyclesToTicks(Cycles(1))); + // Make sure the controller wakes up to service the enqueued message + scheduleEvent(Cycles(1)); + delete pkt->req; delete pkt; + return true; +} + +bool +AbstractController::MemoryPort::trySendTimingReq(PacketPtr pkt) +{ + bool pkt_sent = sendTimingReq(pkt); + if (!pkt_sent) receiverBlocked = true; + return pkt_sent; +} + +void +AbstractController::MemoryPort::recvReqRetry() +{ + receiverBlocked = false; + controller->unblockPort(); } bool AbstractController::MemoryPort::recvTimingResp(PacketPtr pkt) { - controller->recvTimingResp(pkt); - return true; + return controller->recvTimingResp(pkt); } AbstractController::MemoryPort::MemoryPort(const std::string &_name, AbstractController *_controller, const std::string &_label) - : QueuedMasterPort(_name, _controller, reqQueue, snoopRespQueue), - reqQueue(*_controller, *this, _label), - snoopRespQueue(*_controller, *this, _label), - controller(_controller) + : MasterPort(csprintf("%s%s", _name, _label), _controller), + controller(_controller), receiverBlocked(false) { } diff -r b2f995b2099c -r 1b834b01d375 src/mem/slicc/symbols/StateMachine.py --- a/src/mem/slicc/symbols/StateMachine.py Thu Feb 11 15:46:51 2016 -0600 +++ b/src/mem/slicc/symbols/StateMachine.py Thu Feb 11 16:21:52 2016 -0600 @@ -299,7 +299,8 @@ void init(); MessageBuffer *getMandatoryQueue() const; - MessageBuffer *getMemoryQueue() const; + MessageBuffer *getMemReqQueue() const; + MessageBuffer *getMemRespQueue() const; void initNetQueues(); void print(std::ostream& out) const; @@ -670,6 +671,11 @@ if port.code.find("mandatoryQueue_ptr") >= 0: mq_ident = "m_mandatoryQueue_ptr" + memoutq_ident = "NULL" + for param in self.config_parameters: + if param.ident.find("requestToMemory") >= 0: + memoutq_ident = "m_requestToMemory_ptr" + memq_ident = "NULL" for port in self.in_ports: if port.code.find("responseFromMemory_ptr") >= 0: @@ -852,7 +858,13 @@ } MessageBuffer* -$c_ident::getMemoryQueue() const +$c_ident::getMemReqQueue() const +{ + return $memoutq_ident; +} + +MessageBuffer* +$c_ident::getMemRespQueue() const { return $memq_ident; } @@ -1083,6 +1095,10 @@ void ${ident}_Controller::wakeup() { + if (getMemReqQueue() && getMemReqQueue()->isReady(clockEdge())) { + serviceMemoryQueue(); + } + int counter = 0; while (true) { unsigned char rejected[${{len(msg_bufs)}}]; diff -r b2f995b2099c -r 1b834b01d375 src/mem/protocol/RubySlicc_Defines.sm --- a/src/mem/protocol/RubySlicc_Defines.sm Thu Feb 11 15:46:51 2016 -0600 +++ b/src/mem/protocol/RubySlicc_Defines.sm Thu Feb 11 16:21:52 2016 -0600 @@ -32,14 +32,9 @@ NodeID clusterID; Cycles recycle_latency; -// Functions implemented in the AbstractController class for -// making timing access to the memory maintained by the -// memory controllers. -void queueMemoryRead(MachineID id, Addr addr, Cycles latency); -void queueMemoryWrite(MachineID id, Addr addr, Cycles latency, - DataBlock block); -void queueMemoryWritePartial(MachineID id, Addr addr, Cycles latency, - DataBlock block, int size); +// Function to handle port control flow into and out of memory queues +// for AbstractControllers that interface with memory +void sendRetryRespIfBlocked(); // Functions implemented in the AbstractController class for // making functional access to the memory maintained by the