diff --git a/src/mem/ruby/buffers/MessageBuffer.hh b/src/mem/ruby/buffers/MessageBuffer.hh --- a/src/mem/ruby/buffers/MessageBuffer.hh +++ b/src/mem/ruby/buffers/MessageBuffer.hh @@ -141,6 +141,9 @@ void printStats(std::ostream& out); void clearStats() { m_not_avail_count = 0; m_msg_counter = 0; } + void setIncomingLink(int link_id) { m_input_link_id = link_id; } + void setVnet(int net) { m_vnet_id = net; } + private: //added by SS int m_recycle_latency; @@ -183,6 +186,9 @@ bool m_ordering_set; bool m_randomization; Time m_last_arrival_time; + + int m_input_link_id; + int m_vnet_id; }; inline std::ostream& diff --git a/src/mem/ruby/buffers/MessageBuffer.cc b/src/mem/ruby/buffers/MessageBuffer.cc --- a/src/mem/ruby/buffers/MessageBuffer.cc +++ b/src/mem/ruby/buffers/MessageBuffer.cc @@ -56,6 +56,9 @@ m_not_avail_count = 0; m_priority_rank = 0; m_name = name; + + m_input_link_id = 0; + m_vnet_id = 0; } int @@ -211,6 +214,8 @@ msg_ptr->getLastEnqueueTime() + msg_ptr->getDelayedCycles()); msg_ptr->setLastEnqueueTime(arrival_time); + msg_ptr->setIncomingLink(m_input_link_id); + msg_ptr->setVnet(m_vnet_id); // Insert the message into the priority heap MessageBufferNode thisNode(arrival_time, m_msg_counter, message); @@ -226,6 +231,7 @@ // Schedule the wakeup if (m_consumer_ptr != NULL) { g_eventQueue_ptr->scheduleEventAbsolute(m_consumer_ptr, arrival_time); + m_consumer_ptr->storeEventInfo(message, arrival_time); } else { panic("No consumer: %s name: %s\n", *this, m_name); } diff --git a/src/mem/ruby/common/Consumer.hh b/src/mem/ruby/common/Consumer.hh --- a/src/mem/ruby/common/Consumer.hh +++ b/src/mem/ruby/common/Consumer.hh @@ -40,6 +40,7 @@ #include "mem/ruby/common/Global.hh" #include "mem/ruby/eventqueue/RubyEventQueue.hh" +#include "mem/ruby/slicc_interface/Message.hh" class MessageBuffer; @@ -67,6 +68,7 @@ virtual void wakeup() = 0; virtual void print(std::ostream& out) const = 0; + virtual void storeEventInfo(MsgPtr ptr, Time delta) {} const Time& getLastScheduledWakeup() const diff --git a/src/mem/ruby/network/simple/PerfectSwitch.hh b/src/mem/ruby/network/simple/PerfectSwitch.hh --- a/src/mem/ruby/network/simple/PerfectSwitch.hh +++ b/src/mem/ruby/network/simple/PerfectSwitch.hh @@ -39,6 +39,7 @@ #include #include +#include "mem/ruby/buffers/MessageBufferNode.hh" #include "mem/ruby/common/Consumer.hh" #include "mem/ruby/common/Global.hh" #include "mem/ruby/system/NodeID.hh" @@ -69,6 +70,7 @@ int getOutLinks() const { return m_out.size(); } void wakeup(); + void storeEventInfo(MsgPtr message, Time delta); void printStats(std::ostream& out) const; void clearStats(); @@ -92,6 +94,7 @@ int m_round_robin_start; int m_wakeups_wo_switch; SimpleNetwork* m_network_ptr; + std::vector m_pending_message_count; }; inline std::ostream& diff --git a/src/mem/ruby/network/simple/PerfectSwitch.cc b/src/mem/ruby/network/simple/PerfectSwitch.cc --- a/src/mem/ruby/network/simple/PerfectSwitch.cc +++ b/src/mem/ruby/network/simple/PerfectSwitch.cc @@ -54,6 +54,11 @@ m_round_robin_start = 0; m_network_ptr = network_ptr; m_wakeups_wo_switch = 0; + + for(int i = 0;i < m_virtual_networks;++i) + { + m_pending_message_count.push_back(0); + } } void @@ -62,12 +67,15 @@ assert(in.size() == m_virtual_networks); NodeID port = m_in.size(); m_in.push_back(in); + for (int j = 0; j < m_virtual_networks; j++) { m_in[port][j]->setConsumer(this); string desc = csprintf("[Queue from port %s %s %s to PerfectSwitch]", NodeIDToString(m_switch_id), NodeIDToString(port), NodeIDToString(j)); m_in[port][j]->setDescription(desc); + m_in[port][j]->setIncomingLink(port); + m_in[port][j]->setVnet(j); } } @@ -147,168 +155,178 @@ (vnet * decrementer) >= (decrementer * lowest_prio_vnet); vnet -= decrementer) { - // This is for round-robin scheduling - int incoming = m_round_robin_start; - m_round_robin_start++; - if (m_round_robin_start >= m_in.size()) { - m_round_robin_start = 0; - } - - // for all input ports, use round robin scheduling - for (int counter = 0; counter < m_in.size(); counter++) { - // Round robin scheduling - incoming++; - if (incoming >= m_in.size()) { - incoming = 0; + if(m_pending_message_count[vnet] > 0) { + // This is for round-robin scheduling + int incoming = m_round_robin_start; + m_round_robin_start++; + if (m_round_robin_start >= m_in.size()) { + m_round_robin_start = 0; } - // temporary vectors to store the routing results - vector output_links; - vector output_link_destinations; + // for all input ports, use round robin scheduling + for (int counter = 0; counter < m_in.size(); counter++) { + // Round robin scheduling + incoming++; + if (incoming >= m_in.size()) { + incoming = 0; + } - // Is there a message waiting? - while (m_in[incoming][vnet]->isReady()) { - DPRINTF(RubyNetwork, "incoming: %d\n", incoming); + // temporary vectors to store the routing results + vector output_links; + vector output_link_destinations; - // Peek at message - msg_ptr = m_in[incoming][vnet]->peekMsgPtr(); - net_msg_ptr = safe_cast(msg_ptr.get()); - DPRINTF(RubyNetwork, "Message: %s\n", (*net_msg_ptr)); + // Is there a message waiting? + while (m_in[incoming][vnet]->isReady()) { + DPRINTF(RubyNetwork, "incoming: %d\n", incoming); - output_links.clear(); - output_link_destinations.clear(); - NetDest msg_dsts = - net_msg_ptr->getInternalDestination(); + // Peek at message + msg_ptr = m_in[incoming][vnet]->peekMsgPtr(); + net_msg_ptr = safe_cast(msg_ptr.get()); + DPRINTF(RubyNetwork, "Message: %s\n", (*net_msg_ptr)); - // Unfortunately, the token-protocol sends some - // zero-destination messages, so this assert isn't valid - // assert(msg_dsts.count() > 0); + output_links.clear(); + output_link_destinations.clear(); + NetDest msg_dsts = + net_msg_ptr->getInternalDestination(); - assert(m_link_order.size() == m_routing_table.size()); - assert(m_link_order.size() == m_out.size()); + // Unfortunately, the token-protocol sends some + // zero-destination messages, so this assert isn't valid + // assert(msg_dsts.count() > 0); - if (m_network_ptr->getAdaptiveRouting()) { - if (m_network_ptr->isVNetOrdered(vnet)) { - // Don't adaptively route - for (int out = 0; out < m_out.size(); out++) { - m_link_order[out].m_link = out; - m_link_order[out].m_value = 0; + assert(m_link_order.size() == m_routing_table.size()); + assert(m_link_order.size() == m_out.size()); + + if (m_network_ptr->getAdaptiveRouting()) { + if (m_network_ptr->isVNetOrdered(vnet)) { + // Don't adaptively route + for (int out = 0; out < m_out.size(); out++) { + m_link_order[out].m_link = out; + m_link_order[out].m_value = 0; + } + } else { + // Find how clogged each link is + for (int out = 0; out < m_out.size(); out++) { + int out_queue_length = 0; + for (int v = 0; v < m_virtual_networks; v++) { + out_queue_length += m_out[out][v]->getSize(); + } + int value = + (out_queue_length << 8) | (random() & 0xff); + m_link_order[out].m_link = out; + m_link_order[out].m_value = value; + } + + // Look at the most empty link first + sort(m_link_order.begin(), m_link_order.end()); } - } else { - // Find how clogged each link is - for (int out = 0; out < m_out.size(); out++) { - int out_queue_length = 0; - for (int v = 0; v < m_virtual_networks; v++) { - out_queue_length += m_out[out][v]->getSize(); - } - int value = - (out_queue_length << 8) | (random() & 0xff); - m_link_order[out].m_link = out; - m_link_order[out].m_value = value; + } + + for (int i = 0; i < m_routing_table.size(); i++) { + // pick the next link to look at + int link = m_link_order[i].m_link; + NetDest dst = m_routing_table[link]; + DPRINTF(RubyNetwork, "dst: %s\n", dst); + + if (!msg_dsts.intersectionIsNotEmpty(dst)) + continue; + + // Remember what link we're using + output_links.push_back(link); + + // Need to remember which destinations need this + // message in another vector. This Set is the + // intersection of the routing_table entry and the + // current destination set. The intersection must + // not be empty, since we are inside "if" + output_link_destinations.push_back(msg_dsts.AND(dst)); + + // Next, we update the msg_destination not to + // include those nodes that were already handled + // by this link + msg_dsts.removeNetDest(dst); + } + + assert(msg_dsts.count() == 0); + //assert(output_links.size() > 0); + + // Check for resources - for all outgoing queues + bool enough = true; + for (int i = 0; i < output_links.size(); i++) { + int outgoing = output_links[i]; + if (!m_out[outgoing][vnet]->areNSlotsAvailable(1)) + enough = false; + DPRINTF(RubyNetwork, "Checking if node is blocked\n" + "outgoing: %d, vnet: %d, enough: %d\n", + outgoing, vnet, enough); + } + + // There were not enough resources + if (!enough) { + g_eventQueue_ptr->scheduleEvent(this, 1); + DPRINTF(RubyNetwork, "Can't deliver message since a node " + "is blocked\n" + "Message: %s\n", (*net_msg_ptr)); + break; // go to next incoming port + } + + MsgPtr unmodified_msg_ptr; + + if (output_links.size() > 1) { + // If we are sending this message down more than + // one link (size>1), we need to make a copy of + // the message so each branch can have a different + // internal destination we need to create an + // unmodified MsgPtr because the MessageBuffer + // enqueue func will modify the message + + // This magic line creates a private copy of the + // message + unmodified_msg_ptr = msg_ptr->clone(); + } + + // Enqueue it - for all outgoing queues + for (int i=0; i 0) { + // create a private copy of the unmodified + // message + msg_ptr = unmodified_msg_ptr->clone(); } - // Look at the most empty link first - sort(m_link_order.begin(), m_link_order.end()); - } - } + // Change the internal destination set of the + // message so it knows which destinations this + // link is responsible for. + net_msg_ptr = safe_cast(msg_ptr.get()); + net_msg_ptr->getInternalDestination() = + output_link_destinations[i]; - for (int i = 0; i < m_routing_table.size(); i++) { - // pick the next link to look at - int link = m_link_order[i].m_link; - NetDest dst = m_routing_table[link]; - DPRINTF(RubyNetwork, "dst: %s\n", dst); + // Enqeue msg + DPRINTF(RubyNetwork, "Switch: %d enqueuing net msg from " + "inport[%d][%d] to outport [%d][%d] time: %lld.\n", + m_switch_id, incoming, vnet, outgoing, vnet, + g_eventQueue_ptr->getTime()); - if (!msg_dsts.intersectionIsNotEmpty(dst)) - continue; - - // Remember what link we're using - output_links.push_back(link); - - // Need to remember which destinations need this - // message in another vector. This Set is the - // intersection of the routing_table entry and the - // current destination set. The intersection must - // not be empty, since we are inside "if" - output_link_destinations.push_back(msg_dsts.AND(dst)); - - // Next, we update the msg_destination not to - // include those nodes that were already handled - // by this link - msg_dsts.removeNetDest(dst); - } - - assert(msg_dsts.count() == 0); - //assert(output_links.size() > 0); - - // Check for resources - for all outgoing queues - bool enough = true; - for (int i = 0; i < output_links.size(); i++) { - int outgoing = output_links[i]; - if (!m_out[outgoing][vnet]->areNSlotsAvailable(1)) - enough = false; - DPRINTF(RubyNetwork, "Checking if node is blocked\n" - "outgoing: %d, vnet: %d, enough: %d\n", - outgoing, vnet, enough); - } - - // There were not enough resources - if (!enough) { - g_eventQueue_ptr->scheduleEvent(this, 1); - DPRINTF(RubyNetwork, "Can't deliver message since a node " - "is blocked\n" - "Message: %s\n", (*net_msg_ptr)); - break; // go to next incoming port - } - - MsgPtr unmodified_msg_ptr; - - if (output_links.size() > 1) { - // If we are sending this message down more than - // one link (size>1), we need to make a copy of - // the message so each branch can have a different - // internal destination we need to create an - // unmodified MsgPtr because the MessageBuffer - // enqueue func will modify the message - - // This magic line creates a private copy of the - // message - unmodified_msg_ptr = msg_ptr->clone(); - } - - // Enqueue it - for all outgoing queues - for (int i=0; i 0) { - // create a private copy of the unmodified - // message - msg_ptr = unmodified_msg_ptr->clone(); + m_out[outgoing][vnet]->enqueue(msg_ptr); } - // Change the internal destination set of the - // message so it knows which destinations this - // link is responsible for. - net_msg_ptr = safe_cast(msg_ptr.get()); - net_msg_ptr->getInternalDestination() = - output_link_destinations[i]; - - // Enqeue msg - DPRINTF(RubyNetwork, "Switch: %d enqueuing net msg from " - "inport[%d][%d] to outport [%d][%d] time: %lld.\n", - m_switch_id, incoming, vnet, outgoing, vnet, - g_eventQueue_ptr->getTime()); - - m_out[outgoing][vnet]->enqueue(msg_ptr); + // Dequeue msg + m_in[incoming][vnet]->pop(); + m_pending_message_count[vnet]--; } - - // Dequeue msg - m_in[incoming][vnet]->pop(); } } } } void +PerfectSwitch::storeEventInfo(MsgPtr msg_ptr, Time arrival_time) +{ + NetworkMessage* net_msg_ptr = safe_cast(msg_ptr.get()); + m_pending_message_count[net_msg_ptr->getVnet()]++; +} + +void PerfectSwitch::printStats(std::ostream& out) const { out << "PerfectSwitch printStats" << endl; diff --git a/src/mem/ruby/slicc_interface/Message.hh b/src/mem/ruby/slicc_interface/Message.hh --- a/src/mem/ruby/slicc_interface/Message.hh +++ b/src/mem/ruby/slicc_interface/Message.hh @@ -57,6 +57,8 @@ virtual Message* clone() const = 0; virtual void print(std::ostream& out) const = 0; + virtual void setIncomingLink(int) {} + virtual void setVnet(int) {} void setDelayedCycles(const int& cycles) { m_DelayedCycles = cycles; } const int& getDelayedCycles() const {return m_DelayedCycles;} diff --git a/src/mem/ruby/slicc_interface/NetworkMessage.hh b/src/mem/ruby/slicc_interface/NetworkMessage.hh --- a/src/mem/ruby/slicc_interface/NetworkMessage.hh +++ b/src/mem/ruby/slicc_interface/NetworkMessage.hh @@ -82,9 +82,16 @@ virtual void print(std::ostream& out) const = 0; + int getIncomingLink() const { return incoming_link; } + void setIncomingLink(int link) { incoming_link = link; } + int getVnet() const { return vnet; } + void setVnet(int net) { vnet = net; } + private: NetDest m_internal_dest; bool m_internal_dest_valid; + int incoming_link; + int vnet; }; inline std::ostream&