diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1b8f8e0..b9fea22 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,6 @@ target_include_directories(reconverse PRIVATE .) target_sources(reconverse PRIVATE conv-conds.cpp convcore.cpp random.cpp - scheduler.cpp cpuaffinity.cpp mempool.C collectives.cpp + scheduler.cpp scheduler_helpers.cpp cpuaffinity.cpp mempool.C collectives.cpp comm_backend/comm_backend_internal.cpp threads.cpp cldb.rand.cpp cldb.cpp cmirdmautils.cpp conv-rdma.cpp conv-topology.cpp msgmgr.cpp queueing.cpp cmishmem.cpp conv-taskq.cpp) diff --git a/src/convcore.cpp b/src/convcore.cpp index 91972ef..e305769 100644 --- a/src/convcore.cpp +++ b/src/convcore.cpp @@ -137,6 +137,8 @@ void converseRunPe(int rank, int everReturn) { CmiTaskQueueInit(); #endif + CmiQueueRegisterInitThread(); + // init things like cld module, ccs, etc CldModuleInit(CmiMyArgv); @@ -365,6 +367,9 @@ void ConverseInit(int argc, char **argv, CmiStartFn fn, int usched, CmiHandlerTable = new std::vector *[Cmi_mynodesize]; CmiNodeQueue = new ConverseNodeQueue(); + //register queues + CmiQueueRegisterInit(); + _smp_mutex = CmiCreateLock(); CmiMemLock_lock = CmiCreateLock(); diff --git a/src/converse_internal.h b/src/converse_internal.h index dfc820a..0f81f4f 100644 --- a/src/converse_internal.h +++ b/src/converse_internal.h @@ -64,9 +64,12 @@ CmiState *CmiGetState(void); void CmiInitState(int pe); ConverseQueue *CmiGetQueue(int pe); void CrnInit(void); - void CmiPushPE(int destRank, int messageSize, void *msg); +//queue reg init +void CmiQueueRegisterInit(void); +void CmiQueueRegisterInitThread(void); + // node queue ConverseNodeQueue *CmiGetNodeQueue(); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 45eb895..f77b20e 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -1,191 +1,172 @@ #include "scheduler.h" -#include "converse.h" -#include "converse_internal.h" -#include "queue.h" +#if CMK_TASKQUEUE #include "taskqueue.h" -#include - CpvExtern(TaskQueue, CsdTaskQueue); +#endif -/** - * The main scheduler loop for the Charm++ runtime. - */ -void CsdScheduler() { - // get pthread level queue - ConverseQueue *queue = CmiGetQueue(CmiMyRank()); - - // get node level queue - ConverseNodeQueue *nodeQueue = CmiGetNodeQueue(); - - int loop_counter = 0; - - while (CmiStopFlag() == 0) { - - CcdRaiseCondition(CcdSCHEDLOOP); - - #ifdef CMK_USE_SHMEM - CmiIpcBlock* block = CmiPopIpcBlock(CsvAccess(coreIpcManager_)); - if (block != nullptr) { - CmiDeliverIpcBlockMsg(block); - } - #endif +extern std::vector g_handlers; //list of handlers +extern Groups g_groups; //groups of handlers by index +CpvExtern(QueuePollHandlerFn *, poll_handlers); - // poll node queue - if (!nodeQueue->empty()) { - auto result = nodeQueue->pop(); - if (result) { - void *msg = result.value(); +static inline void releaseIdle() { + if (CmiGetIdle()) { + CmiSetIdle(false); + CcdRaiseCondition(CcdPROCESSOR_END_IDLE); + } +} - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } +static inline void setIdle() { + if (!CmiGetIdle()) { + CmiSetIdle(true); + CmiSetIdleTime(CmiWallTimer()); + CcdRaiseCondition(CcdPROCESSOR_BEGIN_IDLE); + } + // if already idle, call still idle and (maybe) long idle + else { + CcdRaiseCondition(CcdPROCESSOR_STILL_IDLE); + if (CmiWallTimer() - CmiGetIdleTime() > 10.0) { + CcdRaiseCondition(CcdPROCESSOR_LONG_IDLE); + } + } +} - // process event - CmiHandleMessage(msg); - } +//poll converse-level node queue +bool pollConverseNodeQueue() { + ConverseNodeQueue *nodeQueue = CmiGetNodeQueue(); + if (!nodeQueue->empty()) { + auto result = nodeQueue->pop(); + if (result) { + void *msg = result.value(); + releaseIdle(); + // process event + CmiHandleMessage(msg); + return true; } + } + return false; +} - // poll thread queue - else if (!queue->empty()) { - // get next event (guaranteed to be there because only single consumer) - void *msg = queue->pop().value(); +//poll converse-level thread queue +bool pollConverseThreadQueue() { + ConverseQueue *queue = CmiGetQueue(CmiMyRank()); + if (!queue->empty()) { + // get next event (guaranteed to be there because only single consumer) + void *msg = queue->pop().value(); + releaseIdle(); + // process event + CmiHandleMessage(msg); + return true; + } + return false; +} - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - +//poll node priority queue +bool pollNodePrioQueue() { + // Try to acquire lock without blocking + if (CmiTryLock(CsvAccess(CsdNodeQueueLock)) == 0) { + if (!QueueEmpty(CsvAccess(CsdNodeQueue))) { + void *msg = QueueTop(CsvAccess(CsdNodeQueue)); + QueuePop(CsvAccess(CsdNodeQueue)); + CmiUnlock(CsvAccess(CsdNodeQueueLock)); + releaseIdle(); // process event CmiHandleMessage(msg); + return true; + } else { + CmiUnlock(CsvAccess(CsdNodeQueueLock)); } + } + return false; +} - // poll node prio queue - else { - // Try to acquire lock without blocking - if (CmiTryLock(CsvAccess(CsdNodeQueueLock)) == 0) { - if (!QueueEmpty(CsvAccess(CsdNodeQueue))) { - void* msg = QueueTop(CsvAccess(CsdNodeQueue)); - QueuePop(CsvAccess(CsdNodeQueue)); - CmiUnlock(CsvAccess(CsdNodeQueueLock)); +//poll thread priority queue +bool pollThreadPrioQueue() { + if (!QueueEmpty(CpvAccess(CsdSchedQueue))) { + void *msg = QueueTop(CpvAccess(CsdSchedQueue)); + QueuePop(CpvAccess(CsdSchedQueue)); + releaseIdle(); + // process event + CmiHandleMessage(msg); + return true; + } + return false; +} - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - - // process event - CmiHandleMessage(msg); - } - else { - CmiUnlock(CsvAccess(CsdNodeQueueLock)); - //empty queue so check thread prio queue - if (!QueueEmpty(CpvAccess(CsdSchedQueue))) { - void *msg = QueueTop(CpvAccess(CsdSchedQueue)); - QueuePop(CpvAccess(CsdSchedQueue)); +bool pollProgress() +{ + if(CmiMyRank() % backend_poll_thread == 0) comm_backend::progress(); + return false; //polling progress doesn't count +} - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - - // process event - CmiHandleMessage(msg); - } else { - #if CMK_TASKQUEUE - // Check local task queue before going idle - void *task_msg = TaskQueuePopLocal(); - if (task_msg != NULL) { - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - // Found a task in our local queue - CmiHandleMessage(task_msg); - } else { - #endif - // the processor is idle - // if not already idle, set idle and raise condition - if (!CmiGetIdle()) { - CmiSetIdle(true); - CmiSetIdleTime(CmiWallTimer()); - CcdRaiseCondition(CcdPROCESSOR_BEGIN_IDLE); - } - // if already idle, call still idle and (maybe) long idle - else { - CcdRaiseCondition(CcdPROCESSOR_STILL_IDLE); - if (CmiWallTimer() - CmiGetIdleTime() > 10.0) { - CcdRaiseCondition(CcdPROCESSOR_LONG_IDLE); - } - } - #if CMK_TASKQUEUE - } - #endif - } - } - } - else { - // Could not acquire node queue lock, skip to thread prio queue - if (!QueueEmpty(CpvAccess(CsdSchedQueue))) { - void *msg = QueueTop(CpvAccess(CsdSchedQueue)); - QueuePop(CpvAccess(CsdSchedQueue)); +#if CMK_TASKQUEUE +bool pollTaskQueue() { + void *task_msg = TaskQueuePopLocal(); + if (task_msg != nullptr) { + releaseIdle(); + CmiHandleMessage(task_msg); + return true; + } + return false; +} +#endif + +void CmiQueueRegisterInitThread() { + std::vector> handlers; + handlers.push_back(std::make_pair(pollConverseNodeQueue, 1)); + handlers.push_back(std::make_pair(pollConverseThreadQueue, 16)); + handlers.push_back(std::make_pair(pollNodePrioQueue, 1)); + handlers.push_back(std::make_pair(pollThreadPrioQueue, 16)); + handlers.push_back(std::make_pair(pollProgress, 4)); +#if CMK_TASKQUEUE + handlers.push_back(std::make_pair(pollTaskQueue, 1)); +#endif + add_list_of_handlers(handlers); +} - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } +//will add queue polling functions +//called at node level (before threads created) +void CmiQueueRegisterInit() { + add_handler(pollConverseNodeQueue, 1); + add_handler(pollConverseThreadQueue, 16); + add_handler(pollNodePrioQueue, 1); + add_handler(pollThreadPrioQueue, 16); + add_handler(pollProgress, backend_poll_freq); +#if CMK_TASKQUEUE + add_handler(pollTaskQueue, 1); +#endif +} - // process event - CmiHandleMessage(msg); +/** + * The main scheduler loop for the Charm++ runtime. + */ +void CsdScheduler() { - } else { - #if CMK_TASKQUEUE - // Check local task queue before going idle - void *task_msg = TaskQueuePopLocal(); - if (task_msg != NULL) { - - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } + uint64_t loop_counter = 0; - // Found a task in our local queue - CmiHandleMessage(task_msg); + while (CmiStopFlag() == 0) { - } else { - #endif - // the processor is idle - // if not already idle, set idle and raise condition - if (!CmiGetIdle()) { - CmiSetIdle(true); - CmiSetIdleTime(CmiWallTimer()); - CcdRaiseCondition(CcdPROCESSOR_BEGIN_IDLE); - } - // if already idle, call still idle and (maybe) long idle - else { - CcdRaiseCondition(CcdPROCESSOR_STILL_IDLE); - if (CmiWallTimer() - CmiGetIdleTime() > 10.0) { - CcdRaiseCondition(CcdPROCESSOR_LONG_IDLE); - } - } - #if CMK_TASKQUEUE - } - #endif + CcdRaiseCondition(CcdSCHEDLOOP); + //always deliver shmem messages first + #ifdef CMK_USE_SHMEM + CmiIpcBlock* block = CmiPopIpcBlock(CsvAccess(coreIpcManager_)); + if (block != nullptr) { + CmiDeliverIpcBlockMsg(block); } - } + #endif + //poll queues + unsigned idx = static_cast(loop_counter & 63ULL); + bool workDone = false; + /* + for (auto fn : g_groups[idx]) { + workDone |= fn(); } - if((CmiMyRank() % backend_poll_thread == 0) && (loop_counter++ == (backend_poll_freq - 1))) - { - loop_counter = 0; - comm_backend::progress(); + */ + workDone |= CpvAccess(poll_handlers)[idx](); + if(!workDone) { + setIdle(); } - CcdCallBacks(); + loop_counter++; } } @@ -195,155 +176,31 @@ void CsdScheduler() { * are empty, not when the scheduler is stopped. */ void CsdSchedulePoll() { - // get pthread level queue - ConverseQueue *queue = CmiGetQueue(CmiMyRank()); - - // get node level queue - ConverseNodeQueue *nodeQueue = CmiGetNodeQueue(); + uint64_t loop_counter = 0; + int current_empty = 0; //number of empty queues while(1){ - CcdCallBacks(); - CcdRaiseCondition(CcdSCHEDLOOP); - - // poll node queue - if (!nodeQueue->empty()) { - auto result = nodeQueue->pop(); - if (result) { - void *msg = result.value(); - - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - - // process event - CmiHandleMessage(msg); - - } + //poll queues + unsigned idx = static_cast(loop_counter & 63ULL); + /* + for (auto fn : g_groups[idx]) { + workDone |= fn(); } - - // poll thread queue - else if (!queue->empty()) { - // get next event (guaranteed to be there because only single consumer) - void *msg = queue->pop().value(); - - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); + */ + bool workDone = CpvAccess(poll_handlers)[idx](); + if(!workDone) { + setIdle(); + current_empty++; + //if all queues empty, return + if(current_empty >= ARRAY_SIZE){ + return; } - - // process event - CmiHandleMessage(msg); - } + CcdCallBacks(); + loop_counter++; - // poll node prio queue - else { - // Try to acquire lock without blocking - if (CmiTryLock(CsvAccess(CsdNodeQueueLock)) == 0) { - if (!QueueEmpty(CsvAccess(CsdNodeQueue))) { - void *msg = QueueTop(CsvAccess(CsdNodeQueue)); - QueuePop(CsvAccess(CsdNodeQueue)); - CmiUnlock(CsvAccess(CsdNodeQueueLock)); - - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - - // process event - CmiHandleMessage(msg); - - } - else { - CmiUnlock(CsvAccess(CsdNodeQueueLock)); - if (!QueueEmpty(CpvAccess(CsdSchedQueue))) { - void *msg = QueueTop(CpvAccess(CsdSchedQueue)); - QueuePop(CpvAccess(CsdSchedQueue)); - - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - - // process event - CmiHandleMessage(msg); - - } - else { - #if CMK_TASKQUEUE - //because idle, check task queue - void *task_msg = TaskQueuePopLocal(); - if (task_msg != NULL) { - - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - - // Found a task in our local queue - CmiHandleMessage(task_msg); - - } - else - { - #endif - comm_backend::progress(); - break; //break when queues are empty - #if CMK_TASKQUEUE - } - #endif - } - } - } - else { - // Could not acquire node queue lock, skip to thread prio queue - if (!QueueEmpty(CpvAccess(CsdSchedQueue))) { - void *msg = QueueTop(CpvAccess(CsdSchedQueue)); - QueuePop(CpvAccess(CsdSchedQueue)); - - // release idle if necessary - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - - // process event - CmiHandleMessage(msg); - - } - else { - #if CMK_TASKQUEUE - //because idle, check task queue - void *task_msg = TaskQueuePopLocal(); - if (task_msg != NULL) { - - if (CmiGetIdle()) { - CmiSetIdle(false); - CcdRaiseCondition(CcdPROCESSOR_END_IDLE); - } - - // Found a task in our local queue - CmiHandleMessage(task_msg); - - } - else - { - #endif - comm_backend::progress(); - break; //break when queues are empty - #if CMK_TASKQUEUE - } - #endif - } - } - } } } @@ -355,7 +212,7 @@ int CsdScheduler(int maxmsgs){ } else CsdSchedulePoll(); //not implementing CsdScheduleCount return 0; - + } void CqsEnqueueGeneral(Queue q, void *Message, int strategy, int priobits, @@ -388,4 +245,3 @@ void CqsEnqueueGeneral(Queue q, void *Message, int strategy, int priobits, void CmiNetworkProgress(){ comm_backend::progress(); } - diff --git a/src/scheduler.h b/src/scheduler.h index a311734..457e7c3 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -1,2 +1,33 @@ +#ifndef _SCHEDULER_H_ +#define _SCHEDULER_H_ +#include "converse.h" +#include "converse_internal.h" +#include "queue.h" +#include +#include +#include +#include + +#define ARRAY_SIZE 64 +using QueuePollHandlerFn = bool(*)(void); //we need a return value to indicate if work was done + +struct QueuePollHandler { + QueuePollHandlerFn fn; + uint64_t mask{0}; // 64-bit mask: bit i == call at loop index i (0..63) + unsigned period{0}; // 1..64, 0 => disabled + unsigned phase{0}; +}; + +using Groups = std::array, 64>; + +void add_handler(QueuePollHandlerFn fn, unsigned period, unsigned phase = 0); + +// Add multiple handlers at once +// pairs of poll handlers and relative frequencies (will be normalized regardless of actual value) +// (frequency/total)*64 +// example: if the frequencies are 8, 1, 16, 1, 4, then they are added up to 30, then normalized to 17, 2, 34, 2, 9 +// then assign to slots based on these normalized values +void add_list_of_handlers(const std::vector>& handlers); void CsdScheduler(); +#endif diff --git a/src/scheduler_helpers.cpp b/src/scheduler_helpers.cpp new file mode 100644 index 0000000..b06ed3f --- /dev/null +++ b/src/scheduler_helpers.cpp @@ -0,0 +1,119 @@ +#include "scheduler.h" + +std::vector g_handlers; //list of handlers +Groups g_groups; //groups of handlers by index +CpvDeclare(QueuePollHandlerFn *, poll_handlers); +CpvDeclare(int*, poll_handler_assigned); +//QueuePollHandlerFn *poll_handlers; // fixed size array + +// default handler used to safely occupy any unassigned slot +static bool pollNoWork() { return false; } + +// Build a 64-bit mask for a period n (1..64) with optional phase (0..n-1) +inline uint64_t make_mask_every_n(unsigned n, unsigned phase = 0) { + if (n == 0) return 0ULL; + if (n == 1) return ~0ULL; + if (n > 64) n = 64; // clamp to 64 + uint64_t mask = 0ULL; + for (unsigned pos = 0; pos < 64; ++pos) { + if (((pos + phase) % n) == 0) mask |= (1ULL << pos); + } + return mask; +} + +// Rebuild groups from current handler masks (in-place). +// Single-threaded callers may call this whenever a handler mask changes. +inline void rebuild_groups() { + // Clear all groups + for (auto &v : g_groups) v.clear(); + + // Populate groups from each handler's mask + for (const auto &h : g_handlers) { + uint64_t m = h.mask; + if (m == 0) continue; + for (unsigned bit = 0; bit < 64; ++bit) { + if ((m >> bit) & 1ULL) { + g_groups[bit].push_back(h.fn); + } + } + } +} + +// Set handler period and phase (period: 1..64, 0 disables). +// Rebuilds groups immediately (cheap relative to hot path). +inline void set_frequency(size_t handlerIndex, unsigned period, unsigned phase = 0) { + if (handlerIndex >= g_handlers.size()) return; + QueuePollHandler &h = g_handlers[handlerIndex]; + + if (period == 0) { + h.period = 0; + h.phase = 0; + h.mask = 0ULL; + } else { + if (period > 64) period = 64; + h.period = period; + h.phase = phase % period; + h.mask = make_mask_every_n(h.period, h.phase); + } + rebuild_groups(); +} + +// Add a handler that will poll a queue at given frequency. +void add_handler(QueuePollHandlerFn fn, unsigned period, unsigned phase) +{ + g_handlers.push_back({fn}); + size_t index = g_handlers.size() - 1; + set_frequency(index, period, phase); +} + +void add_list_of_handlers(const std::vector>& handlers){ + // total frequency + unsigned int total = 0; + for(const auto& handler : handlers){ + total += handler.second; + } + if(total == 0) return; // nothing to add + // loop through handlers and add them to the table + // spread out based on normalized frequency + CpvInitialize(QueuePollHandlerFn *, poll_handlers); + CpvAccess(poll_handlers) = new QueuePollHandlerFn[ARRAY_SIZE]; + CpvInitialize(int*, poll_handler_assigned); + CpvAccess(poll_handler_assigned) = new int[ARRAY_SIZE]; + for(unsigned int i=0; i(total)); //estimate of how many slots this handler should take + //CmiPrintf("Handler %d frequency %u normalized to %ld\n", handler_index, freq, normalized); + if(normalized == 0) normalized = 1; // at least once + // go through loop and find empty slots + // spread out as evenly as possible + unsigned int remaining = normalized; + unsigned int step = ARRAY_SIZE / normalized; + unsigned int index = current_index; + while(remaining > 0){ + //find next empty slot + if(total_assigned >= ARRAY_SIZE){ + break; // all slots assigned + } + while(CpvAccess(poll_handler_assigned)[index] != 0){ + index = (index + 1) % ARRAY_SIZE; + } + CpvAccess(poll_handlers)[index] = handler.first; + CpvAccess(poll_handler_assigned)[index] = 1; + //poll_handlers[index] = handler.first; + total_assigned++; + //CmiPrintf("Adding handler %d at index %d\n", handler_index, index); + remaining--; + index = (index + step) % ARRAY_SIZE; + } + current_index = (current_index + 1) % ARRAY_SIZE; + handler_index++; + } +}