device/gpu: proactive eviction with adaptive percentage threshold

parsec/mca/device/device.c

@@ -301,6 +301,12 @@ no_valid_device: {
 PARSEC_OBJ_CLASS_INSTANCE(parsec_device_module_t, parsec_object_t,
                           NULL, NULL);
 
+#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP) || defined(PARSEC_HAVE_LEVEL_ZERO)
+/* Defined in transfer_gpu.c; registered here so they apply to all GPU backends. */
+extern int32_t parsec_gpu_mem_evict_upper;
+extern int32_t parsec_gpu_mem_evict_lower;
+#endif
+
 int parsec_mca_device_init(void)
 {
     char** parsec_device_list = NULL;
@@ -313,6 +319,18 @@ int parsec_mca_device_init(void)
     PARSEC_OBJ_CONSTRUCT(&parsec_per_device_infos, parsec_info_t);
     PARSEC_OBJ_CONSTRUCT(&parsec_per_stream_infos, parsec_info_t);
 
+#if defined(PARSEC_HAVE_CUDA) || defined(PARSEC_HAVE_HIP) || defined(PARSEC_HAVE_LEVEL_ZERO)
+    (void)parsec_mca_param_reg_int_name("device", "mem_evict_upper",
+                                        "Upper threshold (percentage of total GPU zone capacity) at which proactive "
+                                        "clean-LRU eviction and D2H writeback begin. When the device is truly stalled "
+                                        "(no in-flight evictions and no dirty pages left to queue), the per-device "
+                                        "threshold is stepped down by 5 points toward device_mem_evict_lower.",
+                                        false, false, 95, &parsec_gpu_mem_evict_upper);
+    (void)parsec_mca_param_reg_int_name("device", "mem_evict_lower",
+                                        "Lower bound (percentage of total GPU zone capacity) to which the adaptive "
+                                        "eviction threshold may be reduced after repeated stalls.",
+                                        false, false, 80, &parsec_gpu_mem_evict_lower);
+#endif  /* PARSEC_HAVE_CUDA || PARSEC_HAVE_HIP || PARSEC_HAVE_LEVEL_ZERO */
     (void)parsec_mca_param_reg_int_name("device", "show_capabilities",
                                         "Show the detailed devices capabilities",
                                         false, false, parsec_debug_verbose >= 4 || (parsec_debug_verbose >= 3 && parsec_debug_rank == 0), NULL);

parsec/mca/device/device_gpu.c

@@ -715,6 +715,7 @@ parsec_device_memory_reserve( parsec_device_gpu_module_t* gpu_device,
 #endif
     gpu_device->mem_block_size = eltsize;
     gpu_device->mem_nb_blocks = mem_elem_per_gpu;
+    gpu_device->mem_evict_threshold = parsec_gpu_mem_evict_upper;
 
     return PARSEC_SUCCESS;
 }
@@ -826,6 +827,95 @@ parsec_device_memory_release( parsec_device_gpu_module_t* gpu_device )
     return PARSEC_SUCCESS;
 }
 
+/**
+ * Try to evict one entry from the clean LRU (gpu_mem_lru) by detaching it from
+ * its original data and freeing its zone allocation back to the zone allocator.
+ *
+ * @param[in]  gpu_device       the GPU device whose clean LRU is targeted
+ * @param[in,out] cycling_sentinel  cycle-detector: caller initialises to NULL;
+ *               updated to the first entry that could not be evicted. When
+ *               we pop that entry again we know we have looped the entire LRU
+ *               without finding a free-able entry.
+ *
+ * @return 1 if a zone block was freed, 0 if the LRU is empty or fully cycling.
+ */
+#if !defined(PARSEC_GPU_ALLOC_PER_TILE)
+static int
+parsec_device_try_evict_lru_one( parsec_device_gpu_module_t  *gpu_device,
+                                  parsec_gpu_data_copy_t     **cycling_sentinel )
+{
+    parsec_gpu_data_copy_t *lru_gpu_elem;
+    parsec_data_t *oldmaster;
+
+  retry:
+    lru_gpu_elem = (parsec_gpu_data_copy_t*)parsec_list_pop_front(&gpu_device->gpu_mem_lru);
+    if( NULL == lru_gpu_elem )
+        return 0;
+    PARSEC_LIST_ITEM_SINGLETON(lru_gpu_elem);
+
+    if( *cycling_sentinel == lru_gpu_elem ) {
+        parsec_list_push_front(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)lru_gpu_elem);
+        return 0;
+    }
+
+    /* Dangling reader: the copy is temporarily untracked in the LRU; skip it */
+    if( 0 != lru_gpu_elem->readers )
+        goto retry;
+
+    /* Outstanding object references: not safe to free yet; push back and note cycle */
+    if( lru_gpu_elem->super.super.obj_reference_count > 1 ) {
+        parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super);
+        if( NULL == *cycling_sentinel ) *cycling_sentinel = lru_gpu_elem;
+        goto retry;
+    }
+
+    if( NULL != lru_gpu_elem->original ) {
+        oldmaster = lru_gpu_elem->original;
+        if( !parsec_atomic_trylock(&oldmaster->lock) ) {
+            parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super);
+            if( NULL == *cycling_sentinel ) *cycling_sentinel = lru_gpu_elem;
+            goto retry;
+        }
+        /* Guard against a concurrent d2d reader acquiring the copy */
+        if( !parsec_atomic_cas_int32(&lru_gpu_elem->readers, 0,
+                                      -PARSEC_DEVICE_DATA_COPY_ATOMIC_SENTINEL) ) {
+            parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super);
+            if( NULL == *cycling_sentinel ) *cycling_sentinel = lru_gpu_elem;
+            parsec_atomic_unlock(&oldmaster->lock);
+            goto retry;
+        }
+        int do_unlock = oldmaster->super.obj_reference_count != 1;
+        parsec_data_copy_detach(oldmaster, lru_gpu_elem, gpu_device->super.device_index);
+        parsec_atomic_wmb();
+        if( do_unlock )
+            parsec_atomic_unlock(&oldmaster->lock);
+    }
+
+#if defined(PARSEC_PROF_TRACE)
+    if( (gpu_device->trackable_events & PARSEC_PROFILE_GPU_TRACK_MEM_USE) &&
+        (gpu_device->exec_stream[0]->prof_event_track_enable ||
+         gpu_device->exec_stream[1]->prof_event_track_enable) ) {
+        parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling,
+                                     parsec_gpu_free_memory_key,
+                                     (int64_t)lru_gpu_elem->device_private,
+                                     gpu_device->super.device_index,
+                                     NULL, PARSEC_PROFILING_EVENT_COUNTER);
+        parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling,
+                                     parsec_gpu_use_memory_key_end,
+                                     (uint64_t)lru_gpu_elem->device_private,
+                                     gpu_device->super.device_index, NULL, 0);
+    }
+#endif
+    assert( 0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) );
+    zone_free(gpu_device->memory, (void*)lru_gpu_elem->device_private);
+    lru_gpu_elem->device_private = NULL;
+    gpu_device->super.nb_evictions++;
+    PARSEC_OBJ_RELEASE(lru_gpu_elem);
+    assert( NULL == lru_gpu_elem );
+    return 1;
+}
+#endif  /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */
+
 /**
  * Try to find memory space to move all data on the GPU. We attach a device_elem to
  * a memory_elem as soon as a device_elem is available. If we fail to find enough
@@ -854,6 +944,24 @@ parsec_device_data_reserve_space( parsec_device_gpu_module_t* gpu_device,
 
     (void)copy_readers_update; // potentially unused
 
+#if !defined(PARSEC_GPU_ALLOC_PER_TILE)
+    /* Tier-1 proactive eviction: free clean LRU entries while zone usage exceeds
+     * gpu_device->mem_evict_threshold percent of total capacity.  The threshold
+     * starts at parsec_gpu_mem_evict_upper (default 95%) and is lowered in 5-point
+     * steps (floor: parsec_gpu_mem_evict_lower, default 80%) whenever the device stalled
+     * because the reactive path also failed to find memory. */
+    {
+        size_t total_capacity = (size_t)gpu_device->mem_nb_blocks * gpu_device->mem_block_size;
+        parsec_gpu_data_copy_t *cycling = NULL;
+        while( zone_in_use(gpu_device->memory) * 100 >
+               (size_t)gpu_device->mem_evict_threshold * total_capacity ) {
+            if( !parsec_device_try_evict_lru_one(gpu_device, &cycling) )
+                break;
+            data_avail_epoch++;
+        }
+    }
+#endif  /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */
+
     /**
      * Parse all the input and output flows of data and ensure all have
      * corresponding data on the GPU available.
@@ -2576,6 +2684,62 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module,
                              gpu_device->super.device_index, gpu_device->super.name,
                              parsec_device_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task));
     }
+
+    /* Tier-2 proactive dirty-page writeback: when the clean LRU is empty and zone
+     * memory pressure exceeds the watermark, queue a D2H transfer on exec_stream[1]
+     * now so its latency overlaps with the upcoming H2D stage and kernel execution.
+     * This converts a potential blocking wait (dirty page eviction on the critical
+     * path) into an overlapped background transfer.
+     *
+     * In PARSEC_GPU_ALLOC_PER_TILE mode there is no zone allocator; fall back to the
+     * simple condition of clean LRU being empty. */
+    if( !parsec_list_nolock_is_empty(&gpu_device->gpu_mem_owned_lru) &&
+         parsec_list_nolock_is_empty(&gpu_device->gpu_mem_lru) ) {
+#if !defined(PARSEC_GPU_ALLOC_PER_TILE)
+        {
+            size_t total_capacity = (size_t)gpu_device->mem_nb_blocks * gpu_device->mem_block_size;
+            size_t in_use         = zone_in_use(gpu_device->memory);
+            size_t threshold_bytes = (size_t)gpu_device->mem_evict_threshold * total_capacity / 100;
+            if( in_use > threshold_bytes ) {
+                /* Compute how many more bytes of dirty-page eviction are needed beyond
+                 * what is already in-flight on exec_stream[1]. */
+                size_t needed = in_use - threshold_bytes;
+                size_t still_needed = (needed > gpu_device->mem_evict_in_flight) ?
+                                      (needed - gpu_device->mem_evict_in_flight) : 0;
+                while( still_needed > 0 ) {
+                    size_t selected = 0;
+                    parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es,
+                                                                          still_needed, &selected);
+                    if( NULL == _w2r ) break;
+                    PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream,
+                                         "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, "
+                                         "zone above %d%% threshold; needed %zu, selected %zu bytes",
+                                         gpu_device->super.device_index, gpu_device->super.name,
+                                         gpu_device->mem_evict_threshold, still_needed, selected);
+                    PARSEC_PUSH_TASK(gpu_device->exec_stream[1]->fifo_pending,
+                                     (parsec_list_item_t*)_w2r);
+                    still_needed = (still_needed > selected) ? (still_needed - selected) : 0;
+                }
+            }
+        }
+#else
+        {
+            /* No zone allocator in ALLOC_PER_TILE mode: issue one D2H batch whenever
+             * the clean LRU is empty and the dirty LRU is non-empty. */
+            size_t selected = 0;
+            parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es,
+                                                                   SIZE_MAX, &selected);
+            if( NULL != _w2r ) {
+                PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream,
+                                     "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, selected %zu bytes",
+                                     gpu_device->super.device_index, gpu_device->super.name, selected);
+                PARSEC_PUSH_TASK(gpu_device->exec_stream[1]->fifo_pending,
+                                 (parsec_list_item_t*)_w2r);
+            }
+        }
+#endif  /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */
+    }
+
     rc = parsec_device_progress_stream( gpu_device,
                                         gpu_device->exec_stream[0],
                                         parsec_device_kernel_push,
@@ -2595,8 +2759,19 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module,
         assert(NULL == progress_task);
 
         /* TODO: check this */
-        /* If we can extract data go for it, otherwise try to drain the pending tasks */
-        gpu_task = parsec_gpu_create_w2r_task(gpu_device, es);
+        /* If we can extract data go for it, otherwise try to drain the pending tasks.
+         * Skip if evictions are already in flight to avoid a storm of D2H tasks.
+         * If there are no in-flight evictions and nothing left to queue from the dirty
+         * LRU, we are truly stuck: step the eviction threshold down so tier-1 starts
+         * freeing pages earlier on the next attempt. */
+        if( 0 == gpu_device->mem_evict_in_flight ) {
+            size_t _sel = 0;
+            gpu_task = parsec_gpu_create_w2r_task(gpu_device, es, SIZE_MAX, &_sel);
+            if(gpu_device->mem_evict_threshold - 5 >= parsec_gpu_mem_evict_lower) {
+                /* We had to trigger a proactive D2H writeback so reduce the threshold */
+                gpu_device->mem_evict_threshold -= 5;
+            }
+        }
         if( NULL != gpu_task )
             goto get_data_out_of_device;
     }

parsec/mca/device/device_gpu.h

@@ -245,6 +245,16 @@ struct parsec_device_gpu_module_s {
     parsec_gpu_exec_stream_t **exec_stream;
     size_t                     mem_block_size;
     int64_t                    mem_nb_blocks;
+    int32_t                    mem_evict_threshold; /**< Current eviction threshold (% of total zone
+                                                     *   capacity). Starts at parsec_gpu_mem_evict_upper
+                                                     *   and is stepped down by 5 points (to
+                                                     *   parsec_gpu_mem_evict_lower) each time a task
+                                                     *   stalls waiting for zone memory. */
+    size_t                     mem_evict_in_flight; /**< Bytes of dirty GPU data currently selected for
+                                                     *   D2H eviction (queued or executing on
+                                                     *   exec_stream[1]). Incremented by
+                                                     *   parsec_gpu_create_w2r_task, decremented as each
+                                                     *   copy completes in parsec_gpu_complete_w2r_task. */
 #if defined(PARSEC_PROF_TRACE)
     int                        trackable_events;
 #endif /* PARSEC_PROF_TRACE */
@@ -279,6 +289,8 @@ typedef struct parsec_gpu_workspace_s {
 PARSEC_DECLSPEC extern int parsec_gpu_output_stream;
 PARSEC_DECLSPEC extern int parsec_gpu_verbosity;
 PARSEC_DECLSPEC extern int32_t parsec_gpu_d2h_max_flows;
+PARSEC_DECLSPEC extern int32_t parsec_gpu_mem_evict_upper;
+PARSEC_DECLSPEC extern int32_t parsec_gpu_mem_evict_lower;
 
 /**
  * Debugging functions.
@@ -304,7 +316,8 @@ int parsec_device_free_workspace(parsec_device_gpu_module_t * gpu_device);
 
 /* sort pending task list by number of spaces needed */
 int parsec_device_sort_pending_list(parsec_device_module_t *gpu_device);
-parsec_gpu_task_t* parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, parsec_execution_stream_t *es);
+parsec_gpu_task_t* parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, parsec_execution_stream_t *es,
+                                               size_t required_size, size_t *selected_size);
 int parsec_gpu_complete_w2r_task(parsec_device_gpu_module_t *gpu_device, parsec_gpu_task_t *w2r_task, parsec_execution_stream_t *es);
 
 void parsec_device_enable_debug(void);

parsec/mca/device/transfer_gpu.c

@@ -180,6 +180,16 @@ static const parsec_symbol_t symb_gpu_d2h_task_param = {
 
 int32_t parsec_gpu_d2h_max_flows = 0;
 
+/* Proactive eviction thresholds (percentage of total zone capacity).
+ * Registered as MCA parameters by each GPU backend component.
+ *   mem_evict_upper: initial percentage at which proactive eviction begins (default 95).
+ *   mem_evict_lower: floor to which the per-device threshold may adapt downwards (default 80).
+ * When a task stalls because no zone memory could be freed, the per-device
+ * mem_evict_threshold is lowered by 5 points (clamped to mem_evict_lower) so
+ * future eviction runs start sooner. */
+int32_t parsec_gpu_mem_evict_upper = 95;
+int32_t parsec_gpu_mem_evict_lower = 80;
+
 static const parsec_task_class_t parsec_gpu_d2h_task_class = {
     .name = "GPU D2H data transfer",
     .task_class_id = 0,
@@ -223,16 +233,20 @@ static const parsec_task_class_t parsec_gpu_d2h_task_class = {
  */
 parsec_gpu_task_t*
 parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device,
-                           parsec_execution_stream_t *es)
+                           parsec_execution_stream_t *es,
+                           size_t required_size,
+                           size_t *selected_size)
 {
     parsec_gpu_task_t *w2r_task = NULL;
     parsec_gpu_d2h_task_t *d2h_task = NULL;
     parsec_gpu_data_copy_t *gpu_copy;
     parsec_list_item_t* item = (parsec_list_item_t*)gpu_device->gpu_mem_owned_lru.ghost_element.list_next;
     int nb_cleaned = 0;
+    size_t _selected = 0;
 
-    /* Find a data copy that has no pending users on the GPU, and can be
-     * safely moved back on the main memory */
+    /* Find data copies with no pending GPU readers that can be safely moved back to
+     * main memory.  Stop once nb_cleaned reaches the max-flows cap or we have
+     * accumulated at least required_size bytes. */
     while(nb_cleaned < parsec_gpu_d2h_max_flows) {
         /* Break at the end of the list */
         if( item == &(gpu_device->gpu_mem_owned_lru.ghost_element) ) {
@@ -247,7 +261,7 @@ parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device,
                 d2h_task = (parsec_gpu_d2h_task_t*)parsec_thread_mempool_allocate(es->context_mempool);
                 if( PARSEC_UNLIKELY(NULL == d2h_task) ) { /* we're running out of memory. Bail out. */
                     parsec_atomic_unlock( &gpu_copy->original->lock );
-                    return NULL;
+                    break;
                 }
                 PARSEC_OBJ_CONSTRUCT(d2h_task, parsec_task_t);
             }
@@ -260,17 +274,22 @@ parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device,
             PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream,  "D2H[%d:%s] task %p:\tdata %d -> %p [%p] readers %d",
                                  gpu_device->super.device_index, gpu_device->super.name, (void*)d2h_task,
                                  nb_cleaned, gpu_copy, gpu_copy->original, gpu_copy->readers);
+            _selected += gpu_copy->original->nb_elts;
             nb_cleaned++;
-            if (MAX_PARAM_COUNT == nb_cleaned)
+            if( MAX_PARAM_COUNT == nb_cleaned || _selected >= required_size )
                 break;
         } else {
             parsec_atomic_unlock( &gpu_copy->original->lock );
         }
     }
 
+    *selected_size = _selected;
+
     if( 0 == nb_cleaned )
         return NULL;
 
+    gpu_device->mem_evict_in_flight += _selected;
+
     d2h_task->priority        = INT32_MAX;
     d2h_task->task_class      = &parsec_gpu_d2h_task_class;
     d2h_task->taskpool        = NULL;
@@ -310,6 +329,7 @@ int parsec_gpu_complete_w2r_task(parsec_device_gpu_module_t *gpu_device,
         gpu_copy->readers--;
         gpu_copy->data_transfer_status = PARSEC_DATA_STATUS_COMPLETE_TRANSFER;
         gpu_device->super.data_out_to_host += gpu_copy->original->nb_elts; /* TODO: not hardcoded, use datatype size */
+        gpu_device->mem_evict_in_flight -= gpu_copy->original->nb_elts;
         assert(gpu_copy->readers >= 0);
 
         original = gpu_copy->original;