diff --git a/parsec/mca/device/device.c b/parsec/mca/device/device.c index 8ae216bbe..15136d1ff 100644 --- a/parsec/mca/device/device.c +++ b/parsec/mca/device/device.c @@ -312,6 +312,13 @@ no_valid_device: { PARSEC_OBJ_CLASS_INSTANCE(parsec_device_module_t, parsec_object_t, NULL, NULL); +/* Proactive GPU eviction thresholds, registered as MCA parameters in + * parsec_mca_device_init() below. Defined here (not transfer_gpu.c) so the + * symbols always exist and the parameters can be registered unconditionally, + * regardless of which GPU backends were compiled in. */ +int32_t parsec_gpu_mem_evict_upper = 95; +int32_t parsec_gpu_mem_evict_lower = 80; + int parsec_mca_device_type_supports_batch(uint32_t device_type) { @@ -339,6 +346,39 @@ int parsec_mca_device_init(void) PARSEC_OBJ_CONSTRUCT(&parsec_per_device_infos, parsec_info_t); PARSEC_OBJ_CONSTRUCT(&parsec_per_stream_infos, parsec_info_t); + (void)parsec_mca_param_reg_int_name("device", "mem_evict_upper", + "Upper threshold (percentage of total GPU zone capacity) at which proactive " + "clean-LRU eviction and D2H writeback begin. When the device is truly stalled " + "(no in-flight evictions and no dirty pages left to queue), the per-device " + "threshold is stepped down by 5 points toward device_mem_evict_lower. " + "Valid range [0,100]; must be >= device_mem_evict_lower.", + false, false, 95, &parsec_gpu_mem_evict_upper); + (void)parsec_mca_param_reg_int_name("device", "mem_evict_lower", + "Lower bound (percentage of total GPU zone capacity) to which the adaptive " + "eviction threshold may be reduced after repeated stalls. " + "Valid range [0,100]; must be <= device_mem_evict_upper.", + false, false, 80, &parsec_gpu_mem_evict_lower); + if( 0 < (rc = parsec_mca_param_find("device", NULL, "mem_evict_upper")) ) + parsec_mca_param_lookup_int(rc, &parsec_gpu_mem_evict_upper); + if( 0 < (rc = parsec_mca_param_find("device", NULL, "mem_evict_lower")) ) + parsec_mca_param_lookup_int(rc, &parsec_gpu_mem_evict_lower); + if( parsec_gpu_mem_evict_upper < 0 || parsec_gpu_mem_evict_upper > 100 ) { + parsec_warning("device_mem_evict_upper=%d is out of range [0,100], clamped", + parsec_gpu_mem_evict_upper); + parsec_gpu_mem_evict_upper = parsec_gpu_mem_evict_upper < 0 ? 0 : 100; + } + if( parsec_gpu_mem_evict_lower < 0 || parsec_gpu_mem_evict_lower > 100 ) { + parsec_warning("device_mem_evict_lower=%d is out of range [0,100], clamped", + parsec_gpu_mem_evict_lower); + parsec_gpu_mem_evict_lower = parsec_gpu_mem_evict_lower < 0 ? 0 : 100; + } + if( parsec_gpu_mem_evict_lower > parsec_gpu_mem_evict_upper ) { + parsec_warning("device_mem_evict_lower=%d > device_mem_evict_upper=%d, swapping", + parsec_gpu_mem_evict_lower, parsec_gpu_mem_evict_upper); + int32_t _tmp = parsec_gpu_mem_evict_lower; + parsec_gpu_mem_evict_lower = parsec_gpu_mem_evict_upper; + parsec_gpu_mem_evict_upper = _tmp; + } (void)parsec_mca_param_reg_int_name("device", "show_capabilities", "Show the detailed devices capabilities", false, false, parsec_debug_verbose >= 4 || (parsec_debug_verbose >= 3 && parsec_debug_rank == 0), NULL); diff --git a/parsec/mca/device/device_gpu.c b/parsec/mca/device/device_gpu.c index 67e7ca6c3..a0e5e01b7 100644 --- a/parsec/mca/device/device_gpu.c +++ b/parsec/mca/device/device_gpu.c @@ -986,6 +986,7 @@ parsec_device_memory_reserve( parsec_device_gpu_module_t* gpu_device, #endif gpu_device->mem_block_size = eltsize; gpu_device->mem_nb_blocks = mem_elem_per_gpu; + gpu_device->mem_evict_threshold = parsec_gpu_mem_evict_upper; return PARSEC_SUCCESS; } @@ -1196,6 +1197,95 @@ parsec_device_get_copy( parsec_device_gpu_module_t* gpu_device, parsec_data_copy return PARSEC_SUCCESS; } +/** + * Try to evict one entry from the clean LRU (gpu_mem_lru) by detaching it from + * its original data and freeing its zone allocation back to the zone allocator. + * + * @param[in] gpu_device the GPU device whose clean LRU is targeted + * @param[in,out] cycling_sentinel cycle-detector: caller initialises to NULL; + * updated to the first entry that could not be evicted. When + * we pop that entry again we know we have looped the entire LRU + * without finding a free-able entry. + * + * @return 1 if a zone block was freed, 0 if the LRU is empty or fully cycling. + */ +#if !defined(PARSEC_GPU_ALLOC_PER_TILE) +static int +parsec_device_try_evict_lru_one( parsec_device_gpu_module_t *gpu_device, + parsec_gpu_data_copy_t **cycling_sentinel ) +{ + parsec_gpu_data_copy_t *lru_gpu_elem; + parsec_data_t *oldmaster; + + retry: + lru_gpu_elem = (parsec_gpu_data_copy_t*)parsec_list_pop_front(&gpu_device->gpu_mem_lru); + if( NULL == lru_gpu_elem ) + return 0; + PARSEC_LIST_ITEM_SINGLETON(lru_gpu_elem); + + if( *cycling_sentinel == lru_gpu_elem ) { + parsec_list_push_front(&gpu_device->gpu_mem_lru, (parsec_list_item_t*)lru_gpu_elem); + return 0; + } + + /* Dangling reader: the copy is temporarily untracked in the LRU; skip it */ + if( 0 != lru_gpu_elem->readers ) + goto retry; + + /* Outstanding object references: not safe to free yet; push back and note cycle */ + if( lru_gpu_elem->super.super.obj_reference_count > 1 ) { + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == *cycling_sentinel ) *cycling_sentinel = lru_gpu_elem; + goto retry; + } + + if( NULL != lru_gpu_elem->original ) { + oldmaster = lru_gpu_elem->original; + if( !parsec_atomic_trylock(&oldmaster->lock) ) { + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == *cycling_sentinel ) *cycling_sentinel = lru_gpu_elem; + goto retry; + } + /* Guard against a concurrent d2d reader acquiring the copy */ + if( !parsec_atomic_cas_int32(&lru_gpu_elem->readers, 0, + -PARSEC_DEVICE_DATA_COPY_ATOMIC_SENTINEL) ) { + parsec_list_push_back(&gpu_device->gpu_mem_lru, &lru_gpu_elem->super); + if( NULL == *cycling_sentinel ) *cycling_sentinel = lru_gpu_elem; + parsec_atomic_unlock(&oldmaster->lock); + goto retry; + } + int do_unlock = oldmaster->super.obj_reference_count != 1; + parsec_data_copy_detach(oldmaster, lru_gpu_elem, gpu_device->super.device_index); + parsec_atomic_wmb(); + if( do_unlock ) + parsec_atomic_unlock(&oldmaster->lock); + } + +#if defined(PARSEC_PROF_TRACE) + if( (gpu_device->trackable_events & PARSEC_PROFILE_GPU_TRACK_MEM_USE) && + (gpu_device->exec_stream[0]->prof_event_track_enable || + gpu_device->exec_stream[1]->prof_event_track_enable) ) { + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_free_memory_key, + (int64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, + NULL, PARSEC_PROFILING_EVENT_COUNTER); + parsec_profiling_trace_flags(gpu_device->exec_stream[0]->profiling, + parsec_gpu_use_memory_key_end, + (uint64_t)lru_gpu_elem->device_private, + gpu_device->super.device_index, NULL, 0); + } +#endif + assert( 0 != (lru_gpu_elem->flags & PARSEC_DATA_FLAG_PARSEC_OWNED) ); + zone_free(gpu_device->memory, (void*)lru_gpu_elem->device_private); + lru_gpu_elem->device_private = NULL; + gpu_device->super.nb_evictions++; + PARSEC_OBJ_RELEASE(lru_gpu_elem); + assert( NULL == lru_gpu_elem ); + return 1; +} +#endif /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */ + /** * Try to find memory space to move all data on the GPU. We attach a device_elem to * a memory_elem as soon as a device_elem is available. If we fail to find enough @@ -1233,6 +1323,24 @@ parsec_device_data_reserve_space( parsec_device_gpu_module_t* gpu_device, (void)copy_readers_update; // potentially unused +#if !defined(PARSEC_GPU_ALLOC_PER_TILE) + /* Tier-1 proactive eviction: free clean LRU entries while zone usage exceeds + * gpu_device->mem_evict_threshold percent of total capacity. The threshold + * starts at parsec_gpu_mem_evict_upper (default 95%) and is lowered in 5-point + * steps (floor: parsec_gpu_mem_evict_lower, default 80%) whenever the device stalled + * because the reactive path also failed to find memory. */ + { + size_t total_capacity = (size_t)gpu_device->mem_nb_blocks * gpu_device->mem_block_size; + parsec_gpu_data_copy_t *cycling = NULL; + while( zone_in_use(gpu_device->memory) * 100 > + (size_t)gpu_device->mem_evict_threshold * total_capacity ) { + if( !parsec_device_try_evict_lru_one(gpu_device, &cycling) ) + break; + data_avail_epoch++; + } + } +#endif /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */ + /** * Parse all the input and output flows of data and ensure all have * corresponding data on the GPU available. @@ -3364,6 +3472,20 @@ parsec_device_kernel_cleanout( parsec_device_gpu_module_t *gpu_device, return 0; } +/** + * Returns false if at least one of the execution stream fifos has pending tasks. + * Otherwise, returns true, meaning that the GPU has no new work to schedule into the stream. + */ +static bool gpu_device_exec_streams_fifo_empty( parsec_device_gpu_module_t *gpu_device ) +{ + for (int i = 2; i < gpu_device->num_exec_streams; i++) { + if( !parsec_list_nolock_is_empty(gpu_device->exec_stream[i].fifo_pending) ) { + return false; + } + } + return true; +} + /** * This version is based on 4 streams: one for transfers from the memory to * the GPU, 2 for kernel executions and one for transfers from the GPU into @@ -3442,6 +3564,66 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, gpu_device->super.device_index, gpu_device->super.name, parsec_device_describe_gpu_task(tmp, MAX_TASK_STRLEN, gpu_task)); } + + /* Tier-2 proactive dirty-page writeback: when the clean LRU is empty and zone + * memory pressure exceeds the watermark, queue a D2H transfer on exec_stream[1] + * now so its latency overlaps with the upcoming H2D stage and kernel execution. + * This converts a potential blocking wait (dirty page eviction on the critical + * path) into an overlapped background transfer. + * + * In PARSEC_GPU_ALLOC_PER_TILE mode there is no zone allocator; fall back to the + * simple condition of clean LRU being empty. */ + if( !parsec_list_nolock_is_empty(&gpu_device->gpu_mem_owned_lru) && + parsec_list_nolock_is_empty(&gpu_device->gpu_mem_lru) ) { +#if !defined(PARSEC_GPU_ALLOC_PER_TILE) + { + size_t total_capacity = (size_t)gpu_device->mem_nb_blocks * gpu_device->mem_block_size; + size_t in_use = zone_in_use(gpu_device->memory); + size_t threshold_bytes = (size_t)gpu_device->mem_evict_threshold * total_capacity / 100; + if( in_use > threshold_bytes ) { + /* Compute how many more bytes of dirty-page eviction are needed beyond + * what is already in-flight on exec_stream[1]. */ + size_t needed = in_use - threshold_bytes; + size_t still_needed = (needed > gpu_device->mem_evict_in_flight) ? + (needed - gpu_device->mem_evict_in_flight) : 0; + while( still_needed > 0 ) { + size_t selected = 0; + parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es, + still_needed, &selected); + if( NULL == _w2r ) break; + _w2r->task_type = PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER; + parsec_atomic_fetch_add_int32(&gpu_device->mutex, 1); + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, " + "zone above %d%% threshold; needed %zu, selected %zu bytes", + gpu_device->super.device_index, gpu_device->super.name, + gpu_device->mem_evict_threshold, still_needed, selected); + PARSEC_PUSH_TASK(gpu_device->exec_stream[1]->fifo_pending, + (parsec_list_item_t*)_w2r); + still_needed = (still_needed > selected) ? (still_needed - selected) : 0; + } + } + } +#else + { + /* No zone allocator in ALLOC_PER_TILE mode: issue one D2H batch whenever + * the clean LRU is empty and the dirty LRU is non-empty. */ + size_t selected = 0; + parsec_gpu_task_t *_w2r = parsec_gpu_create_w2r_task(gpu_device, es, + SIZE_MAX, &selected); + if( NULL != _w2r ) { + _w2r->task_type = PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER; + parsec_atomic_fetch_add_int32(&gpu_device->mutex, 1); + PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, + "GPU[%d:%s]: Proactive D2H writeback: clean LRU empty, selected %zu bytes", + gpu_device->super.device_index, gpu_device->super.name, selected); + PARSEC_PUSH_TASK(gpu_device->exec_stream[1]->fifo_pending, + (parsec_list_item_t*)_w2r); + } + } +#endif /* !defined(PARSEC_GPU_ALLOC_PER_TILE) */ + } + rc = parsec_device_progress_stream( gpu_device, es, gpu_device->exec_stream[0], @@ -3462,8 +3644,23 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, assert(NULL == progress_task); /* TODO: check this */ - /* If we can extract data go for it, otherwise try to drain the pending tasks */ - gpu_task = parsec_gpu_create_w2r_task(gpu_device, es); + /* If we can extract data go for it, otherwise try to drain the pending tasks. + * Skip if evictions are already in flight to avoid a storm of D2H tasks. + * If there are no in-flight evictions and nothing left to queue from the dirty + * LRU, we are truly stuck: step the eviction threshold down so tier-1 starts + * freeing pages earlier on the next attempt. + * We don't evict from the dirty LRU if there are pending tasks in the execution streams. + * There is a good chance that memory will become available once the active tasks complete and we still + * have more tasks to execute. */ + if( 0 == gpu_device->mem_evict_in_flight && gpu_device_exec_streams_fifo_empty(gpu_device) ) { + size_t _sel = 0; + gpu_task = parsec_gpu_create_w2r_task(gpu_device, es, SIZE_MAX, &_sel); + if(gpu_device->mem_evict_threshold - 5 >= parsec_gpu_mem_evict_lower) { + /* We had to trigger a reactive D2H writeback so reduce the threshold to + * be more aggressive in the proactive part. */ + gpu_device->mem_evict_threshold -= 5; + } + } if( NULL != gpu_task ) goto get_data_out_of_device; } @@ -3568,9 +3765,26 @@ parsec_device_kernel_scheduler( parsec_device_module_t *module, parsec_task_snprintf(tmp, MAX_TASK_STRLEN, gpu_task->ec)); /* Everything went fine so far, the result is correct and back in the main memory */ PARSEC_LIST_ITEM_SINGLETON(gpu_task); - if (gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2HTRANSFER) { + if (gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2HTRANSFER || + gpu_task->task_type == PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER) { + int _proactive = (gpu_task->task_type == PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER); parsec_gpu_complete_w2r_task(gpu_device, gpu_task, es); + /* gpu_task freed inside parsec_gpu_complete_w2r_task */ gpu_task = progress_task; + if( _proactive ) { + /* Proactive task owned a mutex count; release it now. */ + rc = parsec_atomic_fetch_dec_int32( &(gpu_device->mutex) ); + if( 1 == rc ) { /* I was the last one */ +#if defined(PARSEC_PROF_TRACE) + if( gpu_device->trackable_events & PARSEC_PROFILE_GPU_TRACK_OWN ) + PARSEC_PROFILING_TRACE( es->es_profile, parsec_gpu_own_GPU_key_end, + (unsigned long)es, PROFILE_OBJECT_ID_NULL, NULL ); +#endif + PARSEC_DEBUG_VERBOSE(5, parsec_gpu_output_stream, "GPU[%d:%s]: Leaving GPU management", + gpu_device->super.device_index, gpu_device->super.name); + return PARSEC_HOOK_RETURN_ASYNC; + } + } goto fetch_task_from_shared_queue; } if (gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2D_COMPLETE) { diff --git a/parsec/mca/device/device_gpu.h b/parsec/mca/device/device_gpu.h index b36a40718..ee751600e 100644 --- a/parsec/mca/device/device_gpu.h +++ b/parsec/mca/device/device_gpu.h @@ -275,6 +275,16 @@ struct parsec_device_gpu_module_s { parsec_gpu_exec_stream_t **exec_stream; size_t mem_block_size; int64_t mem_nb_blocks; + int32_t mem_evict_threshold; /**< Current eviction threshold (% of total zone + * capacity). Starts at parsec_gpu_mem_evict_upper + * and is stepped down by 5 points (to + * parsec_gpu_mem_evict_lower) each time a task + * stalls waiting for zone memory. */ + size_t mem_evict_in_flight; /**< Bytes of dirty GPU data currently selected for + * D2H eviction (queued or executing on + * exec_stream[1]). Incremented by + * parsec_gpu_create_w2r_task, decremented as each + * copy completes in parsec_gpu_complete_w2r_task. */ #if defined(PARSEC_PROF_TRACE) int trackable_events; #endif /* PARSEC_PROF_TRACE */ @@ -309,6 +319,8 @@ typedef struct parsec_gpu_workspace_s { PARSEC_DECLSPEC extern int parsec_gpu_output_stream; PARSEC_DECLSPEC extern int parsec_gpu_verbosity; PARSEC_DECLSPEC extern int32_t parsec_gpu_d2h_max_flows; +PARSEC_DECLSPEC extern int32_t parsec_gpu_mem_evict_upper; +PARSEC_DECLSPEC extern int32_t parsec_gpu_mem_evict_lower; /** * Debugging functions. @@ -334,7 +346,8 @@ int parsec_device_free_workspace(parsec_device_gpu_module_t * gpu_device); /* sort pending task list by number of spaces needed */ int parsec_device_sort_pending_list(parsec_device_module_t *gpu_device); -parsec_gpu_task_t* parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, parsec_execution_stream_t *es); +parsec_gpu_task_t* parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, parsec_execution_stream_t *es, + size_t required_size, size_t *selected_size); int parsec_gpu_complete_w2r_task(parsec_device_gpu_module_t *gpu_device, parsec_gpu_task_t *w2r_task, parsec_execution_stream_t *es); /** @@ -366,12 +379,13 @@ void parsec_device_enable_debug(void); char *parsec_device_describe_gpu_task( char *tmp, size_t len, parsec_gpu_task_t *gpu_task ); #endif -#define PARSEC_GPU_TASK_TYPE_KERNEL 0x0000 -#define PARSEC_GPU_TASK_TYPE_D2HTRANSFER 0x1000 -#define PARSEC_GPU_TASK_TYPE_PREFETCH 0x2000 -#define PARSEC_GPU_TASK_TYPE_WARMUP 0x4000 -#define PARSEC_GPU_TASK_TYPE_D2D_COMPLETE 0x8000 -#define PARSEC_GPU_TASK_TYPE_INVALID 0xf000 +#define PARSEC_GPU_TASK_TYPE_KERNEL 0x0000 +#define PARSEC_GPU_TASK_TYPE_D2HTRANSFER 0x1000 +#define PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER 0x1001 /**< Tier-2 proactive D2H: counts in device->mutex */ +#define PARSEC_GPU_TASK_TYPE_PREFETCH 0x2000 +#define PARSEC_GPU_TASK_TYPE_WARMUP 0x4000 +#define PARSEC_GPU_TASK_TYPE_D2D_COMPLETE 0x8000 +#define PARSEC_GPU_TASK_TYPE_INVALID 0xf000 #if defined(PARSEC_PROF_TRACE) #define PARSEC_PROFILE_GPU_TRACK_DATA_IN 0x0001 diff --git a/parsec/mca/device/transfer_gpu.c b/parsec/mca/device/transfer_gpu.c index f0415383b..9f9bb344b 100644 --- a/parsec/mca/device/transfer_gpu.c +++ b/parsec/mca/device/transfer_gpu.c @@ -180,6 +180,10 @@ static const parsec_symbol_t symb_gpu_d2h_task_param = { int32_t parsec_gpu_d2h_max_flows = 0; +/* parsec_gpu_mem_evict_upper and parsec_gpu_mem_evict_lower are defined in + * device.c so they exist even in non-GPU builds and can be registered as MCA + * parameters unconditionally. */ + static const parsec_task_class_t parsec_gpu_d2h_task_class = { .name = "GPU D2H data transfer", .task_class_id = 0, @@ -223,16 +227,20 @@ static const parsec_task_class_t parsec_gpu_d2h_task_class = { */ parsec_gpu_task_t* parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, - parsec_execution_stream_t *es) + parsec_execution_stream_t *es, + size_t required_size, + size_t *selected_size) { parsec_gpu_task_t *w2r_task = NULL; parsec_gpu_d2h_task_t *d2h_task = NULL; parsec_gpu_data_copy_t *gpu_copy; parsec_list_item_t* item = (parsec_list_item_t*)gpu_device->gpu_mem_owned_lru.ghost_element.list_next; int nb_cleaned = 0; + size_t _selected = 0; - /* Find a data copy that has no pending users on the GPU, and can be - * safely moved back on the main memory */ + /* Find data copies with no pending GPU readers that can be safely moved back to + * main memory. Stop once nb_cleaned reaches the max-flows cap or we have + * accumulated at least required_size bytes. */ while(nb_cleaned < parsec_gpu_d2h_max_flows) { /* Break at the end of the list */ if( item == &(gpu_device->gpu_mem_owned_lru.ghost_element) ) { @@ -261,7 +269,7 @@ parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, d2h_task = (parsec_gpu_d2h_task_t*)parsec_thread_mempool_allocate(es->context_mempool); if( PARSEC_UNLIKELY(NULL == d2h_task) ) { /* we're running out of memory. Bail out. */ parsec_atomic_unlock( &gpu_copy->original->lock ); - return NULL; + break; } PARSEC_OBJ_CONSTRUCT(d2h_task, parsec_task_t); } @@ -274,17 +282,22 @@ parsec_gpu_create_w2r_task(parsec_device_gpu_module_t *gpu_device, PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "D2H[%d:%s] task %p:\tdata %d -> %p [%p] readers %d", gpu_device->super.device_index, gpu_device->super.name, (void*)d2h_task, nb_cleaned, gpu_copy, gpu_copy->original, gpu_copy->readers); + _selected += gpu_copy->original->nb_elts; nb_cleaned++; - if (MAX_PARAM_COUNT == nb_cleaned) + if( MAX_PARAM_COUNT == nb_cleaned || _selected >= required_size ) break; } else { parsec_atomic_unlock( &gpu_copy->original->lock ); } } + *selected_size = _selected; + if( 0 == nb_cleaned ) return NULL; + gpu_device->mem_evict_in_flight += _selected; + d2h_task->priority = INT32_MAX; d2h_task->task_class = &parsec_gpu_d2h_task_class; d2h_task->taskpool = NULL; @@ -316,13 +329,15 @@ int parsec_gpu_complete_w2r_task(parsec_device_gpu_module_t *gpu_device, PARSEC_DEBUG_VERBOSE(10, parsec_gpu_output_stream, "D2H[%d:%s] task %p: %d data transferred to host", gpu_device->super.device_index, gpu_device->super.name, (void*)task, task->locals[0].value); - assert(gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2HTRANSFER); + assert(gpu_task->task_type == PARSEC_GPU_TASK_TYPE_D2HTRANSFER || + gpu_task->task_type == PARSEC_GPU_TASK_TYPE_PROACTIVE_D2HTRANSFER); for( int i = 0; i < task->locals[0].value; i++ ) { gpu_copy = task->data[i].data_out; parsec_atomic_lock(&gpu_copy->original->lock); gpu_copy->readers--; gpu_copy->data_transfer_status = PARSEC_DATA_STATUS_COMPLETE_TRANSFER; gpu_device->super.data_out_to_host += gpu_copy->original->span; /* TODO: not hardcoded, use datatype size */ + gpu_device->mem_evict_in_flight -= gpu_copy->original->nb_elts; assert(gpu_copy->readers >= 0); original = gpu_copy->original;