79#define CS_MALLOC(_ptr, _ni, _type) \
80_ptr = (_type *) cs_mem_malloc(_ni, sizeof(_type), \
81 #_ptr, __FILE__, __LINE__)
99#define CS_MALLOC_HD(_ptr, _ni, _type, _mode) \
100_ptr = (_type *) cs_mem_malloc_hd(_mode, _ni, sizeof(_type), \
101 #_ptr, __FILE__, __LINE__)
115#define CS_REALLOC(_ptr, _ni, _type) \
116_ptr = (_type *) cs_mem_realloc(_ptr, _ni, sizeof(_type), \
117 #_ptr, __FILE__, __LINE__)
135#define CS_REALLOC_HD(_ptr, _ni, _type, _mode) \
136_ptr = (_type *) cs_mem_realloc_hd(_ptr, _mode, _ni, sizeof(_type), \
137 #_ptr, __FILE__, __LINE__)
155#define CS_FREE(_ptr) \
156cs_mem_free(_ptr, #_ptr, __FILE__, __LINE__), _ptr = NULL
171#define CS_MEMALIGN(_ptr, _align, _ni, _type) \
172_ptr = (_type *) cs_mem_memalign(_align, _ni, sizeof(_type), \
173 #_ptr, __FILE__, __LINE__)
179#if defined(HAVE_ACCEL)
187#define cs_alloc_mode CS_ALLOC_HOST
188#define cs_alloc_mode_read_mostly CS_ALLOC_HOST
189#define cs_alloc_mode_device CS_ALLOC_HOST
266 const char *var_name,
267 const char *file_name,
270#if defined(HAVE_ACCEL)
297 const char *var_name,
298 const char *file_name,
307 const char *var_name,
308 const char *file_name,
312 return cs_mem_malloc(ni, size, var_name, file_name, line_num);
341 const char *var_name,
342 const char *file_name,
375#if defined(HAVE_ACCEL)
378cs_mem_realloc_hd(
void *ptr,
382 const char *var_name,
383 const char *file_name,
389cs_mem_realloc_hd(
void *ptr,
393 const char *var_name,
394 const char *file_name,
398 return cs_mem_realloc(ptr, ni, size, var_name, file_name, line_num);
424 const char *var_name,
425 const char *file_name,
454 const char *var_name,
455 const char *file_name,
519#if defined(HAVE_OPENMP_TARGET)
530cs_mem_set_omp_target_device_id(
int device_id);
560 uint64_t *n_reallocs,
562 uint64_t *n_current);
568#if defined(__cplusplus)
570#if defined(HAVE_ACCEL)
586cs_copy_h2d(
void *dest,
604cs_copy_d2h(
void *dest,
622cs_copy_d2d(
void *dest,
644#if defined(HAVE_ACCEL)
659#if defined(HAVE_ACCEL)
689#if defined(HAVE_ACCEL)
696inline static const void *
704#if defined(HAVE_ACCEL)
713 return (
const T *)ptr_v;
729#if defined(HAVE_ACCEL)
756#if defined(HAVE_ACCEL)
788#if defined(HAVE_ACCEL)
797#define cs_associate_device_ptr(_host_ptr, _ni, _size);
812#if defined(HAVE_ACCEL)
819#define cs_disassociate_device_ptr(_host_ptr);
836#if defined(HAVE_ACCEL)
844#define cs_set_alloc_mode(_host_ptr, mode);
861#if defined(HAVE_ACCEL)
875#define cs_set_alloc_mode_r(_host_ptr, mode);
887#if defined(HAVE_ACCEL)
894#define cs_mem_advise_set_read_mostly(ptr);
906#if defined(HAVE_ACCEL)
939#if defined(HAVE_ACCEL)
972#if defined(HAVE_ACCEL)
1006#if defined(HAVE_ACCEL)
1040#if defined(HAVE_ACCEL)
1074#if defined(HAVE_ACCEL)
1108#if defined(HAVE_ACCEL)
1136#if defined(HAVE_ACCEL)
1146 [[maybe_unused]]
size_t size)
1165#if defined(HAVE_ACCEL)
1190#if defined(HAVE_ACCEL)
1204#if defined(HAVE_ACCEL)
1215cs_mem_device_pool_set_active(
bool status);
1231cs_mem_device_pool_set_max_capacity(
size_t size);
1242cs_mem_device_pool_set_max_tries(
short int n_tries);
1251cs_mem_device_pool_clear(
void);
void() bft_error_handler_t(const char *const file_name, const int line_num, const int sys_error_code, const char *const format, va_list arg_ptr)
Function pointer to opaque error handler.
Definition: bft_error.h:52
#define BEGIN_C_DECLS
Definition: cs_defs.h:554
#define CS_UNUSED(x)
Definition: cs_defs.h:543
#define END_C_DECLS
Definition: cs_defs.h:555
@ p
Definition: cs_field_pointer.h:67
int cs_mem_have_memalign(void)
Indicate if a memory aligned allocation variant is available.
Definition: cs_mem.cpp:2321
void * cs_mem_memalign(size_t alignment, size_t ni, size_t size, const char *var_name, const char *file_name, int line_num)
Allocate aligned memory for ni elements of size bytes.
Definition: cs_mem.cpp:2133
size_t cs_mem_size_max(void)
Return maximum theoretical dynamic memory allocated.
Definition: cs_mem.cpp:2218
int cs_mem_initialized(void)
Indicates if cs_mem_...() functions are initialized.
Definition: cs_mem.cpp:1885
void * cs_mem_malloc(size_t ni, size_t size, const char *var_name, const char *file_name, int line_num)
Allocate memory for ni elements of size bytes.
Definition: cs_mem.cpp:1914
bft_error_handler_t * cs_mem_error_handler_get(void)
Returns the error handler associated with the cs_mem_...() functions.
Definition: cs_mem.cpp:2339
void cs_mem_init(const char *log_file_name)
Initialize memory handling.
Definition: cs_mem.cpp:1714
size_t cs_mem_size_current(void)
Return current theoretical dynamic memory allocated.
Definition: cs_mem.cpp:2204
void cs_mem_end(void)
End memory handling.
Definition: cs_mem.cpp:1808
void * cs_mem_realloc(void *ptr, size_t ni, size_t size, const char *var_name, const char *file_name, int line_num)
Reallocate memory for ni elements of size bytes.
Definition: cs_mem.cpp:1973
void cs_mem_error_handler_set(bft_error_handler_t *handler)
Definition: cs_mem.cpp:2358
void * cs_mem_free(void *ptr, const char *var_name, const char *file_name, int line_num)
Free allocated memory.
Definition: cs_mem.cpp:2066
static void cs_sync_d2h_start(void *ptr)
Start synchronization of data from device to host.
Definition: cs_mem.h:1048
static void cs_sync_h2d_start(const void *ptr)
Start synchronization of data from host to device.
Definition: cs_mem.h:980
#define cs_mem_advise_set_read_mostly(ptr)
Advise memory system that a given allocation will be mostly read.
Definition: cs_mem.h:894
#define cs_alloc_mode_device
Definition: cs_mem.h:189
#define cs_set_alloc_mode_r(_host_ptr, mode)
Set allocation mode for an already allocated pointer using pass by reference semantics for the pointe...
Definition: cs_mem.h:875
static bool cs_mem_is_device_ptr(const void *ptr)
Check if a pointer is a device (or shared) pointer.
Definition: cs_mem.h:764
static void cs_prefetch_d2h(void *ptr, size_t size)
Prefetch data from device to host.
Definition: cs_mem.h:1174
static void cs_mem_advise_unset_read_mostly(void *ptr)
Advise memory system that a given allocation will be mostly read.
Definition: cs_mem.h:914
static const void * cs_get_device_ptr_const(const void *ptr)
Return matching device pointer for a given constant pointer.
Definition: cs_mem.h:697
#define cs_associate_device_ptr(_host_ptr, _ni, _size)
Associate device memory with a given host memory pointer.
Definition: cs_mem.h:797
static void cs_mem_hd_async_wait(void)
Wait for asynchronous memory operations between device and pinned host memory to finish.
Definition: cs_mem.h:1198
#define cs_set_alloc_mode(_host_ptr, mode)
Set allocation mode for an already allocated pointer.
Definition: cs_mem.h:844
static void cs_sync_d2h(void *ptr)
Synchronize data from device to host.
Definition: cs_mem.h:1014
#define cs_disassociate_device_ptr(_host_ptr)
Detach device memory from a given host memory pointer.
Definition: cs_mem.h:819
#define cs_alloc_mode
Definition: cs_mem.h:187
#define cs_alloc_mode_read_mostly
Definition: cs_mem.h:188
static void cs_prefetch_h2d(const void *ptr, size_t size)
Prefetch data from host to device.
Definition: cs_mem.h:1145
int cs_mem_stats(uint64_t *alloc_cur, uint64_t *alloc_max, uint64_t *n_allocs, uint64_t *n_reallocs, uint64_t *n_frees, uint64_t *n_current)
Return memory allocation stats, if available.
Definition: cs_mem.cpp:2272
static cs_alloc_mode_t cs_check_device_ptr(const void *ptr)
Check if a pointer is associated with a device.
Definition: cs_mem.h:737
static void cs_sync_d2h_if_needed_start(void *ptr)
Start synchronization of data from device to host, only if needed.
Definition: cs_mem.h:1116
static void cs_sync_d2h_if_needed(void *ptr)
Synchronize data from device to host, only if needed.
Definition: cs_mem.h:1082
static void cs_sync_h2d(const void *ptr)
Synchronize data from host to device.
Definition: cs_mem.h:947
static void * cs_get_device_ptr(void *ptr)
Return matching device pointer for a given pointer.
Definition: cs_mem.h:652
cs_alloc_mode_t
Definition: cs_mem.h:50
@ CS_ALLOC_HOST
Definition: cs_mem.h:52
@ CS_ALLOC_HOST_DEVICE_PINNED
Definition: cs_mem.h:54
@ CS_ALLOC_HOST_DEVICE_SHARED
Definition: cs_mem.h:57
@ CS_ALLOC_HOST_DEVICE
Definition: cs_mem.h:53
@ CS_ALLOC_DEVICE
Definition: cs_mem.h:59