cs_mem.h File Reference

#include "base/cs_defs.h"
#include "bft/bft_error.h"

Include dependency graph for cs_mem.h:

Go to the source code of this file.

Macros
#define	CS_MALLOC(_ptr, _ni, _type)
	Allocate memory for _ni elements of type _type. More...
#define	CS_MALLOC_HD(_ptr, _ni, _type, _mode)
#define	CS_REALLOC(_ptr, _ni, _type)
	Reallocate memory for _ni elements of type _type. More...
#define	CS_REALLOC_HD(_ptr, _ni, _type, _mode)
#define	CS_FREE(_ptr)   cs_mem_free(_ptr, #_ptr, __FILE__, __LINE__), _ptr = NULL
#define	CS_MEMALIGN(_ptr, _align, _ni, _type)
	Allocate aligned memory for _ni elements of type _type. More...
#define	cs_alloc_mode   CS_ALLOC_HOST
#define	cs_alloc_mode_read_mostly   CS_ALLOC_HOST
#define	cs_alloc_mode_device   CS_ALLOC_HOST
#define	cs_associate_device_ptr(_host_ptr, _ni, _size)   ;
	Associate device memory with a given host memory pointer. More...
#define	cs_disassociate_device_ptr(_host_ptr)   ;
	Detach device memory from a given host memory pointer. More...
#define	cs_set_alloc_mode(_host_ptr, mode)   ;
	Set allocation mode for an already allocated pointer. More...
#define	cs_set_alloc_mode_r(_host_ptr, mode)   ;
	Set allocation mode for an already allocated pointer using pass by reference semantics for the pointer (C++ only). More...
#define	cs_mem_advise_set_read_mostly(ptr)   ;
	Advise memory system that a given allocation will be mostly read. More...

Enumerations
enum	cs_alloc_mode_t {   CS_ALLOC_HOST , CS_ALLOC_HOST_DEVICE , CS_ALLOC_HOST_DEVICE_PINNED , CS_ALLOC_HOST_DEVICE_SHARED ,   CS_ALLOC_DEVICE }

Functions
int	cs_mem_stats (uint64_t *alloc_cur, uint64_t *alloc_max, uint64_t *n_allocs, uint64_t *n_reallocs, uint64_t *n_frees, uint64_t *n_current)
	Return memory allocation stats, if available. More...
static void *	cs_get_device_ptr (void *ptr)
	Return matching device pointer for a given pointer. More...
static const void *	cs_get_device_ptr_const (const void *ptr)
	Return matching device pointer for a given constant pointer. More...
static cs_alloc_mode_t	cs_check_device_ptr (const void *ptr)
	Check if a pointer is associated with a device. More...
static bool	cs_mem_is_device_ptr (const void *ptr)
	Check if a pointer is a device (or shared) pointer. More...
static void	cs_mem_advise_unset_read_mostly (void *ptr)
	Advise memory system that a given allocation will be mostly read. More...
static void	cs_sync_h2d (const void *ptr)
	Synchronize data from host to device. More...
static void	cs_sync_h2d_start (const void *ptr)
	Start synchronization of data from host to device. More...
static void	cs_sync_d2h (void *ptr)
	Synchronize data from device to host. More...
static void	cs_sync_d2h_start (void *ptr)
	Start synchronization of data from device to host. More...
static void	cs_sync_d2h_if_needed (void *ptr)
	Synchronize data from device to host, only if needed. More...
static void	cs_sync_d2h_if_needed_start (void *ptr)
	Start synchronization of data from device to host, only if needed. More...
static void	cs_prefetch_h2d (const void *ptr, size_t size)
	Prefetch data from host to device. More...
static void	cs_prefetch_d2h (void *ptr, size_t size)
	Prefetch data from device to host. More...
static void	cs_mem_hd_async_wait (void)
	Wait for asynchronous memory operations between device and pinned host memory to finish. More...

Macro Definition Documentation

cs_alloc_mode

#define cs_alloc_mode   CS_ALLOC_HOST

cs_alloc_mode_device

#define cs_alloc_mode_device   CS_ALLOC_HOST

cs_alloc_mode_read_mostly

#define cs_alloc_mode_read_mostly   CS_ALLOC_HOST

cs_associate_device_ptr

#define cs_associate_device_ptr	(		_host_ptr,
			_ni,
			_size
	)		;

Associate device memory with a given host memory pointer.

If the host memory is already associated with the device, the existing device pointer is returned. Otherwise, a new device allocation is called and returned.

Parameters

[in]	host_ptr	host pointer
[in]	ni	number of elements
[in]	size	element size

Returns

pointer to allocated memory.

cs_disassociate_device_ptr

#define cs_disassociate_device_ptr

(

_host_ptr

)

;

Detach device memory from a given host memory pointer.

If the host memory is shared with the device (i.e. using CS_ALLOC_SHARED), device memory stays shared.

Parameters

[in]

host_ptr

host pointer

CS_FREE

#define CS_FREE

(

_ptr

)

cs_mem_free(_ptr, #_ptr, __FILE__, __LINE__), _ptr = NULL

CS_MALLOC

CS_MALLOC	(		_ptr,
			_ni,
			_type
	)

Value:

_ptr = (_type *) cs_mem_malloc(_ni, sizeof(_type), \
                               #_ptr, __FILE__, __LINE__)

Allocate memory for _ni elements of type _type.

This macro calls cs_mem_malloc(), automatically setting the allocated variable name and source file name and line arguments.

Parameters

[out]	_ptr	pointer to allocated memory.
[in]	_ni	number of elements.
[in]	_type	element type.

CS_MALLOC_HD

#define CS_MALLOC_HD	(		_ptr,
			_ni,
			_type,
			_mode
	)

Value:

_ptr = (_type *) cs_mem_malloc_hd(_mode, _ni, sizeof(_type), \
                                  #_ptr, __FILE__, __LINE__)

cs_mem_advise_set_read_mostly

#define cs_mem_advise_set_read_mostly

(

ptr

)

;

Advise memory system that a given allocation will be mostly read.

Parameters

[in]

ptr

pointer to allocation

CS_MEMALIGN

CS_MEMALIGN	(		_ptr,
			_align,
			_ni,
			_type
	)

Value:

_ptr = (_type *) cs_mem_memalign(_align, _ni, sizeof(_type), \
                                 #_ptr, __FILE__, __LINE__)

Allocate aligned memory for _ni elements of type _type.

This macro calls cs_mem_memalign(), automatically setting the allocated variable name and source file name and line arguments.

Parameters

[out]	_ptr	pointer to allocated memory.
[in]	_align	alignment.
[in]	_ni	number of elements.
[in]	_type	element type.

CS_REALLOC

CS_REALLOC	(		_ptr,
			_ni,
			_type
	)

Value:

_ptr = (_type *) cs_mem_realloc(_ptr, _ni, sizeof(_type), \
                                #_ptr, __FILE__, __LINE__)

Reallocate memory for _ni elements of type _type.

This macro calls cs_mem_realloc(), automatically setting the allocated variable name and source file name and line arguments.

Parameters

[in,out]	_ptr	pointer to allocated memory.
[in]	_ni	number of elements.
[in]	_type	element type.

CS_REALLOC_HD

#define CS_REALLOC_HD	(		_ptr,
			_ni,
			_type,
			_mode
	)

Value:

_ptr = (_type *) cs_mem_realloc_hd(_ptr, _mode, _ni, sizeof(_type), \
                                   #_ptr, __FILE__, __LINE__)

cs_set_alloc_mode

#define cs_set_alloc_mode	(		_host_ptr,
			mode
	)		;

Set allocation mode for an already allocated pointer.

If the allocation mode is different from the previous one, the associated memory will be reallocated with the desired mode, and the previous allocation freed.

Parameters

[in,out]	host_ptr	pointer to host pointer to modify
[in]	mode	desired allocation mode

cs_set_alloc_mode_r

#define cs_set_alloc_mode_r	(		_host_ptr,
			mode
	)		;

Set allocation mode for an already allocated pointer using pass by reference semantics for the pointer (C++ only).

In C++, this function is preferred to direct use of cs_set_alloc_mode, as it allows a more concise syntax and does not require additional casting.

Parameters

[in,out]	host_ptr	reference to host pointer to modify
[in]	mode	desired allocation mode

Enumeration Type Documentation

cs_alloc_mode_t

enum cs_alloc_mode_t

Allocation modes for accelerated code.

Enumerator
CS_ALLOC_HOST	allocation on host only
CS_ALLOC_HOST_DEVICE	allocation on host and device
CS_ALLOC_HOST_DEVICE_PINNED	allocation on host and device, using page-locked memory on host if possible
CS_ALLOC_HOST_DEVICE_SHARED	allocation on host and device, using mapped/shared memory
CS_ALLOC_DEVICE	allocation on device only

Function Documentation

cs_check_device_ptr()

static cs_alloc_mode_t cs_check_device_ptr ( const void *  ptr )

inlinestatic

Check if a pointer is associated with a device.

If separate pointers are used on the host and device, the host pointer should be used with this function.

Returns

allocation mode associated with pointer

cs_get_device_ptr()

static void * cs_get_device_ptr ( void *  ptr )

inlinestatic

Return matching device pointer for a given pointer.

If separate pointers are used on the host and device, the host pointer should be used with this function.

If memory is not allocated on device yet at the call site, it will be allocated automatically by this function.

Parameters

[in]

ptr

pointer

Returns

pointer to device memory.

cs_get_device_ptr_const()

static const void * cs_get_device_ptr_const ( const void *  ptr )

inlinestatic

Return matching device pointer for a given constant pointer.

If separate pointers are used on the host and device, the host pointer should be used with this function.

If memory is not allocated on device yet at the call site, it will be allocated automatically by this function.

Parameters

[in]

ptr

pointer

Returns

pointer to device memory.

cs_mem_advise_unset_read_mostly()

static void cs_mem_advise_unset_read_mostly ( void *  ptr )

inlinestatic

Advise memory system that a given allocation will be mostly read.

Parameters

[in]

ptr

pointer to allocation

cs_mem_hd_async_wait()

static void cs_mem_hd_async_wait ( void  )

inlinestatic

Wait for asynchronous memory operations between device and pinned host memory to finish.

cs_mem_is_device_ptr()

static bool cs_mem_is_device_ptr ( const void *  ptr )

inlinestatic

Check if a pointer is a device (or shared) pointer.

Parameters

[in]

ptr

pointer to device data

Returns

true if the pointer is usable from the device or null, false if available on host only or if query failed.

cs_mem_stats()

int cs_mem_stats	(	uint64_t *	alloc_cur,
		uint64_t *	alloc_max,
		uint64_t *	n_allocs,
		uint64_t *	n_reallocs,
		uint64_t *	n_frees,
		uint64_t *	n_current
	)

Return memory allocation stats, if available.

Availability of statistics depends on the cs_mem_init options.

Parameters

[out]	alloc_cur	current allocation size, or nullptr
[out]	alloc_max	max allocation size, or nullptr
[out]	n_allocs	total number of allocations, or nullptr
[out]	n_reallocs	total number of reallocations, or nullptr
[out]	n_frees	total number of frees, or nullptr
[out]	n_current	total number of current allocations, or nullptr

\return 1 if stats are available, O otherwise.

Return memory allocation stats, if available.

Availability of statistics depends on the cs_mem_init options.

With acceleration, alloc_cur and alloc_max should have 4 entries each 0: host 1: pinned (included in host) 2: unified (included in host and device) 3: device

Parameters

[out]	alloc_cur	current allocation sizes, or nullptr
[out]	alloc_max	max allocation sizes, or nullptr
[out]	n_allocs	total number of allocations, or nullptr
[out]	n_reallocs	total number of reallocations, or nullptr
[out]	n_frees	total number of frees, or nullptr
[out]	n_current	total number of current allocations, or nullptr

Returns

1 if stats are available, O otherwise.

cs_prefetch_d2h()

static void cs_prefetch_d2h ( void *  ptr, size_t  size  )

inlinestatic

Prefetch data from device to host.

This function should only be used on arrays using shared host and device memory, shuch as those allocated using CS_ALLOC_HOST_DEVICE_SHARED. It should be usable on a subset of such an array.

Parameters

[in,out]	ptr	pointer to data to prefetch
[in]	size	number of bytes to prefetch

cs_prefetch_h2d()

static void cs_prefetch_h2d ( const void *  ptr, size_t  size  )

inlinestatic

Prefetch data from host to device.

This function should only be used on arrays using shared host and device memory, shuch as those allocated using CS_ALLOC_HOST_DEVICE_SHARED. It should be usable on a subset of such an array.

Parameters

[in,out]	ptr	pointer to data to prefetch
[in]	size	number of bytes to prefetch

cs_sync_d2h()

static void cs_sync_d2h ( void *  ptr )

inlinestatic

Synchronize data from device to host.

If separate allocations are used on the host and device (mode == CS_ALLOC_HOST_DEVICE), the host pointer should be passed to this function.

Depending on the allocaton type, this can imply a copy, data prefetch, or a no-op.

This function assumes the provided pointer was allocated using CS_MALLOC_HD or CS_REALLOC_HD, as it uses the associated mapping to determine associated metadata.

Parameters

[in,out]

ptr

pointer to values to copy or prefetch

cs_sync_d2h_if_needed()

static void cs_sync_d2h_if_needed ( void *  ptr )

inlinestatic

Synchronize data from device to host, only if needed.

If separate allocations are used on the host and device (mode == CS_ALLOC_HOST_DEVICE), the host pointer should be passed to this function.

Depending on the allocation type, this can imply a copy, data prefetch, or a no-op.

No operation occurs if the provided pointer was not allocated using CS_MALLOC_HD or CS_REALLOC_HD, as it uses the associated mapping to determine associated metadata.

Parameters

[in,out]

ptr

pointer to values to copy or prefetch

cs_sync_d2h_if_needed_start()

static void cs_sync_d2h_if_needed_start ( void *  ptr )

inlinestatic

Start synchronization of data from device to host, only if needed.

If separate allocations are used on the host and device (mode == CS_ALLOC_HOST_DEVICE), the host pointer should be passed to this function.

Depending on the allocation type, this can imply a copy, data prefetch, or a no-op.

No operation occurs if the provided pointer was not allocated using CS_MALLOC_HD or CS_REALLOC_HD, as it uses the associated mapping to determine associated metadata.

Parameters

[in,out]

ptr

pointer to values to copy or prefetch

cs_sync_d2h_start()

static void cs_sync_d2h_start ( void *  ptr )

inlinestatic

Start synchronization of data from device to host.

If separate allocations are used on the host and device (mode == CS_ALLOC_HOST_DEVICE), the host pointer should be passed to this function.

Depending on the allocaton type, this can imply a copy, data prefetch, or a no-op.

This function assumes the provided pointer was allocated using CS_MALLOC_HD or CS_REALLOC_HD, as it uses the associated mapping to determine associated metadata.

Parameters

[in,out]

ptr

pointer to values to copy or prefetch

cs_sync_h2d()

static void cs_sync_h2d ( const void *  ptr )

inlinestatic

Synchronize data from host to device.

If separate pointers are used on the host and device, the host pointer should be used with this function.

Depending on the allocation type, this can imply a copy, data prefetch, or a no-op.

This function assumes the provided pointer was allocated using CS_MALLOC_HD or CS_REALLOC_HD, as it uses the associated mapping to determine associated metadata.

Parameters

[in,out]

ptr

host pointer to values to copy or prefetch

cs_sync_h2d_start()

static void cs_sync_h2d_start ( const void *  ptr )

inlinestatic

Start synchronization of data from host to device.

If separate pointers are used on the host and device, the host pointer should be used with this function.

Depending on the allocation type, this can imply a copy, data prefetch, or a no-op.

This function assumes the provided pointer was allocated using CS_MALLOC_HD or CS_REALLOC_HD, as it uses the associated mapping to determine associated metadata.

Parameters

[in,out]

ptr

host pointer to values to copy or prefetch