Alexandria  2.25.0
SDC-CH common library for the Euclid project
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Classes | Functions | Variables
Euclid::NdArray Namespace Reference

Classes

struct  NpyDtype
 
struct  NpyDtype< int8_t >
 
struct  NpyDtype< int16_t >
 
struct  NpyDtype< int32_t >
 
struct  NpyDtype< int64_t >
 
struct  NpyDtype< uint8_t >
 
struct  NpyDtype< uint16_t >
 
struct  NpyDtype< uint32_t >
 
struct  NpyDtype< uint64_t >
 
struct  NpyDtype< float >
 
struct  NpyDtype< double >
 
class  MappedContainer
 
class  NdArray
 

Functions

void parseSingleValue (const std::string &descr, bool &big_endian, std::string &dtype)
 
void parseFieldValues (const std::string &descr, bool &big_endian, std::vector< std::string > &attrs, std::string &dtype)
 
void parseNpyDict (const std::string &header, bool &fortran_order, bool &big_endian, std::string &dtype, std::vector< size_t > &shape, std::vector< std::string > &attrs, size_t &n_elements)
 
void readNpyHeader (std::istream &input, std::string &dtype, std::vector< size_t > &shape, std::vector< std::string > &attrs, size_t &n_elements)
 
std::string npyShape (std::vector< size_t > shape)
 
std::string typeDescription (const std::string &type, const std::vector< std::string > &attrs)
 
template<typename T >
void writeNpyHeader (std::ostream &out, std::vector< size_t > shape, const std::vector< std::string > &attrs)
 
template<typename T >
void writeNpy (std::ostream &out, const NdArray< T > &array)
 
template<typename T >
NdArray< T > readNpy (std::istream &input)
 
template<typename T >
void writeNpy (const boost::filesystem::path &path, const NdArray< T > &array)
 
template<typename T >
NdArray< T > readNpy (const boost::filesystem::path &path)
 
template<typename T >
NdArray< T > mmapNpy (const boost::filesystem::path &path, boost::iostreams::mapped_file_base::mapmode mode=boost::iostreams::mapped_file_base::readwrite, size_t max_size=0)
 
template<typename T >
NdArray< T > createMmapNpy (const boost::filesystem::path &path, const std::vector< size_t > &shape, const std::vector< std::string > &attr_names, size_t max_size=0)
 
template<typename T >
NdArray< T > createMmapNpy (const boost::filesystem::path &path, const std::vector< size_t > &shape, size_t max_size=0)
 
template<typename T >
std::ostreamoperator<< (std::ostream &out, const NdArray< T > &ndarray)
 
std::vector< std::size_tunravel_index (std::size_t index, const std::vector< std::size_t > &shape)
 
template<typename T >
sum (const NdArray< T > &array)
 
template<typename T >
NdArray< T > sum (const NdArray< T > &array, int axis)
 
template<typename T , typename Iterator >
NdArray< T > trapz (const NdArray< T > &array, const Iterator kbegin, const Iterator kend, int axis)
 
template<typename T >
std::vector< std::size_targmax (const NdArray< T > &array)
 
template<typename T >
std::vector< std::size_targmin (const NdArray< T > &array)
 
template<typename T >
void sort (NdArray< T > &array, const std::vector< std::string > &attrs)
 

Variables

constexpr const char NPY_MAGIC [] = {'\x93', 'N', 'U', 'M', 'P', 'Y'}
 
constexpr const char * ENDIAN_MARKER = "<"
 
constexpr const uint8_t NPY_VERSION [] = {'\x02', '\x00'}
 

Function Documentation

template<typename T >
std::vector<std::size_t> Euclid::NdArray::argmax ( const NdArray< T > &  array)

Return the coordinates for the maximum element

Template Parameters
TNdArray type
Parameters
arrayThe ndarray
Returns
Coordinates for the element with the highest value
template<typename T >
std::vector<std::size_t> Euclid::NdArray::argmin ( const NdArray< T > &  array)

Return the coordinates for the minimum element

Template Parameters
TNdArray type
Parameters
arrayThe ndarray
Returns
Coordinates for the element with the lowest value
template<typename T >
NdArray<T> Euclid::NdArray::createMmapNpy ( const boost::filesystem::path path,
const std::vector< size_t > &  shape,
const std::vector< std::string > &  attr_names,
size_t  max_size = 0 
)

Create using mmap an NdArray backed by a numpy file

Template Parameters
TNdArray cell type
Parameters
pathOutput path
shapeNdArray shape
attr_namesAttribute names
max_sizeMaximum size of the file. If you are going to call NdArray<T>::concatenate, mind this parameter!
Returns
A new NdArray
template<typename T >
NdArray<T> Euclid::NdArray::createMmapNpy ( const boost::filesystem::path path,
const std::vector< size_t > &  shape,
size_t  max_size = 0 
)

Create using mmap an NdArray backed by a numpy file

Template Parameters
TNdArray cell type
Parameters
pathOutput path
shapeNdArray shape
max_sizeMaximum size of the file. If you are going to call NdArray<T>::concatenate, mind this parameter!
Returns
A new NdArray

Definition at line 86 of file NpyMmap.h.

template<typename T >
NdArray<T> Euclid::NdArray::mmapNpy ( const boost::filesystem::path path,
boost::iostreams::mapped_file_base::mapmode  mode = boost::iostreams::mapped_file_base::readwrite,
size_t  max_size = 0 
)

Open using mmap an existing numpy file

Template Parameters
TNdArray cell type
Parameters
pathInput path
modeOpen mode. By default read/write, so changes are persisted to disk. boost::iostreams::mapped_file_base::priv enabled a Copy-On-Write, so the memory can be modified but the changes will not persist
max_sizeMaximum size of the file. If you are going to call NdArray<T>::concatenate, mind this parameter!
Returns
A new NdArray
Note
The underlying numpy format is expected to match the template type T
If you open in read-only mode, assign to a const NdArray to avoid accidental writes
std::string Euclid::NdArray::npyShape ( std::vector< size_t >  shape)
inline

Generate a string that represents an NdArray shape vector as a Python tuple

Parameters
shapeA string with the Python representation of a tuple

Definition at line 168 of file NpyCommon.h.

References s, and std::stringstream::str().

Referenced by writeNpyHeader().

Here is the call graph for this function:

template<typename T >
std::ostream& Euclid::NdArray::operator<< ( std::ostream out,
const NdArray< T > &  ndarray 
)

Serialize a NdArray

void Euclid::NdArray::parseFieldValues ( const std::string descr,
bool &  big_endian,
std::vector< std::string > &  attrs,
std::string dtype 
)
inline

Parse the description field from npy arrays with named fields, which are stored as the string representation of a list of tuples (name, dtype). i.e: [('a', '<i4'), ('b', '<i4')]

Exceptions
std::invalid_argumentNdArrays only support uniform types, so this method will fail if there are mixed types on the npy file

Definition at line 30 of file NpyCommon.cpp.

References std::string::begin(), std::vector< T >::emplace_back(), std::string::empty(), std::string::end(), and parseSingleValue().

Referenced by parseNpyDict().

Here is the call graph for this function:

void Euclid::NdArray::parseNpyDict ( const std::string header,
bool &  fortran_order,
bool &  big_endian,
std::string dtype,
std::vector< size_t > &  shape,
std::vector< std::string > &  attrs,
size_t &  n_elements 
)
inline

Parse the dictionary serialized on the npy file

Parameters
headerString representation of the dictionary
fortran_order[out] Put here if the numpy array follows the Fortran convention
big_endian[out] Put here if the numpy array layout is big-endian
dtypePut here the read dtype
shape[out] Put here the read shape
attrs[out]Put here the attribute names
n_elements[out] Total number of elements (multiplication of shape)

Definition at line 55 of file NpyCommon.cpp.

References std::accumulate(), std::vector< T >::begin(), std::vector< T >::end(), std::string::find(), parseFieldValues(), parseSingleValue(), and std::string::substr().

Referenced by readNpyHeader().

Here is the call graph for this function:

void Euclid::NdArray::parseSingleValue ( const std::string descr,
bool &  big_endian,
std::string dtype 
)

Parse a single dtype description (i.e. '<f8')

Definition at line 25 of file NpyCommon.cpp.

References std::string::front(), and std::string::substr().

Referenced by parseFieldValues(), and parseNpyDict().

Here is the call graph for this function:

template<typename T >
NdArray<T> Euclid::NdArray::readNpy ( std::istream input)

Read an NdArray from a file following numpy format

Template Parameters
TNdArray cell type
ContainerNdArray container type
Parameters
inputInput stream
Returns
A new NdArray
Note
The underlying numpy format is expected to match the template type T
template<typename T >
NdArray<T> Euclid::NdArray::readNpy ( const boost::filesystem::path path)

Read an NdArray from a file following numpy format

Template Parameters
TNdArray cell type
ContainerNdArray container type
Parameters
pathInput path
Returns
A new NdArray
Note
The underlying numpy format is expected to match the template type T

Definition at line 93 of file Npy.h.

void Euclid::NdArray::readNpyHeader ( std::istream input,
std::string dtype,
std::vector< size_t > &  shape,
std::vector< std::string > &  attrs,
size_t &  n_elements 
)

Read the npy header

Parameters
inputInput stream
dtype[out] Put here the read dtype
shape[out] Put here the read shape
attrs[out] Put here the attribute names
n_elements[out] Total number of elements (multiplication of shape)
Returns

Definition at line 81 of file NpyCommon.cpp.

References std::memcmp(), NPY_MAGIC, parseNpyDict(), and std::istream::read().

Here is the call graph for this function:

template<typename T >
void Euclid::NdArray::sort ( NdArray< T > &  array,
const std::vector< std::string > &  attrs 
)

Sort in place a 2D NdArray based on the attribute names

Template Parameters
TNdArray type
Parameters
arrayThe ndarray
attrsList of attributes to use for the sorting
template<typename T >
T Euclid::NdArray::sum ( const NdArray< T > &  array)

Sum all elements in an ndarray

Parameters
arrayThe ndarray
template<typename T >
NdArray<T> Euclid::NdArray::sum ( const NdArray< T > &  array,
int  axis 
)

Sum elements in an ndarray along the given axis

Parameters
arrayThe ndarray
axisThe axis. 0 is the first. Negative values index from the end.
Returns
Another NdArray with one axis less
template<typename T , typename Iterator >
NdArray<T> Euclid::NdArray::trapz ( const NdArray< T > &  array,
const Iterator  kbegin,
const Iterator  kend,
int  axis 
)

Integrate elements in an ndarray along the given axis

Parameters
arrayThe ndarray
kbeginIterator to the beginning of the knot values
kendIterator to the end of the knot values
axisThe axis. 0 is the first. Negative values index from the end.
Returns
std::string Euclid::NdArray::typeDescription ( const std::string type,
const std::vector< std::string > &  attrs 
)
inline

Definition at line 178 of file NpyCommon.h.

References std::vector< T >::empty(), ENDIAN_MARKER, and std::stringstream::str().

Referenced by writeNpyHeader().

Here is the call graph for this function:

std::vector< std::size_t > Euclid::NdArray::unravel_index ( std::size_t  index,
const std::vector< std::size_t > &  shape 
)

Given an "flat" index (i.e a memory offset), return the appropriate coordinates for an ndarray of the given shape

Parameters
indexFlat index
shapeArray shape
Returns
The coordinates
Exceptions
std::out_of_rangeIf the index goes beyond the size of an ndarray with the given shape

Definition at line 24 of file Operations.cpp.

References std::vector< T >::size().

Referenced by Euclid::MathUtils::createSamplerFromGrid().

Here is the call graph for this function:

template<typename T >
void Euclid::NdArray::writeNpy ( std::ostream out,
const NdArray< T > &  array 
)

Write an NdArray to a file following numpy format

See Also
https://numpy.org/devdocs/reference/generated/numpy.lib.format.html
Template Parameters
TNdArray cell type
ContainerNdArray container type
Parameters
outOutput stream
arrayNdArray to write

Referenced by writeNpy().

template<typename T >
void Euclid::NdArray::writeNpy ( const boost::filesystem::path path,
const NdArray< T > &  array 
)
See Also
https://numpy.org/devdocs/reference/generated/numpy.lib.format.html
Template Parameters
TNdArray cell type
ContainerNdArray container type
Parameters
pathOutput path
arrayNdArray to write

Definition at line 74 of file Npy.h.

References writeNpy().

Here is the call graph for this function:

template<typename T >
void Euclid::NdArray::writeNpyHeader ( std::ostream out,
std::vector< size_t >  shape,
const std::vector< std::string > &  attrs 
)

Variable Documentation

constexpr const char* Euclid::NdArray::ENDIAN_MARKER = "<"

Endianness marker for the numpy array

Definition at line 43 of file NpyCommon.h.

Referenced by typeDescription().

constexpr const char Euclid::NdArray::NPY_MAGIC[] = {'\x93', 'N', 'U', 'M', 'P', 'Y'}

Magic string for .npy files

Definition at line 37 of file NpyCommon.h.

Referenced by readNpyHeader(), and writeNpyHeader().

constexpr const uint8_t Euclid::NdArray::NPY_VERSION[] = {'\x02', '\x00'}

We write arrays following 2.0 version (32 bits header size)

Definition at line 161 of file NpyCommon.h.

Referenced by writeNpyHeader().