Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Core][MPI] Adding MinLocAll and MaxLocAll to DataCommunicator #11712

Merged
merged 7 commits into from
Nov 8, 2023
Merged

[Core][MPI] Adding MinLocAll and MaxLocAll to DataCommunicator #11712

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
218acb5
[Core][MPI] Adding `MinLocAll` and `MaxLocAll` to `DataCommunicator`
loumalouomega Oct 23, 2023
bb95bac
Moving definition to MPIMessage
loumalouomega Oct 23, 2023
cde37b7
MinLoc/MaxLoc only for types where it makes sense
jcotela Oct 24, 2023
6bbfa9d
Adding doc
loumalouomega Oct 26, 2023
32d932d
Better explained
loumalouomega Oct 26, 2023
85f5cc8
Merge branch 'master' into core/mpi/adding-min-loc-all-and-max-loc-all
loumalouomega Nov 8, 2023
4e46de9
Adding `testAllReduceLocValueOperations`
loumalouomega Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 38 additions & 26 deletions kratos/includes/data_communicator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,32 +80,37 @@ virtual void Max(const std::vector<__VA_ARGS__>& rLocalValues, std::vector<__VA_
* The returned value is defined on all ranks.
*/
#ifndef KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_INTERFACE_FOR_TYPE
#define KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_INTERFACE_FOR_TYPE(...) \
virtual __VA_ARGS__ SumAll(const __VA_ARGS__& rLocalValue) const { return rLocalValue; } \
virtual std::vector<__VA_ARGS__> SumAll(const std::vector<__VA_ARGS__>& rLocalValues) const { \
return rLocalValues; \
} \
virtual void SumAll(const std::vector<__VA_ARGS__>& rLocalValues, std::vector<__VA_ARGS__>& rGlobalValues) const { \
KRATOS_DATA_COMMUNICATOR_DEBUG_SIZE_CHECK(rLocalValues.size(), rGlobalValues.size(), "SumAll"); \
rGlobalValues = SumAll(rLocalValues); \
} \
virtual __VA_ARGS__ MinAll(const __VA_ARGS__& rLocalValue) const { return rLocalValue; } \
virtual std::vector<__VA_ARGS__> MinAll(const std::vector<__VA_ARGS__>& rLocalValues) const { \
return rLocalValues; \
} \
virtual void MinAll(const std::vector<__VA_ARGS__>& rLocalValues, std::vector<__VA_ARGS__>& rGlobalValues) const { \
KRATOS_DATA_COMMUNICATOR_DEBUG_SIZE_CHECK(rLocalValues.size(), rGlobalValues.size(), "MinAll"); \
rGlobalValues = MinAll(rLocalValues); \
} \
virtual __VA_ARGS__ MaxAll(const __VA_ARGS__& rLocalValue) const { return rLocalValue; } \
virtual std::vector<__VA_ARGS__> MaxAll(const std::vector<__VA_ARGS__>& rLocalValues) const { \
return rLocalValues; \
} \
virtual void MaxAll(const std::vector<__VA_ARGS__>& rLocalValues, std::vector<__VA_ARGS__>& rGlobalValues) const { \
KRATOS_DATA_COMMUNICATOR_DEBUG_SIZE_CHECK(rLocalValues.size(), rGlobalValues.size(), "MaxAll"); \
rGlobalValues = MaxAll(rLocalValues); \
} \
#define KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_INTERFACE_FOR_TYPE(...) \
virtual __VA_ARGS__ SumAll(const __VA_ARGS__& rLocalValue) const { return rLocalValue; } \
virtual std::vector<__VA_ARGS__> SumAll(const std::vector<__VA_ARGS__>& rLocalValues) const { \
return rLocalValues; \
} \
virtual void SumAll(const std::vector<__VA_ARGS__>& rLocalValues, std::vector<__VA_ARGS__>& rGlobalValues) const { \
KRATOS_DATA_COMMUNICATOR_DEBUG_SIZE_CHECK(rLocalValues.size(), rGlobalValues.size(), "SumAll"); \
rGlobalValues = SumAll(rLocalValues); \
} \
virtual __VA_ARGS__ MinAll(const __VA_ARGS__& rLocalValue) const { return rLocalValue; } \
virtual std::vector<__VA_ARGS__> MinAll(const std::vector<__VA_ARGS__>& rLocalValues) const { \
return rLocalValues; \
} \
virtual void MinAll(const std::vector<__VA_ARGS__>& rLocalValues, std::vector<__VA_ARGS__>& rGlobalValues) const { \
KRATOS_DATA_COMMUNICATOR_DEBUG_SIZE_CHECK(rLocalValues.size(), rGlobalValues.size(), "MinAll"); \
rGlobalValues = MinAll(rLocalValues); \
} \
virtual __VA_ARGS__ MaxAll(const __VA_ARGS__& rLocalValue) const { return rLocalValue; } \
virtual std::vector<__VA_ARGS__> MaxAll(const std::vector<__VA_ARGS__>& rLocalValues) const { \
return rLocalValues; \
} \
virtual void MaxAll(const std::vector<__VA_ARGS__>& rLocalValues, std::vector<__VA_ARGS__>& rGlobalValues) const { \
KRATOS_DATA_COMMUNICATOR_DEBUG_SIZE_CHECK(rLocalValues.size(), rGlobalValues.size(), "MaxAll"); \
rGlobalValues = MaxAll(rLocalValues); \
}
#endif

#ifndef KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE
#define KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(...) \
virtual std::pair<__VA_ARGS__, int> MinLocAll(const __VA_ARGS__& rLocalValue) const { return std::pair<__VA_ARGS__, int>(rLocalValue, 0); } \
virtual std::pair<__VA_ARGS__, int> MaxLocAll(const __VA_ARGS__& rLocalValue) const { return std::pair<__VA_ARGS__, int>(rLocalValue, 0); }
#endif

// Compute the partial sum of the given quantity from rank 0 to the current rank (included).
Expand All @@ -115,7 +120,7 @@ virtual void MaxAll(const std::vector<__VA_ARGS__>& rLocalValues, std::vector<__
*/
#ifndef KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_SCANSUM_INTERFACE_FOR_TYPE
#define KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_SCANSUM_INTERFACE_FOR_TYPE(...) \
virtual __VA_ARGS__ ScanSum(const __VA_ARGS__& rLocalValue) const { return rLocalValue; } \
virtual __VA_ARGS__ ScanSum(const __VA_ARGS__& rLocalValue) const { return rLocalValue; } \
virtual std::vector<__VA_ARGS__> ScanSum(const std::vector<__VA_ARGS__>& rLocalValues) const { \
return rLocalValues; \
} \
Expand Down Expand Up @@ -394,6 +399,13 @@ class KRATOS_API(KRATOS_CORE) DataCommunicator
KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_PUBLIC_INTERFACE_FOR_TYPE(Vector)
KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_PUBLIC_INTERFACE_FOR_TYPE(Matrix)

// MinLoc and MaxLoc AllReduce operations
KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(char)
KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(int)
KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(unsigned int)
KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(long unsigned int)
KRATOS_BASE_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(double)

// Reduce operations

virtual bool AndReduce(
Expand Down
27 changes: 27 additions & 0 deletions kratos/mpi/includes/mpi_data_communicator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@ void MaxAll(const std::vector<__VA_ARGS__>& rLocalValues, std::vector<__VA_ARGS_

#endif

#ifndef KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE
#define KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(...) \
std::pair<__VA_ARGS__, int> MinLocAll(const __VA_ARGS__& rLocalValue) const override; \
std::pair<__VA_ARGS__, int> MaxLocAll(const __VA_ARGS__& rLocalValue) const override;
#endif

#ifndef KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_SCANSUM_INTERFACE_FOR_TYPE
#define KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_SCANSUM_INTERFACE_FOR_TYPE(...) \
__VA_ARGS__ ScanSum(const __VA_ARGS__& rLocalValue) const override; \
Expand Down Expand Up @@ -217,6 +223,13 @@ class KRATOS_API(KRATOS_MPI_CORE) MPIDataCommunicator: public DataCommunicator
KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_PUBLIC_INTERFACE_FOR_TYPE(Vector)
KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_PUBLIC_INTERFACE_FOR_TYPE(Matrix)

// MinLoc and MaxLoc AllReduce operations
KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(char)
KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(int)
KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(unsigned int)
KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(long unsigned int)
KRATOS_MPI_DATA_COMMUNICATOR_DECLARE_ALLREDUCE_LOC_INTERFACE_FOR_TYPE(double)

// Reduce operations

bool AndReduce(
Expand Down Expand Up @@ -380,6 +393,20 @@ class KRATOS_API(KRATOS_MPI_CORE) MPIDataCommunicator: public DataCommunicator
const std::vector<TDataType>& rLocalValues,
MPI_Op Operation) const;

/**
* @brief Performs an AllReduce operation with location information (the partition where the reduced value was found).
* @details This function performs an AllReduce operation on a pair of data and an integer location using the specified MPI operation. The AllReduce operation combines the data from all processes and stores the result in the pair's first element. The location information (integer) is the partition where the reduced value was found.
* @tparam TDataType The data type of the pair's first element.
* @param rLocalValues A pair containing the local data and location information to be reduced.
* @param Operation The MPI operation to use for the reduction.
* @return A pair where the first element contains the result of the AllReduce operation, and the second element is the partition where the reduced value was found.
*/
template<class TDataType>
std::pair<TDataType, int> AllReduceDetailWithLocation(
const std::pair<TDataType, int>& rLocalValues,
loumalouomega marked this conversation as resolved.
Show resolved Hide resolved
MPI_Op Operation
) const;

template<class TDataType> void ScanDetail(
const TDataType& rLocalValues,
TDataType& rReducedValues,
Expand Down
44 changes: 44 additions & 0 deletions kratos/mpi/includes/mpi_message.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,15 @@

#pragma once

// System includes
#include <string>
#include <vector>
#include <type_traits>

// External includes
#include "mpi.h"

// Project includes
#include "containers/array_1d.h"
#include "containers/flags.h"
#include "utilities/data_type_traits.h"
Expand Down Expand Up @@ -84,6 +88,46 @@ template<> struct MPIDataType<int64_t>
}
};

template<> struct MPIDataType<std::pair<char, int>>
{
static inline MPI_Datatype DataType()
{
return MPI_2INT;
}
};

template<> struct MPIDataType<std::pair<int, int>>
{
static inline MPI_Datatype DataType()
{
return MPI_2INT;
}
};

template<> struct MPIDataType<std::pair<unsigned int, int>>
{
static inline MPI_Datatype DataType()
{
return MPI_2INT;
}
};

template<> struct MPIDataType<std::pair<long unsigned int, int>>
{
static inline MPI_Datatype DataType()
{
return MPI_LONG_INT;
}
};

template<> struct MPIDataType<std::pair<double, int>>
{
static inline MPI_Datatype DataType()
{
return MPI_DOUBLE_INT;
}
};

}

template<class TDataType> class MPIMessage
Expand Down
Loading