19#ifndef TPFA_LINEARIZER_GPU_HH
20#define TPFA_LINEARIZER_GPU_HH
36#include <opm/common/Exceptions.hpp>
37#include <opm/common/utility/gpuDecorators.hpp>
39#include <dune/common/fvector.hh>
51 static constexpr bool value =
true;
57template <
class TypeTag>
62template <
class TypeTag>
69 static constexpr int numEq = getPropValue<TypeTag, Properties::NumEq>();
70 static constexpr bool runAssemblyOnGpu = getPropValue<TypeTag, Properties::RunAssemblyOnGpu>();
71 using MatrixBlockCPU =
typename SparseMatrixAdapter::MatrixBlock;
72 using VectorBlockCPU = Dune::FieldVector<Scalar, numEq>;
80 using MatrixBlock = std::conditional_t<runAssemblyOnGpu, MatrixBlockGPU, MatrixBlockCPU>;
81 using VectorBlock = std::conditional_t<runAssemblyOnGpu, VectorBlockGPU, VectorBlockCPU>;
82 using ADVectorBlock = std::conditional_t<runAssemblyOnGpu, ADVectorBlockGPU, ADVectorBlockCPU>;
93 gpuBufferDiagMatAddress_.reset();
96 gpuSystemActive_ =
false;
99 void createMatrix([[maybe_unused]]
const SparseMatrixAdapter& jacobian)
101 if constexpr (runAssemblyOnGpu) {
103#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
104 gpuJacobian_ = std::make_unique<gpuistl::GpuSparseMatrixWrapper<Scalar>>(
106 gpuBufferDiagMatAddress_ = std::make_unique<gpuistl::GpuBuffer<MatrixBlockGPU*>>(
107 gpuistl::detail::getDiagPtrsTyped<MatrixBlockGPU>(*gpuJacobian_));
114 if constexpr (runAssemblyOnGpu) {
116#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
117 gpuJacobian_->setToZero();
124 return gpuSystemActive_;
130#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
131 return gpuParams_->gpuJacobian();
133 OPM_THROW(std::logic_error,
"GPU matrix requires GPU support");
140#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
141 return gpuParams_->residualBuffer();
143 OPM_THROW(std::logic_error,
"GPU residual requires GPU support");
150#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
151 return gpuParams_->flattenedResidual();
153 OPM_THROW(std::logic_error,
"GPU residual requires GPU support");
159 if constexpr (runAssemblyOnGpu) {
161#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
162 if (gpuSystemActive_) {
163 gpuParams_->copyJacobianToHost(jacobian, *gpuJacobian_);
171 gpuSystemActive_ =
false;
172 if constexpr (runAssemblyOnGpu) {
174#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
177 gpuSystemActive_ = Parameters::Get<Parameters::EnableGpuAssemblySolverBridge>()
178 && linearizer.simulator_().gridView().comm().size() == 1
179 && linearizer.problem_().wellModel().numLocalWellsEnd() == 0
180 && linearizer.simulator_().model().newtonMethod().linearSolver().hasGpuSolver();
183 return gpuSystemActive_;
186 template <
class SubDomainType>
188 [[maybe_unused]]
const SubDomainType& domain,
189 [[maybe_unused]]
unsigned int numCells,
190 [[maybe_unused]] Scalar dt,
191 [[maybe_unused]]
bool dispersionActive)
194#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
195 const bool useAssemblyLinearsolveBridge = gpuSystemActive_;
197 if constexpr (std::is_same_v<SubDomainType, FullDomain<>>) {
199 int constexpr blockSize = 256;
203 auto params = std::make_unique<GpuParams>(domain,
204 linearizer.neighborInfo_,
206 *gpuBufferDiagMatAddress_,
207 linearizer.jacobian_->istlMatrix(),
208 linearizer.residual_,
209 linearizer.boundaryInfo_,
211 linearizer.problem_(),
213 gpuParams_ = GpuParamsOwner(params.release(), [](GpuParams* ptr) {
214 std::default_delete<GpuParams> {}(ptr);
217 gpuParams_->refreshForLinearization(
218 linearizer.model_(), linearizer.boundaryInfo_, numCells);
221 launchLinearization<typename GpuParams::LocalResidualGPU>(
223 gpuParams_->domainView(),
224 gpuParams_->neighborInfoView(),
225 gpuParams_->diagMatAddressView(),
226 gpuParams_->residualView(),
227 gpuParams_->modelView(),
230 gpuParams_->flowProblemView());
232 if (gpuParams_->boundaryInfoSize() > 0) {
233 auto boundaryInfoView = gpuParams_->boundaryInfoView();
234 linearize_bc_threadsafe<TpfaLinearizerGpu<TypeTag>,
235 typename GpuParams::GPUBOIQ,
236 decltype(gpuParams_->modelView()),
237 typename GpuParams::LocalResidualGPU>
238 <<<((gpuParams_->boundaryInfoSize() + blockSize - 1) / blockSize), blockSize>>>(
239 gpuParams_->diagMatAddressView(),
240 gpuParams_->residualView(),
242 gpuParams_->modelView(),
243 gpuParams_->flowProblemView());
246 if (useAssemblyLinearsolveBridge) {
247 linearizeSourceTerms(linearizer, numCells, domain);
251 gpuParams_->copyResidualToHost(linearizer.residual_, numCells);
253 gpuParams_->copyResidualToHost(linearizer.residual_, numCells);
254 gpuParams_->copyJacobianToHost(*linearizer.jacobian_, *gpuJacobian_);
257 OPM_THROW(std::logic_error,
"Only FullDomain is supported on GPU");
260 OPM_THROW(std::logic_error,
261 "Trying to run GPU assembly without compiling with GPU support");
264#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
267 template <
class LocalIntensiveQuantities,
268 class LocalResidualT,
274 OPM_HOST_DEVICE
static void
275 linearize_bc_threadsafe_single_cell(DiagPtrType diagonalAddresses,
277 const BoundaryInfoT boundaryInfoElement,
282 VectorBlock residualContribution(0.0);
283 MatrixBlock jacobianContribution(0.0);
284 ADVectorBlock flux(0.0);
285 const unsigned cell = boundaryInfoElement.cell;
286 const LocalIntensiveQuantities& insideIntensiveQuantities
287 = model.intensiveQuantities(cell, 0);
290 LocalResidualT::computeBoundaryFlux(
291 flux, problem, boundaryInfoElement.bcdata, insideIntensiveQuantities, cell);
292 flux *= boundaryInfoElement.bcdata.faceArea;
294 residualContribution, jacobianContribution, flux);
295 auto* residualBlock = &(residual.data()[cell]);
296 for (
int i = 0; i < numEq; ++i) {
297 atomicAdd(&((*residualBlock)[i]), residualContribution[i]);
299 auto* matrixBlock = diagonalAddresses[cell];
300 for (
int row = 0; row < jacobianContribution.size(); ++row) {
301 for (
int col = 0; col < jacobianContribution.size(); ++col) {
302 Scalar* element = &((*matrixBlock)[row][col]);
303 atomicAdd(element, jacobianContribution[row][col]);
313 static void requireGpuSupport()
315 if constexpr (!runAssemblyOnGpu) {
316 OPM_THROW(std::logic_error,
"GPU assembly is disabled for this type tag");
319 OPM_THROW(std::logic_error,
"GPU assembly requires CUDA or HIP support");
320#elif !OPM_IS_COMPILING_WITH_GPU_COMPILER
321 OPM_THROW(std::logic_error,
"GPU assembly requires compilation with a GPU compiler");
325#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
326 template <
class LocalResidualT,
330 class NeighborSparseTable,
333 void launchLinearization(
const unsigned int numCells,
334 const DomainType& domain,
335 const NeighborSparseTable& neighborInfo,
336 DiagPtrType& diagMatAddress,
337 ResidualType& residual,
338 const ModelClass& model,
340 [[maybe_unused]]
bool dispersionActive,
341 const ProblemT& problem)
343 static_assert(!getPropValue<TypeTag, Properties::EnableBioeffects>(),
344 "Bioeffects not yet supported on GPU");
345 assert(!dispersionActive &&
"Dispersion not yet supported on GPU");
346 int constexpr blockSize = 256;
347 kernel_linearize<TpfaLinearizer<TypeTag>,
355 ProblemT><<<((numCells + blockSize - 1) / blockSize), blockSize>>>(
356 numCells, domain, neighborInfo, diagMatAddress, residual, model, dt, problem);
358 template <
class SubDomainType>
359 void linearizeSourceTerms(TpfaLinearizer<TypeTag>& linearizer,
360 unsigned int numCells,
361 const SubDomainType& domain)
365 std::vector<unsigned> indices;
366 std::vector<VectorBlockCPU> residualContributions;
367 std::vector<MatrixBlockCPU> jacobianContributions;
369 constexpr std::size_t reservedSize = 20;
370 indices.reserve(reservedSize);
371 residualContributions.reserve(reservedSize);
372 jacobianContributions.reserve(reservedSize);
373 for (
unsigned ii = 0; ii < numCells; ++ii) {
374 const unsigned cell = domain.cells[ii];
375 ADVectorBlockCPU source(0.0);
376 const auto& intQuantsIn = linearizer.model_().intensiveQuantities(cell, 0);
377 const double volume = linearizer.model_().dofTotalVolume(cell);
378 if (linearizer.separateSparseSourceTerms_) {
379 LocalResidual::computeSourceDense(
380 source, linearizer.problem_(), intQuantsIn, cell, 0);
382 LocalResidual::computeSource(source, linearizer.problem_(), intQuantsIn, cell, 0);
385 VectorBlockCPU residualContribution(0.0);
386 MatrixBlockCPU jacobianContribution(0.0);
388 residualContribution, jacobianContribution, source);
389 bool nonzero =
false;
390 for (
unsigned eq = 0; eq < numEq; ++eq) {
391 nonzero = nonzero || residualContribution[eq] != 0.0;
392 for (
unsigned pv = 0; pv < numEq; ++pv) {
393 nonzero = nonzero || jacobianContribution[eq][pv] != 0.0;
397 indices.push_back(cell);
398 residualContributions.push_back(residualContribution);
399 jacobianContributions.push_back(jacobianContribution);
402 gpuParams_->applySourceContributions(indices, residualContributions, jacobianContributions);
407 std::unique_ptr<gpuistl::GpuSparseMatrixWrapper<Scalar>> gpuJacobian_;
408 std::unique_ptr<gpuistl::GpuBuffer<MatrixBlockGPU*>> gpuBufferDiagMatAddress_;
410 using GpuParams = TpfaLinearizerGpuParams<TypeTag>;
411 using GpuParamsOwner = std::unique_ptr<GpuParams, void (*)(GpuParams*)>;
412 GpuParamsOwner gpuParams_ {
nullptr,
nullptr};
414 bool gpuSystemActive_ =
false;
Declares the properties required by the black oil model.
Definition: tpfalinearizergpu.hh:64
void linearize(TpfaLinearizer< TypeTag > &linearizer, const SubDomainType &domain, unsigned int numCells, Scalar dt, bool dispersionActive)
Definition: tpfalinearizergpu.hh:187
VectorBlockCPU VectorBlock
Definition: tpfalinearizergpu.hh:85
void resetSystem()
Definition: tpfalinearizergpu.hh:112
decltype(auto) gpuJacobian()
Definition: tpfalinearizergpu.hh:127
void eraseMatrix()
Definition: tpfalinearizergpu.hh:89
ADVectorBlockCPU ADVectorBlock
Definition: tpfalinearizergpu.hh:86
MatrixBlockCPU MatrixBlock
Definition: tpfalinearizergpu.hh:84
void copyJacobianToHost(SparseMatrixAdapter &jacobian)
Definition: tpfalinearizergpu.hh:157
decltype(auto) gpuResidual()
Definition: tpfalinearizergpu.hh:137
bool gpuSystemActive() const
Definition: tpfalinearizergpu.hh:122
bool updateSolverBridge(TpfaLinearizer< TypeTag > &linearizer)
Definition: tpfalinearizergpu.hh:169
void createMatrix(const SparseMatrixAdapter &jacobian)
Definition: tpfalinearizergpu.hh:99
decltype(auto) flattenedGpuResidual()
Definition: tpfalinearizergpu.hh:147
The common code for the linearizers of non-linear systems of equations.
Definition: tpfalinearizer.hh:103
static OPM_HOST_DEVICE void setResAndJacobi(VectorBlockType &res, MatrixBlockType &bMat, const ADVectorBlockType &resid)
Definition: tpfalinearizer.hh:761
static GpuSparseMatrixWrapper< T, ForceLegacy > fromMatrix(const MatrixType &matrix, bool copyNonZeroElementsDirectly=false)
fromMatrix creates a new matrix with the same block size and values as the given matrix
Definition: GpuSparseMatrixWrapper.hpp:184
A small fixed-size square matrix class for use in CUDA kernels.
Definition: MiniMatrix.hpp:37
Definition: MiniVector.hpp:50
Declare the properties used by the infrastructure code of the finite volume discretizations.
Declares the properties required by the black oil model.
@ NONE
Definition: DeferredLogger.hpp:46
Definition: blackoilnewtonmethodparams.hpp:31
Definition: blackoilbioeffectsmodules.hh:45
typename Properties::Detail::GetPropImpl< TypeTag, Property >::type::type GetPropType
get the type alias defined in the property (equivalent to old macro GET_PROP_TYPE(....
Definition: propertysystem.hh:233
This file provides the infrastructure to retrieve run-time parameters.
Definition: tpfalinearizergpu.hh:50
static constexpr bool value
Definition: tpfalinearizergpu.hh:51
GPU parameter setup class for TpfaLinearizer.