tpfalinearizergpu.hh
Go to the documentation of this file.
1/*
2 Copyright 2026 Equinor AS
3
4 This file is part of the Open Porous Media project (OPM).
5
6 OPM is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
10
11 OPM is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with OPM. If not, see <http://www.gnu.org/licenses/>.
18*/
19#ifndef TPFA_LINEARIZER_GPU_HH
20#define TPFA_LINEARIZER_GPU_HH
21
27
28#if HAVE_CUDA
34#endif
35
36#include <opm/common/Exceptions.hpp>
37#include <opm/common/utility/gpuDecorators.hpp>
38
39#include <dune/common/fvector.hh>
40
41#include <cassert>
42#include <cstddef>
43#include <memory>
44#include <stdexcept>
45#include <type_traits>
46#include <vector>
47
48namespace Opm::Parameters
49{
51 static constexpr bool value = true;
52};
53} // namespace Opm::Parameters
54
55namespace Opm
56{
57template <class TypeTag>
58class TpfaLinearizer;
59
60// Owns the GPU assembly lifecycle. The layout is identical in host and device
61// translation units; only GPU compilers instantiate kernel-dependent methods.
62template <class TypeTag>
64{
69 static constexpr int numEq = getPropValue<TypeTag, Properties::NumEq>();
70 static constexpr bool runAssemblyOnGpu = getPropValue<TypeTag, Properties::RunAssemblyOnGpu>();
71 using MatrixBlockCPU = typename SparseMatrixAdapter::MatrixBlock;
72 using VectorBlockCPU = Dune::FieldVector<Scalar, numEq>;
73 using ADVectorBlockCPU = GetPropType<TypeTag, Properties::RateVector>;
74
75public:
76#if HAVE_CUDA
77 using MatrixBlockGPU = gpuistl::MiniMatrix<Scalar, numEq>;
78 using VectorBlockGPU = gpuistl::MiniVector<Scalar, numEq>;
79 using ADVectorBlockGPU = gpuistl::MiniVector<Evaluation, numEq>;
80 using MatrixBlock = std::conditional_t<runAssemblyOnGpu, MatrixBlockGPU, MatrixBlockCPU>;
81 using VectorBlock = std::conditional_t<runAssemblyOnGpu, VectorBlockGPU, VectorBlockCPU>;
82 using ADVectorBlock = std::conditional_t<runAssemblyOnGpu, ADVectorBlockGPU, ADVectorBlockCPU>;
83#else
84 using MatrixBlock = MatrixBlockCPU;
85 using VectorBlock = VectorBlockCPU;
86 using ADVectorBlock = ADVectorBlockCPU;
87#endif
88
90 {
91#if HAVE_CUDA
92 gpuParams_.reset();
93 gpuBufferDiagMatAddress_.reset();
94 gpuJacobian_.reset();
95#endif
96 gpuSystemActive_ = false;
97 }
98
99 void createMatrix([[maybe_unused]] const SparseMatrixAdapter& jacobian)
100 {
101 if constexpr (runAssemblyOnGpu) {
102 requireGpuSupport();
103#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
104 gpuJacobian_ = std::make_unique<gpuistl::GpuSparseMatrixWrapper<Scalar>>(
106 gpuBufferDiagMatAddress_ = std::make_unique<gpuistl::GpuBuffer<MatrixBlockGPU*>>(
107 gpuistl::detail::getDiagPtrsTyped<MatrixBlockGPU>(*gpuJacobian_));
108#endif
109 }
110 }
111
113 {
114 if constexpr (runAssemblyOnGpu) {
115 requireGpuSupport();
116#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
117 gpuJacobian_->setToZero();
118#endif
119 }
120 }
121
122 [[nodiscard]] bool gpuSystemActive() const
123 {
124 return gpuSystemActive_;
125 }
126
127 decltype(auto) gpuJacobian()
128 {
129 requireGpuSupport();
130#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
131 return gpuParams_->gpuJacobian();
132#else
133 OPM_THROW(std::logic_error, "GPU matrix requires GPU support");
134#endif
135 }
136
137 decltype(auto) gpuResidual()
138 {
139 requireGpuSupport();
140#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
141 return gpuParams_->residualBuffer();
142#else
143 OPM_THROW(std::logic_error, "GPU residual requires GPU support");
144#endif
145 }
146
147 decltype(auto) flattenedGpuResidual()
148 {
149 requireGpuSupport();
150#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
151 return gpuParams_->flattenedResidual();
152#else
153 OPM_THROW(std::logic_error, "GPU residual requires GPU support");
154#endif
155 }
156
157 void copyJacobianToHost([[maybe_unused]] SparseMatrixAdapter& jacobian)
158 {
159 if constexpr (runAssemblyOnGpu) {
160 requireGpuSupport();
161#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
162 if (gpuSystemActive_) {
163 gpuParams_->copyJacobianToHost(jacobian, *gpuJacobian_);
164 }
165#endif
166 }
167 }
168
169 [[nodiscard]] bool updateSolverBridge([[maybe_unused]] TpfaLinearizer<TypeTag>& linearizer)
170 {
171 gpuSystemActive_ = false;
172 if constexpr (runAssemblyOnGpu) {
173 requireGpuSupport();
174#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
175 // The bridge requires a serial run without wells and a GPU linear solver.
176 // Multiple ranks still need a host matrix for makeOverlapRowsInvalid.
177 gpuSystemActive_ = Parameters::Get<Parameters::EnableGpuAssemblySolverBridge>()
178 && linearizer.simulator_().gridView().comm().size() == 1
179 && linearizer.problem_().wellModel().numLocalWellsEnd() == 0
180 && linearizer.simulator_().model().newtonMethod().linearSolver().hasGpuSolver();
181#endif
182 }
183 return gpuSystemActive_;
184 }
185
186 template <class SubDomainType>
187 void linearize([[maybe_unused]] TpfaLinearizer<TypeTag>& linearizer,
188 [[maybe_unused]] const SubDomainType& domain,
189 [[maybe_unused]] unsigned int numCells,
190 [[maybe_unused]] Scalar dt,
191 [[maybe_unused]] bool dispersionActive)
192 {
193 requireGpuSupport();
194#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
195 const bool useAssemblyLinearsolveBridge = gpuSystemActive_;
196 // GPU assembly currently supports only the full domain.
197 if constexpr (std::is_same_v<SubDomainType, FullDomain<>>) {
198
199 int constexpr blockSize = 256; // Experimentally this is a good value for multiple
200 // GPUs. Autotune this later.
201
202 if (!gpuParams_) {
203 auto params = std::make_unique<GpuParams>(domain,
204 linearizer.neighborInfo_,
205 *gpuJacobian_,
206 *gpuBufferDiagMatAddress_,
207 linearizer.jacobian_->istlMatrix(),
208 linearizer.residual_,
209 linearizer.boundaryInfo_,
210 linearizer.model_(),
211 linearizer.problem_(),
212 numCells);
213 gpuParams_ = GpuParamsOwner(params.release(), [](GpuParams* ptr) {
214 std::default_delete<GpuParams> {}(ptr);
215 });
216 } else {
217 gpuParams_->refreshForLinearization(
218 linearizer.model_(), linearizer.boundaryInfo_, numCells);
219 }
220
221 launchLinearization<typename GpuParams::LocalResidualGPU>(
222 numCells,
223 gpuParams_->domainView(),
224 gpuParams_->neighborInfoView(),
225 gpuParams_->diagMatAddressView(),
226 gpuParams_->residualView(),
227 gpuParams_->modelView(),
228 dt,
229 dispersionActive,
230 gpuParams_->flowProblemView());
231
232 if (gpuParams_->boundaryInfoSize() > 0) {
233 auto boundaryInfoView = gpuParams_->boundaryInfoView();
234 linearize_bc_threadsafe<TpfaLinearizerGpu<TypeTag>,
235 typename GpuParams::GPUBOIQ,
236 decltype(gpuParams_->modelView()),
237 typename GpuParams::LocalResidualGPU>
238 <<<((gpuParams_->boundaryInfoSize() + blockSize - 1) / blockSize), blockSize>>>(
239 gpuParams_->diagMatAddressView(),
240 gpuParams_->residualView(),
241 boundaryInfoView,
242 gpuParams_->modelView(),
243 gpuParams_->flowProblemView());
244 }
245
246 if (useAssemblyLinearsolveBridge) {
247 linearizeSourceTerms(linearizer, numCells, domain);
248
249 // The convergence report consumes the residual on the host,
250 // but the numerical Jacobian remains device authoritative.
251 gpuParams_->copyResidualToHost(linearizer.residual_, numCells);
252 } else {
253 gpuParams_->copyResidualToHost(linearizer.residual_, numCells);
254 gpuParams_->copyJacobianToHost(*linearizer.jacobian_, *gpuJacobian_);
255 }
256 } else {
257 OPM_THROW(std::logic_error, "Only FullDomain is supported on GPU");
258 }
259#else
260 OPM_THROW(std::logic_error,
261 "Trying to run GPU assembly without compiling with GPU support");
262#endif
263 }
264#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
265 // Thread-safe on the GPU, but floating-point atomic additions make the numerical
266 // result depend on the order in which boundary faces are calculated.
267 template <class LocalIntensiveQuantities,
268 class LocalResidualT,
269 class ModelT,
270 class DiagPtrType,
271 class ResidualT,
272 class BoundaryInfoT,
273 class ProblemT>
274 OPM_HOST_DEVICE static void
275 linearize_bc_threadsafe_single_cell(DiagPtrType diagonalAddresses,
276 ResidualT residual,
277 const BoundaryInfoT boundaryInfoElement,
278 ModelT model,
279 ProblemT problem)
280 {
281 if (boundaryInfoElement.bcdata.type != BCType::NONE) {
282 VectorBlock residualContribution(0.0);
283 MatrixBlock jacobianContribution(0.0);
284 ADVectorBlock flux(0.0);
285 const unsigned cell = boundaryInfoElement.cell;
286 const LocalIntensiveQuantities& insideIntensiveQuantities
287 = model.intensiveQuantities(cell, /*timeIdx*/ 0);
288 // LocalResidualT carries the device fluid system. The linearizer's
289 // TypeTag still names the host fluid system, so it cannot gate this call.
290 LocalResidualT::computeBoundaryFlux(
291 flux, problem, boundaryInfoElement.bcdata, insideIntensiveQuantities, cell);
292 flux *= boundaryInfoElement.bcdata.faceArea;
294 residualContribution, jacobianContribution, flux);
295 auto* residualBlock = &(residual.data()[cell]);
296 for (int i = 0; i < numEq; ++i) {
297 atomicAdd(&((*residualBlock)[i]), residualContribution[i]);
298 }
299 auto* matrixBlock = diagonalAddresses[cell];
300 for (int row = 0; row < jacobianContribution.size(); ++row) {
301 for (int col = 0; col < jacobianContribution.size(); ++col) {
302 Scalar* element = &((*matrixBlock)[row][col]);
303 atomicAdd(element, jacobianContribution[row][col]);
304 }
305 }
306 }
307 }
308#endif
309
310private:
311 // CPU assembly also calls the lifecycle methods, which return before this
312 // check. Actual GPU operations must never silently succeed without support.
313 static void requireGpuSupport()
314 {
315 if constexpr (!runAssemblyOnGpu) {
316 OPM_THROW(std::logic_error, "GPU assembly is disabled for this type tag");
317 }
318#if !HAVE_CUDA
319 OPM_THROW(std::logic_error, "GPU assembly requires CUDA or HIP support");
320#elif !OPM_IS_COMPILING_WITH_GPU_COMPILER
321 OPM_THROW(std::logic_error, "GPU assembly requires compilation with a GPU compiler");
322#endif
323 }
324
325#if HAVE_CUDA && OPM_IS_COMPILING_WITH_GPU_COMPILER
326 template <class LocalResidualT,
327 class ModelClass,
328 class DiagPtrType,
329 class DomainType,
330 class NeighborSparseTable,
331 class ResidualType,
332 class ProblemT>
333 void launchLinearization(const unsigned int numCells,
334 const DomainType& domain,
335 const NeighborSparseTable& neighborInfo,
336 DiagPtrType& diagMatAddress,
337 ResidualType& residual,
338 const ModelClass& model,
339 Scalar dt,
340 [[maybe_unused]] bool dispersionActive,
341 const ProblemT& problem)
342 {
343 static_assert(!getPropValue<TypeTag, Properties::EnableBioeffects>(),
344 "Bioeffects not yet supported on GPU");
345 assert(!dispersionActive && "Dispersion not yet supported on GPU");
346 int constexpr blockSize = 256;
347 kernel_linearize<TpfaLinearizer<TypeTag>,
348 ModelClass,
349 LocalResidualT,
350 DiagPtrType,
351 Scalar,
352 DomainType,
353 NeighborSparseTable,
354 ResidualType,
355 ProblemT><<<((numCells + blockSize - 1) / blockSize), blockSize>>>(
356 numCells, domain, neighborInfo, diagMatAddress, residual, model, dt, problem);
357 }
358 template <class SubDomainType>
359 void linearizeSourceTerms(TpfaLinearizer<TypeTag>& linearizer,
360 unsigned int numCells,
361 const SubDomainType& domain)
362 {
363 // Evaluate the same CPU source calculation as the host path, uploading
364 // only nonzero residual and diagonal-block contributions to gpuParams_.
365 std::vector<unsigned> indices;
366 std::vector<VectorBlockCPU> residualContributions;
367 std::vector<MatrixBlockCPU> jacobianContributions;
368 // Small initial capacity; larger source sets grow normally.
369 constexpr std::size_t reservedSize = 20;
370 indices.reserve(reservedSize);
371 residualContributions.reserve(reservedSize);
372 jacobianContributions.reserve(reservedSize);
373 for (unsigned ii = 0; ii < numCells; ++ii) {
374 const unsigned cell = domain.cells[ii];
375 ADVectorBlockCPU source(0.0);
376 const auto& intQuantsIn = linearizer.model_().intensiveQuantities(cell, 0);
377 const double volume = linearizer.model_().dofTotalVolume(cell);
378 if (linearizer.separateSparseSourceTerms_) {
379 LocalResidual::computeSourceDense(
380 source, linearizer.problem_(), intQuantsIn, cell, 0);
381 } else {
382 LocalResidual::computeSource(source, linearizer.problem_(), intQuantsIn, cell, 0);
383 }
384 source *= -volume;
385 VectorBlockCPU residualContribution(0.0);
386 MatrixBlockCPU jacobianContribution(0.0);
388 residualContribution, jacobianContribution, source);
389 bool nonzero = false;
390 for (unsigned eq = 0; eq < numEq; ++eq) {
391 nonzero = nonzero || residualContribution[eq] != 0.0;
392 for (unsigned pv = 0; pv < numEq; ++pv) {
393 nonzero = nonzero || jacobianContribution[eq][pv] != 0.0;
394 }
395 }
396 if (nonzero) {
397 indices.push_back(cell);
398 residualContributions.push_back(residualContribution);
399 jacobianContributions.push_back(jacobianContribution);
400 }
401 }
402 gpuParams_->applySourceContributions(indices, residualContributions, jacobianContributions);
403 }
404#endif
405
406#if HAVE_CUDA
407 std::unique_ptr<gpuistl::GpuSparseMatrixWrapper<Scalar>> gpuJacobian_;
408 std::unique_ptr<gpuistl::GpuBuffer<MatrixBlockGPU*>> gpuBufferDiagMatAddress_;
409 // Type-erased destruction permits an incomplete state in host translation units.
410 using GpuParams = TpfaLinearizerGpuParams<TypeTag>;
411 using GpuParamsOwner = std::unique_ptr<GpuParams, void (*)(GpuParams*)>;
412 GpuParamsOwner gpuParams_ {nullptr, nullptr};
413#endif
414 bool gpuSystemActive_ = false;
415};
416} // namespace Opm
417#endif
Declares the properties required by the black oil model.
Definition: tpfalinearizergpu.hh:64
void linearize(TpfaLinearizer< TypeTag > &linearizer, const SubDomainType &domain, unsigned int numCells, Scalar dt, bool dispersionActive)
Definition: tpfalinearizergpu.hh:187
VectorBlockCPU VectorBlock
Definition: tpfalinearizergpu.hh:85
void resetSystem()
Definition: tpfalinearizergpu.hh:112
decltype(auto) gpuJacobian()
Definition: tpfalinearizergpu.hh:127
void eraseMatrix()
Definition: tpfalinearizergpu.hh:89
ADVectorBlockCPU ADVectorBlock
Definition: tpfalinearizergpu.hh:86
MatrixBlockCPU MatrixBlock
Definition: tpfalinearizergpu.hh:84
void copyJacobianToHost(SparseMatrixAdapter &jacobian)
Definition: tpfalinearizergpu.hh:157
decltype(auto) gpuResidual()
Definition: tpfalinearizergpu.hh:137
bool gpuSystemActive() const
Definition: tpfalinearizergpu.hh:122
bool updateSolverBridge(TpfaLinearizer< TypeTag > &linearizer)
Definition: tpfalinearizergpu.hh:169
void createMatrix(const SparseMatrixAdapter &jacobian)
Definition: tpfalinearizergpu.hh:99
decltype(auto) flattenedGpuResidual()
Definition: tpfalinearizergpu.hh:147
The common code for the linearizers of non-linear systems of equations.
Definition: tpfalinearizer.hh:103
static OPM_HOST_DEVICE void setResAndJacobi(VectorBlockType &res, MatrixBlockType &bMat, const ADVectorBlockType &resid)
Definition: tpfalinearizer.hh:761
static GpuSparseMatrixWrapper< T, ForceLegacy > fromMatrix(const MatrixType &matrix, bool copyNonZeroElementsDirectly=false)
fromMatrix creates a new matrix with the same block size and values as the given matrix
Definition: GpuSparseMatrixWrapper.hpp:184
A small fixed-size square matrix class for use in CUDA kernels.
Definition: MiniMatrix.hpp:37
Definition: MiniVector.hpp:50
Declare the properties used by the infrastructure code of the finite volume discretizations.
Declares the properties required by the black oil model.
@ NONE
Definition: DeferredLogger.hpp:46
Definition: blackoilnewtonmethodparams.hpp:31
Definition: blackoilbioeffectsmodules.hh:45
typename Properties::Detail::GetPropImpl< TypeTag, Property >::type::type GetPropType
get the type alias defined in the property (equivalent to old macro GET_PROP_TYPE(....
Definition: propertysystem.hh:233
This file provides the infrastructure to retrieve run-time parameters.
Definition: tpfalinearizergpu.hh:50
static constexpr bool value
Definition: tpfalinearizergpu.hh:51
GPU parameter setup class for TpfaLinearizer.