16#ifndef OPM_COMMON_PARALLEL_VARIABLESIZECOMMUNICATOR_HH
17#define OPM_COMMON_PARALLEL_VARIABLESIZECOMMUNICATOR_HH
33#include <dune/common/parallel/interface.hh>
34#include <dune/common/parallel/mpitraits.hh>
35#include <dune/common/unused.hh>
58template<
class T,
class Allocator=std::allocator<T> >
66 explicit MessageBuffer(
int size)
67 : buffer_(new T[size]), size_(size), position_(0)
73 explicit MessageBuffer(
const MessageBuffer& o)
74 : buffer_(new T[o.size_]), size_(o.size_), position_(o.position_)
86 void write(
const T&
data)
88 assert(position_<size_);
89 buffer_[position_++]=
data;
98 assert(position_<size_);
99 data=buffer_[position_++];
118 return position_==size_;
126 bool hasSpaceForItems(
int noItems)
128 return position_+noItems<=size_;
134 std::size_t size()
const
138 std::size_t position()
const
163 std::size_t position_;
166using Dune::InterfaceInformation;
167using Dune::Interface;
172class InterfaceTracker
180 InterfaceTracker(
int rank, InterfaceInformation info, std::size_t fixedsize=0,
181 bool allocateSizes=
false)
182 :
fixedSize(fixedsize),rank_(rank), index_(), interface_(info), sizes_()
186 sizes_.resize(info.size());
193 void moveToNextIndex()
196 assert(index_<=interface_.size());
206 assert(index_<=interface_.size());
212 bool finished()
const
214 return index_==interface_.size();
217 void skipZeroIndices()
220 while(sizes_.size() && index_!=interface_.size() &&!size())
228 std::size_t index()
const
230 return interface_[index_];
235 std::size_t size()
const
237 assert(sizes_.size());
238 return sizes_[index_];
243 std::size_t* getSizesPointer()
253 return !interface_.size();
260 std::size_t indicesLeft()
const
262 return interface_.size()-index_;
278 std::size_t offset()
const
288 InterfaceInformation interface_;
289 std::vector<std::size_t> sizes_;
332template<
class Allocator=std::allocator<std::pair<InterfaceInformation,InterfaceInformation> > >
340 typedef std::map<int,std::pair<InterfaceInformation,InterfaceInformation>,
342 typename Allocator::template rebind<std::pair<const int,std::pair<InterfaceInformation,InterfaceInformation> > >::other>
InterfaceMap;
344#ifndef DUNE_PARALLEL_MAX_COMMUNICATION_BUFFER_SIZE
352 : maxBufferSize_(32768), interface_(&inf)
354 MPI_Comm_dup(comm, &communicator_);
361 : maxBufferSize_(32768), interface_(&inf.interfaces())
363 MPI_Comm_dup(inf.communicator(), &communicator_);
373 : maxBufferSize_(DUNE_PARALLEL_MAX_COMMUNICATION_BUFFER_SIZE),
376 MPI_Comm_dup(comm, &communicator_);
383 : maxBufferSize_(DUNE_PARALLEL_MAX_COMMUNICATION_BUFFER_SIZE),
384 interface_(&inf.interfaces())
386 MPI_Comm_dup(inf.communicator(), &communicator_);
396 : maxBufferSize_(max_buffer_size), interface_(&inf)
398 MPI_Comm_dup(comm, &communicator_);
407 : maxBufferSize_(max_buffer_size), interface_(&inf.interfaces())
409 MPI_Comm_dup(inf.communicator(), &communicator_);
414 MPI_Comm_free(&communicator_);
437 template<
class DataHandle>
440 communicate<true>(handle);
462 template<
class DataHandle>
465 communicate<false>(handle);
469 template<
bool FORWARD,
class DataHandle>
470 void communicateSizes(DataHandle& handle,
471 std::vector<InterfaceTracker>& recv_trackers);
479 template<
bool forward,
class DataHandle>
480 void communicate(DataHandle& handle);
490 template<
bool FORWARD,
class DataHandle>
491 void setupInterfaceTrackers(DataHandle& handle,
492 std::vector<InterfaceTracker>& send_trackers,
493 std::vector<InterfaceTracker>& recv_trackers);
501 template<
bool FORWARD,
class DataHandle>
502 [[maybe_unused]]
void communicateFixedSize(DataHandle& handle);
510 template<
bool FORWARD,
class DataHandle>
511 void communicateVariableSize(DataHandle& handle);
518 std::size_t maxBufferSize_;
532 MPI_Comm communicator_;
541template<
class DataHandle>
545 typedef std::size_t DataType;
547 SizeDataHandle(DataHandle&
data,
548 std::vector<InterfaceTracker>& trackers)
549 : data_(
data), trackers_(trackers), index_()
555 std::size_t size(std::size_t i)
557 DUNE_UNUSED_PARAMETER(i);
561 void gather(B& buf,
int i)
563 buf.write(data_.size(i));
565 void setReceivingIndex(std::size_t i)
569 std::size_t* getSizesPointer()
571 return trackers_[index_].getSizesPointer();
576 std::vector<InterfaceTracker>& trackers_;
581void setReceivingIndex(T&,
int)
585void setReceivingIndex(SizeDataHandle<T>& t,
int i)
587 t.setReceivingIndex(i);
596template<
bool FORWARD>
597struct InterfaceInformationChooser
602 static const InterfaceInformation&
603 getSend(
const std::pair<InterfaceInformation,InterfaceInformation>& info)
611 static const InterfaceInformation&
612 getReceive(
const std::pair<InterfaceInformation,InterfaceInformation>& info)
619struct InterfaceInformationChooser<false>
621 static const InterfaceInformation&
622 getSend(
const std::pair<InterfaceInformation,InterfaceInformation>& info)
627 static const InterfaceInformation&
628 getReceive(
const std::pair<InterfaceInformation,InterfaceInformation>& info)
639template<
class DataHandle>
643 int operator()(DataHandle& handle, InterfaceTracker& tracker,
644 MessageBuffer<typename DataHandle::DataType>&
buffer,
647 DUNE_UNUSED_PARAMETER(i);
648 return operator()(handle,tracker,
buffer);
658 int operator()(DataHandle& handle, InterfaceTracker& tracker,
659 MessageBuffer<typename DataHandle::DataType>&
buffer)
const
661 if(tracker.fixedSize)
664 std::size_t noIndices=std::min(
buffer.size()/tracker.fixedSize, tracker.indicesLeft());
665 for(std::size_t i=0; i< noIndices; ++i)
667 handle.gather(
buffer, tracker.index());
668 tracker.moveToNextIndex();
670 return noIndices*tracker.fixedSize;
675 tracker.skipZeroIndices();
676 while(!tracker.finished())
677 if(
buffer.hasSpaceForItems(handle.size(tracker.index())))
679 assert(std::size_t(packed) ==
buffer.position());
680 handle.gather(
buffer, tracker.index());
681 packed+=handle.size(tracker.index());
682 assert(std::size_t(packed) <=
buffer.size());
683 assert(std::size_t(packed) ==
buffer.position());
684 tracker.moveToNextIndex();
698template<
class DataHandle>
708 bool operator()(DataHandle& handle, InterfaceTracker& tracker,
709 MessageBuffer<typename DataHandle::DataType>&
buffer,
712 if(tracker.fixedSize)
714 std::size_t noIndices=std::min(
buffer.size()/tracker.fixedSize, tracker.indicesLeft());
716 for(std::size_t i=0; i< noIndices; ++i)
718 handle.scatter(
buffer, tracker.index(), tracker.fixedSize);
719 tracker.moveToNextIndex();
721 return tracker.finished();
726 for(
int unpacked=0;unpacked<count;)
728 assert(!tracker.finished());
729 assert(
buffer.hasSpaceForItems(tracker.size()));
730 handle.scatter(
buffer, tracker.index(), tracker.size());
731 unpacked+=tracker.size();
732 tracker.moveToNextIndex();
734 return tracker.finished();
743template<
class DataHandle>
744struct UnpackSizeEntries{
753 bool operator()(SizeDataHandle<DataHandle>& handle, InterfaceTracker& tracker,
754 MessageBuffer<
typename SizeDataHandle<DataHandle>::DataType>&
buffer)
const
756 std::size_t noIndices=std::min(
buffer.size(), tracker.indicesLeft());
757 std::copy(
static_cast<std::size_t*
>(
buffer),
static_cast<std::size_t*
>(
buffer)+noIndices,
758 handle.getSizesPointer()+tracker.offset());
759 tracker.increment(noIndices);
762 bool operator()(SizeDataHandle<DataHandle>& handle, InterfaceTracker& tracker,
763 MessageBuffer<
typename SizeDataHandle<DataHandle>::DataType>&
buffer,
int)
const
765 return operator()(handle,tracker,
buffer);
776[[maybe_unused]]
void sendFixedSize(std::vector<InterfaceTracker>& send_trackers,
777 std::vector<MPI_Request>& send_requests,
778 std::vector<InterfaceTracker>& recv_trackers,
779 std::vector<MPI_Request>& recv_requests,
780 MPI_Comm communicator)
782 typedef std::vector<InterfaceTracker>::iterator TIter;
783 std::vector<MPI_Request>::iterator mIter=recv_requests.begin();
785 for(TIter iter=recv_trackers.begin(), end=recv_trackers.end(); iter!=end;
788 MPI_Irecv(&(iter->fixedSize), 1, Dune::MPITraits<std::size_t>::getType(),
789 iter->rank(), 933881, communicator, &(*mIter));
793 std::vector<MPI_Request>::iterator mIter1=send_requests.begin();
794 for(TIter iter=send_trackers.begin(), end=send_trackers.end();
798 MPI_Issend(&(iter->fixedSize), 1, Dune::MPITraits<std::size_t>::getType(),
799 iter->rank(), 933881, communicator, &(*mIter1));
808template<
class DataHandle>
809struct SetupSendRequest{
810 void operator()(DataHandle& handle,
811 InterfaceTracker& tracker,
812 MessageBuffer<typename DataHandle::DataType>&
buffer,
813 MPI_Request& request,
817 int size=PackEntries<DataHandle>()(handle, tracker,
buffer);
819 while(!tracker.finished() && !handle.size(tracker.index()))
820 tracker.moveToNextIndex();
823 assert(std::size_t(size) <=
buffer.size());
824 MPI_Issend(
buffer, size, Dune::MPITraits<typename DataHandle::DataType>::getType(),
825 tracker.rank(), 933399, comm, &request);
835template<
class DataHandle>
836struct SetupRecvRequest{
837 void operator()(DataHandle& ,
838 InterfaceTracker& tracker,
839 MessageBuffer<typename DataHandle::DataType>&
buffer,
840 MPI_Request& request,
844 if(tracker.indicesLeft())
845 MPI_Irecv(
buffer,
buffer.size(), Dune::MPITraits<typename DataHandle::DataType>::getType(),
846 tracker.rank(), 933399, comm, &request);
853template<
class DataHandle>
854struct NullPackUnpackFunctor
856 int operator()(DataHandle&, InterfaceTracker&,
857 MessageBuffer<typename DataHandle::DataType>&,
int)
861 int operator()(DataHandle&, InterfaceTracker&,
862 MessageBuffer<typename DataHandle::DataType>&)
882template<
class DataHandle,
class BufferFunctor,
class CommunicationFunctor>
883std::size_t checkAndContinue(DataHandle& handle,
884 std::vector<InterfaceTracker>& trackers,
885 std::vector<MPI_Request>& requests,
886 std::vector<MPI_Request>& requests2,
887 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
889 BufferFunctor buffer_func,
890 CommunicationFunctor comm_func,
894 std::size_t size=requests.size();
895 std::vector<MPI_Status> statuses(size);
897 std::vector<int> indices(size, -1);
899 MPI_Testsome(size, &(requests[0]), &no_completed, &(indices[0]), &(statuses[0]));
900 indices.resize(no_completed);
901 for(std::vector<int>::iterator index=indices.begin(), end=indices.end();
904 InterfaceTracker& tracker=trackers[*index];
905 setReceivingIndex(handle, *index);
910 MPI_Get_count(&(statuses[index-indices.begin()]),
911 Dune::MPITraits<typename DataHandle::DataType>::getType(),
914 buffer_func(handle, tracker, buffers[*index], count);
916 buffer_func(handle, tracker, buffers[*index]);
917 tracker.skipZeroIndices();
918 if(!tracker.finished()){
920 comm_func(handle, tracker, buffers[*index], requests2[*index], comm);
921 tracker.skipZeroIndices();
939template<
class DataHandle>
940std::size_t receiveSizeAndSetupReceive(DataHandle& handle,
941 std::vector<InterfaceTracker>& trackers,
942 std::vector<MPI_Request>& size_requests,
943 std::vector<MPI_Request>& data_requests,
944 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
947 return checkAndContinue(handle, trackers, size_requests, data_requests, buffers, comm,
948 NullPackUnpackFunctor<DataHandle>(), SetupRecvRequest<DataHandle>(),
false);
959template<
class DataHandle>
960std::size_t checkSendAndContinueSending(DataHandle& handle,
961 std::vector<InterfaceTracker>& trackers,
962 std::vector<MPI_Request>& requests,
963 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
966 return checkAndContinue(handle, trackers, requests, requests, buffers, comm,
967 NullPackUnpackFunctor<DataHandle>(), SetupSendRequest<DataHandle>());
978template<
class DataHandle>
979std::size_t checkReceiveAndContinueReceiving(DataHandle& handle,
980 std::vector<InterfaceTracker>& trackers,
981 std::vector<MPI_Request>& requests,
982 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
985 return checkAndContinue(handle, trackers, requests, requests, buffers, comm,
986 UnpackEntries<DataHandle>(), SetupRecvRequest<DataHandle>(),
987 true, !handle.fixedsize());
991[[maybe_unused]]
bool validRecvRequests(
const std::vector<MPI_Request> reqs)
993 for(std::vector<MPI_Request>::const_iterator i=reqs.begin(), end=reqs.end();
995 if(*i!=MPI_REQUEST_NULL)
1010template<
class DataHandle,
class Functor>
1011std::size_t setupRequests(DataHandle& handle,
1012 std::vector<InterfaceTracker>& trackers,
1013 std::vector<MessageBuffer<typename DataHandle::DataType> >& buffers,
1014 std::vector<MPI_Request>& requests,
1015 const Functor& setupFunctor,
1016 MPI_Comm communicator)
1018 typedef typename std::vector<InterfaceTracker>::iterator TIter;
1019 typename std::vector<MessageBuffer<typename DataHandle::DataType> >::iterator
1020 biter=buffers.begin();
1021 typename std::vector<MPI_Request>::iterator riter=requests.begin();
1022 std::size_t complete=0;
1023 for(TIter titer=trackers.begin(), end=trackers.end(); titer!=end; ++titer, ++biter, ++riter)
1025 setupFunctor(handle, *titer, *biter, *riter, communicator);
1026 complete+=titer->finished();
1032template<
class Allocator>
1033template<
bool FORWARD,
class DataHandle>
1034void VariableSizeCommunicator<Allocator>::setupInterfaceTrackers(DataHandle& handle,
1035 std::vector<InterfaceTracker>& send_trackers,
1036 std::vector<InterfaceTracker>& recv_trackers)
1038 if(interface_->size()==0)
1040 send_trackers.reserve(interface_->size());
1041 recv_trackers.reserve(interface_->size());
1044 if(handle.fixedsize())
1048 typedef typename InterfaceMap::const_iterator IIter;
1049 for(IIter inf=interface_->begin(), end=interface_->end(); inf!=end; ++inf)
1052 if(handle.fixedsize() && InterfaceInformationChooser<FORWARD>::getSend(inf->second).size())
1053 fixedsize=handle.size(InterfaceInformationChooser<FORWARD>::getSend(inf->second)[0]);
1054 assert(!handle.fixedsize()||fixedsize>0);
1055 send_trackers.push_back(InterfaceTracker(inf->first,
1056 InterfaceInformationChooser<FORWARD>::getSend(inf->second), fixedsize));
1057 recv_trackers.push_back(InterfaceTracker(inf->first,
1058 InterfaceInformationChooser<FORWARD>::getReceive(inf->second), fixedsize, fixedsize==0));
1062template<
class Allocator>
1063template<
bool FORWARD,
class DataHandle>
1064void VariableSizeCommunicator<Allocator>::communicateFixedSize(DataHandle& handle)
1066 std::vector<MPI_Request> size_send_req(interface_->size());
1067 std::vector<MPI_Request> size_recv_req(interface_->size());
1069 std::vector<InterfaceTracker> send_trackers;
1070 std::vector<InterfaceTracker> recv_trackers;
1071 setupInterfaceTrackers<FORWARD>(handle,send_trackers, recv_trackers);
1072 sendFixedSize(send_trackers, size_send_req, recv_trackers, size_recv_req, communicator_);
1074 std::vector<MPI_Request> data_send_req(interface_->size(), MPI_REQUEST_NULL);
1075 std::vector<MPI_Request> data_recv_req(interface_->size(), MPI_REQUEST_NULL);
1076 typedef typename DataHandle::DataType DataType;
1077 std::vector<MessageBuffer<DataType> > send_buffers(interface_->size(), MessageBuffer<DataType>(maxBufferSize_)),
1078 recv_buffers(interface_->size(), MessageBuffer<DataType>(maxBufferSize_));
1081 setupRequests(handle, send_trackers, send_buffers, data_send_req,
1082 SetupSendRequest<DataHandle>(), communicator_);
1084 std::size_t no_size_to_recv, no_to_send, no_to_recv, old_size;
1085 no_size_to_recv = no_to_send = no_to_recv = old_size = interface_->size();
1088 typedef typename std::vector<InterfaceTracker>::const_iterator Iter;
1089 for(Iter i=recv_trackers.begin(), end=recv_trackers.end(); i!=end; ++i)
1092 for(Iter i=send_trackers.begin(), end=send_trackers.end(); i!=end; ++i)
1096 while(no_size_to_recv+no_to_send+no_to_recv)
1100 no_size_to_recv -= receiveSizeAndSetupReceive(handle,recv_trackers, size_recv_req,
1101 data_recv_req, recv_buffers,
1106 no_to_send -= checkSendAndContinueSending(handle, send_trackers, data_send_req,
1107 send_buffers, communicator_);
1108 if(validRecvRequests(data_recv_req))
1110 no_to_recv -= checkReceiveAndContinueReceiving(handle, recv_trackers, data_recv_req,
1111 recv_buffers, communicator_);
1116 MPI_Waitall(size_send_req.size(), &(size_send_req[0]), MPI_STATUSES_IGNORE);
1120template<
class Allocator>
1121template<
bool FORWARD,
class DataHandle>
1122void VariableSizeCommunicator<Allocator>::communicateSizes(DataHandle& handle,
1123 std::vector<InterfaceTracker>& data_recv_trackers)
1125 std::vector<InterfaceTracker> send_trackers;
1126 std::vector<InterfaceTracker> recv_trackers;
1127 std::size_t size = interface_->size();
1128 std::vector<MPI_Request> send_requests(size, MPI_REQUEST_NULL);
1129 std::vector<MPI_Request> recv_requests(size, MPI_REQUEST_NULL);
1130 std::vector<MessageBuffer<std::size_t> >
1131 send_buffers(size, MessageBuffer<std::size_t>(maxBufferSize_)),
1132 recv_buffers(size, MessageBuffer<std::size_t>(maxBufferSize_));
1133 SizeDataHandle<DataHandle> size_handle(handle,data_recv_trackers);
1134 setupInterfaceTrackers<FORWARD>(size_handle,send_trackers, recv_trackers);
1135 setupRequests(size_handle, send_trackers, send_buffers, send_requests,
1136 SetupSendRequest<SizeDataHandle<DataHandle> >(), communicator_);
1137 setupRequests(size_handle, recv_trackers, recv_buffers, recv_requests,
1138 SetupRecvRequest<SizeDataHandle<DataHandle> >(), communicator_);
1141 auto valid_req_func =
1142 [](
const MPI_Request& req) {
return req != MPI_REQUEST_NULL; };
1144 auto size_to_send = std::count_if(send_requests.begin(), send_requests.end(),
1146 auto size_to_recv = std::count_if(recv_requests.begin(), recv_requests.end(),
1149 while(size_to_send+size_to_recv)
1153 checkSendAndContinueSending(size_handle, send_trackers, send_requests,
1154 send_buffers, communicator_);
1160 checkAndContinue(size_handle, recv_trackers, recv_requests, recv_requests,
1161 recv_buffers, communicator_, UnpackSizeEntries<DataHandle>(),
1162 SetupRecvRequest<SizeDataHandle<DataHandle> >());
1166template<
class Allocator>
1167template<
bool FORWARD,
class DataHandle>
1168void VariableSizeCommunicator<Allocator>::communicateVariableSize(DataHandle& handle)
1171 std::vector<InterfaceTracker> send_trackers;
1172 std::vector<InterfaceTracker> recv_trackers;
1173 setupInterfaceTrackers<FORWARD>(handle, send_trackers, recv_trackers);
1175 std::vector<MPI_Request> send_requests(interface_->size(), MPI_REQUEST_NULL);
1176 std::vector<MPI_Request> recv_requests(interface_->size(), MPI_REQUEST_NULL);
1177 typedef typename DataHandle::DataType DataType;
1178 std::vector<MessageBuffer<DataType> >
1179 send_buffers(interface_->size(), MessageBuffer<DataType>(maxBufferSize_)),
1180 recv_buffers(interface_->size(), MessageBuffer<DataType>(maxBufferSize_));
1182 communicateSizes<FORWARD>(handle, recv_trackers);
1184 setupRequests(handle, send_trackers, send_buffers, send_requests,
1185 SetupSendRequest<DataHandle>(), communicator_);
1186 setupRequests(handle, recv_trackers, recv_buffers, recv_requests,
1187 SetupRecvRequest<DataHandle>(), communicator_);
1190 auto valid_req_func =
1191 [](
const MPI_Request& req) {
return req != MPI_REQUEST_NULL;};
1193 auto no_to_send = std::count_if(send_requests.begin(), send_requests.end(),
1195 auto no_to_recv = std::count_if(recv_requests.begin(), recv_requests.end(),
1197 while(no_to_send+no_to_recv)
1201 no_to_send -= checkSendAndContinueSending(handle, send_trackers, send_requests,
1202 send_buffers, communicator_);
1205 no_to_recv -= checkReceiveAndContinueReceiving(handle, recv_trackers, recv_requests,
1206 recv_buffers, communicator_);
1210template<
class Allocator>
1211template<
bool FORWARD,
class DataHandle>
1212void VariableSizeCommunicator<Allocator>::communicate(DataHandle& handle)
1214 if( interface_->size() == 0)
1219 if(handle.fixedsize())
1220 communicateFixedSize<FORWARD>(handle);
1222 communicateVariableSize<FORWARD>(handle);
DataHandle & data
Definition: CpGridData.hpp:1085
mover::MoveBuffer< typename DataHandle::DataType > & buffer
Definition: CpGridData.hpp:1103
std::size_t fixedSize
The number of data items per index if it is fixed, 0 otherwise.
Definition: VariableSizeCommunicator.hpp:267
A buffered communicator where the amount of data sent does not have to be known a priori.
Definition: VariableSizeCommunicator.hpp:334
void backward(DataHandle &handle)
Communicate backwards.
Definition: VariableSizeCommunicator.hpp:463
VariableSizeCommunicator(const Interface &inf, std::size_t max_buffer_size)
Creates a communicator with a specific maximum buffer size.
Definition: VariableSizeCommunicator.hpp:406
VariableSizeCommunicator(MPI_Comm comm, const InterfaceMap &inf, std::size_t max_buffer_size)
Creates a communicator with a specific maximum buffer size.
Definition: VariableSizeCommunicator.hpp:395
std::map< int, std::pair< InterfaceInformation, InterfaceInformation >, std::less< int >, typename Allocator::template rebind< std::pair< const int, std::pair< InterfaceInformation, InterfaceInformation > > >::other > InterfaceMap
The type of the map from process number to InterfaceInformation for sending and receiving to and from...
Definition: VariableSizeCommunicator.hpp:342
void forward(DataHandle &handle)
Communicate forward.
Definition: VariableSizeCommunicator.hpp:438
~VariableSizeCommunicator()
Definition: VariableSizeCommunicator.hpp:412
VariableSizeCommunicator(const Interface &inf)
Creates a communicator with the default maximum buffer size.
Definition: VariableSizeCommunicator.hpp:360
VariableSizeCommunicator(MPI_Comm comm, const InterfaceMap &inf)
Creates a communicator with the default maximum buffer size.
Definition: VariableSizeCommunicator.hpp:351
T * increment(T *cc, int i, int dim)
Increment an iterator over an array that reresents a dense row-major matrix with dims columns.
Definition: GridHelpers.hpp:354
Holds the implementation of the CpGrid as a pimple.
Definition: CellQuadrature.hpp:26