1 #ifndef DUNE_SPGRID_COMMUNICATION_HH
2 #define DUNE_SPGRID_COMMUNICATION_HH
4 #include <dune/common/hybridutilities.hh>
5 #include <dune/common/parallel/communication.hh>
6 #include <dune/common/parallel/mpicommunication.hh>
7 #include <dune/common/parallel/mpitraits.hh>
8 #include <dune/common/visibility.hh>
10 #include <dune/grid/common/exceptions.hh>
11 #include <dune/grid/common/datahandleif.hh>
22 template<
class Comm >
52 return comm( MPI_COMM_WORLD );
64 static unsigned char counter = 0;
65 return int( counter++ ) + 1536;
75 template<
class Gr
id,
class DataHandle >
83 typedef typename DataHandle::DataType
DataType;
93 InterfaceType iftype, CommunicationDirection dir );
100 bool pending ()
const {
return bool( interface_ ); }
106 DataHandle &dataHandle_;
108 CommunicationDirection dir_;
111 std::vector< WriteBuffer > writeBuffers_;
112 std::vector< ReadBuffer > readBuffers_;
120 template<
class Gr
id,
class DataHandle >
123 InterfaceType iftype, CommunicationDirection dir )
124 : gridLevel_( gridLevel ),
125 dataHandle_( dataHandle ),
126 interface_( &gridLevel.commInterface( iftype ) ),
131 for(
int codim = 0; codim <= dimension; ++codim )
132 fixedSize_ &= !dataHandle_.contains( dimension, codim ) || dataHandle_.fixedsize( dimension, codim );
134 const std::size_t numLinks = interface_->size();
135 readBuffers_.reserve( numLinks );
139 for(
typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
141 readBuffers_.emplace_back( gridLevel.
grid().comm() );
142 std::size_t size = 0;
144 Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [
this, &partitionList, &size ] (
auto codim ) {
147 if( !dataHandle_.contains( dimension, codim ) )
150 const Iterator end( gridLevel_, partitionList,
typename Iterator::End() );
151 for( Iterator it( gridLevel_, partitionList,
typename Iterator::Begin() ); it != end; ++it )
152 size += dataHandle_.size( *it );
155 readBuffers_.back().receive( it->rank(), tag_, size );
159 writeBuffers_.reserve( numLinks );
160 for(
typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
162 writeBuffers_.emplace_back( gridLevel.
grid().comm() );
164 Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [
this, &partitionList ] (
auto codim ) {
167 if( !dataHandle_.contains( dimension, codim ) )
170 const bool fixedSize = dataHandle_.fixedsize( dimension, codim );
171 const Iterator end( gridLevel_, partitionList,
typename Iterator::End() );
172 for( Iterator it( gridLevel_, partitionList,
typename Iterator::Begin() ); it != end; ++it )
174 const auto &entity = *it;
176 writeBuffers_.back().write(
static_cast< int >( dataHandle_.size( entity ) ) );
178 const std::size_t posBeforeGather = writeBuffers_.back().position();
180 dataHandle_.gather( writeBuffers_.back(), entity );
182 const std::size_t posAfterGather = writeBuffers_.back().position();
183 const std::size_t sizeInBytes = dataHandle_.size( entity ) *
sizeof(
DataType );
184 if( posAfterGather - posBeforeGather != sizeInBytes )
185 DUNE_THROW( GridError,
"Number of bytes written (" << (posAfterGather - posBeforeGather) <<
") does not coincide with reported size (" << sizeInBytes <<
")" );
189 writeBuffers_.back().send( it->rank(), tag_ );
194 template<
class Gr
id,
class DataHandle >
196 : gridLevel_( other.gridLevel_ ),
197 dataHandle_( other.dataHandle_ ),
198 interface_( other.interface_ ),
201 fixedSize_( other.fixedSize_ ),
202 writeBuffers_( std::move( other.writeBuffers_ ) ),
203 readBuffers_( std::move( other.readBuffers_ ) )
205 other.interface_ =
nullptr;
209 template<
class Gr
id,
class DataHandle >
215 const std::size_t numLinks = interface_->size();
219 for( std::size_t i = 0; i < numLinks; ++i )
221 readBuffers_.emplace_back( gridLevel_.grid().comm() );
222 readBuffers_.back().receive( tag_ );
226 for( std::size_t i = 0; i < numLinks; ++i )
228 const typename std::vector< ReadBuffer >::iterator buffer = waitAny( readBuffers_ );
229 for(
typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
231 if( it->rank() == buffer->rank() )
233 const PartitionList &partitionList = it->receiveList( dir_ );
234 Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [
this, &partitionList, buffer ] (
auto codim ) {
237 if( !dataHandle_.contains( dimension, codim ) )
240 const bool fixedSize = dataHandle_.fixedsize( dimension, codim );
241 const Iterator end( gridLevel_, partitionList,
typename Iterator::End() );
242 for( Iterator it( gridLevel_, partitionList,
typename Iterator::Begin() ); it != end; ++it )
244 const auto &entity = *it;
248 buffer->read( size );
250 size = dataHandle_.size( entity );
252 const std::size_t posBeforeGather = buffer->position();
254 dataHandle_.scatter( *buffer, entity, size );
256 const std::size_t posAfterGather = buffer->position();
257 const std::size_t sizeInBytes =
static_cast< std::size_t
>( size ) *
sizeof(
DataType );
258 if( posAfterGather - posBeforeGather != sizeInBytes )
259 DUNE_THROW( GridError,
"Number of bytes read (" << (posAfterGather - posBeforeGather) <<
") does not coincide with reported size (" << sizeInBytes <<
")" );
267 readBuffers_.clear();
269 for(
typename std::vector< WriteBuffer >::iterator it = writeBuffers_.begin(); it != writeBuffers_.end(); ++it )
271 writeBuffers_.clear();
273 interface_ =
nullptr;
Definition: iostream.hh:7
DUNE_EXPORT int getCommTag()
Definition: communication.hh:62
Definition: communication.hh:24
Dune::CollectiveCommunication< Comm > CollectiveCommunication
Definition: communication.hh:25
static CollectiveCommunication comm(const C &)
Definition: communication.hh:28
static CollectiveCommunication defaultComm()
Definition: communication.hh:33
static CollectiveCommunication defaultComm()
Definition: communication.hh:50
static CollectiveCommunication comm(const MPI_Comm &mpiComm)
Definition: communication.hh:45
Dune::CollectiveCommunication< MPI_Comm > CollectiveCommunication
Definition: communication.hh:43
Definition: communication.hh:77
static const int dimension
Definition: communication.hh:78
bool pending() const
Definition: communication.hh:100
DataHandle::DataType DataType
Definition: communication.hh:83
GridLevel::CommInterface Interface
Definition: communication.hh:85
SPCommunication(const SPCommunication &)=delete
void wait()
Definition: communication.hh:210
SPCommunication(const GridLevel &gridLevel, DataHandle &dataHandle, InterfaceType iftype, CommunicationDirection dir)
Definition: communication.hh:122
SPPartitionList< dimension > PartitionList
Definition: communication.hh:81
SPGridLevel< Grid > GridLevel
Definition: communication.hh:80
~SPCommunication()
Definition: communication.hh:98
Definition: gridlevel.hh:35
const Grid & grid() const
Definition: gridlevel.hh:82
Definition: iterator.hh:19
Definition: linkage.hh:54
NodeContainer::const_iterator Iterator
Definition: linkage.hh:60
Definition: messagebuffer.hh:91
Definition: messagebuffer.hh:204
Definition: partitionlist.hh:16