dune-spgrid  2.7
communication.hh
Go to the documentation of this file.
1 #ifndef DUNE_SPGRID_COMMUNICATION_HH
2 #define DUNE_SPGRID_COMMUNICATION_HH
3 
4 #include <dune/common/hybridutilities.hh>
5 #include <dune/common/parallel/communication.hh>
6 #include <dune/common/parallel/mpicommunication.hh>
7 #include <dune/common/parallel/mpitraits.hh>
8 #include <dune/common/visibility.hh>
9 
10 #include <dune/grid/common/exceptions.hh>
11 #include <dune/grid/common/datahandleif.hh>
12 
15 
16 namespace Dune
17 {
18 
19  // SPCommunicationTraits
20  // ---------------------
21 
22  template< class Comm >
24  {
25  typedef Dune::CollectiveCommunication< Comm > CollectiveCommunication;
26 
27  template< class C >
28  static CollectiveCommunication comm ( const C & )
29  {
30  return defaultComm();
31  }
32 
34  {
35  return CollectiveCommunication();
36  }
37  };
38 
39 #if HAVE_MPI
40  template<>
41  struct SPCommunicationTraits< MPI_Comm >
42  {
43  typedef Dune::CollectiveCommunication< MPI_Comm > CollectiveCommunication;
44 
45  static CollectiveCommunication comm ( const MPI_Comm &mpiComm )
46  {
47  return CollectiveCommunication( mpiComm );
48  }
49 
51  {
52  return comm( MPI_COMM_WORLD );
53  }
54  };
55 #endif // #if HAVE_MPI
56 
57 
58 
59  namespace __SPGrid
60  {
61 
62  DUNE_EXPORT inline int getCommTag ()
63  {
64  static unsigned char counter = 0;
65  return int( counter++ ) + 1536;
66  }
67 
68  } // namespace __SPGrid
69 
70 
71 
72  // SPCommunication
73  // ---------------
74 
75  template< class Grid, class DataHandle >
77  {
78  static const int dimension = Grid::dimension;
79 
82 
83  typedef typename DataHandle::DataType DataType;
84 
86 
87  private:
90 
91  public:
92  SPCommunication ( const GridLevel &gridLevel, DataHandle &dataHandle,
93  InterfaceType iftype, CommunicationDirection dir );
94 
95  SPCommunication ( const SPCommunication & ) = delete;
96  SPCommunication ( SPCommunication &&other );
97 
99 
100  bool pending () const { return bool( interface_ ); }
101 
102  void wait ();
103 
104  private:
105  const GridLevel &gridLevel_;
106  DataHandle &dataHandle_;
107  const Interface *interface_;
108  CommunicationDirection dir_;
109  int tag_;
110  bool fixedSize_;
111  std::vector< WriteBuffer > writeBuffers_;
112  std::vector< ReadBuffer > readBuffers_;
113  };
114 
115 
116 
117  // Implementation of SPCommunication
118  // ---------------------------------
119 
120  template< class Grid, class DataHandle >
122  ::SPCommunication ( const GridLevel &gridLevel, DataHandle &dataHandle,
123  InterfaceType iftype, CommunicationDirection dir )
124  : gridLevel_( gridLevel ),
125  dataHandle_( dataHandle ),
126  interface_( &gridLevel.commInterface( iftype ) ),
127  dir_( dir ),
128  tag_( __SPGrid::getCommTag() ),
129  fixedSize_( true )
130  {
131  for( int codim = 0; codim <= dimension; ++codim )
132  fixedSize_ &= !dataHandle_.contains( dimension, codim ) || dataHandle_.fixedsize( dimension, codim );
133 
134  const std::size_t numLinks = interface_->size();
135  readBuffers_.reserve( numLinks );
136 
137  if( fixedSize_ )
138  {
139  for( typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
140  {
141  readBuffers_.emplace_back( gridLevel.grid().comm() );
142  std::size_t size = 0;
143  const PartitionList &partitionList = it->receiveList( dir );
144  Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [ this, &partitionList, &size ] ( auto codim ) {
146 
147  if( !dataHandle_.contains( dimension, codim ) )
148  return;
149 
150  const Iterator end( gridLevel_, partitionList, typename Iterator::End() );
151  for( Iterator it( gridLevel_, partitionList, typename Iterator::Begin() ); it != end; ++it )
152  size += dataHandle_.size( *it );
153  } );
154  size *= sizeof( DataType );
155  readBuffers_.back().receive( it->rank(), tag_, size );
156  }
157  }
158 
159  writeBuffers_.reserve( numLinks );
160  for( typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
161  {
162  writeBuffers_.emplace_back( gridLevel.grid().comm() );
163  const PartitionList &partitionList = it->sendList( dir );
164  Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [ this, &partitionList ] ( auto codim ) {
166 
167  if( !dataHandle_.contains( dimension, codim ) )
168  return;
169 
170  const bool fixedSize = dataHandle_.fixedsize( dimension, codim );
171  const Iterator end( gridLevel_, partitionList, typename Iterator::End() );
172  for( Iterator it( gridLevel_, partitionList, typename Iterator::Begin() ); it != end; ++it )
173  {
174  const auto &entity = *it;
175  if( !fixedSize )
176  writeBuffers_.back().write( static_cast< int >( dataHandle_.size( entity ) ) );
177 #ifndef NDEBUG
178  const std::size_t posBeforeGather = writeBuffers_.back().position();
179 #endif // #ifndef NDEBUG
180  dataHandle_.gather( writeBuffers_.back(), entity );
181 #ifndef NDEBUG
182  const std::size_t posAfterGather = writeBuffers_.back().position();
183  const std::size_t sizeInBytes = dataHandle_.size( entity ) * sizeof( DataType );
184  if( posAfterGather - posBeforeGather != sizeInBytes )
185  DUNE_THROW( GridError, "Number of bytes written (" << (posAfterGather - posBeforeGather) << ") does not coincide with reported size (" << sizeInBytes << ")" );
186 #endif // #ifndef NDEBUG
187  }
188  } );
189  writeBuffers_.back().send( it->rank(), tag_ );
190  }
191  }
192 
193 
194  template< class Grid, class DataHandle >
196  : gridLevel_( other.gridLevel_ ),
197  dataHandle_( other.dataHandle_ ),
198  interface_( other.interface_ ),
199  dir_( other.dir_ ),
200  tag_( other.tag_ ),
201  fixedSize_( other.fixedSize_ ),
202  writeBuffers_( std::move( other.writeBuffers_ ) ),
203  readBuffers_( std::move( other.readBuffers_ ) )
204  {
205  other.interface_ = nullptr;
206  }
207 
208 
209  template< class Grid, class DataHandle >
211  {
212  if( !pending() )
213  return;
214 
215  const std::size_t numLinks = interface_->size();
216 
217  if( !fixedSize_ )
218  {
219  for( std::size_t i = 0; i < numLinks; ++i )
220  {
221  readBuffers_.emplace_back( gridLevel_.grid().comm() );
222  readBuffers_.back().receive( tag_ );
223  }
224  }
225 
226  for( std::size_t i = 0; i < numLinks; ++i )
227  {
228  const typename std::vector< ReadBuffer >::iterator buffer = waitAny( readBuffers_ );
229  for( typename Interface::Iterator it = interface_->begin(); it != interface_->end(); ++it )
230  {
231  if( it->rank() == buffer->rank() )
232  {
233  const PartitionList &partitionList = it->receiveList( dir_ );
234  Hybrid::forEach( std::make_integer_sequence< int, dimension+1 >(), [ this, &partitionList, buffer ] ( auto codim ) {
236 
237  if( !dataHandle_.contains( dimension, codim ) )
238  return;
239 
240  const bool fixedSize = dataHandle_.fixedsize( dimension, codim );
241  const Iterator end( gridLevel_, partitionList, typename Iterator::End() );
242  for( Iterator it( gridLevel_, partitionList, typename Iterator::Begin() ); it != end; ++it )
243  {
244  const auto &entity = *it;
245 
246  int size;
247  if( !fixedSize )
248  buffer->read( size );
249  else
250  size = dataHandle_.size( entity );
251 #ifndef NDEBUG
252  const std::size_t posBeforeGather = buffer->position();
253 #endif // #ifndef NDEBUG
254  dataHandle_.scatter( *buffer, entity, size );
255 #ifndef NDEBUG
256  const std::size_t posAfterGather = buffer->position();
257  const std::size_t sizeInBytes = static_cast< std::size_t >( size ) * sizeof( DataType );
258  if( posAfterGather - posBeforeGather != sizeInBytes )
259  DUNE_THROW( GridError, "Number of bytes read (" << (posAfterGather - posBeforeGather) << ") does not coincide with reported size (" << sizeInBytes << ")" );
260 #endif // #ifndef NDEBUG
261  }
262  } );
263  break;
264  }
265  }
266  }
267  readBuffers_.clear();
268 
269  for( typename std::vector< WriteBuffer >::iterator it = writeBuffers_.begin(); it != writeBuffers_.end(); ++it )
270  it->wait();
271  writeBuffers_.clear();
272 
273  interface_ = nullptr;
274  }
275 
276 } // namespace Dune
277 
278 #endif // #ifndef DUNE_SPGRID_COMMUNICATION_HH
Definition: iostream.hh:7
DUNE_EXPORT int getCommTag()
Definition: communication.hh:62
Definition: communication.hh:24
Dune::CollectiveCommunication< Comm > CollectiveCommunication
Definition: communication.hh:25
static CollectiveCommunication comm(const C &)
Definition: communication.hh:28
static CollectiveCommunication defaultComm()
Definition: communication.hh:33
static CollectiveCommunication defaultComm()
Definition: communication.hh:50
static CollectiveCommunication comm(const MPI_Comm &mpiComm)
Definition: communication.hh:45
Dune::CollectiveCommunication< MPI_Comm > CollectiveCommunication
Definition: communication.hh:43
Definition: communication.hh:77
static const int dimension
Definition: communication.hh:78
bool pending() const
Definition: communication.hh:100
DataHandle::DataType DataType
Definition: communication.hh:83
GridLevel::CommInterface Interface
Definition: communication.hh:85
SPCommunication(const SPCommunication &)=delete
void wait()
Definition: communication.hh:210
SPCommunication(const GridLevel &gridLevel, DataHandle &dataHandle, InterfaceType iftype, CommunicationDirection dir)
Definition: communication.hh:122
SPPartitionList< dimension > PartitionList
Definition: communication.hh:81
SPGridLevel< Grid > GridLevel
Definition: communication.hh:80
~SPCommunication()
Definition: communication.hh:98
Definition: gridlevel.hh:35
const Grid & grid() const
Definition: gridlevel.hh:82
Definition: iterator.hh:19
Definition: linkage.hh:54
NodeContainer::const_iterator Iterator
Definition: linkage.hh:60
Definition: messagebuffer.hh:91
Definition: messagebuffer.hh:204
Definition: partitionlist.hh:16