Actual source code: data_ex.c

  1: /*
  2: Build a few basic tools to help with partitioned domains.

  4: 1)
  5: On each processor, have a DomainExchangerTopology.
  6: This is a doubly-connected edge list which enumerates the
  7: communication paths between connected processors. By numbering
  8: these paths we can always uniquely assign message identifiers.

 10:         edge
 11:          10
 12: proc  --------->  proc
 13:  0    <--------    1
 14:          11
 15:         twin

 17: Eg: Proc 0 send to proc 1 with message id is 10. To receive the correct
 18: message, proc 1 looks for the edge connected to proc 0, and then the
 19: message id comes from the twin of that edge

 21: 2)
 22: A DomainExchangerArrayPacker.
 23: A little function which given a piece of data, will memcpy the data into
 24: an array (which will be sent to procs) into the correct place.

 26: On Proc 1 we sent data to procs 0,2,3. The data is on different lengths.
 27: All data gets jammed into single array. Need to "jam" data into correct locations
 28: The Packer knows how much is to going to each processor and keeps track of the inserts
 29: so as to avoid ever packing TOO much into one slot, and inevatbly corrupting some memory

 31: data to 0    data to 2       data to 3

 33: |--------|-----------------|--|

 35: User has to unpack message themselves. I can get you the pointer for each i
 36: entry, but you'll have to cast it to the appropriate data type.

 38: Phase A: Build topology

 40: Phase B: Define message lengths

 42: Phase C: Pack data

 44: Phase D: Send data

 46: + Constructor
 47: DMSwarmDataExCreate()
 48: + Phase A
 49: DMSwarmDataExTopologyInitialize()
 50: DMSwarmDataExTopologyAddNeighbour()
 51: DMSwarmDataExTopologyAddNeighbour()
 52: DMSwarmDataExTopologyFinalize()
 53: + Phase B
 54: DMSwarmDataExZeroAllSendCount()
 55: DMSwarmDataExAddToSendCount()
 56: DMSwarmDataExAddToSendCount()
 57: DMSwarmDataExAddToSendCount()
 58: + Phase C
 59: DMSwarmDataExPackInitialize()
 60: DMSwarmDataExPackData()
 61: DMSwarmDataExPackData()
 62: DMSwarmDataExPackFinalize()
 63: +Phase D
 64: DMSwarmDataExBegin()
 65:  ... perform any calculations ...
 66: DMSwarmDataExEnd()

 68: ... user calls any getters here ...

 70: */
 71: #include <petscvec.h>
 72: #include <petscmat.h>

 74: #include "../src/dm/impls/swarm/data_ex.h"

 76: const char *status_names[] = {"initialized", "finalized", "unknown"};

 78: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerTopologySetup;
 79: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerBegin;
 80: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerEnd;
 81: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerSendCount;
 82: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerPack;

 84: PetscErrorCode DMSwarmDataExCreate(MPI_Comm comm, const PetscInt count, DMSwarmDataEx *ex)
 85: {
 86:   DMSwarmDataEx d;

 88:   PetscNew(&d);
 89:   MPI_Comm_dup(comm, &d->comm);
 90:   MPI_Comm_rank(d->comm, &d->rank);

 92:   d->instance = count;

 94:   d->topology_status        = DEOBJECT_STATE_UNKNOWN;
 95:   d->message_lengths_status = DEOBJECT_STATE_UNKNOWN;
 96:   d->packer_status          = DEOBJECT_STATE_UNKNOWN;
 97:   d->communication_status   = DEOBJECT_STATE_UNKNOWN;

 99:   d->n_neighbour_procs = -1;
100:   d->neighbour_procs   = NULL;

102:   d->messages_to_be_sent      = NULL;
103:   d->message_offsets          = NULL;
104:   d->messages_to_be_recvieved = NULL;

106:   d->unit_message_size   = (size_t)-1;
107:   d->send_message        = NULL;
108:   d->send_message_length = -1;
109:   d->recv_message        = NULL;
110:   d->recv_message_length = -1;
111:   d->total_pack_cnt      = -1;
112:   d->pack_cnt            = NULL;

114:   d->send_tags = NULL;
115:   d->recv_tags = NULL;

117:   d->_stats    = NULL;
118:   d->_requests = NULL;
119:   *ex          = d;
120:   return 0;
121: }

123: /*
124:     This code is horrible, who let it get into main.

126:     Should be printing to a viewer, should not be using PETSC_COMM_WORLD

128: */
129: PetscErrorCode DMSwarmDataExView(DMSwarmDataEx d)
130: {
131:   PetscMPIInt p;

133:   PetscPrintf(PETSC_COMM_WORLD, "DMSwarmDataEx: instance=%" PetscInt_FMT "\n", d->instance);
134:   PetscPrintf(PETSC_COMM_WORLD, "  topology status:        %s \n", status_names[d->topology_status]);
135:   PetscPrintf(PETSC_COMM_WORLD, "  message lengths status: %s \n", status_names[d->message_lengths_status]);
136:   PetscPrintf(PETSC_COMM_WORLD, "  packer status status:   %s \n", status_names[d->packer_status]);
137:   PetscPrintf(PETSC_COMM_WORLD, "  communication status:   %s \n", status_names[d->communication_status]);

139:   if (d->topology_status == DEOBJECT_FINALIZED) {
140:     PetscPrintf(PETSC_COMM_WORLD, "  Topology:\n");
141:     PetscSynchronizedPrintf(PETSC_COMM_WORLD, "    [%d] neighbours: %d \n", d->rank, d->n_neighbour_procs);
142:     for (p = 0; p < d->n_neighbour_procs; p++) PetscSynchronizedPrintf(PETSC_COMM_WORLD, "    [%d]   neighbour[%d] = %d \n", d->rank, p, d->neighbour_procs[p]);
143:     PetscSynchronizedFlush(PETSC_COMM_WORLD, stdout);
144:   }

146:   if (d->message_lengths_status == DEOBJECT_FINALIZED) {
147:     PetscPrintf(PETSC_COMM_WORLD, "  Message lengths:\n");
148:     PetscSynchronizedPrintf(PETSC_COMM_WORLD, "    [%d] atomic size: %ld \n", d->rank, (long int)d->unit_message_size);
149:     for (p = 0; p < d->n_neighbour_procs; p++) {
150:       PetscSynchronizedPrintf(PETSC_COMM_WORLD, "    [%d] >>>>> ( %" PetscInt_FMT " units :: tag = %d) >>>>> [%d] \n", d->rank, d->messages_to_be_sent[p], d->send_tags[p], d->neighbour_procs[p]);
151:     }
152:     for (p = 0; p < d->n_neighbour_procs; p++) {
153:       PetscSynchronizedPrintf(PETSC_COMM_WORLD, "    [%d] <<<<< ( %" PetscInt_FMT " units :: tag = %d) <<<<< [%d] \n", d->rank, d->messages_to_be_recvieved[p], d->recv_tags[p], d->neighbour_procs[p]);
154:     }
155:     PetscSynchronizedFlush(PETSC_COMM_WORLD, stdout);
156:   }
157:   if (d->packer_status == DEOBJECT_FINALIZED) { }
158:   if (d->communication_status == DEOBJECT_FINALIZED) { }
159:   return 0;
160: }

162: PetscErrorCode DMSwarmDataExDestroy(DMSwarmDataEx d)
163: {
164:   MPI_Comm_free(&d->comm);
165:   if (d->neighbour_procs) PetscFree(d->neighbour_procs);
166:   if (d->messages_to_be_sent) PetscFree(d->messages_to_be_sent);
167:   if (d->message_offsets) PetscFree(d->message_offsets);
168:   if (d->messages_to_be_recvieved) PetscFree(d->messages_to_be_recvieved);
169:   if (d->send_message) PetscFree(d->send_message);
170:   if (d->recv_message) PetscFree(d->recv_message);
171:   if (d->pack_cnt) PetscFree(d->pack_cnt);
172:   if (d->send_tags) PetscFree(d->send_tags);
173:   if (d->recv_tags) PetscFree(d->recv_tags);
174:   if (d->_stats) PetscFree(d->_stats);
175:   if (d->_requests) PetscFree(d->_requests);
176:   PetscFree(d);
177:   return 0;
178: }

180: /* === Phase A === */

182: PetscErrorCode DMSwarmDataExTopologyInitialize(DMSwarmDataEx d)
183: {
184:   d->topology_status   = DEOBJECT_INITIALIZED;
185:   d->n_neighbour_procs = 0;
186:   PetscFree(d->neighbour_procs);
187:   PetscFree(d->messages_to_be_sent);
188:   PetscFree(d->message_offsets);
189:   PetscFree(d->messages_to_be_recvieved);
190:   PetscFree(d->pack_cnt);
191:   PetscFree(d->send_tags);
192:   PetscFree(d->recv_tags);
193:   return 0;
194: }

196: PetscErrorCode DMSwarmDataExTopologyAddNeighbour(DMSwarmDataEx d, const PetscMPIInt proc_id)
197: {
198:   PetscMPIInt n, found;
199:   PetscMPIInt size;


204:   /* error on negative entries */
206:   /* error on ranks larger than number of procs in communicator */
207:   MPI_Comm_size(d->comm, &size);
209:   if (d->n_neighbour_procs == 0) PetscMalloc1(1, &d->neighbour_procs);
210:   /* check for proc_id */
211:   found = 0;
212:   for (n = 0; n < d->n_neighbour_procs; n++) {
213:     if (d->neighbour_procs[n] == proc_id) found = 1;
214:   }
215:   if (found == 0) { /* add it to list */
216:     PetscRealloc(sizeof(PetscMPIInt) * (d->n_neighbour_procs + 1), &d->neighbour_procs);
217:     d->neighbour_procs[d->n_neighbour_procs] = proc_id;
218:     d->n_neighbour_procs++;
219:   }
220:   return 0;
221: }

223: /*
224: counter: the index of the communication object
225: N: the number of processors
226: r0: rank of sender
227: r1: rank of receiver

229: procs = { 0, 1, 2, 3 }

231: 0 ==> 0         e=0
232: 0 ==> 1         e=1
233: 0 ==> 2         e=2
234: 0 ==> 3         e=3

236: 1 ==> 0         e=4
237: 1 ==> 1         e=5
238: 1 ==> 2         e=6
239: 1 ==> 3         e=7

241: 2 ==> 0         e=8
242: 2 ==> 1         e=9
243: 2 ==> 2         e=10
244: 2 ==> 3         e=11

246: 3 ==> 0         e=12
247: 3 ==> 1         e=13
248: 3 ==> 2         e=14
249: 3 ==> 3         e=15

251: If we require that proc A sends to proc B, then the SEND tag index will be given by
252:   N * rank(A) + rank(B) + offset
253: If we require that proc A will receive from proc B, then the RECV tag index will be given by
254:   N * rank(B) + rank(A) + offset

256: */
257: static void _get_tags(PetscInt counter, PetscMPIInt N, PetscMPIInt r0, PetscMPIInt r1, PetscMPIInt *_st, PetscMPIInt *_rt)
258: {
259:   PetscMPIInt st, rt;

261:   st   = N * r0 + r1 + N * N * counter;
262:   rt   = N * r1 + r0 + N * N * counter;
263:   *_st = st;
264:   *_rt = rt;
265: }

267: /*
268: Makes the communication map symmetric
269: */
270: PetscErrorCode _DMSwarmDataExCompleteCommunicationMap(MPI_Comm comm, PetscMPIInt n, PetscMPIInt proc_neighbours[], PetscMPIInt *n_new, PetscMPIInt **proc_neighbours_new)
271: {
272:   Mat                A;
273:   PetscInt           i, j, nc;
274:   PetscInt           n_, *proc_neighbours_;
275:   PetscInt           rank_;
276:   PetscMPIInt        size, rank;
277:   PetscScalar       *vals;
278:   const PetscInt    *cols;
279:   const PetscScalar *red_vals;
280:   PetscMPIInt        _n_new, *_proc_neighbours_new;

282:   n_ = n;
283:   PetscMalloc(sizeof(PetscInt) * n_, &proc_neighbours_);
284:   for (i = 0; i < n_; ++i) proc_neighbours_[i] = proc_neighbours[i];
285:   MPI_Comm_size(comm, &size);
286:   MPI_Comm_rank(comm, &rank);
287:   rank_ = rank;

289:   MatCreate(comm, &A);
290:   MatSetSizes(A, PETSC_DECIDE, PETSC_DECIDE, size, size);
291:   MatSetType(A, MATAIJ);
292:   MatSeqAIJSetPreallocation(A, 1, NULL);
293:   MatMPIAIJSetPreallocation(A, n_, NULL, n_, NULL);
294:   MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
295:   /* Build original map */
296:   PetscMalloc1(n_, &vals);
297:   for (i = 0; i < n_; ++i) vals[i] = 1.0;
298:   MatSetValues(A, 1, &rank_, n_, proc_neighbours_, vals, INSERT_VALUES);
299:   MatAssemblyBegin(A, MAT_FLUSH_ASSEMBLY);
300:   MatAssemblyEnd(A, MAT_FLUSH_ASSEMBLY);
301:   /* Now force all other connections if they are not already there */
302:   /* It's more efficient to do them all at once */
303:   for (i = 0; i < n_; ++i) vals[i] = 2.0;
304:   MatSetValues(A, n_, proc_neighbours_, 1, &rank_, vals, INSERT_VALUES);
305:   MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);
306:   MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);
307:   /*
308:   PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);
309:   MatView(A,PETSC_VIEWER_STDOUT_WORLD);
310:   PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD);
311: */
312:   if ((n_new != NULL) && (proc_neighbours_new != NULL)) {
313:     MatGetRow(A, rank_, &nc, &cols, &red_vals);
314:     _n_new = (PetscMPIInt)nc;
315:     PetscMalloc1(_n_new, &_proc_neighbours_new);
316:     for (j = 0; j < nc; ++j) _proc_neighbours_new[j] = (PetscMPIInt)cols[j];
317:     MatRestoreRow(A, rank_, &nc, &cols, &red_vals);
318:     *n_new               = (PetscMPIInt)_n_new;
319:     *proc_neighbours_new = (PetscMPIInt *)_proc_neighbours_new;
320:   }
321:   MatDestroy(&A);
322:   PetscFree(vals);
323:   PetscFree(proc_neighbours_);
324:   MPI_Barrier(comm);
325:   return 0;
326: }

328: PetscErrorCode DMSwarmDataExTopologyFinalize(DMSwarmDataEx d)
329: {
330:   PetscMPIInt symm_nn, *symm_procs, r0, n, st, rt, size;


334:   PetscLogEventBegin(DMSWARM_DataExchangerTopologySetup, 0, 0, 0, 0);
335:   /* given information about all my neighbours, make map symmetric */
336:   _DMSwarmDataExCompleteCommunicationMap(d->comm, d->n_neighbour_procs, d->neighbour_procs, &symm_nn, &symm_procs);
337:   /* update my arrays */
338:   PetscFree(d->neighbour_procs);
339:   d->n_neighbour_procs = symm_nn;
340:   d->neighbour_procs   = symm_procs;
341:   /* allocates memory */
342:   if (!d->messages_to_be_sent) PetscMalloc1(d->n_neighbour_procs + 1, &d->messages_to_be_sent);
343:   if (!d->message_offsets) PetscMalloc1(d->n_neighbour_procs + 1, &d->message_offsets);
344:   if (!d->messages_to_be_recvieved) PetscMalloc1(d->n_neighbour_procs + 1, &d->messages_to_be_recvieved);
345:   if (!d->pack_cnt) PetscMalloc(sizeof(PetscInt) * d->n_neighbour_procs, &d->pack_cnt);
346:   if (!d->_stats) PetscMalloc(sizeof(MPI_Status) * 2 * d->n_neighbour_procs, &d->_stats);
347:   if (!d->_requests) PetscMalloc(sizeof(MPI_Request) * 2 * d->n_neighbour_procs, &d->_requests);
348:   if (!d->send_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->send_tags);
349:   if (!d->recv_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->recv_tags);
350:   /* compute message tags */
351:   MPI_Comm_size(d->comm, &size);
352:   r0 = d->rank;
353:   for (n = 0; n < d->n_neighbour_procs; ++n) {
354:     PetscMPIInt r1 = d->neighbour_procs[n];

356:     _get_tags(d->instance, size, r0, r1, &st, &rt);
357:     d->send_tags[n] = (int)st;
358:     d->recv_tags[n] = (int)rt;
359:   }
360:   d->topology_status = DEOBJECT_FINALIZED;
361:   PetscLogEventEnd(DMSWARM_DataExchangerTopologySetup, 0, 0, 0, 0);
362:   return 0;
363: }

365: /* === Phase B === */
366: PetscErrorCode _DMSwarmDataExConvertProcIdToLocalIndex(DMSwarmDataEx de, PetscMPIInt proc_id, PetscMPIInt *local)
367: {
368:   PetscMPIInt i, np;

370:   np     = de->n_neighbour_procs;
371:   *local = -1;
372:   for (i = 0; i < np; ++i) {
373:     if (proc_id == de->neighbour_procs[i]) {
374:       *local = i;
375:       break;
376:     }
377:   }
378:   return 0;
379: }

381: PetscErrorCode DMSwarmDataExInitializeSendCount(DMSwarmDataEx de)
382: {
383:   PetscMPIInt i;

386:   PetscLogEventBegin(DMSWARM_DataExchangerSendCount, 0, 0, 0, 0);
387:   de->message_lengths_status = DEOBJECT_INITIALIZED;
388:   for (i = 0; i < de->n_neighbour_procs; ++i) de->messages_to_be_sent[i] = 0;
389:   return 0;
390: }

392: /*
393: 1) only allows counters to be set on neighbouring cpus
394: */
395: PetscErrorCode DMSwarmDataExAddToSendCount(DMSwarmDataEx de, const PetscMPIInt proc_id, const PetscInt count)
396: {
397:   PetscMPIInt local_val;


402:   _DMSwarmDataExConvertProcIdToLocalIndex(de, proc_id, &local_val);

405:   de->messages_to_be_sent[local_val] = de->messages_to_be_sent[local_val] + count;
406:   return 0;
407: }

409: PetscErrorCode DMSwarmDataExFinalizeSendCount(DMSwarmDataEx de)
410: {

413:   de->message_lengths_status = DEOBJECT_FINALIZED;
414:   PetscLogEventEnd(DMSWARM_DataExchangerSendCount, 0, 0, 0, 0);
415:   return 0;
416: }

418: /* === Phase C === */
419: /*
420:   zero out all send counts
421:   free send and recv buffers
422:   zeros out message length
423:   zeros out all counters
424:   zero out packed data counters
425: */
426: PetscErrorCode _DMSwarmDataExInitializeTmpStorage(DMSwarmDataEx de)
427: {
428:   PetscMPIInt i, np;

430:   np = de->n_neighbour_procs;
431:   for (i = 0; i < np; ++i) {
432:     /*  de->messages_to_be_sent[i] = -1; */
433:     de->messages_to_be_recvieved[i] = -1;
434:   }
435:   PetscFree(de->send_message);
436:   PetscFree(de->recv_message);
437:   return 0;
438: }

440: /*
441:    Zeros out pack data counters
442:    Ensures mesaage length is set
443:    Checks send counts properly initialized
444:    allocates space for pack data
445: */
446: PetscErrorCode DMSwarmDataExPackInitialize(DMSwarmDataEx de, size_t unit_message_size)
447: {
448:   PetscMPIInt i, np;
449:   PetscInt    total;

453:   PetscLogEventBegin(DMSWARM_DataExchangerPack, 0, 0, 0, 0);
454:   de->packer_status = DEOBJECT_INITIALIZED;
455:   _DMSwarmDataExInitializeTmpStorage(de);
456:   np                    = de->n_neighbour_procs;
457:   de->unit_message_size = unit_message_size;
458:   total                 = 0;
459:   for (i = 0; i < np; ++i) {
460:     if (de->messages_to_be_sent[i] == -1) {
461:       PetscMPIInt proc_neighour = de->neighbour_procs[i];
462:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ORDER, "Messages_to_be_sent[neighbour_proc=%d] is un-initialised. Call DMSwarmDataExSetSendCount() first", (int)proc_neighour);
463:     }
464:     total = total + de->messages_to_be_sent[i];
465:   }
466:   /* create space for the data to be sent */
467:   PetscMalloc(unit_message_size * (total + 1), &de->send_message);
468:   /* initialize memory */
469:   PetscMemzero(de->send_message, unit_message_size * (total + 1));
470:   /* set total items to send */
471:   de->send_message_length = total;
472:   de->message_offsets[0]  = 0;
473:   total                   = de->messages_to_be_sent[0];
474:   for (i = 1; i < np; ++i) {
475:     de->message_offsets[i] = total;
476:     total                  = total + de->messages_to_be_sent[i];
477:   }
478:   /* init the packer counters */
479:   de->total_pack_cnt = 0;
480:   for (i = 0; i < np; ++i) de->pack_cnt[i] = 0;
481:   return 0;
482: }

484: /*
485:     Ensures data gets been packed appropriately and no overlaps occur
486: */
487: PetscErrorCode DMSwarmDataExPackData(DMSwarmDataEx de, PetscMPIInt proc_id, PetscInt n, void *data)
488: {
489:   PetscMPIInt local;
490:   PetscInt    insert_location;
491:   void       *dest;


497:   _DMSwarmDataExConvertProcIdToLocalIndex(de, proc_id, &local);

501:   /* copy memory */
502:   insert_location = de->message_offsets[local] + de->pack_cnt[local];
503:   dest            = ((char *)de->send_message) + de->unit_message_size * insert_location;
504:   PetscMemcpy(dest, data, de->unit_message_size * n);
505:   /* increment counter */
506:   de->pack_cnt[local] = de->pack_cnt[local] + n;
507:   return 0;
508: }

510: /*
511: *) Ensures all data has been packed
512: */
513: PetscErrorCode DMSwarmDataExPackFinalize(DMSwarmDataEx de)
514: {
515:   PetscMPIInt i, np;
516:   PetscInt    total;

519:   np = de->n_neighbour_procs;
520:   for (i = 0; i < np; ++i) {
522:   }
523:   /* init */
524:   for (i = 0; i < np; ++i) de->messages_to_be_recvieved[i] = -1;
525:   /* figure out the recv counts here */
526:   for (i = 0; i < np; ++i) MPI_Isend(&de->messages_to_be_sent[i], 1, MPIU_INT, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
527:   for (i = 0; i < np; ++i) MPI_Irecv(&de->messages_to_be_recvieved[i], 1, MPIU_INT, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np + i]);
528:   MPI_Waitall(2 * np, de->_requests, de->_stats);
529:   /* create space for the data to be recvieved */
530:   total = 0;
531:   for (i = 0; i < np; ++i) total = total + de->messages_to_be_recvieved[i];
532:   PetscMalloc(de->unit_message_size * (total + 1), &de->recv_message);
533:   /* initialize memory */
534:   PetscMemzero(de->recv_message, de->unit_message_size * (total + 1));
535:   /* set total items to receive */
536:   de->recv_message_length  = total;
537:   de->packer_status        = DEOBJECT_FINALIZED;
538:   de->communication_status = DEOBJECT_INITIALIZED;
539:   PetscLogEventEnd(DMSWARM_DataExchangerPack, 0, 0, 0, 0);
540:   return 0;
541: }

543: /* do the actual message passing */
544: PetscErrorCode DMSwarmDataExBegin(DMSwarmDataEx de)
545: {
546:   PetscMPIInt i, np;
547:   void       *dest;
548:   PetscInt    length;

555:   PetscLogEventBegin(DMSWARM_DataExchangerBegin, 0, 0, 0, 0);
556:   np = de->n_neighbour_procs;
557:   /* == NON BLOCKING == */
558:   for (i = 0; i < np; ++i) {
559:     length = de->messages_to_be_sent[i] * de->unit_message_size;
560:     dest   = ((char *)de->send_message) + de->unit_message_size * de->message_offsets[i];
561:     MPI_Isend(dest, length, MPI_CHAR, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
562:   }
563:   PetscLogEventEnd(DMSWARM_DataExchangerBegin, 0, 0, 0, 0);
564:   return 0;
565: }

567: /* do the actual message passing now */
568: PetscErrorCode DMSwarmDataExEnd(DMSwarmDataEx de)
569: {
570:   PetscMPIInt i, np;
571:   PetscInt    total;
572:   PetscInt   *message_recv_offsets;
573:   void       *dest;
574:   PetscInt    length;

578:   PetscLogEventBegin(DMSWARM_DataExchangerEnd, 0, 0, 0, 0);
579:   np = de->n_neighbour_procs;
580:   PetscMalloc1(np + 1, &message_recv_offsets);
581:   message_recv_offsets[0] = 0;
582:   total                   = de->messages_to_be_recvieved[0];
583:   for (i = 1; i < np; ++i) {
584:     message_recv_offsets[i] = total;
585:     total                   = total + de->messages_to_be_recvieved[i];
586:   }
587:   /* == NON BLOCKING == */
588:   for (i = 0; i < np; ++i) {
589:     length = de->messages_to_be_recvieved[i] * de->unit_message_size;
590:     dest   = ((char *)de->recv_message) + de->unit_message_size * message_recv_offsets[i];
591:     MPI_Irecv(dest, length, MPI_CHAR, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np + i]);
592:   }
593:   MPI_Waitall(2 * np, de->_requests, de->_stats);
594:   PetscFree(message_recv_offsets);
595:   de->communication_status = DEOBJECT_FINALIZED;
596:   PetscLogEventEnd(DMSWARM_DataExchangerEnd, 0, 0, 0, 0);
597:   return 0;
598: }

600: PetscErrorCode DMSwarmDataExGetSendData(DMSwarmDataEx de, PetscInt *length, void **send)
601: {
603:   *length = de->send_message_length;
604:   *send   = de->send_message;
605:   return 0;
606: }

608: PetscErrorCode DMSwarmDataExGetRecvData(DMSwarmDataEx de, PetscInt *length, void **recv)
609: {
611:   *length = de->recv_message_length;
612:   *recv   = de->recv_message;
613:   return 0;
614: }

616: PetscErrorCode DMSwarmDataExTopologyGetNeighbours(DMSwarmDataEx de, PetscMPIInt *n, PetscMPIInt *neigh[])
617: {
618:   if (n) *n = de->n_neighbour_procs;
619:   if (neigh) *neigh = de->neighbour_procs;
620:   return 0;
621: }