Actual source code: data_ex.c
1: /*
2: Build a few basic tools to help with partitioned domains.
4: 1)
5: On each processor, have a DomainExchangerTopology.
6: This is a doubly-connected edge list which enumerates the
7: communication paths between connected processors. By numbering
8: these paths we can always uniquely assign message identifiers.
10: edge
11: 10
12: proc ---------> proc
13: 0 <-------- 1
14: 11
15: twin
17: Eg: Proc 0 send to proc 1 with message id is 10. To receive the correct
18: message, proc 1 looks for the edge connected to proc 0, and then the
19: message id comes from the twin of that edge
21: 2)
22: A DomainExchangerArrayPacker.
23: A little function which given a piece of data, will memcpy the data into
24: an array (which will be sent to procs) into the correct place.
26: On Proc 1 we sent data to procs 0,2,3. The data is on different lengths.
27: All data gets jammed into single array. Need to "jam" data into correct locations
28: The Packer knows how much is to going to each processor and keeps track of the inserts
29: so as to avoid ever packing TOO much into one slot, and inevatbly corrupting some memory
31: data to 0 data to 2 data to 3
33: |--------|-----------------|--|
35: User has to unpack message themselves. I can get you the pointer for each i
36: entry, but you'll have to cast it to the appropriate data type.
38: Phase A: Build topology
40: Phase B: Define message lengths
42: Phase C: Pack data
44: Phase D: Send data
46: + Constructor
47: DMSwarmDataExCreate()
48: + Phase A
49: DMSwarmDataExTopologyInitialize()
50: DMSwarmDataExTopologyAddNeighbour()
51: DMSwarmDataExTopologyAddNeighbour()
52: DMSwarmDataExTopologyFinalize()
53: + Phase B
54: DMSwarmDataExZeroAllSendCount()
55: DMSwarmDataExAddToSendCount()
56: DMSwarmDataExAddToSendCount()
57: DMSwarmDataExAddToSendCount()
58: + Phase C
59: DMSwarmDataExPackInitialize()
60: DMSwarmDataExPackData()
61: DMSwarmDataExPackData()
62: DMSwarmDataExPackFinalize()
63: +Phase D
64: DMSwarmDataExBegin()
65: ... perform any calculations ...
66: DMSwarmDataExEnd()
68: ... user calls any getters here ...
70: */
71: #include <petscvec.h>
72: #include <petscmat.h>
74: #include "../src/dm/impls/swarm/data_ex.h"
76: const char *status_names[] = {"initialized", "finalized", "unknown"};
78: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerTopologySetup;
79: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerBegin;
80: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerEnd;
81: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerSendCount;
82: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerPack;
84: PetscErrorCode DMSwarmDataExCreate(MPI_Comm comm, const PetscInt count, DMSwarmDataEx *ex)
85: {
86: DMSwarmDataEx d;
88: PetscNew(&d);
89: MPI_Comm_dup(comm, &d->comm);
90: MPI_Comm_rank(d->comm, &d->rank);
92: d->instance = count;
94: d->topology_status = DEOBJECT_STATE_UNKNOWN;
95: d->message_lengths_status = DEOBJECT_STATE_UNKNOWN;
96: d->packer_status = DEOBJECT_STATE_UNKNOWN;
97: d->communication_status = DEOBJECT_STATE_UNKNOWN;
99: d->n_neighbour_procs = -1;
100: d->neighbour_procs = NULL;
102: d->messages_to_be_sent = NULL;
103: d->message_offsets = NULL;
104: d->messages_to_be_recvieved = NULL;
106: d->unit_message_size = (size_t)-1;
107: d->send_message = NULL;
108: d->send_message_length = -1;
109: d->recv_message = NULL;
110: d->recv_message_length = -1;
111: d->total_pack_cnt = -1;
112: d->pack_cnt = NULL;
114: d->send_tags = NULL;
115: d->recv_tags = NULL;
117: d->_stats = NULL;
118: d->_requests = NULL;
119: *ex = d;
120: return 0;
121: }
123: /*
124: This code is horrible, who let it get into main.
126: Should be printing to a viewer, should not be using PETSC_COMM_WORLD
128: */
129: PetscErrorCode DMSwarmDataExView(DMSwarmDataEx d)
130: {
131: PetscMPIInt p;
133: PetscPrintf(PETSC_COMM_WORLD, "DMSwarmDataEx: instance=%" PetscInt_FMT "\n", d->instance);
134: PetscPrintf(PETSC_COMM_WORLD, " topology status: %s \n", status_names[d->topology_status]);
135: PetscPrintf(PETSC_COMM_WORLD, " message lengths status: %s \n", status_names[d->message_lengths_status]);
136: PetscPrintf(PETSC_COMM_WORLD, " packer status status: %s \n", status_names[d->packer_status]);
137: PetscPrintf(PETSC_COMM_WORLD, " communication status: %s \n", status_names[d->communication_status]);
139: if (d->topology_status == DEOBJECT_FINALIZED) {
140: PetscPrintf(PETSC_COMM_WORLD, " Topology:\n");
141: PetscSynchronizedPrintf(PETSC_COMM_WORLD, " [%d] neighbours: %d \n", d->rank, d->n_neighbour_procs);
142: for (p = 0; p < d->n_neighbour_procs; p++) PetscSynchronizedPrintf(PETSC_COMM_WORLD, " [%d] neighbour[%d] = %d \n", d->rank, p, d->neighbour_procs[p]);
143: PetscSynchronizedFlush(PETSC_COMM_WORLD, stdout);
144: }
146: if (d->message_lengths_status == DEOBJECT_FINALIZED) {
147: PetscPrintf(PETSC_COMM_WORLD, " Message lengths:\n");
148: PetscSynchronizedPrintf(PETSC_COMM_WORLD, " [%d] atomic size: %ld \n", d->rank, (long int)d->unit_message_size);
149: for (p = 0; p < d->n_neighbour_procs; p++) {
150: PetscSynchronizedPrintf(PETSC_COMM_WORLD, " [%d] >>>>> ( %" PetscInt_FMT " units :: tag = %d) >>>>> [%d] \n", d->rank, d->messages_to_be_sent[p], d->send_tags[p], d->neighbour_procs[p]);
151: }
152: for (p = 0; p < d->n_neighbour_procs; p++) {
153: PetscSynchronizedPrintf(PETSC_COMM_WORLD, " [%d] <<<<< ( %" PetscInt_FMT " units :: tag = %d) <<<<< [%d] \n", d->rank, d->messages_to_be_recvieved[p], d->recv_tags[p], d->neighbour_procs[p]);
154: }
155: PetscSynchronizedFlush(PETSC_COMM_WORLD, stdout);
156: }
157: if (d->packer_status == DEOBJECT_FINALIZED) { }
158: if (d->communication_status == DEOBJECT_FINALIZED) { }
159: return 0;
160: }
162: PetscErrorCode DMSwarmDataExDestroy(DMSwarmDataEx d)
163: {
164: MPI_Comm_free(&d->comm);
165: if (d->neighbour_procs) PetscFree(d->neighbour_procs);
166: if (d->messages_to_be_sent) PetscFree(d->messages_to_be_sent);
167: if (d->message_offsets) PetscFree(d->message_offsets);
168: if (d->messages_to_be_recvieved) PetscFree(d->messages_to_be_recvieved);
169: if (d->send_message) PetscFree(d->send_message);
170: if (d->recv_message) PetscFree(d->recv_message);
171: if (d->pack_cnt) PetscFree(d->pack_cnt);
172: if (d->send_tags) PetscFree(d->send_tags);
173: if (d->recv_tags) PetscFree(d->recv_tags);
174: if (d->_stats) PetscFree(d->_stats);
175: if (d->_requests) PetscFree(d->_requests);
176: PetscFree(d);
177: return 0;
178: }
180: /* === Phase A === */
182: PetscErrorCode DMSwarmDataExTopologyInitialize(DMSwarmDataEx d)
183: {
184: d->topology_status = DEOBJECT_INITIALIZED;
185: d->n_neighbour_procs = 0;
186: PetscFree(d->neighbour_procs);
187: PetscFree(d->messages_to_be_sent);
188: PetscFree(d->message_offsets);
189: PetscFree(d->messages_to_be_recvieved);
190: PetscFree(d->pack_cnt);
191: PetscFree(d->send_tags);
192: PetscFree(d->recv_tags);
193: return 0;
194: }
196: PetscErrorCode DMSwarmDataExTopologyAddNeighbour(DMSwarmDataEx d, const PetscMPIInt proc_id)
197: {
198: PetscMPIInt n, found;
199: PetscMPIInt size;
204: /* error on negative entries */
206: /* error on ranks larger than number of procs in communicator */
207: MPI_Comm_size(d->comm, &size);
209: if (d->n_neighbour_procs == 0) PetscMalloc1(1, &d->neighbour_procs);
210: /* check for proc_id */
211: found = 0;
212: for (n = 0; n < d->n_neighbour_procs; n++) {
213: if (d->neighbour_procs[n] == proc_id) found = 1;
214: }
215: if (found == 0) { /* add it to list */
216: PetscRealloc(sizeof(PetscMPIInt) * (d->n_neighbour_procs + 1), &d->neighbour_procs);
217: d->neighbour_procs[d->n_neighbour_procs] = proc_id;
218: d->n_neighbour_procs++;
219: }
220: return 0;
221: }
223: /*
224: counter: the index of the communication object
225: N: the number of processors
226: r0: rank of sender
227: r1: rank of receiver
229: procs = { 0, 1, 2, 3 }
231: 0 ==> 0 e=0
232: 0 ==> 1 e=1
233: 0 ==> 2 e=2
234: 0 ==> 3 e=3
236: 1 ==> 0 e=4
237: 1 ==> 1 e=5
238: 1 ==> 2 e=6
239: 1 ==> 3 e=7
241: 2 ==> 0 e=8
242: 2 ==> 1 e=9
243: 2 ==> 2 e=10
244: 2 ==> 3 e=11
246: 3 ==> 0 e=12
247: 3 ==> 1 e=13
248: 3 ==> 2 e=14
249: 3 ==> 3 e=15
251: If we require that proc A sends to proc B, then the SEND tag index will be given by
252: N * rank(A) + rank(B) + offset
253: If we require that proc A will receive from proc B, then the RECV tag index will be given by
254: N * rank(B) + rank(A) + offset
256: */
257: static void _get_tags(PetscInt counter, PetscMPIInt N, PetscMPIInt r0, PetscMPIInt r1, PetscMPIInt *_st, PetscMPIInt *_rt)
258: {
259: PetscMPIInt st, rt;
261: st = N * r0 + r1 + N * N * counter;
262: rt = N * r1 + r0 + N * N * counter;
263: *_st = st;
264: *_rt = rt;
265: }
267: /*
268: Makes the communication map symmetric
269: */
270: PetscErrorCode _DMSwarmDataExCompleteCommunicationMap(MPI_Comm comm, PetscMPIInt n, PetscMPIInt proc_neighbours[], PetscMPIInt *n_new, PetscMPIInt **proc_neighbours_new)
271: {
272: Mat A;
273: PetscInt i, j, nc;
274: PetscInt n_, *proc_neighbours_;
275: PetscInt rank_;
276: PetscMPIInt size, rank;
277: PetscScalar *vals;
278: const PetscInt *cols;
279: const PetscScalar *red_vals;
280: PetscMPIInt _n_new, *_proc_neighbours_new;
282: n_ = n;
283: PetscMalloc(sizeof(PetscInt) * n_, &proc_neighbours_);
284: for (i = 0; i < n_; ++i) proc_neighbours_[i] = proc_neighbours[i];
285: MPI_Comm_size(comm, &size);
286: MPI_Comm_rank(comm, &rank);
287: rank_ = rank;
289: MatCreate(comm, &A);
290: MatSetSizes(A, PETSC_DECIDE, PETSC_DECIDE, size, size);
291: MatSetType(A, MATAIJ);
292: MatSeqAIJSetPreallocation(A, 1, NULL);
293: MatMPIAIJSetPreallocation(A, n_, NULL, n_, NULL);
294: MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
295: /* Build original map */
296: PetscMalloc1(n_, &vals);
297: for (i = 0; i < n_; ++i) vals[i] = 1.0;
298: MatSetValues(A, 1, &rank_, n_, proc_neighbours_, vals, INSERT_VALUES);
299: MatAssemblyBegin(A, MAT_FLUSH_ASSEMBLY);
300: MatAssemblyEnd(A, MAT_FLUSH_ASSEMBLY);
301: /* Now force all other connections if they are not already there */
302: /* It's more efficient to do them all at once */
303: for (i = 0; i < n_; ++i) vals[i] = 2.0;
304: MatSetValues(A, n_, proc_neighbours_, 1, &rank_, vals, INSERT_VALUES);
305: MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY);
306: MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY);
307: /*
308: PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);
309: MatView(A,PETSC_VIEWER_STDOUT_WORLD);
310: PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD);
311: */
312: if ((n_new != NULL) && (proc_neighbours_new != NULL)) {
313: MatGetRow(A, rank_, &nc, &cols, &red_vals);
314: _n_new = (PetscMPIInt)nc;
315: PetscMalloc1(_n_new, &_proc_neighbours_new);
316: for (j = 0; j < nc; ++j) _proc_neighbours_new[j] = (PetscMPIInt)cols[j];
317: MatRestoreRow(A, rank_, &nc, &cols, &red_vals);
318: *n_new = (PetscMPIInt)_n_new;
319: *proc_neighbours_new = (PetscMPIInt *)_proc_neighbours_new;
320: }
321: MatDestroy(&A);
322: PetscFree(vals);
323: PetscFree(proc_neighbours_);
324: MPI_Barrier(comm);
325: return 0;
326: }
328: PetscErrorCode DMSwarmDataExTopologyFinalize(DMSwarmDataEx d)
329: {
330: PetscMPIInt symm_nn, *symm_procs, r0, n, st, rt, size;
334: PetscLogEventBegin(DMSWARM_DataExchangerTopologySetup, 0, 0, 0, 0);
335: /* given information about all my neighbours, make map symmetric */
336: _DMSwarmDataExCompleteCommunicationMap(d->comm, d->n_neighbour_procs, d->neighbour_procs, &symm_nn, &symm_procs);
337: /* update my arrays */
338: PetscFree(d->neighbour_procs);
339: d->n_neighbour_procs = symm_nn;
340: d->neighbour_procs = symm_procs;
341: /* allocates memory */
342: if (!d->messages_to_be_sent) PetscMalloc1(d->n_neighbour_procs + 1, &d->messages_to_be_sent);
343: if (!d->message_offsets) PetscMalloc1(d->n_neighbour_procs + 1, &d->message_offsets);
344: if (!d->messages_to_be_recvieved) PetscMalloc1(d->n_neighbour_procs + 1, &d->messages_to_be_recvieved);
345: if (!d->pack_cnt) PetscMalloc(sizeof(PetscInt) * d->n_neighbour_procs, &d->pack_cnt);
346: if (!d->_stats) PetscMalloc(sizeof(MPI_Status) * 2 * d->n_neighbour_procs, &d->_stats);
347: if (!d->_requests) PetscMalloc(sizeof(MPI_Request) * 2 * d->n_neighbour_procs, &d->_requests);
348: if (!d->send_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->send_tags);
349: if (!d->recv_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->recv_tags);
350: /* compute message tags */
351: MPI_Comm_size(d->comm, &size);
352: r0 = d->rank;
353: for (n = 0; n < d->n_neighbour_procs; ++n) {
354: PetscMPIInt r1 = d->neighbour_procs[n];
356: _get_tags(d->instance, size, r0, r1, &st, &rt);
357: d->send_tags[n] = (int)st;
358: d->recv_tags[n] = (int)rt;
359: }
360: d->topology_status = DEOBJECT_FINALIZED;
361: PetscLogEventEnd(DMSWARM_DataExchangerTopologySetup, 0, 0, 0, 0);
362: return 0;
363: }
365: /* === Phase B === */
366: PetscErrorCode _DMSwarmDataExConvertProcIdToLocalIndex(DMSwarmDataEx de, PetscMPIInt proc_id, PetscMPIInt *local)
367: {
368: PetscMPIInt i, np;
370: np = de->n_neighbour_procs;
371: *local = -1;
372: for (i = 0; i < np; ++i) {
373: if (proc_id == de->neighbour_procs[i]) {
374: *local = i;
375: break;
376: }
377: }
378: return 0;
379: }
381: PetscErrorCode DMSwarmDataExInitializeSendCount(DMSwarmDataEx de)
382: {
383: PetscMPIInt i;
386: PetscLogEventBegin(DMSWARM_DataExchangerSendCount, 0, 0, 0, 0);
387: de->message_lengths_status = DEOBJECT_INITIALIZED;
388: for (i = 0; i < de->n_neighbour_procs; ++i) de->messages_to_be_sent[i] = 0;
389: return 0;
390: }
392: /*
393: 1) only allows counters to be set on neighbouring cpus
394: */
395: PetscErrorCode DMSwarmDataExAddToSendCount(DMSwarmDataEx de, const PetscMPIInt proc_id, const PetscInt count)
396: {
397: PetscMPIInt local_val;
402: _DMSwarmDataExConvertProcIdToLocalIndex(de, proc_id, &local_val);
405: de->messages_to_be_sent[local_val] = de->messages_to_be_sent[local_val] + count;
406: return 0;
407: }
409: PetscErrorCode DMSwarmDataExFinalizeSendCount(DMSwarmDataEx de)
410: {
413: de->message_lengths_status = DEOBJECT_FINALIZED;
414: PetscLogEventEnd(DMSWARM_DataExchangerSendCount, 0, 0, 0, 0);
415: return 0;
416: }
418: /* === Phase C === */
419: /*
420: zero out all send counts
421: free send and recv buffers
422: zeros out message length
423: zeros out all counters
424: zero out packed data counters
425: */
426: PetscErrorCode _DMSwarmDataExInitializeTmpStorage(DMSwarmDataEx de)
427: {
428: PetscMPIInt i, np;
430: np = de->n_neighbour_procs;
431: for (i = 0; i < np; ++i) {
432: /* de->messages_to_be_sent[i] = -1; */
433: de->messages_to_be_recvieved[i] = -1;
434: }
435: PetscFree(de->send_message);
436: PetscFree(de->recv_message);
437: return 0;
438: }
440: /*
441: Zeros out pack data counters
442: Ensures mesaage length is set
443: Checks send counts properly initialized
444: allocates space for pack data
445: */
446: PetscErrorCode DMSwarmDataExPackInitialize(DMSwarmDataEx de, size_t unit_message_size)
447: {
448: PetscMPIInt i, np;
449: PetscInt total;
453: PetscLogEventBegin(DMSWARM_DataExchangerPack, 0, 0, 0, 0);
454: de->packer_status = DEOBJECT_INITIALIZED;
455: _DMSwarmDataExInitializeTmpStorage(de);
456: np = de->n_neighbour_procs;
457: de->unit_message_size = unit_message_size;
458: total = 0;
459: for (i = 0; i < np; ++i) {
460: if (de->messages_to_be_sent[i] == -1) {
461: PetscMPIInt proc_neighour = de->neighbour_procs[i];
462: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ORDER, "Messages_to_be_sent[neighbour_proc=%d] is un-initialised. Call DMSwarmDataExSetSendCount() first", (int)proc_neighour);
463: }
464: total = total + de->messages_to_be_sent[i];
465: }
466: /* create space for the data to be sent */
467: PetscMalloc(unit_message_size * (total + 1), &de->send_message);
468: /* initialize memory */
469: PetscMemzero(de->send_message, unit_message_size * (total + 1));
470: /* set total items to send */
471: de->send_message_length = total;
472: de->message_offsets[0] = 0;
473: total = de->messages_to_be_sent[0];
474: for (i = 1; i < np; ++i) {
475: de->message_offsets[i] = total;
476: total = total + de->messages_to_be_sent[i];
477: }
478: /* init the packer counters */
479: de->total_pack_cnt = 0;
480: for (i = 0; i < np; ++i) de->pack_cnt[i] = 0;
481: return 0;
482: }
484: /*
485: Ensures data gets been packed appropriately and no overlaps occur
486: */
487: PetscErrorCode DMSwarmDataExPackData(DMSwarmDataEx de, PetscMPIInt proc_id, PetscInt n, void *data)
488: {
489: PetscMPIInt local;
490: PetscInt insert_location;
491: void *dest;
497: _DMSwarmDataExConvertProcIdToLocalIndex(de, proc_id, &local);
501: /* copy memory */
502: insert_location = de->message_offsets[local] + de->pack_cnt[local];
503: dest = ((char *)de->send_message) + de->unit_message_size * insert_location;
504: PetscMemcpy(dest, data, de->unit_message_size * n);
505: /* increment counter */
506: de->pack_cnt[local] = de->pack_cnt[local] + n;
507: return 0;
508: }
510: /*
511: *) Ensures all data has been packed
512: */
513: PetscErrorCode DMSwarmDataExPackFinalize(DMSwarmDataEx de)
514: {
515: PetscMPIInt i, np;
516: PetscInt total;
519: np = de->n_neighbour_procs;
520: for (i = 0; i < np; ++i) {
522: }
523: /* init */
524: for (i = 0; i < np; ++i) de->messages_to_be_recvieved[i] = -1;
525: /* figure out the recv counts here */
526: for (i = 0; i < np; ++i) MPI_Isend(&de->messages_to_be_sent[i], 1, MPIU_INT, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
527: for (i = 0; i < np; ++i) MPI_Irecv(&de->messages_to_be_recvieved[i], 1, MPIU_INT, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np + i]);
528: MPI_Waitall(2 * np, de->_requests, de->_stats);
529: /* create space for the data to be recvieved */
530: total = 0;
531: for (i = 0; i < np; ++i) total = total + de->messages_to_be_recvieved[i];
532: PetscMalloc(de->unit_message_size * (total + 1), &de->recv_message);
533: /* initialize memory */
534: PetscMemzero(de->recv_message, de->unit_message_size * (total + 1));
535: /* set total items to receive */
536: de->recv_message_length = total;
537: de->packer_status = DEOBJECT_FINALIZED;
538: de->communication_status = DEOBJECT_INITIALIZED;
539: PetscLogEventEnd(DMSWARM_DataExchangerPack, 0, 0, 0, 0);
540: return 0;
541: }
543: /* do the actual message passing */
544: PetscErrorCode DMSwarmDataExBegin(DMSwarmDataEx de)
545: {
546: PetscMPIInt i, np;
547: void *dest;
548: PetscInt length;
555: PetscLogEventBegin(DMSWARM_DataExchangerBegin, 0, 0, 0, 0);
556: np = de->n_neighbour_procs;
557: /* == NON BLOCKING == */
558: for (i = 0; i < np; ++i) {
559: length = de->messages_to_be_sent[i] * de->unit_message_size;
560: dest = ((char *)de->send_message) + de->unit_message_size * de->message_offsets[i];
561: MPI_Isend(dest, length, MPI_CHAR, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
562: }
563: PetscLogEventEnd(DMSWARM_DataExchangerBegin, 0, 0, 0, 0);
564: return 0;
565: }
567: /* do the actual message passing now */
568: PetscErrorCode DMSwarmDataExEnd(DMSwarmDataEx de)
569: {
570: PetscMPIInt i, np;
571: PetscInt total;
572: PetscInt *message_recv_offsets;
573: void *dest;
574: PetscInt length;
578: PetscLogEventBegin(DMSWARM_DataExchangerEnd, 0, 0, 0, 0);
579: np = de->n_neighbour_procs;
580: PetscMalloc1(np + 1, &message_recv_offsets);
581: message_recv_offsets[0] = 0;
582: total = de->messages_to_be_recvieved[0];
583: for (i = 1; i < np; ++i) {
584: message_recv_offsets[i] = total;
585: total = total + de->messages_to_be_recvieved[i];
586: }
587: /* == NON BLOCKING == */
588: for (i = 0; i < np; ++i) {
589: length = de->messages_to_be_recvieved[i] * de->unit_message_size;
590: dest = ((char *)de->recv_message) + de->unit_message_size * message_recv_offsets[i];
591: MPI_Irecv(dest, length, MPI_CHAR, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np + i]);
592: }
593: MPI_Waitall(2 * np, de->_requests, de->_stats);
594: PetscFree(message_recv_offsets);
595: de->communication_status = DEOBJECT_FINALIZED;
596: PetscLogEventEnd(DMSWARM_DataExchangerEnd, 0, 0, 0, 0);
597: return 0;
598: }
600: PetscErrorCode DMSwarmDataExGetSendData(DMSwarmDataEx de, PetscInt *length, void **send)
601: {
603: *length = de->send_message_length;
604: *send = de->send_message;
605: return 0;
606: }
608: PetscErrorCode DMSwarmDataExGetRecvData(DMSwarmDataEx de, PetscInt *length, void **recv)
609: {
611: *length = de->recv_message_length;
612: *recv = de->recv_message;
613: return 0;
614: }
616: PetscErrorCode DMSwarmDataExTopologyGetNeighbours(DMSwarmDataEx de, PetscMPIInt *n, PetscMPIInt *neigh[])
617: {
618: if (n) *n = de->n_neighbour_procs;
619: if (neigh) *neigh = de->neighbour_procs;
620: return 0;
621: }