Actual source code: trajmemory.c
1: #include <petsc/private/tsimpl.h>
2: #include <petscsys.h>
3: #if defined(PETSC_HAVE_REVOLVE)
4: #include <revolve_c.h>
6: /* Limit Revolve to 32-bits */
7: #define PETSC_REVOLVE_INT_MAX 2147483647
9: typedef int PetscRevolveInt;
11: static inline PetscErrorCode PetscRevolveIntCast(PetscInt a, PetscRevolveInt *b)
12: {
13: #if defined(PETSC_USE_64BIT_INDICES)
14: *b = 0;
16: #endif
17: *b = (PetscRevolveInt)(a);
18: return 0;
19: }
20: #endif
21: #if defined(PETSC_HAVE_CAMS)
22: #include <offline_schedule.h>
23: #endif
25: PetscLogEvent TSTrajectory_DiskWrite, TSTrajectory_DiskRead;
26: static PetscErrorCode TSTrajectorySet_Memory(TSTrajectory, TS, PetscInt, PetscReal, Vec);
28: typedef enum {
29: NONE,
30: TWO_LEVEL_NOREVOLVE,
31: TWO_LEVEL_REVOLVE,
32: TWO_LEVEL_TWO_REVOLVE,
33: REVOLVE_OFFLINE,
34: REVOLVE_ONLINE,
35: REVOLVE_MULTISTAGE,
36: CAMS_OFFLINE
37: } SchedulerType;
39: typedef enum {
40: UNSET = -1,
41: SOLUTIONONLY = 0,
42: STAGESONLY = 1,
43: SOLUTION_STAGES = 2
44: } CheckpointType;
46: typedef enum {
47: TJ_REVOLVE,
48: TJ_CAMS,
49: TJ_PETSC
50: } TSTrajectoryMemoryType;
51: static const char *const TSTrajectoryMemoryTypes[] = {"REVOLVE", "CAMS", "PETSC", "TSTrajectoryMemoryType", "TJ_", NULL};
53: #define HaveSolution(m) ((m) == SOLUTIONONLY || (m) == SOLUTION_STAGES)
54: #define HaveStages(m) ((m) == STAGESONLY || (m) == SOLUTION_STAGES)
56: typedef struct _StackElement {
57: PetscInt stepnum;
58: Vec X;
59: Vec *Y;
60: PetscReal time;
61: PetscReal timeprev; /* for no solution_only mode */
62: PetscReal timenext; /* for solution_only mode */
63: CheckpointType cptype;
64: } *StackElement;
66: #if defined(PETSC_HAVE_REVOLVE)
67: typedef struct _RevolveCTX {
68: PetscBool reverseonestep;
69: PetscRevolveInt where;
70: PetscRevolveInt snaps_in;
71: PetscRevolveInt stepsleft;
72: PetscRevolveInt check;
73: PetscRevolveInt oldcapo;
74: PetscRevolveInt capo;
75: PetscRevolveInt fine;
76: PetscRevolveInt info;
77: } RevolveCTX;
78: #endif
80: #if defined(PETSC_HAVE_CAMS)
81: typedef struct _CAMSCTX {
82: PetscInt lastcheckpointstep;
83: PetscInt lastcheckpointtype;
84: PetscInt num_units_avail;
85: PetscInt endstep;
86: PetscInt num_stages;
87: PetscInt nextcheckpointstep;
88: PetscInt nextcheckpointtype; /* (0) solution only (1) stages (2) solution+stages */
89: PetscInt info;
90: } CAMSCTX;
91: #endif
93: typedef struct _Stack {
94: PetscInt stacksize;
95: PetscInt top;
96: StackElement *container;
97: PetscInt nallocated;
98: PetscInt numY;
99: PetscBool solution_only;
100: PetscBool use_dram;
101: } Stack;
103: typedef struct _DiskStack {
104: PetscInt stacksize;
105: PetscInt top;
106: PetscInt *container;
107: } DiskStack;
109: typedef struct _TJScheduler {
110: SchedulerType stype;
111: TSTrajectoryMemoryType tj_memory_type;
112: #if defined(PETSC_HAVE_REVOLVE)
113: RevolveCTX *rctx, *rctx2;
114: PetscBool use_online;
115: PetscInt store_stride;
116: #endif
117: #if defined(PETSC_HAVE_CAMS)
118: CAMSCTX *actx;
119: #endif
120: PetscBool recompute;
121: PetscBool skip_trajectory;
122: PetscBool save_stack;
123: PetscInt max_units_ram; /* maximum checkpointing units in RAM */
124: PetscInt max_units_disk; /* maximum checkpointing units on disk */
125: PetscInt max_cps_ram; /* maximum checkpoints in RAM */
126: PetscInt max_cps_disk; /* maximum checkpoints on disk */
127: PetscInt stride;
128: PetscInt total_steps; /* total number of steps */
129: Stack stack;
130: DiskStack diskstack;
131: PetscViewer viewer;
132: } TJScheduler;
134: static PetscErrorCode TurnForwardWithStepsize(TS ts, PetscReal nextstepsize)
135: {
136: /* reverse the direction */
137: TSSetTimeStep(ts, nextstepsize);
138: return 0;
139: }
141: static PetscErrorCode TurnForward(TS ts)
142: {
143: PetscReal stepsize;
145: /* reverse the direction */
146: TSGetTimeStep(ts, &stepsize);
147: TSSetTimeStep(ts, -stepsize);
148: return 0;
149: }
151: static PetscErrorCode TurnBackward(TS ts)
152: {
153: PetscReal stepsize;
155: if (!ts->trajectory->adjoint_solve_mode) return 0;
156: /* reverse the direction */
157: stepsize = ts->ptime_prev - ts->ptime;
158: TSSetTimeStep(ts, stepsize);
159: return 0;
160: }
162: static PetscErrorCode ElementCreate(TS ts, CheckpointType cptype, Stack *stack, StackElement *e)
163: {
164: Vec X;
165: Vec *Y;
167: if (stack->top < stack->stacksize - 1 && stack->container[stack->top + 1]) {
168: *e = stack->container[stack->top + 1];
169: if (HaveSolution(cptype) && !(*e)->X) {
170: TSGetSolution(ts, &X);
171: VecDuplicate(X, &(*e)->X);
172: }
173: if (cptype == 1 && (*e)->X) VecDestroy(&(*e)->X);
174: if (HaveStages(cptype) && !(*e)->Y) {
175: TSGetStages(ts, &stack->numY, &Y);
176: if (stack->numY) VecDuplicateVecs(Y[0], stack->numY, &(*e)->Y);
177: }
178: if (cptype == 0 && (*e)->Y) VecDestroyVecs(stack->numY, &(*e)->Y);
179: (*e)->cptype = cptype;
180: return 0;
181: }
182: if (stack->use_dram) PetscMallocSetDRAM();
183: PetscNew(e);
184: if (HaveSolution(cptype)) {
185: TSGetSolution(ts, &X);
186: VecDuplicate(X, &(*e)->X);
187: }
188: if (HaveStages(cptype)) {
189: TSGetStages(ts, &stack->numY, &Y);
190: if (stack->numY) VecDuplicateVecs(Y[0], stack->numY, &(*e)->Y);
191: }
192: if (stack->use_dram) PetscMallocResetDRAM();
193: stack->nallocated++;
194: (*e)->cptype = cptype;
195: return 0;
196: }
198: static PetscErrorCode ElementSet(TS ts, Stack *stack, StackElement *e, PetscInt stepnum, PetscReal time, Vec X)
199: {
200: Vec *Y;
201: PetscInt i;
202: PetscReal timeprev;
204: if (HaveSolution((*e)->cptype)) VecCopy(X, (*e)->X);
205: if (HaveStages((*e)->cptype)) {
206: TSGetStages(ts, &stack->numY, &Y);
207: for (i = 0; i < stack->numY; i++) VecCopy(Y[i], (*e)->Y[i]);
208: }
209: (*e)->stepnum = stepnum;
210: (*e)->time = time;
211: /* for consistency */
212: if (stepnum == 0) {
213: (*e)->timeprev = (*e)->time - ts->time_step;
214: } else {
215: TSGetPrevTime(ts, &timeprev);
216: (*e)->timeprev = timeprev;
217: }
218: return 0;
219: }
221: static PetscErrorCode ElementDestroy(Stack *stack, StackElement e)
222: {
223: if (stack->use_dram) PetscMallocSetDRAM();
224: VecDestroy(&e->X);
225: if (e->Y) VecDestroyVecs(stack->numY, &e->Y);
226: PetscFree(e);
227: if (stack->use_dram) PetscMallocResetDRAM();
228: stack->nallocated--;
229: return 0;
230: }
232: static PetscErrorCode StackResize(Stack *stack, PetscInt newsize)
233: {
234: StackElement *newcontainer;
235: PetscInt i;
237: PetscCalloc1(newsize * sizeof(StackElement), &newcontainer);
238: for (i = 0; i < stack->stacksize; i++) newcontainer[i] = stack->container[i];
239: PetscFree(stack->container);
240: stack->container = newcontainer;
241: stack->stacksize = newsize;
242: return 0;
243: }
245: static PetscErrorCode StackPush(Stack *stack, StackElement e)
246: {
248: stack->container[++stack->top] = e;
249: return 0;
250: }
252: static PetscErrorCode StackPop(Stack *stack, StackElement *e)
253: {
254: *e = NULL;
256: *e = stack->container[stack->top--];
257: return 0;
258: }
260: static PetscErrorCode StackTop(Stack *stack, StackElement *e)
261: {
262: *e = stack->container[stack->top];
263: return 0;
264: }
266: static PetscErrorCode StackInit(Stack *stack, PetscInt size, PetscInt ny)
267: {
268: stack->top = -1;
269: stack->numY = ny;
271: if (!stack->container) PetscCalloc1(size, &stack->container);
272: return 0;
273: }
275: static PetscErrorCode StackDestroy(Stack *stack)
276: {
277: const PetscInt n = stack->nallocated;
279: if (!stack->container) return 0;
281: for (PetscInt i = 0; i < n; i++) ElementDestroy(stack, stack->container[i]);
282: PetscFree(stack->container);
283: return 0;
284: }
286: static PetscErrorCode StackFind(Stack *stack, StackElement *e, PetscInt index)
287: {
288: *e = NULL;
290: *e = stack->container[index];
291: return 0;
292: }
294: static PetscErrorCode WriteToDisk(PetscBool stifflyaccurate, PetscInt stepnum, PetscReal time, PetscReal timeprev, Vec X, Vec *Y, PetscInt numY, CheckpointType cptype, PetscViewer viewer)
295: {
296: PetscViewerBinaryWrite(viewer, &stepnum, 1, PETSC_INT);
297: if (HaveSolution(cptype)) VecView(X, viewer);
298: if (HaveStages(cptype)) {
299: for (PetscInt i = 0; i < numY; i++) {
300: /* For stiffly accurate TS methods, the last stage Y[ns-1] is the same as the solution X, thus does not need to be saved again. */
301: if (stifflyaccurate && i == numY - 1 && HaveSolution(cptype)) continue;
302: VecView(Y[i], viewer);
303: }
304: }
305: PetscViewerBinaryWrite(viewer, &time, 1, PETSC_REAL);
306: PetscViewerBinaryWrite(viewer, &timeprev, 1, PETSC_REAL);
307: return 0;
308: }
310: static PetscErrorCode ReadFromDisk(PetscBool stifflyaccurate, PetscInt *stepnum, PetscReal *time, PetscReal *timeprev, Vec X, Vec *Y, PetscInt numY, CheckpointType cptype, PetscViewer viewer)
311: {
312: PetscViewerBinaryRead(viewer, stepnum, 1, NULL, PETSC_INT);
313: if (HaveSolution(cptype)) VecLoad(X, viewer);
314: if (HaveStages(cptype)) {
315: for (PetscInt i = 0; i < numY; i++) {
316: /* For stiffly accurate TS methods, the last stage Y[ns-1] is the same as the solution X, thus does not need to be loaded again. */
317: if (stifflyaccurate && i == numY - 1 && HaveSolution(cptype)) continue;
318: VecLoad(Y[i], viewer);
319: }
320: }
321: PetscViewerBinaryRead(viewer, time, 1, NULL, PETSC_REAL);
322: PetscViewerBinaryRead(viewer, timeprev, 1, NULL, PETSC_REAL);
323: return 0;
324: }
326: static PetscErrorCode StackDumpAll(TSTrajectory tj, TS ts, Stack *stack, PetscInt id)
327: {
328: Vec *Y;
329: PetscInt ndumped, cptype_int;
330: StackElement e = NULL;
331: TJScheduler *tjsch = (TJScheduler *)tj->data;
332: char filename[PETSC_MAX_PATH_LEN];
333: MPI_Comm comm;
335: PetscObjectGetComm((PetscObject)ts, &comm);
336: if (tj->monitor) {
337: PetscViewerASCIIPushTab(tj->monitor);
338: PetscViewerASCIIPrintf(tj->monitor, "Dump stack id %" PetscInt_FMT " to file\n", id);
339: PetscViewerASCIIPopTab(tj->monitor);
340: }
341: PetscSNPrintf(filename, sizeof(filename), "%s/TS-STACK%06" PetscInt_FMT ".bin", tj->dirname, id);
342: PetscViewerFileSetName(tjsch->viewer, filename);
343: PetscViewerSetUp(tjsch->viewer);
344: ndumped = stack->top + 1;
345: PetscViewerBinaryWrite(tjsch->viewer, &ndumped, 1, PETSC_INT);
346: for (PetscInt i = 0; i < ndumped; i++) {
347: e = stack->container[i];
348: cptype_int = (PetscInt)e->cptype;
349: PetscViewerBinaryWrite(tjsch->viewer, &cptype_int, 1, PETSC_INT);
350: PetscLogEventBegin(TSTrajectory_DiskWrite, tj, ts, 0, 0);
351: WriteToDisk(ts->stifflyaccurate, e->stepnum, e->time, e->timeprev, e->X, e->Y, stack->numY, e->cptype, tjsch->viewer);
352: PetscLogEventEnd(TSTrajectory_DiskWrite, tj, ts, 0, 0);
353: ts->trajectory->diskwrites++;
354: StackPop(stack, &e);
355: }
356: /* save the last step for restart, the last step is in memory when using single level schemes, but not necessarily the case for multi level schemes */
357: TSGetStages(ts, &stack->numY, &Y);
358: PetscLogEventBegin(TSTrajectory_DiskWrite, tj, ts, 0, 0);
359: WriteToDisk(ts->stifflyaccurate, ts->steps, ts->ptime, ts->ptime_prev, ts->vec_sol, Y, stack->numY, SOLUTION_STAGES, tjsch->viewer);
360: PetscLogEventEnd(TSTrajectory_DiskWrite, tj, ts, 0, 0);
361: ts->trajectory->diskwrites++;
362: return 0;
363: }
365: static PetscErrorCode StackLoadAll(TSTrajectory tj, TS ts, Stack *stack, PetscInt id)
366: {
367: Vec *Y;
368: PetscInt i, nloaded, cptype_int;
369: StackElement e;
370: PetscViewer viewer;
371: char filename[PETSC_MAX_PATH_LEN];
373: if (tj->monitor) {
374: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
375: PetscViewerASCIIPrintf(tj->monitor, "Load stack from file\n");
376: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
377: }
378: PetscSNPrintf(filename, sizeof filename, "%s/TS-STACK%06" PetscInt_FMT ".bin", tj->dirname, id);
379: PetscViewerBinaryOpen(PetscObjectComm((PetscObject)tj), filename, FILE_MODE_READ, &viewer);
380: PetscViewerBinarySetSkipInfo(viewer, PETSC_TRUE);
381: PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE);
382: PetscViewerBinaryRead(viewer, &nloaded, 1, NULL, PETSC_INT);
383: for (i = 0; i < nloaded; i++) {
384: PetscViewerBinaryRead(viewer, &cptype_int, 1, NULL, PETSC_INT);
385: ElementCreate(ts, (CheckpointType)cptype_int, stack, &e);
386: StackPush(stack, e);
387: PetscLogEventBegin(TSTrajectory_DiskRead, tj, ts, 0, 0);
388: ReadFromDisk(ts->stifflyaccurate, &e->stepnum, &e->time, &e->timeprev, e->X, e->Y, stack->numY, e->cptype, viewer);
389: PetscLogEventEnd(TSTrajectory_DiskRead, tj, ts, 0, 0);
390: ts->trajectory->diskreads++;
391: }
392: /* load the last step into TS */
393: TSGetStages(ts, &stack->numY, &Y);
394: PetscLogEventBegin(TSTrajectory_DiskRead, tj, ts, 0, 0);
395: ReadFromDisk(ts->stifflyaccurate, &ts->steps, &ts->ptime, &ts->ptime_prev, ts->vec_sol, Y, stack->numY, SOLUTION_STAGES, viewer);
396: PetscLogEventEnd(TSTrajectory_DiskRead, tj, ts, 0, 0);
397: ts->trajectory->diskreads++;
398: TurnBackward(ts);
399: PetscViewerDestroy(&viewer);
400: return 0;
401: }
403: #if defined(PETSC_HAVE_REVOLVE)
404: static PetscErrorCode StackLoadLast(TSTrajectory tj, TS ts, Stack *stack, PetscInt id)
405: {
406: Vec *Y;
407: PetscInt size;
408: PetscViewer viewer;
409: char filename[PETSC_MAX_PATH_LEN];
410: #if defined(PETSC_HAVE_MPIIO)
411: PetscBool usempiio;
412: #endif
413: int fd;
414: off_t off, offset;
416: if (tj->monitor) {
417: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
418: PetscViewerASCIIPrintf(tj->monitor, "Load last stack element from file\n");
419: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
420: }
421: TSGetStages(ts, &stack->numY, &Y);
422: VecGetSize(Y[0], &size);
423: /* VecView writes to file two extra int's for class id and number of rows */
424: off = -((stack->solution_only ? 0 : stack->numY) + 1) * (size * PETSC_BINARY_SCALAR_SIZE + 2 * PETSC_BINARY_INT_SIZE) - PETSC_BINARY_INT_SIZE - 2 * PETSC_BINARY_SCALAR_SIZE;
426: PetscSNPrintf(filename, sizeof filename, "%s/TS-STACK%06" PetscInt_FMT ".bin", tj->dirname, id);
427: PetscViewerBinaryOpen(PetscObjectComm((PetscObject)tj), filename, FILE_MODE_READ, &viewer);
428: PetscViewerBinarySetSkipInfo(viewer, PETSC_TRUE);
429: PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE);
430: #if defined(PETSC_HAVE_MPIIO)
431: PetscViewerBinaryGetUseMPIIO(viewer, &usempiio);
432: if (usempiio) {
433: PetscViewerBinaryGetMPIIODescriptor(viewer, (MPI_File *)&fd);
434: PetscBinarySynchronizedSeek(PetscObjectComm((PetscObject)tj), fd, off, PETSC_BINARY_SEEK_END, &offset);
435: } else {
436: #endif
437: PetscViewerBinaryGetDescriptor(viewer, &fd);
438: PetscBinarySeek(fd, off, PETSC_BINARY_SEEK_END, &offset);
439: #if defined(PETSC_HAVE_MPIIO)
440: }
441: #endif
442: /* load the last step into TS */
443: PetscLogEventBegin(TSTrajectory_DiskRead, tj, ts, 0, 0);
444: ReadFromDisk(ts->stifflyaccurate, &ts->steps, &ts->ptime, &ts->ptime_prev, ts->vec_sol, Y, stack->numY, SOLUTION_STAGES, viewer);
445: PetscLogEventEnd(TSTrajectory_DiskRead, tj, ts, 0, 0);
446: ts->trajectory->diskreads++;
447: PetscViewerDestroy(&viewer);
448: TurnBackward(ts);
449: return 0;
450: }
451: #endif
453: static PetscErrorCode DumpSingle(TSTrajectory tj, TS ts, Stack *stack, PetscInt id)
454: {
455: Vec *Y;
456: PetscInt stepnum;
457: TJScheduler *tjsch = (TJScheduler *)tj->data;
458: char filename[PETSC_MAX_PATH_LEN];
459: MPI_Comm comm;
461: PetscObjectGetComm((PetscObject)ts, &comm);
462: if (tj->monitor) {
463: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
464: PetscViewerASCIIPrintf(tj->monitor, "Dump a single point from file\n");
465: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
466: }
467: TSGetStepNumber(ts, &stepnum);
468: PetscSNPrintf(filename, sizeof(filename), "%s/TS-CPS%06" PetscInt_FMT ".bin", tj->dirname, id);
469: PetscViewerFileSetName(tjsch->viewer, filename);
470: PetscViewerSetUp(tjsch->viewer);
472: TSGetStages(ts, &stack->numY, &Y);
473: PetscLogEventBegin(TSTrajectory_DiskWrite, tj, ts, 0, 0);
474: WriteToDisk(ts->stifflyaccurate, stepnum, ts->ptime, ts->ptime_prev, ts->vec_sol, Y, stack->numY, SOLUTION_STAGES, tjsch->viewer);
475: PetscLogEventEnd(TSTrajectory_DiskWrite, tj, ts, 0, 0);
476: ts->trajectory->diskwrites++;
477: return 0;
478: }
480: static PetscErrorCode LoadSingle(TSTrajectory tj, TS ts, Stack *stack, PetscInt id)
481: {
482: Vec *Y;
483: PetscViewer viewer;
484: char filename[PETSC_MAX_PATH_LEN];
486: if (tj->monitor) {
487: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
488: PetscViewerASCIIPrintf(tj->monitor, "Load a single point from file\n");
489: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
490: }
491: PetscSNPrintf(filename, sizeof filename, "%s/TS-CPS%06" PetscInt_FMT ".bin", tj->dirname, id);
492: PetscViewerBinaryOpen(PetscObjectComm((PetscObject)tj), filename, FILE_MODE_READ, &viewer);
493: PetscViewerBinarySetSkipInfo(viewer, PETSC_TRUE);
494: PetscViewerPushFormat(viewer, PETSC_VIEWER_NATIVE);
495: TSGetStages(ts, &stack->numY, &Y);
496: PetscLogEventBegin(TSTrajectory_DiskRead, tj, ts, 0, 0);
497: ReadFromDisk(ts->stifflyaccurate, &ts->steps, &ts->ptime, &ts->ptime_prev, ts->vec_sol, Y, stack->numY, SOLUTION_STAGES, viewer);
498: PetscLogEventEnd(TSTrajectory_DiskRead, tj, ts, 0, 0);
499: ts->trajectory->diskreads++;
500: PetscViewerDestroy(&viewer);
501: return 0;
502: }
504: static PetscErrorCode UpdateTS(TS ts, Stack *stack, StackElement e, PetscInt stepnum, PetscBool adjoint_mode)
505: {
506: Vec *Y;
507: PetscInt i;
509: /* In adjoint mode we do not need to copy solution if the stepnum is the same */
510: if (!adjoint_mode || (HaveSolution(e->cptype) && e->stepnum != stepnum)) VecCopy(e->X, ts->vec_sol);
511: if (HaveStages(e->cptype)) {
512: TSGetStages(ts, &stack->numY, &Y);
513: if (e->stepnum && e->stepnum == stepnum) {
514: for (i = 0; i < stack->numY; i++) VecCopy(e->Y[i], Y[i]);
515: } else if (ts->stifflyaccurate) {
516: VecCopy(e->Y[stack->numY - 1], ts->vec_sol);
517: }
518: }
519: if (adjoint_mode) {
520: TSSetTimeStep(ts, e->timeprev - e->time); /* stepsize will be negative */
521: } else {
522: TSSetTimeStep(ts, e->time - e->timeprev); /* stepsize will be positive */
523: }
524: ts->ptime = e->time;
525: ts->ptime_prev = e->timeprev;
526: return 0;
527: }
529: static PetscErrorCode ReCompute(TS ts, TJScheduler *tjsch, PetscInt stepnumbegin, PetscInt stepnumend)
530: {
531: Stack *stack = &tjsch->stack;
532: PetscInt i;
534: tjsch->recompute = PETSC_TRUE; /* hints TSTrajectorySet() that it is in recompute mode */
535: TSSetStepNumber(ts, stepnumbegin); /* global step number */
536: for (i = stepnumbegin; i < stepnumend; i++) { /* assume fixed step size */
537: if (stack->solution_only && !tjsch->skip_trajectory) { /* revolve online need this */
538: /* don't use the public interface as it will update the TSHistory: this need a better fix */
539: TSTrajectorySet_Memory(ts->trajectory, ts, ts->steps, ts->ptime, ts->vec_sol);
540: }
541: TSMonitor(ts, ts->steps, ts->ptime, ts->vec_sol);
542: TSStep(ts);
543: if (!stack->solution_only && !tjsch->skip_trajectory) {
544: /* don't use the public interface as it will update the TSHistory: this need a better fix */
545: TSTrajectorySet_Memory(ts->trajectory, ts, ts->steps, ts->ptime, ts->vec_sol);
546: }
547: TSEventHandler(ts);
548: if (!ts->steprollback) TSPostStep(ts);
549: }
550: TurnBackward(ts);
551: ts->trajectory->recomps += stepnumend - stepnumbegin; /* recomputation counter */
552: TSSetStepNumber(ts, stepnumend);
553: tjsch->recompute = PETSC_FALSE; /* reset the flag for recompute mode */
554: return 0;
555: }
557: static PetscErrorCode TopLevelStore(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscInt localstepnum, PetscInt laststridesize, PetscBool *done)
558: {
559: Stack *stack = &tjsch->stack;
560: DiskStack *diskstack = &tjsch->diskstack;
561: PetscInt stridenum;
563: *done = PETSC_FALSE;
564: stridenum = stepnum / tjsch->stride;
565: /* make sure saved checkpoint id starts from 1
566: skip last stride when using stridenum+1
567: skip first stride when using stridenum */
568: if (stack->solution_only) {
569: if (tjsch->save_stack) {
570: if (localstepnum == tjsch->stride - 1 && stepnum < tjsch->total_steps - laststridesize) { /* current step will be saved without going through stack */
571: StackDumpAll(tj, ts, stack, stridenum + 1);
572: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum + 1;
573: *done = PETSC_TRUE;
574: }
575: } else {
576: if (localstepnum == 0 && stepnum < tjsch->total_steps - laststridesize) {
577: DumpSingle(tj, ts, stack, stridenum + 1);
578: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum + 1;
579: *done = PETSC_TRUE;
580: }
581: }
582: } else {
583: if (tjsch->save_stack) {
584: if (localstepnum == 0 && stepnum < tjsch->total_steps && stepnum != 0) { /* skip the first stride */
585: StackDumpAll(tj, ts, stack, stridenum);
586: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum;
587: *done = PETSC_TRUE;
588: }
589: } else {
590: if (localstepnum == 1 && stepnum < tjsch->total_steps - laststridesize) {
591: DumpSingle(tj, ts, stack, stridenum + 1);
592: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum + 1;
593: *done = PETSC_TRUE;
594: }
595: }
596: }
597: return 0;
598: }
600: static PetscErrorCode TSTrajectoryMemorySet_N(TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscReal time, Vec X)
601: {
602: Stack *stack = &tjsch->stack;
603: StackElement e;
604: CheckpointType cptype;
606: /* skip the last step */
607: if (ts->reason) { /* only affect the forward run */
608: /* update total_steps in the end of forward run */
609: if (stepnum != tjsch->total_steps) tjsch->total_steps = stepnum;
610: if (stack->solution_only) {
611: /* get rid of the solution at second last step */
612: StackPop(stack, &e);
613: }
614: return 0;
615: }
616: /* do not save trajectory at the recompute stage for solution_only mode */
617: if (stack->solution_only && tjsch->recompute) return 0;
618: /* skip the first step for no_solution_only mode */
619: if (!stack->solution_only && stepnum == 0) return 0;
621: /* resize the stack */
622: if (stack->top + 1 == stack->stacksize) StackResize(stack, 2 * stack->stacksize);
623: /* update timenext for the previous step; necessary for step adaptivity */
624: if (stack->top > -1) {
625: StackTop(stack, &e);
626: e->timenext = ts->ptime;
627: }
629: cptype = stack->solution_only ? SOLUTIONONLY : STAGESONLY;
630: ElementCreate(ts, cptype, stack, &e);
631: ElementSet(ts, stack, &e, stepnum, time, X);
632: StackPush(stack, e);
633: return 0;
634: }
636: static PetscErrorCode TSTrajectoryMemorySet_N_2(TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscReal time, Vec X)
637: {
638: Stack *stack = &tjsch->stack;
639: StackElement e;
640: CheckpointType cptype;
642: if (stack->top + 1 == stack->stacksize) StackResize(stack, 2 * stack->stacksize);
643: /* update timenext for the previous step; necessary for step adaptivity */
644: if (stack->top > -1) {
645: StackTop(stack, &e);
646: e->timenext = ts->ptime;
647: }
649: cptype = stack->solution_only ? SOLUTIONONLY : SOLUTION_STAGES; /* Always include solution in a checkpoint in non-adjoint mode */
650: ElementCreate(ts, cptype, stack, &e);
651: ElementSet(ts, stack, &e, stepnum, time, X);
652: StackPush(stack, e);
653: return 0;
654: }
656: static PetscErrorCode TSTrajectoryMemoryGet_N(TS ts, TJScheduler *tjsch, PetscInt stepnum)
657: {
658: Stack *stack = &tjsch->stack;
659: StackElement e;
660: PetscInt ns;
662: /* If TSTrajectoryGet() is called after TSAdjointSolve() converges (e.g. outside the while loop in TSAdjointSolve()), skip getting the checkpoint. */
663: if (ts->reason) return 0;
664: if (stepnum == tjsch->total_steps) {
665: TurnBackward(ts);
666: return 0;
667: }
668: /* restore a checkpoint */
669: StackTop(stack, &e);
670: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
671: TSGetStages(ts, &ns, PETSC_IGNORE);
672: if (stack->solution_only && ns) { /* recompute one step */
673: TurnForwardWithStepsize(ts, e->timenext - e->time);
674: ReCompute(ts, tjsch, e->stepnum, stepnum);
675: }
676: StackPop(stack, &e);
677: return 0;
678: }
680: static PetscErrorCode TSTrajectoryMemoryGet_N_2(TS ts, TJScheduler *tjsch, PetscInt stepnum)
681: {
682: Stack *stack = &tjsch->stack;
683: StackElement e = NULL;
685: StackFind(stack, &e, stepnum);
687: UpdateTS(ts, stack, e, stepnum, PETSC_FALSE);
688: return 0;
689: }
691: static PetscErrorCode TSTrajectoryMemorySet_TLNR(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscReal time, Vec X)
692: {
693: Stack *stack = &tjsch->stack;
694: PetscInt localstepnum, laststridesize;
695: StackElement e;
696: PetscBool done;
697: CheckpointType cptype;
699: if (!stack->solution_only && stepnum == 0) return 0;
700: if (stack->solution_only && stepnum == tjsch->total_steps) return 0;
701: if (tjsch->save_stack && tjsch->recompute) return 0;
703: localstepnum = stepnum % tjsch->stride;
704: /* (stridesize-1) checkpoints are saved in each stride; an extra point is added by StackDumpAll() */
705: laststridesize = tjsch->total_steps % tjsch->stride;
706: if (!laststridesize) laststridesize = tjsch->stride;
708: if (!tjsch->recompute) {
709: TopLevelStore(tj, ts, tjsch, stepnum, localstepnum, laststridesize, &done);
710: if (!tjsch->save_stack && stepnum < tjsch->total_steps - laststridesize) return 0;
711: }
712: if (!stack->solution_only && localstepnum == 0) return 0; /* skip last point in each stride at recompute stage or last stride */
713: if (stack->solution_only && localstepnum == tjsch->stride - 1) return 0; /* skip last step in each stride at recompute stage or last stride */
715: cptype = stack->solution_only ? SOLUTIONONLY : STAGESONLY;
716: ElementCreate(ts, cptype, stack, &e);
717: ElementSet(ts, stack, &e, stepnum, time, X);
718: StackPush(stack, e);
719: return 0;
720: }
722: static PetscErrorCode TSTrajectoryMemoryGet_TLNR(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum)
723: {
724: Stack *stack = &tjsch->stack;
725: PetscInt id, localstepnum, laststridesize;
726: StackElement e;
728: if (stepnum == tjsch->total_steps) {
729: TurnBackward(ts);
730: return 0;
731: }
733: localstepnum = stepnum % tjsch->stride;
734: laststridesize = tjsch->total_steps % tjsch->stride;
735: if (!laststridesize) laststridesize = tjsch->stride;
736: if (stack->solution_only) {
737: /* fill stack with info */
738: if (localstepnum == 0 && tjsch->total_steps - stepnum >= laststridesize) {
739: id = stepnum / tjsch->stride;
740: if (tjsch->save_stack) {
741: StackLoadAll(tj, ts, stack, id);
742: tjsch->skip_trajectory = PETSC_TRUE;
743: TurnForward(ts);
744: ReCompute(ts, tjsch, id * tjsch->stride - 1, id * tjsch->stride);
745: tjsch->skip_trajectory = PETSC_FALSE;
746: } else {
747: LoadSingle(tj, ts, stack, id);
748: TurnForward(ts);
749: ReCompute(ts, tjsch, (id - 1) * tjsch->stride, id * tjsch->stride);
750: }
751: return 0;
752: }
753: /* restore a checkpoint */
754: StackPop(stack, &e);
755: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
756: tjsch->skip_trajectory = PETSC_TRUE;
757: TurnForward(ts);
758: ReCompute(ts, tjsch, e->stepnum, stepnum);
759: tjsch->skip_trajectory = PETSC_FALSE;
760: } else {
761: CheckpointType cptype = STAGESONLY;
762: /* fill stack with info */
763: if (localstepnum == 0 && tjsch->total_steps - stepnum >= laststridesize) {
764: id = stepnum / tjsch->stride;
765: if (tjsch->save_stack) {
766: StackLoadAll(tj, ts, stack, id);
767: } else {
768: LoadSingle(tj, ts, stack, id);
769: ElementCreate(ts, cptype, stack, &e);
770: ElementSet(ts, stack, &e, (id - 1) * tjsch->stride + 1, ts->ptime, ts->vec_sol);
771: StackPush(stack, e);
772: TurnForward(ts);
773: ReCompute(ts, tjsch, e->stepnum, id * tjsch->stride);
774: }
775: return 0;
776: }
777: /* restore a checkpoint */
778: StackPop(stack, &e);
779: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
780: }
781: return 0;
782: }
784: #if defined(PETSC_HAVE_REVOLVE)
785: static PetscErrorCode printwhattodo(PetscViewer viewer, PetscRevolveInt whattodo, RevolveCTX *rctx, PetscRevolveInt shift)
786: {
787: if (!viewer) return 0;
789: switch (whattodo) {
790: case 1:
791: PetscViewerASCIIPrintf(viewer, "Advance from %d to %d\n", rctx->oldcapo + shift, rctx->capo + shift);
792: break;
793: case 2:
794: PetscViewerASCIIPrintf(viewer, "Store in checkpoint number %d (located in RAM)\n", rctx->check);
795: break;
796: case 3:
797: PetscViewerASCIIPrintf(viewer, "First turn: Initialize adjoints and reverse first step\n");
798: break;
799: case 4:
800: PetscViewerASCIIPrintf(viewer, "Forward and reverse one step\n");
801: break;
802: case 5:
803: PetscViewerASCIIPrintf(viewer, "Restore in checkpoint number %d (located in RAM)\n", rctx->check);
804: break;
805: case 7:
806: PetscViewerASCIIPrintf(viewer, "Store in checkpoint number %d (located on disk)\n", rctx->check);
807: break;
808: case 8:
809: PetscViewerASCIIPrintf(viewer, "Restore in checkpoint number %d (located on disk)\n", rctx->check);
810: break;
811: case -1:
812: PetscViewerASCIIPrintf(viewer, "Error!");
813: break;
814: }
815: return 0;
816: }
818: static PetscErrorCode printwhattodo2(PetscViewer viewer, PetscRevolveInt whattodo, RevolveCTX *rctx, PetscRevolveInt shift)
819: {
820: if (!viewer) return 0;
822: switch (whattodo) {
823: case 1:
824: PetscViewerASCIIPrintf(viewer, "[Top Level] Advance from stride %d to stride %d\n", rctx->oldcapo + shift, rctx->capo + shift);
825: break;
826: case 2:
827: PetscViewerASCIIPrintf(viewer, "[Top Level] Store in checkpoint number %d\n", rctx->check);
828: break;
829: case 3:
830: PetscViewerASCIIPrintf(viewer, "[Top Level] First turn: Initialize adjoints and reverse first stride\n");
831: break;
832: case 4:
833: PetscViewerASCIIPrintf(viewer, "[Top Level] Forward and reverse one stride\n");
834: break;
835: case 5:
836: PetscViewerASCIIPrintf(viewer, "[Top Level] Restore in checkpoint number %d\n", rctx->check);
837: break;
838: case 7:
839: PetscViewerASCIIPrintf(viewer, "[Top Level] Store in top-level checkpoint number %d\n", rctx->check);
840: break;
841: case 8:
842: PetscViewerASCIIPrintf(viewer, "[Top Level] Restore in top-level checkpoint number %d\n", rctx->check);
843: break;
844: case -1:
845: PetscViewerASCIIPrintf(viewer, "[Top Level] Error!");
846: break;
847: }
848: return 0;
849: }
851: static PetscErrorCode InitRevolve(PetscInt fine, PetscInt snaps, RevolveCTX *rctx)
852: {
853: PetscRevolveInt rsnaps, rfine;
855: PetscRevolveIntCast(snaps, &rsnaps);
856: PetscRevolveIntCast(fine, &rfine);
857: revolve_reset();
858: revolve_create_offline(rfine, rsnaps);
859: rctx->snaps_in = rsnaps;
860: rctx->fine = rfine;
861: rctx->check = 0;
862: rctx->capo = 0;
863: rctx->reverseonestep = PETSC_FALSE;
864: /* check stepsleft? */
865: return 0;
866: }
868: static PetscErrorCode FastForwardRevolve(RevolveCTX *rctx)
869: {
870: PetscRevolveInt whattodo;
872: whattodo = 0;
873: while (whattodo != 3) { /* we have to fast forward revolve to the beginning of the backward sweep due to unfriendly revolve interface */
874: whattodo = revolve_action(&rctx->check, &rctx->capo, &rctx->fine, rctx->snaps_in, &rctx->info, &rctx->where);
875: }
876: return 0;
877: }
879: static PetscErrorCode ApplyRevolve(PetscViewer viewer, SchedulerType stype, RevolveCTX *rctx, PetscRevolveInt total_steps, PetscRevolveInt stepnum, PetscRevolveInt localstepnum, PetscBool toplevel, PetscInt *store)
880: {
881: PetscRevolveInt shift, whattodo;
883: *store = 0;
884: if (rctx->stepsleft > 0) { /* advance the solution without checkpointing anything as Revolve requires */
885: rctx->stepsleft--;
886: return 0;
887: }
888: /* let Revolve determine what to do next */
889: shift = stepnum - localstepnum;
890: rctx->oldcapo = rctx->capo;
891: rctx->capo = localstepnum;
893: if (!toplevel) whattodo = revolve_action(&rctx->check, &rctx->capo, &rctx->fine, rctx->snaps_in, &rctx->info, &rctx->where);
894: else whattodo = revolve2_action(&rctx->check, &rctx->capo, &rctx->fine, rctx->snaps_in, &rctx->info, &rctx->where);
895: if (stype == REVOLVE_ONLINE && whattodo == 8) whattodo = 5;
896: if (stype == REVOLVE_ONLINE && whattodo == 7) whattodo = 2;
897: if (!toplevel) printwhattodo(viewer, whattodo, rctx, shift);
898: else printwhattodo2(viewer, whattodo, rctx, shift);
900: if (whattodo == 1) { /* advance some time steps */
901: if (stype == REVOLVE_ONLINE && rctx->capo >= total_steps - 1) {
902: revolve_turn(total_steps, &rctx->capo, &rctx->fine);
903: if (!toplevel) printwhattodo(viewer, whattodo, rctx, shift);
904: else printwhattodo2(viewer, whattodo, rctx, shift);
905: }
906: rctx->stepsleft = rctx->capo - rctx->oldcapo - 1;
907: }
908: if (whattodo == 3 || whattodo == 4) { /* ready for a reverse step */
909: rctx->reverseonestep = PETSC_TRUE;
910: }
911: if (whattodo == 5) { /* restore a checkpoint and ask Revolve what to do next */
912: rctx->oldcapo = rctx->capo;
913: if (!toplevel) whattodo = revolve_action(&rctx->check, &rctx->capo, &rctx->fine, rctx->snaps_in, &rctx->info, &rctx->where); /* must return 1 or 3 or 4*/
914: else whattodo = revolve2_action(&rctx->check, &rctx->capo, &rctx->fine, rctx->snaps_in, &rctx->info, &rctx->where);
915: if (!toplevel) printwhattodo(viewer, whattodo, rctx, shift);
916: else printwhattodo2(viewer, whattodo, rctx, shift);
917: if (whattodo == 3 || whattodo == 4) rctx->reverseonestep = PETSC_TRUE;
918: if (whattodo == 1) rctx->stepsleft = rctx->capo - rctx->oldcapo;
919: }
920: if (whattodo == 7) { /* save the checkpoint to disk */
921: *store = 2;
922: rctx->oldcapo = rctx->capo;
923: whattodo = revolve_action(&rctx->check, &rctx->capo, &rctx->fine, rctx->snaps_in, &rctx->info, &rctx->where); /* must return 1 */
924: printwhattodo(viewer, whattodo, rctx, shift);
925: rctx->stepsleft = rctx->capo - rctx->oldcapo - 1;
926: }
927: if (whattodo == 2) { /* store a checkpoint to RAM and ask Revolve how many time steps to advance next */
928: *store = 1;
929: rctx->oldcapo = rctx->capo;
930: if (!toplevel) whattodo = revolve_action(&rctx->check, &rctx->capo, &rctx->fine, rctx->snaps_in, &rctx->info, &rctx->where); /* must return 1 */
931: else whattodo = revolve2_action(&rctx->check, &rctx->capo, &rctx->fine, rctx->snaps_in, &rctx->info, &rctx->where);
932: if (!toplevel) printwhattodo(viewer, whattodo, rctx, shift);
933: else printwhattodo2(viewer, whattodo, rctx, shift);
934: if (stype == REVOLVE_ONLINE && rctx->capo >= total_steps - 1) {
935: revolve_turn(total_steps, &rctx->capo, &rctx->fine);
936: printwhattodo(viewer, whattodo, rctx, shift);
937: }
938: rctx->stepsleft = rctx->capo - rctx->oldcapo - 1;
939: }
940: return 0;
941: }
943: static PetscErrorCode TSTrajectoryMemorySet_ROF(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscReal time, Vec X)
944: {
945: Stack *stack = &tjsch->stack;
946: PetscInt store;
947: StackElement e;
948: PetscRevolveInt rtotal_steps, rstepnum;
949: CheckpointType cptype;
951: if (!stack->solution_only && stepnum == 0) return 0;
952: if (stack->solution_only && stepnum == tjsch->total_steps) return 0;
953: PetscRevolveIntCast(tjsch->total_steps, &rtotal_steps);
954: PetscRevolveIntCast(stepnum, &rstepnum);
955: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx, rtotal_steps, rstepnum, rstepnum, PETSC_FALSE, &store);
956: if (store == 1) {
958: cptype = stack->solution_only ? SOLUTIONONLY : SOLUTION_STAGES;
959: ElementCreate(ts, cptype, stack, &e);
960: ElementSet(ts, stack, &e, stepnum, time, X);
961: StackPush(stack, e);
962: }
963: return 0;
964: }
966: static PetscErrorCode TSTrajectoryMemoryGet_ROF(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum)
967: {
968: Stack *stack = &tjsch->stack;
969: PetscInt store;
970: PetscRevolveInt whattodo, shift, rtotal_steps, rstepnum;
971: StackElement e;
973: if (stepnum == 0 || stepnum == tjsch->total_steps) {
974: TurnBackward(ts);
975: tjsch->rctx->reverseonestep = PETSC_FALSE;
976: return 0;
977: }
978: /* restore a checkpoint */
979: StackTop(stack, &e);
980: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
981: PetscRevolveIntCast(tjsch->total_steps, &rtotal_steps);
982: PetscRevolveIntCast(stepnum, &rstepnum);
983: if (stack->solution_only) { /* start with restoring a checkpoint */
984: tjsch->rctx->capo = rstepnum;
985: tjsch->rctx->oldcapo = tjsch->rctx->capo;
986: shift = 0;
987: whattodo = revolve_action(&tjsch->rctx->check, &tjsch->rctx->capo, &tjsch->rctx->fine, tjsch->rctx->snaps_in, &tjsch->rctx->info, &tjsch->rctx->where);
988: printwhattodo(tj->monitor, whattodo, tjsch->rctx, shift);
989: } else { /* 2 revolve actions: restore a checkpoint and then advance */
990: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx, rtotal_steps, rstepnum, rstepnum, PETSC_FALSE, &store);
991: if (tj->monitor) {
992: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
993: PetscViewerASCIIPrintf(tj->monitor, "Skip the step from %d to %d (stage values already checkpointed)\n", tjsch->rctx->oldcapo, tjsch->rctx->oldcapo + 1);
994: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
995: }
996: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
997: }
998: if (stack->solution_only || (!stack->solution_only && e->stepnum < stepnum)) {
999: TurnForward(ts);
1000: ReCompute(ts, tjsch, e->stepnum, stepnum);
1001: }
1002: if ((stack->solution_only && e->stepnum + 1 == stepnum) || (!stack->solution_only && e->stepnum == stepnum)) StackPop(stack, &e);
1003: tjsch->rctx->reverseonestep = PETSC_FALSE;
1004: return 0;
1005: }
1007: static PetscErrorCode TSTrajectoryMemorySet_RON(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscReal time, Vec X)
1008: {
1009: Stack *stack = &tjsch->stack;
1010: Vec *Y;
1011: PetscInt i, store;
1012: PetscReal timeprev;
1013: StackElement e;
1014: RevolveCTX *rctx = tjsch->rctx;
1015: PetscRevolveInt rtotal_steps, rstepnum;
1016: CheckpointType cptype;
1018: if (!stack->solution_only && stepnum == 0) return 0;
1019: if (stack->solution_only && stepnum == tjsch->total_steps) return 0;
1020: PetscRevolveIntCast(tjsch->total_steps, &rtotal_steps);
1021: PetscRevolveIntCast(stepnum, &rstepnum);
1022: ApplyRevolve(tj->monitor, tjsch->stype, rctx, rtotal_steps, rstepnum, rstepnum, PETSC_FALSE, &store);
1023: if (store == 1) {
1024: if (rctx->check != stack->top + 1) { /* overwrite some non-top checkpoint in the stack */
1025: StackFind(stack, &e, rctx->check);
1026: if (HaveSolution(e->cptype)) VecCopy(X, e->X);
1027: if (HaveStages(e->cptype)) {
1028: TSGetStages(ts, &stack->numY, &Y);
1029: for (i = 0; i < stack->numY; i++) VecCopy(Y[i], e->Y[i]);
1030: }
1031: e->stepnum = stepnum;
1032: e->time = time;
1033: TSGetPrevTime(ts, &timeprev);
1034: e->timeprev = timeprev;
1035: } else {
1037: cptype = stack->solution_only ? SOLUTIONONLY : SOLUTION_STAGES;
1038: ElementCreate(ts, cptype, stack, &e);
1039: ElementSet(ts, stack, &e, stepnum, time, X);
1040: StackPush(stack, e);
1041: }
1042: }
1043: return 0;
1044: }
1046: static PetscErrorCode TSTrajectoryMemoryGet_RON(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum)
1047: {
1048: Stack *stack = &tjsch->stack;
1049: PetscRevolveInt whattodo, shift, rstepnum;
1050: StackElement e;
1052: if (stepnum == 0 || stepnum == tjsch->total_steps) {
1053: TurnBackward(ts);
1054: tjsch->rctx->reverseonestep = PETSC_FALSE;
1055: return 0;
1056: }
1057: PetscRevolveIntCast(stepnum, &rstepnum);
1058: tjsch->rctx->capo = rstepnum;
1059: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1060: shift = 0;
1061: whattodo = revolve_action(&tjsch->rctx->check, &tjsch->rctx->capo, &tjsch->rctx->fine, tjsch->rctx->snaps_in, &tjsch->rctx->info, &tjsch->rctx->where); /* whattodo=restore */
1062: if (whattodo == 8) whattodo = 5;
1063: printwhattodo(tj->monitor, whattodo, tjsch->rctx, shift);
1064: /* restore a checkpoint */
1065: StackFind(stack, &e, tjsch->rctx->check);
1066: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
1067: if (!stack->solution_only) { /* whattodo must be 5 */
1068: /* ask Revolve what to do next */
1069: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1070: whattodo = revolve_action(&tjsch->rctx->check, &tjsch->rctx->capo, &tjsch->rctx->fine, tjsch->rctx->snaps_in, &tjsch->rctx->info, &tjsch->rctx->where); /* must return 1 or 3 or 4*/
1071: printwhattodo(tj->monitor, whattodo, tjsch->rctx, shift);
1072: if (whattodo == 3 || whattodo == 4) tjsch->rctx->reverseonestep = PETSC_TRUE;
1073: if (whattodo == 1) tjsch->rctx->stepsleft = tjsch->rctx->capo - tjsch->rctx->oldcapo;
1074: if (tj->monitor) {
1075: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
1076: PetscViewerASCIIPrintf(tj->monitor, "Skip the step from %d to %d (stage values already checkpointed)\n", tjsch->rctx->oldcapo, tjsch->rctx->oldcapo + 1);
1077: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
1078: }
1079: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
1080: }
1081: if (stack->solution_only || (!stack->solution_only && e->stepnum < stepnum)) {
1082: TurnForward(ts);
1083: ReCompute(ts, tjsch, e->stepnum, stepnum);
1084: }
1085: tjsch->rctx->reverseonestep = PETSC_FALSE;
1086: return 0;
1087: }
1089: static PetscErrorCode TSTrajectoryMemorySet_TLR(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscReal time, Vec X)
1090: {
1091: Stack *stack = &tjsch->stack;
1092: PetscInt store, localstepnum, laststridesize;
1093: StackElement e;
1094: PetscBool done = PETSC_FALSE;
1095: PetscRevolveInt rtotal_steps, rstepnum, rlocalstepnum;
1096: CheckpointType cptype;
1098: if (!stack->solution_only && stepnum == 0) return 0;
1099: if (stack->solution_only && stepnum == tjsch->total_steps) return 0;
1101: localstepnum = stepnum % tjsch->stride;
1102: laststridesize = tjsch->total_steps % tjsch->stride;
1103: if (!laststridesize) laststridesize = tjsch->stride;
1105: if (!tjsch->recompute) {
1106: TopLevelStore(tj, ts, tjsch, stepnum, localstepnum, laststridesize, &done);
1107: /* revolve is needed for the last stride; different starting points for last stride between solutin_only and !solutin_only */
1108: if (!stack->solution_only && !tjsch->save_stack && stepnum <= tjsch->total_steps - laststridesize) return 0;
1109: if (stack->solution_only && !tjsch->save_stack && stepnum < tjsch->total_steps - laststridesize) return 0;
1110: }
1111: if (tjsch->save_stack && done) {
1112: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1113: return 0;
1114: }
1115: if (laststridesize < tjsch->stride) {
1116: if (stack->solution_only && stepnum == tjsch->total_steps - laststridesize && !tjsch->recompute) { /* step tjsch->total_steps-laststridesize-1 is skipped, but the next step is not */
1117: InitRevolve(laststridesize, tjsch->max_cps_ram, tjsch->rctx);
1118: }
1119: if (!stack->solution_only && stepnum == tjsch->total_steps - laststridesize + 1 && !tjsch->recompute) { /* step tjsch->total_steps-laststridesize is skipped, but the next step is not */
1120: InitRevolve(laststridesize, tjsch->max_cps_ram, tjsch->rctx);
1121: }
1122: }
1123: PetscRevolveIntCast(tjsch->total_steps, &rtotal_steps);
1124: PetscRevolveIntCast(stepnum, &rstepnum);
1125: PetscRevolveIntCast(localstepnum, &rlocalstepnum);
1126: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx, rtotal_steps, rstepnum, rlocalstepnum, PETSC_FALSE, &store);
1127: if (store == 1) {
1129: cptype = stack->solution_only ? SOLUTIONONLY : SOLUTION_STAGES;
1130: ElementCreate(ts, cptype, stack, &e);
1131: ElementSet(ts, stack, &e, stepnum, time, X);
1132: StackPush(stack, e);
1133: }
1134: return 0;
1135: }
1137: static PetscErrorCode TSTrajectoryMemoryGet_TLR(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum)
1138: {
1139: Stack *stack = &tjsch->stack;
1140: PetscRevolveInt whattodo, shift, rstepnum, rlocalstepnum, rtotal_steps;
1141: PetscInt localstepnum, stridenum, laststridesize, store;
1142: StackElement e;
1143: CheckpointType cptype;
1145: localstepnum = stepnum % tjsch->stride;
1146: stridenum = stepnum / tjsch->stride;
1147: if (stepnum == tjsch->total_steps) {
1148: TurnBackward(ts);
1149: tjsch->rctx->reverseonestep = PETSC_FALSE;
1150: return 0;
1151: }
1152: laststridesize = tjsch->total_steps % tjsch->stride;
1153: if (!laststridesize) laststridesize = tjsch->stride;
1154: PetscRevolveIntCast(tjsch->total_steps, &rtotal_steps);
1155: PetscRevolveIntCast(stepnum, &rstepnum);
1156: PetscRevolveIntCast(localstepnum, &rlocalstepnum);
1157: if (stack->solution_only) {
1158: /* fill stack */
1159: if (localstepnum == 0 && stepnum <= tjsch->total_steps - laststridesize) {
1160: if (tjsch->save_stack) {
1161: StackLoadAll(tj, ts, stack, stridenum);
1162: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1163: FastForwardRevolve(tjsch->rctx);
1164: tjsch->skip_trajectory = PETSC_TRUE;
1165: TurnForward(ts);
1166: ReCompute(ts, tjsch, stridenum * tjsch->stride - 1, stridenum * tjsch->stride);
1167: tjsch->skip_trajectory = PETSC_FALSE;
1168: } else {
1169: LoadSingle(tj, ts, stack, stridenum);
1170: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1171: TurnForward(ts);
1172: ReCompute(ts, tjsch, (stridenum - 1) * tjsch->stride, stridenum * tjsch->stride);
1173: }
1174: return 0;
1175: }
1176: /* restore a checkpoint */
1177: StackTop(stack, &e);
1178: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
1179: /* start with restoring a checkpoint */
1180: tjsch->rctx->capo = rstepnum;
1181: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1182: shift = rstepnum - rlocalstepnum;
1183: whattodo = revolve_action(&tjsch->rctx->check, &tjsch->rctx->capo, &tjsch->rctx->fine, tjsch->rctx->snaps_in, &tjsch->rctx->info, &tjsch->rctx->where);
1184: printwhattodo(tj->monitor, whattodo, tjsch->rctx, shift);
1185: TurnForward(ts);
1186: ReCompute(ts, tjsch, e->stepnum, stepnum);
1187: if (e->stepnum + 1 == stepnum) StackPop(stack, &e);
1188: } else {
1189: /* fill stack with info */
1190: if (localstepnum == 0 && tjsch->total_steps - stepnum >= laststridesize) {
1191: if (tjsch->save_stack) {
1192: StackLoadAll(tj, ts, stack, stridenum);
1193: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1194: FastForwardRevolve(tjsch->rctx);
1195: } else {
1196: PetscRevolveInt rnum;
1197: LoadSingle(tj, ts, stack, stridenum);
1198: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1199: PetscRevolveIntCast((stridenum - 1) * tjsch->stride + 1, &rnum);
1200: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx, rtotal_steps, rnum, 1, PETSC_FALSE, &store);
1201: if (tj->monitor) {
1202: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
1203: PetscCall(
1204: PetscViewerASCIIPrintf(tj->monitor, "Skip the step from %" PetscInt_FMT " to %" PetscInt_FMT " (stage values already checkpointed)\n", (stridenum - 1) * tjsch->stride + tjsch->rctx->oldcapo, (stridenum - 1) * tjsch->stride + tjsch->rctx->oldcapo + 1));
1205: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
1206: }
1207: cptype = SOLUTION_STAGES;
1208: ElementCreate(ts, cptype, stack, &e);
1209: ElementSet(ts, stack, &e, (stridenum - 1) * tjsch->stride + 1, ts->ptime, ts->vec_sol);
1210: StackPush(stack, e);
1211: TurnForward(ts);
1212: ReCompute(ts, tjsch, e->stepnum, stridenum * tjsch->stride);
1213: }
1214: return 0;
1215: }
1216: /* restore a checkpoint */
1217: StackTop(stack, &e);
1218: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
1219: /* 2 revolve actions: restore a checkpoint and then advance */
1220: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx, rtotal_steps, rstepnum, rlocalstepnum, PETSC_FALSE, &store);
1221: if (tj->monitor) {
1222: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
1223: PetscViewerASCIIPrintf(tj->monitor, "Skip the step from %" PetscInt_FMT " to %" PetscInt_FMT " (stage values already checkpointed)\n", stepnum - localstepnum + tjsch->rctx->oldcapo, stepnum - localstepnum + tjsch->rctx->oldcapo + 1);
1224: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
1225: }
1226: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
1227: if (e->stepnum < stepnum) {
1228: TurnForward(ts);
1229: ReCompute(ts, tjsch, e->stepnum, stepnum);
1230: }
1231: if (e->stepnum == stepnum) StackPop(stack, &e);
1232: }
1233: tjsch->rctx->reverseonestep = PETSC_FALSE;
1234: return 0;
1235: }
1237: static PetscErrorCode TSTrajectoryMemorySet_TLTR(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscReal time, Vec X)
1238: {
1239: Stack *stack = &tjsch->stack;
1240: PetscInt store, localstepnum, stridenum, laststridesize;
1241: StackElement e;
1242: PetscBool done = PETSC_FALSE;
1243: PetscRevolveInt rlocalstepnum, rstepnum, rtotal_steps;
1245: if (!stack->solution_only && stepnum == 0) return 0;
1246: if (stack->solution_only && stepnum == tjsch->total_steps) return 0;
1248: localstepnum = stepnum % tjsch->stride; /* index at the bottom level (inside a stride) */
1249: stridenum = stepnum / tjsch->stride; /* index at the top level */
1250: laststridesize = tjsch->total_steps % tjsch->stride;
1251: if (!laststridesize) laststridesize = tjsch->stride;
1252: if (stack->solution_only && localstepnum == 0 && !tjsch->rctx2->reverseonestep) {
1253: PetscRevolveIntCast((tjsch->total_steps + tjsch->stride - 1) / tjsch->stride, &rtotal_steps);
1254: PetscRevolveIntCast(stridenum, &rstepnum);
1255: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx2, rtotal_steps, rstepnum, rstepnum, PETSC_TRUE, &tjsch->store_stride);
1256: if (laststridesize < tjsch->stride && stepnum == tjsch->total_steps - laststridesize) InitRevolve(laststridesize, tjsch->max_cps_ram, tjsch->rctx);
1257: }
1258: if (!stack->solution_only && localstepnum == 1 && !tjsch->rctx2->reverseonestep) {
1259: PetscRevolveIntCast((tjsch->total_steps + tjsch->stride - 1) / tjsch->stride, &rtotal_steps);
1260: PetscRevolveIntCast(stridenum, &rstepnum);
1261: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx2, rtotal_steps, rstepnum, rstepnum, PETSC_TRUE, &tjsch->store_stride);
1262: if (laststridesize < tjsch->stride && stepnum == tjsch->total_steps - laststridesize + 1) InitRevolve(laststridesize, tjsch->max_cps_ram, tjsch->rctx);
1263: }
1264: if (tjsch->store_stride) {
1265: TopLevelStore(tj, ts, tjsch, stepnum, localstepnum, laststridesize, &done);
1266: if (done) {
1267: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1268: return 0;
1269: }
1270: }
1271: if (stepnum < tjsch->total_steps - laststridesize) {
1272: if (tjsch->save_stack && !tjsch->store_stride && !tjsch->rctx2->reverseonestep) return 0; /* store or forward-and-reverse at top level trigger revolve at bottom level */
1273: if (!tjsch->save_stack && !tjsch->rctx2->reverseonestep) return 0; /* store operation does not require revolve be called at bottom level */
1274: }
1275: /* Skipping stepnum=0 for !stack->only is enough for TLR, but not for TLTR. Here we skip the first step for each stride so that the top-level revolve is applied (always at localstepnum=1) ahead of the bottom-level revolve */
1276: if (!stack->solution_only && localstepnum == 0 && stepnum != tjsch->total_steps && !tjsch->recompute) return 0;
1277: PetscRevolveIntCast(tjsch->total_steps, &rtotal_steps);
1278: PetscRevolveIntCast(stepnum, &rstepnum);
1279: PetscRevolveIntCast(localstepnum, &rlocalstepnum);
1280: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx, rtotal_steps, rstepnum, rlocalstepnum, PETSC_FALSE, &store);
1281: if (store == 1) {
1282: CheckpointType cptype;
1284: cptype = stack->solution_only ? SOLUTIONONLY : SOLUTION_STAGES;
1285: ElementCreate(ts, cptype, stack, &e);
1286: ElementSet(ts, stack, &e, stepnum, time, X);
1287: StackPush(stack, e);
1288: }
1289: return 0;
1290: }
1292: static PetscErrorCode TSTrajectoryMemoryGet_TLTR(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum)
1293: {
1294: Stack *stack = &tjsch->stack;
1295: DiskStack *diskstack = &tjsch->diskstack;
1296: PetscInt localstepnum, stridenum, restoredstridenum, laststridesize, store;
1297: StackElement e;
1298: PetscRevolveInt whattodo, shift;
1299: PetscRevolveInt rtotal_steps, rstepnum, rlocalstepnum;
1301: localstepnum = stepnum % tjsch->stride;
1302: stridenum = stepnum / tjsch->stride;
1303: if (stepnum == tjsch->total_steps) {
1304: TurnBackward(ts);
1305: tjsch->rctx->reverseonestep = PETSC_FALSE;
1306: return 0;
1307: }
1308: laststridesize = tjsch->total_steps % tjsch->stride;
1309: if (!laststridesize) laststridesize = tjsch->stride;
1310: /*
1311: Last stride can be adjoined directly. All the other strides require that the stack in memory be ready before an adjoint step is taken (at the end of each stride). The following two cases need to be addressed differently:
1312: Case 1 (save_stack)
1313: Restore a disk checkpoint; update TS with the last element in the restored data; recompute to the current point.
1314: Case 2 (!save_stack)
1315: Restore a disk checkpoint; update TS with the restored point; recompute to the current point.
1316: */
1317: if (localstepnum == 0 && stepnum <= tjsch->total_steps - laststridesize) {
1318: /* restore the top element in the stack for disk checkpoints */
1319: restoredstridenum = diskstack->container[diskstack->top];
1320: tjsch->rctx2->reverseonestep = PETSC_FALSE;
1321: /* top-level revolve must be applied before current step, just like the solution_only mode for single-level revolve */
1322: if (!tjsch->save_stack && stack->solution_only) { /* start with restoring a checkpoint */
1323: PetscRevolveIntCast(stridenum, &rstepnum);
1324: tjsch->rctx2->capo = rstepnum;
1325: tjsch->rctx2->oldcapo = tjsch->rctx2->capo;
1326: shift = 0;
1327: whattodo = revolve2_action(&tjsch->rctx2->check, &tjsch->rctx2->capo, &tjsch->rctx2->fine, tjsch->rctx2->snaps_in, &tjsch->rctx2->info, &tjsch->rctx2->where);
1328: printwhattodo2(tj->monitor, whattodo, tjsch->rctx2, shift);
1329: } else { /* 2 revolve actions: restore a checkpoint and then advance */
1330: PetscRevolveIntCast((tjsch->total_steps + tjsch->stride - 1) / tjsch->stride, &rtotal_steps);
1331: PetscRevolveIntCast(stridenum, &rstepnum);
1332: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx2, rtotal_steps, rstepnum, rstepnum, PETSC_TRUE, &tjsch->store_stride);
1333: if (tj->monitor) {
1334: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
1335: PetscViewerASCIIPrintf(tj->monitor, "[Top Level] Skip the stride from %d to %d (stage values already checkpointed)\n", tjsch->rctx2->oldcapo, tjsch->rctx2->oldcapo + 1);
1336: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
1337: }
1338: if (!tjsch->rctx2->reverseonestep && tjsch->rctx2->stepsleft > 0) tjsch->rctx2->stepsleft--;
1339: }
1340: /* fill stack */
1341: if (stack->solution_only) {
1342: if (tjsch->save_stack) {
1343: if (restoredstridenum < stridenum) {
1344: StackLoadLast(tj, ts, stack, restoredstridenum);
1345: } else {
1346: StackLoadAll(tj, ts, stack, restoredstridenum);
1347: }
1348: /* recompute one step ahead */
1349: tjsch->skip_trajectory = PETSC_TRUE;
1350: TurnForward(ts);
1351: ReCompute(ts, tjsch, stridenum * tjsch->stride - 1, stridenum * tjsch->stride);
1352: tjsch->skip_trajectory = PETSC_FALSE;
1353: if (restoredstridenum < stridenum) {
1354: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1355: TurnForward(ts);
1356: ReCompute(ts, tjsch, restoredstridenum * tjsch->stride, stepnum);
1357: } else { /* stack ready, fast forward revolve status */
1358: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1359: FastForwardRevolve(tjsch->rctx);
1360: }
1361: } else {
1362: LoadSingle(tj, ts, stack, restoredstridenum);
1363: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1364: TurnForward(ts);
1365: ReCompute(ts, tjsch, (restoredstridenum - 1) * tjsch->stride, stepnum);
1366: }
1367: } else {
1368: if (tjsch->save_stack) {
1369: if (restoredstridenum < stridenum) {
1370: StackLoadLast(tj, ts, stack, restoredstridenum);
1371: /* reset revolve */
1372: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1373: TurnForward(ts);
1374: ReCompute(ts, tjsch, restoredstridenum * tjsch->stride, stepnum);
1375: } else { /* stack ready, fast forward revolve status */
1376: StackLoadAll(tj, ts, stack, restoredstridenum);
1377: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1378: FastForwardRevolve(tjsch->rctx);
1379: }
1380: } else {
1381: LoadSingle(tj, ts, stack, restoredstridenum);
1382: InitRevolve(tjsch->stride, tjsch->max_cps_ram, tjsch->rctx);
1383: /* push first element to stack */
1384: if (tjsch->store_stride || tjsch->rctx2->reverseonestep) {
1385: CheckpointType cptype = SOLUTION_STAGES;
1386: shift = (restoredstridenum - 1) * tjsch->stride - localstepnum;
1387: PetscRevolveIntCast(tjsch->total_steps, &rtotal_steps);
1388: PetscRevolveIntCast((restoredstridenum - 1) * tjsch->stride + 1, &rstepnum);
1389: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx, rtotal_steps, rstepnum, 1, PETSC_FALSE, &store);
1390: if (tj->monitor) {
1391: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
1392: PetscViewerASCIIPrintf(tj->monitor, "Skip the step from %" PetscInt_FMT " to %" PetscInt_FMT " (stage values already checkpointed)\n", (restoredstridenum - 1) * tjsch->stride, (restoredstridenum - 1) * tjsch->stride + 1);
1393: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
1394: }
1395: ElementCreate(ts, cptype, stack, &e);
1396: ElementSet(ts, stack, &e, (restoredstridenum - 1) * tjsch->stride + 1, ts->ptime, ts->vec_sol);
1397: StackPush(stack, e);
1398: }
1399: TurnForward(ts);
1400: ReCompute(ts, tjsch, (restoredstridenum - 1) * tjsch->stride + 1, stepnum);
1401: }
1402: }
1403: if (restoredstridenum == stridenum) diskstack->top--;
1404: tjsch->rctx->reverseonestep = PETSC_FALSE;
1405: return 0;
1406: }
1408: if (stack->solution_only) {
1409: /* restore a checkpoint */
1410: StackTop(stack, &e);
1411: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
1412: /* start with restoring a checkpoint */
1413: PetscRevolveIntCast(stepnum, &rstepnum);
1414: PetscRevolveIntCast(localstepnum, &rlocalstepnum);
1415: tjsch->rctx->capo = rstepnum;
1416: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1417: shift = rstepnum - rlocalstepnum;
1418: whattodo = revolve_action(&tjsch->rctx->check, &tjsch->rctx->capo, &tjsch->rctx->fine, tjsch->rctx->snaps_in, &tjsch->rctx->info, &tjsch->rctx->where);
1419: printwhattodo(tj->monitor, whattodo, tjsch->rctx, shift);
1420: TurnForward(ts);
1421: ReCompute(ts, tjsch, e->stepnum, stepnum);
1422: if (e->stepnum + 1 == stepnum) StackPop(stack, &e);
1423: } else {
1424: PetscRevolveInt rlocalstepnum;
1425: /* restore a checkpoint */
1426: StackTop(stack, &e);
1427: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
1428: /* 2 revolve actions: restore a checkpoint and then advance */
1429: PetscRevolveIntCast(tjsch->total_steps, &rtotal_steps);
1430: PetscRevolveIntCast(stridenum, &rstepnum);
1431: PetscRevolveIntCast(localstepnum, &rlocalstepnum);
1432: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx, rtotal_steps, rstepnum, rlocalstepnum, PETSC_FALSE, &store);
1433: if (tj->monitor) {
1434: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
1435: PetscViewerASCIIPrintf(tj->monitor, "Skip the step from %" PetscInt_FMT " to %" PetscInt_FMT " (stage values already checkpointed)\n", stepnum - localstepnum + tjsch->rctx->oldcapo, stepnum - localstepnum + tjsch->rctx->oldcapo + 1);
1436: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
1437: }
1438: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
1439: if (e->stepnum < stepnum) {
1440: TurnForward(ts);
1441: ReCompute(ts, tjsch, e->stepnum, stepnum);
1442: }
1443: if (e->stepnum == stepnum) StackPop(stack, &e);
1444: }
1445: tjsch->rctx->reverseonestep = PETSC_FALSE;
1446: return 0;
1447: }
1449: static PetscErrorCode TSTrajectoryMemorySet_RMS(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscReal time, Vec X)
1450: {
1451: Stack *stack = &tjsch->stack;
1452: PetscInt store;
1453: StackElement e;
1454: PetscRevolveInt rtotal_steps, rstepnum;
1456: if (!stack->solution_only && stepnum == 0) return 0;
1457: if (stack->solution_only && stepnum == tjsch->total_steps) return 0;
1458: PetscRevolveIntCast(tjsch->total_steps, &rtotal_steps);
1459: PetscRevolveIntCast(stepnum, &rstepnum);
1460: ApplyRevolve(tj->monitor, tjsch->stype, tjsch->rctx, rtotal_steps, rstepnum, rstepnum, PETSC_FALSE, &store);
1461: if (store == 1) {
1462: CheckpointType cptype;
1464: cptype = stack->solution_only ? SOLUTIONONLY : SOLUTION_STAGES;
1465: ElementCreate(ts, cptype, stack, &e);
1466: ElementSet(ts, stack, &e, stepnum, time, X);
1467: StackPush(stack, e);
1468: } else if (store == 2) {
1469: DumpSingle(tj, ts, stack, tjsch->rctx->check + 1);
1470: }
1471: return 0;
1472: }
1474: static PetscErrorCode TSTrajectoryMemoryGet_RMS(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum)
1475: {
1476: Stack *stack = &tjsch->stack;
1477: PetscRevolveInt whattodo, shift, rstepnum;
1478: PetscInt restart;
1479: PetscBool ondisk;
1480: StackElement e;
1482: if (stepnum == 0 || stepnum == tjsch->total_steps) {
1483: TurnBackward(ts);
1484: tjsch->rctx->reverseonestep = PETSC_FALSE;
1485: return 0;
1486: }
1487: PetscRevolveIntCast(stepnum, &rstepnum);
1488: tjsch->rctx->capo = rstepnum;
1489: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1490: shift = 0;
1491: whattodo = revolve_action(&tjsch->rctx->check, &tjsch->rctx->capo, &tjsch->rctx->fine, tjsch->rctx->snaps_in, &tjsch->rctx->info, &tjsch->rctx->where); /* whattodo=restore */
1492: printwhattodo(tj->monitor, whattodo, tjsch->rctx, shift);
1493: /* restore a checkpoint */
1494: restart = tjsch->rctx->capo;
1495: if (!tjsch->rctx->where) {
1496: ondisk = PETSC_TRUE;
1497: LoadSingle(tj, ts, stack, tjsch->rctx->check + 1);
1498: TurnBackward(ts);
1499: } else {
1500: ondisk = PETSC_FALSE;
1501: StackTop(stack, &e);
1502: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
1503: }
1504: if (!stack->solution_only) { /* whattodo must be 5 or 8 */
1505: /* ask Revolve what to do next */
1506: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1507: whattodo = revolve_action(&tjsch->rctx->check, &tjsch->rctx->capo, &tjsch->rctx->fine, tjsch->rctx->snaps_in, &tjsch->rctx->info, &tjsch->rctx->where); /* must return 1 or 3 or 4*/
1508: printwhattodo(tj->monitor, whattodo, tjsch->rctx, shift);
1509: if (whattodo == 3 || whattodo == 4) tjsch->rctx->reverseonestep = PETSC_TRUE;
1510: if (whattodo == 1) tjsch->rctx->stepsleft = tjsch->rctx->capo - tjsch->rctx->oldcapo;
1511: if (tj->monitor) {
1512: PetscViewerASCIIAddTab(tj->monitor, ((PetscObject)tj)->tablevel);
1513: PetscViewerASCIIPrintf(tj->monitor, "Skip the step from %d to %d (stage values already checkpointed)\n", tjsch->rctx->oldcapo, tjsch->rctx->oldcapo + 1);
1514: PetscViewerASCIISubtractTab(tj->monitor, ((PetscObject)tj)->tablevel);
1515: }
1516: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
1517: restart++; /* skip one step */
1518: }
1519: if (stack->solution_only || (!stack->solution_only && restart < stepnum)) {
1520: TurnForward(ts);
1521: ReCompute(ts, tjsch, restart, stepnum);
1522: }
1523: if (!ondisk && ((stack->solution_only && e->stepnum + 1 == stepnum) || (!stack->solution_only && e->stepnum == stepnum))) StackPop(stack, &e);
1524: tjsch->rctx->reverseonestep = PETSC_FALSE;
1525: return 0;
1526: }
1527: #endif
1529: #if defined(PETSC_HAVE_CAMS)
1530: /* Optimal offline adjoint checkpointing for multistage time integration methods */
1531: static PetscErrorCode TSTrajectoryMemorySet_AOF(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum, PetscReal time, Vec X)
1532: {
1533: Stack *stack = &tjsch->stack;
1534: StackElement e;
1536: /* skip if no checkpoint to use. This also avoids an error when num_units_avail=0 */
1537: if (tjsch->actx->nextcheckpointstep == -1) return 0;
1538: if (stepnum == 0) { /* When placing the first checkpoint, no need to change the units available */
1539: if (stack->solution_only) {
1540: offline_ca(tjsch->actx->lastcheckpointstep, tjsch->actx->num_units_avail, tjsch->actx->endstep, &tjsch->actx->nextcheckpointstep);
1541: } else {
1542: /* First two arguments must be -1 when first time calling cams */
1543: offline_cams(tjsch->actx->lastcheckpointstep, tjsch->actx->lastcheckpointtype, tjsch->actx->num_units_avail, tjsch->actx->endstep, tjsch->actx->num_stages, &tjsch->actx->nextcheckpointstep, &tjsch->actx->nextcheckpointtype);
1544: }
1545: }
1547: if (stack->solution_only && stepnum == tjsch->total_steps) return 0;
1549: if (tjsch->actx->nextcheckpointstep == stepnum) {
1552: if (tjsch->actx->nextcheckpointtype == 2) { /* solution + stage values */
1553: if (tj->monitor) PetscViewerASCIIPrintf(tj->monitor, "Store in checkpoint number %" PetscInt_FMT " with stage values and solution (located in RAM)\n", stepnum);
1554: ElementCreate(ts, SOLUTION_STAGES, stack, &e);
1555: ElementSet(ts, stack, &e, stepnum, time, X);
1556: }
1557: if (tjsch->actx->nextcheckpointtype == 1) {
1558: if (tj->monitor) PetscViewerASCIIPrintf(tj->monitor, "Store in checkpoint number %" PetscInt_FMT " with stage values (located in RAM)\n", stepnum);
1559: ElementCreate(ts, STAGESONLY, stack, &e);
1560: ElementSet(ts, stack, &e, stepnum, time, X);
1561: }
1562: if (tjsch->actx->nextcheckpointtype == 0) { /* solution only */
1563: if (tj->monitor) PetscViewerASCIIPrintf(tj->monitor, "Store in checkpoint number %" PetscInt_FMT " (located in RAM)\n", stepnum);
1564: ElementCreate(ts, SOLUTIONONLY, stack, &e);
1565: ElementSet(ts, stack, &e, stepnum, time, X);
1566: }
1567: StackPush(stack, e);
1569: tjsch->actx->lastcheckpointstep = stepnum;
1570: if (stack->solution_only) {
1571: offline_ca(tjsch->actx->lastcheckpointstep, tjsch->actx->num_units_avail, tjsch->actx->endstep, &tjsch->actx->nextcheckpointstep);
1572: tjsch->actx->num_units_avail--;
1573: } else {
1574: tjsch->actx->lastcheckpointtype = tjsch->actx->nextcheckpointtype;
1575: offline_cams(tjsch->actx->lastcheckpointstep, tjsch->actx->lastcheckpointtype, tjsch->actx->num_units_avail, tjsch->actx->endstep, tjsch->actx->num_stages, &tjsch->actx->nextcheckpointstep, &tjsch->actx->nextcheckpointtype);
1576: if (tjsch->actx->lastcheckpointtype == 2) tjsch->actx->num_units_avail -= tjsch->actx->num_stages + 1;
1577: if (tjsch->actx->lastcheckpointtype == 1) tjsch->actx->num_units_avail -= tjsch->actx->num_stages;
1578: if (tjsch->actx->lastcheckpointtype == 0) tjsch->actx->num_units_avail--;
1579: }
1580: }
1581: return 0;
1582: }
1584: static PetscErrorCode TSTrajectoryMemoryGet_AOF(TSTrajectory tj, TS ts, TJScheduler *tjsch, PetscInt stepnum)
1585: {
1586: Stack *stack = &tjsch->stack;
1587: StackElement e;
1588: PetscInt estepnum;
1590: if (stepnum == 0 || stepnum == tjsch->total_steps) {
1591: TurnBackward(ts);
1592: return 0;
1593: }
1594: /* Restore a checkpoint */
1595: StackTop(stack, &e);
1596: estepnum = e->stepnum;
1597: if (estepnum == stepnum && e->cptype == SOLUTIONONLY) { /* discard the checkpoint if not useful (corner case) */
1598: StackPop(stack, &e);
1599: tjsch->actx->num_units_avail++;
1600: StackTop(stack, &e);
1601: estepnum = e->stepnum;
1602: }
1603: /* Update TS with stage values if an adjoint step can be taken immediately */
1604: if (HaveStages(e->cptype)) {
1605: if (tj->monitor) PetscViewerASCIIPrintf(tj->monitor, "Restore in checkpoint number %" PetscInt_FMT " with stage values (located in RAM)\n", e->stepnum);
1606: if (e->cptype == STAGESONLY) tjsch->actx->num_units_avail += tjsch->actx->num_stages;
1607: if (e->cptype == SOLUTION_STAGES) tjsch->actx->num_units_avail += tjsch->actx->num_stages + 1;
1608: } else {
1609: if (tj->monitor) PetscViewerASCIIPrintf(tj->monitor, "Restore in checkpoint number %" PetscInt_FMT " (located in RAM)\n", e->stepnum);
1610: tjsch->actx->num_units_avail++;
1611: }
1612: UpdateTS(ts, stack, e, stepnum, PETSC_TRUE);
1613: /* Query the scheduler */
1614: tjsch->actx->lastcheckpointstep = estepnum;
1615: tjsch->actx->endstep = stepnum;
1616: if (stack->solution_only) { /* start with restoring a checkpoint */
1617: offline_ca(tjsch->actx->lastcheckpointstep, tjsch->actx->num_units_avail, tjsch->actx->endstep, &tjsch->actx->nextcheckpointstep);
1618: } else { /* 2 revolve actions: restore a checkpoint and then advance */
1619: tjsch->actx->lastcheckpointtype = e->cptype;
1620: offline_cams(tjsch->actx->lastcheckpointstep, tjsch->actx->lastcheckpointtype, tjsch->actx->num_units_avail, tjsch->actx->endstep, tjsch->actx->num_stages, &tjsch->actx->nextcheckpointstep, &tjsch->actx->nextcheckpointtype);
1621: }
1622: /* Discard the checkpoint if not needed, decrease the number of available checkpoints if it still stays in stack */
1623: if (HaveStages(e->cptype)) {
1624: if (estepnum == stepnum) {
1625: StackPop(stack, &e);
1626: } else {
1627: if (e->cptype == STAGESONLY) tjsch->actx->num_units_avail -= tjsch->actx->num_stages;
1628: if (e->cptype == SOLUTION_STAGES) tjsch->actx->num_units_avail -= tjsch->actx->num_stages + 1;
1629: }
1630: } else {
1631: if (estepnum + 1 == stepnum) {
1632: StackPop(stack, &e);
1633: } else {
1634: tjsch->actx->num_units_avail--;
1635: }
1636: }
1637: /* Recompute from the restored checkpoint */
1638: if (stack->solution_only || (!stack->solution_only && estepnum < stepnum)) {
1639: TurnForward(ts);
1640: ReCompute(ts, tjsch, estepnum, stepnum);
1641: }
1642: return 0;
1643: }
1644: #endif
1646: static PetscErrorCode TSTrajectorySet_Memory(TSTrajectory tj, TS ts, PetscInt stepnum, PetscReal time, Vec X)
1647: {
1648: TJScheduler *tjsch = (TJScheduler *)tj->data;
1650: if (!tjsch->recompute) { /* use global stepnum in the forward sweep */
1651: TSGetStepNumber(ts, &stepnum);
1652: }
1653: /* for consistency */
1654: if (!tjsch->recompute && stepnum == 0) ts->ptime_prev = ts->ptime - ts->time_step;
1655: switch (tjsch->stype) {
1656: case NONE:
1657: if (tj->adjoint_solve_mode) {
1658: TSTrajectoryMemorySet_N(ts, tjsch, stepnum, time, X);
1659: } else {
1660: TSTrajectoryMemorySet_N_2(ts, tjsch, stepnum, time, X);
1661: }
1662: break;
1663: case TWO_LEVEL_NOREVOLVE:
1665: TSTrajectoryMemorySet_TLNR(tj, ts, tjsch, stepnum, time, X);
1666: break;
1667: #if defined(PETSC_HAVE_REVOLVE)
1668: case TWO_LEVEL_REVOLVE:
1670: TSTrajectoryMemorySet_TLR(tj, ts, tjsch, stepnum, time, X);
1671: break;
1672: case TWO_LEVEL_TWO_REVOLVE:
1674: TSTrajectoryMemorySet_TLTR(tj, ts, tjsch, stepnum, time, X);
1675: break;
1676: case REVOLVE_OFFLINE:
1678: TSTrajectoryMemorySet_ROF(tj, ts, tjsch, stepnum, time, X);
1679: break;
1680: case REVOLVE_ONLINE:
1682: TSTrajectoryMemorySet_RON(tj, ts, tjsch, stepnum, time, X);
1683: break;
1684: case REVOLVE_MULTISTAGE:
1686: TSTrajectoryMemorySet_RMS(tj, ts, tjsch, stepnum, time, X);
1687: break;
1688: #endif
1689: #if defined(PETSC_HAVE_CAMS)
1690: case CAMS_OFFLINE:
1692: TSTrajectoryMemorySet_AOF(tj, ts, tjsch, stepnum, time, X);
1693: break;
1694: #endif
1695: default:
1696: break;
1697: }
1698: return 0;
1699: }
1701: static PetscErrorCode TSTrajectoryGet_Memory(TSTrajectory tj, TS ts, PetscInt stepnum, PetscReal *t)
1702: {
1703: TJScheduler *tjsch = (TJScheduler *)tj->data;
1705: if (tj->adjoint_solve_mode && stepnum == 0) {
1706: TSTrajectoryReset(tj); /* reset TSTrajectory so users do not need to reset TSTrajectory */
1707: return 0;
1708: }
1709: switch (tjsch->stype) {
1710: case NONE:
1711: if (tj->adjoint_solve_mode) {
1712: TSTrajectoryMemoryGet_N(ts, tjsch, stepnum);
1713: } else {
1714: TSTrajectoryMemoryGet_N_2(ts, tjsch, stepnum);
1715: }
1716: break;
1717: case TWO_LEVEL_NOREVOLVE:
1719: TSTrajectoryMemoryGet_TLNR(tj, ts, tjsch, stepnum);
1720: break;
1721: #if defined(PETSC_HAVE_REVOLVE)
1722: case TWO_LEVEL_REVOLVE:
1724: TSTrajectoryMemoryGet_TLR(tj, ts, tjsch, stepnum);
1725: break;
1726: case TWO_LEVEL_TWO_REVOLVE:
1728: TSTrajectoryMemoryGet_TLTR(tj, ts, tjsch, stepnum);
1729: break;
1730: case REVOLVE_OFFLINE:
1732: TSTrajectoryMemoryGet_ROF(tj, ts, tjsch, stepnum);
1733: break;
1734: case REVOLVE_ONLINE:
1736: TSTrajectoryMemoryGet_RON(tj, ts, tjsch, stepnum);
1737: break;
1738: case REVOLVE_MULTISTAGE:
1740: TSTrajectoryMemoryGet_RMS(tj, ts, tjsch, stepnum);
1741: break;
1742: #endif
1743: #if defined(PETSC_HAVE_CAMS)
1744: case CAMS_OFFLINE:
1746: TSTrajectoryMemoryGet_AOF(tj, ts, tjsch, stepnum);
1747: break;
1748: #endif
1749: default:
1750: break;
1751: }
1752: return 0;
1753: }
1755: PETSC_UNUSED static PetscErrorCode TSTrajectorySetStride_Memory(TSTrajectory tj, PetscInt stride)
1756: {
1757: TJScheduler *tjsch = (TJScheduler *)tj->data;
1759: tjsch->stride = stride;
1760: return 0;
1761: }
1763: static PetscErrorCode TSTrajectorySetMaxCpsRAM_Memory(TSTrajectory tj, PetscInt max_cps_ram)
1764: {
1765: TJScheduler *tjsch = (TJScheduler *)tj->data;
1767: tjsch->max_cps_ram = max_cps_ram;
1768: return 0;
1769: }
1771: static PetscErrorCode TSTrajectorySetMaxCpsDisk_Memory(TSTrajectory tj, PetscInt max_cps_disk)
1772: {
1773: TJScheduler *tjsch = (TJScheduler *)tj->data;
1775: tjsch->max_cps_disk = max_cps_disk;
1776: return 0;
1777: }
1779: static PetscErrorCode TSTrajectorySetMaxUnitsRAM_Memory(TSTrajectory tj, PetscInt max_units_ram)
1780: {
1781: TJScheduler *tjsch = (TJScheduler *)tj->data;
1784: tjsch->max_units_ram = max_units_ram;
1785: return 0;
1786: }
1788: static PetscErrorCode TSTrajectorySetMaxUnitsDisk_Memory(TSTrajectory tj, PetscInt max_units_disk)
1789: {
1790: TJScheduler *tjsch = (TJScheduler *)tj->data;
1793: tjsch->max_units_ram = max_units_disk;
1794: return 0;
1795: }
1797: static PetscErrorCode TSTrajectoryMemorySetType_Memory(TSTrajectory tj, TSTrajectoryMemoryType tj_memory_type)
1798: {
1799: TJScheduler *tjsch = (TJScheduler *)tj->data;
1802: tjsch->tj_memory_type = tj_memory_type;
1803: return 0;
1804: }
1806: #if defined(PETSC_HAVE_REVOLVE)
1807: PETSC_UNUSED static PetscErrorCode TSTrajectorySetRevolveOnline(TSTrajectory tj, PetscBool use_online)
1808: {
1809: TJScheduler *tjsch = (TJScheduler *)tj->data;
1811: tjsch->use_online = use_online;
1812: return 0;
1813: }
1814: #endif
1816: PETSC_UNUSED static PetscErrorCode TSTrajectorySetSaveStack(TSTrajectory tj, PetscBool save_stack)
1817: {
1818: TJScheduler *tjsch = (TJScheduler *)tj->data;
1820: tjsch->save_stack = save_stack;
1821: return 0;
1822: }
1824: PETSC_UNUSED static PetscErrorCode TSTrajectorySetUseDRAM(TSTrajectory tj, PetscBool use_dram)
1825: {
1826: TJScheduler *tjsch = (TJScheduler *)tj->data;
1828: tjsch->stack.use_dram = use_dram;
1829: return 0;
1830: }
1832: /*@C
1833: TSTrajectoryMemorySetType - sets the software that is used to generate the checkpointing schedule.
1835: Logically Collective
1837: Input Parameters:
1838: + tj - the `TSTrajectory` context
1839: - tj_memory_type - Revolve or CAMS
1841: Options Database Key:
1842: . -ts_trajectory_memory_type <tj_memory_type> - petsc, revolve, cams
1844: Level: intermediate
1846: .seealso: [](chapter_ts), `TSTrajectory`, `TSTrajectorySetMaxUnitsRAM()`, `TSTrajectoryMemoryType`
1847: @*/
1848: PetscErrorCode TSTrajectoryMemorySetType(TSTrajectory tj, TSTrajectoryMemoryType tj_memory_type)
1849: {
1850: PetscTryMethod(tj, "TSTrajectoryMemorySetType_C", (TSTrajectory, TSTrajectoryMemoryType), (tj, tj_memory_type));
1851: return 0;
1852: }
1854: /*@C
1855: TSTrajectorySetMaxCpsRAM - Set maximum number of checkpoints in RAM
1857: Logically collective
1859: Input Parameter:
1860: . tj - tstrajectory context
1862: Output Parameter:
1863: . max_cps_ram - maximum number of checkpoints in RAM
1865: Level: intermediate
1867: .seealso: [](chapter_ts), `TSTrajectory`, `TSTrajectorySetMaxUnitsRAM()`
1868: @*/
1869: PetscErrorCode TSTrajectorySetMaxCpsRAM(TSTrajectory tj, PetscInt max_cps_ram)
1870: {
1871: PetscUseMethod(tj, "TSTrajectorySetMaxCpsRAM_C", (TSTrajectory, PetscInt), (tj, max_cps_ram));
1872: return 0;
1873: }
1875: /*@C
1876: TSTrajectorySetMaxCpsDisk - Set maximum number of checkpoints on disk
1878: Logically collective
1880: Input Parameter:
1881: . tj - tstrajectory context
1883: Output Parameter:
1884: . max_cps_disk - maximum number of checkpoints on disk
1886: Level: intermediate
1888: .seealso: [](chapter_ts), `TSTrajectory`, `TSTrajectorySetMaxUnitsDisk()`, `TSTrajectorySetMaxUnitsRAM()`
1889: @*/
1890: PetscErrorCode TSTrajectorySetMaxCpsDisk(TSTrajectory tj, PetscInt max_cps_disk)
1891: {
1892: PetscUseMethod(tj, "TSTrajectorySetMaxCpsDisk_C", (TSTrajectory, PetscInt), (tj, max_cps_disk));
1893: return 0;
1894: }
1896: /*@C
1897: TSTrajectorySetMaxUnitsRAM - Set maximum number of checkpointing units in RAM
1899: Logically collective
1901: Input Parameter:
1902: . tj - tstrajectory context
1904: Output Parameter:
1905: . max_units_ram - maximum number of checkpointing units in RAM
1907: Level: intermediate
1909: .seealso: [](chapter_ts), `TSTrajectory`, `TSTrajectorySetMaxCpsRAM()`
1910: @*/
1911: PetscErrorCode TSTrajectorySetMaxUnitsRAM(TSTrajectory tj, PetscInt max_units_ram)
1912: {
1913: PetscUseMethod(tj, "TSTrajectorySetMaxUnitsRAM_C", (TSTrajectory, PetscInt), (tj, max_units_ram));
1914: return 0;
1915: }
1917: /*@C
1918: TSTrajectorySetMaxUnitsDisk - Set maximum number of checkpointing units on disk
1920: Logically collective
1922: Input Parameter:
1923: . tj - tstrajectory context
1925: Output Parameter:
1926: . max_units_disk - maximum number of checkpointing units on disk
1928: Level: intermediate
1930: .seealso: [](chapter_ts), `TSTrajectory`, `TSTrajectorySetMaxCpsDisk()`
1931: @*/
1932: PetscErrorCode TSTrajectorySetMaxUnitsDisk(TSTrajectory tj, PetscInt max_units_disk)
1933: {
1934: PetscUseMethod(tj, "TSTrajectorySetMaxUnitsDisk_C", (TSTrajectory, PetscInt), (tj, max_units_disk));
1935: return 0;
1936: }
1938: static PetscErrorCode TSTrajectorySetFromOptions_Memory(TSTrajectory tj, PetscOptionItems *PetscOptionsObject)
1939: {
1940: TJScheduler *tjsch = (TJScheduler *)tj->data;
1941: PetscEnum etmp;
1942: PetscInt max_cps_ram, max_cps_disk, max_units_ram, max_units_disk;
1943: PetscBool flg;
1945: PetscOptionsHeadBegin(PetscOptionsObject, "Memory based TS trajectory options");
1946: {
1947: PetscOptionsInt("-ts_trajectory_max_cps_ram", "Maximum number of checkpoints in RAM", "TSTrajectorySetMaxCpsRAM", tjsch->max_cps_ram, &max_cps_ram, &flg);
1948: if (flg) TSTrajectorySetMaxCpsRAM(tj, max_cps_ram);
1949: PetscOptionsInt("-ts_trajectory_max_cps_disk", "Maximum number of checkpoints on disk", "TSTrajectorySetMaxCpsDisk", tjsch->max_cps_disk, &max_cps_disk, &flg);
1950: if (flg) TSTrajectorySetMaxCpsDisk(tj, max_cps_disk);
1951: PetscOptionsInt("-ts_trajectory_max_units_ram", "Maximum number of checkpointing units in RAM", "TSTrajectorySetMaxUnitsRAM", tjsch->max_units_ram, &max_units_ram, &flg);
1952: if (flg) TSTrajectorySetMaxUnitsRAM(tj, max_units_ram);
1953: PetscOptionsInt("-ts_trajectory_max_units_disk", "Maximum number of checkpointing units on disk", "TSTrajectorySetMaxUnitsDisk", tjsch->max_units_disk, &max_units_disk, &flg);
1954: if (flg) TSTrajectorySetMaxUnitsDisk(tj, max_units_disk);
1955: PetscOptionsInt("-ts_trajectory_stride", "Stride to save checkpoints to file", "TSTrajectorySetStride", tjsch->stride, &tjsch->stride, NULL);
1956: #if defined(PETSC_HAVE_REVOLVE)
1957: PetscOptionsBool("-ts_trajectory_revolve_online", "Trick TS trajectory into using online mode of revolve", "TSTrajectorySetRevolveOnline", tjsch->use_online, &tjsch->use_online, NULL);
1958: #endif
1959: PetscOptionsBool("-ts_trajectory_save_stack", "Save all stack to disk", "TSTrajectorySetSaveStack", tjsch->save_stack, &tjsch->save_stack, NULL);
1960: PetscOptionsBool("-ts_trajectory_use_dram", "Use DRAM for checkpointing", "TSTrajectorySetUseDRAM", tjsch->stack.use_dram, &tjsch->stack.use_dram, NULL);
1961: PetscOptionsEnum("-ts_trajectory_memory_type", "Checkpointing scchedule software to use", "TSTrajectoryMemorySetType", TSTrajectoryMemoryTypes, (PetscEnum)(int)(tjsch->tj_memory_type), &etmp, &flg);
1962: if (flg) TSTrajectoryMemorySetType(tj, (TSTrajectoryMemoryType)etmp);
1963: }
1964: PetscOptionsHeadEnd();
1965: return 0;
1966: }
1968: static PetscErrorCode TSTrajectorySetUp_Memory(TSTrajectory tj, TS ts)
1969: {
1970: TJScheduler *tjsch = (TJScheduler *)tj->data;
1971: Stack *stack = &tjsch->stack;
1972: #if defined(PETSC_HAVE_REVOLVE)
1973: RevolveCTX *rctx, *rctx2;
1974: DiskStack *diskstack = &tjsch->diskstack;
1975: PetscInt diskblocks;
1976: #endif
1977: PetscInt numY, total_steps;
1978: PetscBool fixedtimestep;
1980: if (ts->adapt) {
1981: PetscObjectTypeCompare((PetscObject)ts->adapt, TSADAPTNONE, &fixedtimestep);
1982: } else {
1983: fixedtimestep = PETSC_TRUE;
1984: }
1985: total_steps = (PetscInt)(PetscCeilReal((ts->max_time - ts->ptime) / ts->time_step));
1986: total_steps = total_steps < 0 ? PETSC_MAX_INT : total_steps;
1987: if (fixedtimestep) tjsch->total_steps = PetscMin(ts->max_steps, total_steps);
1989: tjsch->stack.solution_only = tj->solution_only;
1990: TSGetStages(ts, &numY, PETSC_IGNORE);
1991: if (stack->solution_only) {
1992: if (tjsch->max_units_ram) tjsch->max_cps_ram = tjsch->max_units_ram;
1993: else tjsch->max_units_ram = tjsch->max_cps_ram;
1994: if (tjsch->max_units_disk) tjsch->max_cps_disk = tjsch->max_units_disk;
1995: } else {
1996: if (tjsch->max_units_ram) tjsch->max_cps_ram = (ts->stifflyaccurate) ? tjsch->max_units_ram / numY : tjsch->max_units_ram / (numY + 1);
1997: else tjsch->max_units_ram = (ts->stifflyaccurate) ? numY * tjsch->max_cps_ram : (numY + 1) * tjsch->max_cps_ram;
1998: if (tjsch->max_units_disk) tjsch->max_cps_disk = (ts->stifflyaccurate) ? tjsch->max_units_disk / numY : tjsch->max_units_disk / (numY + 1);
1999: else tjsch->max_units_disk = (ts->stifflyaccurate) ? numY * tjsch->max_cps_disk : (numY + 1) * tjsch->max_cps_disk;
2000: }
2001: if (tjsch->max_cps_ram > 0) stack->stacksize = tjsch->max_units_ram; /* maximum stack size. Could be overallocated. */
2003: /* Determine the scheduler type */
2004: if (tjsch->stride > 1) { /* two level mode */
2006: if (tjsch->max_cps_disk <= 1 && tjsch->max_cps_ram > 1 && tjsch->max_cps_ram <= tjsch->stride - 1) tjsch->stype = TWO_LEVEL_REVOLVE; /* use revolve_offline for each stride */
2007: if (tjsch->max_cps_disk > 1 && tjsch->max_cps_ram > 1 && tjsch->max_cps_ram <= tjsch->stride - 1) tjsch->stype = TWO_LEVEL_TWO_REVOLVE; /* use revolve_offline for each stride */
2008: if (tjsch->max_cps_disk <= 1 && (tjsch->max_cps_ram >= tjsch->stride || tjsch->max_cps_ram == -1)) tjsch->stype = TWO_LEVEL_NOREVOLVE; /* can also be handled by TWO_LEVEL_REVOLVE */
2009: } else { /* single level mode */
2010: if (fixedtimestep) {
2011: if (tjsch->max_cps_ram >= tjsch->total_steps - 1 || tjsch->max_cps_ram == -1) tjsch->stype = NONE; /* checkpoint all */
2012: else { /* choose the schedule software for offline checkpointing */ switch (tjsch->tj_memory_type) {
2013: case TJ_PETSC:
2014: tjsch->stype = NONE;
2015: break;
2016: case TJ_CAMS:
2017: tjsch->stype = CAMS_OFFLINE;
2018: break;
2019: case TJ_REVOLVE:
2020: tjsch->stype = (tjsch->max_cps_disk > 1) ? REVOLVE_MULTISTAGE : REVOLVE_OFFLINE;
2021: break;
2022: default:
2023: break;
2024: }
2025: }
2026: } else tjsch->stype = NONE; /* checkpoint all for adaptive time step */
2027: #if defined(PETSC_HAVE_REVOLVE)
2028: if (tjsch->use_online) tjsch->stype = REVOLVE_ONLINE; /* trick into online (for testing purpose only) */
2029: #endif
2031: }
2032: if (tjsch->stype >= CAMS_OFFLINE) {
2033: #ifndef PETSC_HAVE_CAMS
2034: SETERRQ(PetscObjectComm((PetscObject)ts), PETSC_ERR_SUP, "CAMS is needed when there is not enough memory to checkpoint all time steps according to the user's settings, please reconfigure with the additional option --download-cams.");
2035: #else
2036: CAMSCTX *actx;
2037: PetscInt ns = 0;
2038: if (stack->solution_only) {
2039: offline_ca_create(tjsch->total_steps, tjsch->max_cps_ram);
2040: } else {
2041: TSGetStages(ts, &ns, PETSC_IGNORE);
2042: offline_cams_create(tjsch->total_steps, tjsch->max_units_ram, ns, ts->stifflyaccurate);
2043: }
2044: PetscNew(&actx);
2045: actx->lastcheckpointstep = -1; /* -1 can trigger the initialization of CAMS */
2046: actx->lastcheckpointtype = -1; /* -1 can trigger the initialization of CAMS */
2047: actx->endstep = tjsch->total_steps;
2048: actx->num_units_avail = tjsch->max_units_ram;
2049: actx->num_stages = ns;
2050: tjsch->actx = actx;
2051: #endif
2052: } else if (tjsch->stype > TWO_LEVEL_NOREVOLVE) {
2053: #ifndef PETSC_HAVE_REVOLVE
2054: SETERRQ(PetscObjectComm((PetscObject)ts), PETSC_ERR_SUP, "revolve is needed when there is not enough memory to checkpoint all time steps according to the user's settings, please reconfigure with the additional option --download-revolve.");
2055: #else
2056: PetscRevolveInt rfine, rsnaps, rsnaps2;
2058: switch (tjsch->stype) {
2059: case TWO_LEVEL_REVOLVE:
2060: PetscRevolveIntCast(tjsch->stride, &rfine);
2061: PetscRevolveIntCast(tjsch->max_cps_ram, &rsnaps);
2062: revolve_create_offline(rfine, rsnaps);
2063: break;
2064: case TWO_LEVEL_TWO_REVOLVE:
2065: diskblocks = tjsch->save_stack ? tjsch->max_cps_disk / (tjsch->max_cps_ram + 1) : tjsch->max_cps_disk; /* The block size depends on whether the stack is saved. */
2066: diskstack->stacksize = diskblocks;
2067: PetscRevolveIntCast(tjsch->stride, &rfine);
2068: PetscRevolveIntCast(tjsch->max_cps_ram, &rsnaps);
2069: revolve_create_offline(rfine, rsnaps);
2070: PetscRevolveIntCast((tjsch->total_steps + tjsch->stride - 1) / tjsch->stride, &rfine);
2071: PetscRevolveIntCast(diskblocks, &rsnaps);
2072: revolve2_create_offline(rfine, rsnaps);
2073: PetscNew(&rctx2);
2074: rctx2->snaps_in = rsnaps;
2075: rctx2->reverseonestep = PETSC_FALSE;
2076: rctx2->check = 0;
2077: rctx2->oldcapo = 0;
2078: rctx2->capo = 0;
2079: rctx2->info = 2;
2080: rctx2->fine = rfine;
2081: tjsch->rctx2 = rctx2;
2082: diskstack->top = -1;
2083: PetscMalloc1(diskstack->stacksize, &diskstack->container);
2084: break;
2085: case REVOLVE_OFFLINE:
2086: PetscRevolveIntCast(tjsch->total_steps, &rfine);
2087: PetscRevolveIntCast(tjsch->max_cps_ram, &rsnaps);
2088: revolve_create_offline(rfine, rsnaps);
2089: break;
2090: case REVOLVE_ONLINE:
2091: stack->stacksize = tjsch->max_cps_ram;
2092: PetscRevolveIntCast(tjsch->max_cps_ram, &rsnaps);
2093: revolve_create_online(rsnaps);
2094: break;
2095: case REVOLVE_MULTISTAGE:
2096: PetscRevolveIntCast(tjsch->total_steps, &rfine);
2097: PetscRevolveIntCast(tjsch->max_cps_ram, &rsnaps);
2098: PetscRevolveIntCast(tjsch->max_cps_ram + tjsch->max_cps_disk, &rsnaps2);
2099: revolve_create_multistage(rfine, rsnaps2, rsnaps);
2100: break;
2101: default:
2102: break;
2103: }
2104: PetscNew(&rctx);
2105: PetscRevolveIntCast(tjsch->max_cps_ram, &rsnaps);
2106: rctx->snaps_in = rsnaps; /* for theta methods snaps_in=2*max_cps_ram */
2107: rctx->reverseonestep = PETSC_FALSE;
2108: rctx->check = 0;
2109: rctx->oldcapo = 0;
2110: rctx->capo = 0;
2111: rctx->info = 2;
2112: if (tjsch->stride > 1) {
2113: PetscRevolveIntCast(tjsch->stride, &rfine);
2114: } else {
2115: PetscRevolveIntCast(tjsch->total_steps, &rfine);
2116: }
2117: rctx->fine = rfine;
2118: tjsch->rctx = rctx;
2119: if (tjsch->stype == REVOLVE_ONLINE) rctx->fine = -1;
2120: #endif
2121: } else {
2122: if (tjsch->stype == TWO_LEVEL_NOREVOLVE) stack->stacksize = tjsch->stride - 1; /* need tjsch->stride-1 at most */
2123: if (tjsch->stype == NONE) {
2124: if (fixedtimestep) stack->stacksize = stack->solution_only ? tjsch->total_steps : tjsch->total_steps - 1;
2125: else { /* adaptive time step */ /* if max_cps_ram is not specified, use maximal allowed number of steps for stack size */
2126: if (tjsch->max_cps_ram == -1) stack->stacksize = ts->max_steps < PETSC_MAX_INT ? ts->max_steps : 10000;
2127: tjsch->total_steps = stack->solution_only ? stack->stacksize : stack->stacksize + 1; /* will be updated as time integration advances */
2128: }
2129: }
2130: }
2132: if ((tjsch->stype >= TWO_LEVEL_NOREVOLVE && tjsch->stype < REVOLVE_OFFLINE) || tjsch->stype == REVOLVE_MULTISTAGE) { /* these types need to use disk */
2133: TSTrajectorySetUp_Basic(tj, ts);
2134: }
2136: stack->stacksize = PetscMax(stack->stacksize, 1);
2137: tjsch->recompute = PETSC_FALSE;
2138: StackInit(stack, stack->stacksize, numY);
2139: return 0;
2140: }
2142: static PetscErrorCode TSTrajectoryReset_Memory(TSTrajectory tj)
2143: {
2144: #if defined(PETSC_HAVE_REVOLVE) || defined(PETSC_HAVE_CAMS)
2145: TJScheduler *tjsch = (TJScheduler *)tj->data;
2146: #endif
2148: #if defined(PETSC_HAVE_REVOLVE)
2149: if (tjsch->stype > TWO_LEVEL_NOREVOLVE) {
2150: revolve_reset();
2151: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) {
2152: revolve2_reset();
2153: PetscFree(tjsch->diskstack.container);
2154: }
2155: }
2156: if (tjsch->stype > TWO_LEVEL_NOREVOLVE) {
2157: PetscFree(tjsch->rctx);
2158: PetscFree(tjsch->rctx2);
2159: }
2160: #endif
2161: #if defined(PETSC_HAVE_CAMS)
2162: if (tjsch->stype == CAMS_OFFLINE) {
2163: if (tjsch->stack.solution_only) offline_ca_destroy();
2164: else offline_ca_destroy();
2165: PetscFree(tjsch->actx);
2166: }
2167: #endif
2168: return 0;
2169: }
2171: static PetscErrorCode TSTrajectoryDestroy_Memory(TSTrajectory tj)
2172: {
2173: TJScheduler *tjsch = (TJScheduler *)tj->data;
2175: StackDestroy(&tjsch->stack);
2176: PetscViewerDestroy(&tjsch->viewer);
2177: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectorySetMaxCpsRAM_C", NULL);
2178: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectorySetMaxCpsDisk_C", NULL);
2179: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectorySetMaxUnitsRAM_C", NULL);
2180: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectorySetMaxUnitsDisk_C", NULL);
2181: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectoryMemorySetType_C", NULL);
2182: PetscFree(tjsch);
2183: return 0;
2184: }
2186: /*MC
2187: TSTRAJECTORYMEMORY - Stores each solution of the ODE/ADE in memory
2189: Level: intermediate
2191: .seealso: [](chapter_ts), `TSTrajectoryCreate()`, `TS`, `TSTrajectorySetType()`, `TSTrajectoryType`, `TSTrajectory`
2192: M*/
2193: PETSC_EXTERN PetscErrorCode TSTrajectoryCreate_Memory(TSTrajectory tj, TS ts)
2194: {
2195: TJScheduler *tjsch;
2197: tj->ops->set = TSTrajectorySet_Memory;
2198: tj->ops->get = TSTrajectoryGet_Memory;
2199: tj->ops->setup = TSTrajectorySetUp_Memory;
2200: tj->ops->setfromoptions = TSTrajectorySetFromOptions_Memory;
2201: tj->ops->reset = TSTrajectoryReset_Memory;
2202: tj->ops->destroy = TSTrajectoryDestroy_Memory;
2204: PetscNew(&tjsch);
2205: tjsch->stype = NONE;
2206: tjsch->max_cps_ram = -1; /* -1 indicates that it is not set */
2207: tjsch->max_cps_disk = -1; /* -1 indicates that it is not set */
2208: tjsch->stride = 0; /* if not zero, two-level checkpointing will be used */
2209: #if defined(PETSC_HAVE_REVOLVE)
2210: tjsch->use_online = PETSC_FALSE;
2211: #endif
2212: tjsch->save_stack = PETSC_TRUE;
2214: tjsch->stack.solution_only = tj->solution_only;
2215: PetscViewerCreate(PetscObjectComm((PetscObject)tj), &tjsch->viewer);
2216: PetscViewerSetType(tjsch->viewer, PETSCVIEWERBINARY);
2217: PetscViewerPushFormat(tjsch->viewer, PETSC_VIEWER_NATIVE);
2218: PetscViewerFileSetMode(tjsch->viewer, FILE_MODE_WRITE);
2220: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectorySetMaxCpsRAM_C", TSTrajectorySetMaxCpsRAM_Memory);
2221: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectorySetMaxCpsDisk_C", TSTrajectorySetMaxCpsDisk_Memory);
2222: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectorySetMaxUnitsRAM_C", TSTrajectorySetMaxUnitsRAM_Memory);
2223: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectorySetMaxUnitsDisk_C", TSTrajectorySetMaxUnitsDisk_Memory);
2224: PetscObjectComposeFunction((PetscObject)tj, "TSTrajectoryMemorySetType_C", TSTrajectoryMemorySetType_Memory);
2225: tj->data = tjsch;
2226: return 0;
2227: }