Actual source code: trajmemory.c
petsc-3.7.3 2016-08-01
1: #include <petsc/private/tsimpl.h> /*I "petscts.h" I*/
2: #include <petscsys.h>
3: #ifdef PETSC_HAVE_REVOLVE
4: #include <revolve_c.h>
5: #endif
7: PetscLogEvent TSTrajectory_DiskWrite, TSTrajectory_DiskRead;
9: typedef enum {NONE,TWO_LEVEL_NOREVOLVE,TWO_LEVEL_REVOLVE,TWO_LEVEL_TWO_REVOLVE,REVOLVE_OFFLINE,REVOLVE_ONLINE,REVOLVE_MULTISTAGE} SchedulerType;
11: typedef struct _StackElement {
12: PetscInt stepnum;
13: Vec X;
14: Vec *Y;
15: PetscReal time;
16: PetscReal timeprev; /* for no solution_only mode */
17: PetscReal timenext; /* for solution_only mode */
18: } *StackElement;
20: #ifdef PETSC_HAVE_REVOLVE
21: typedef struct _RevolveCTX {
22: PetscBool reverseonestep;
23: PetscInt where;
24: PetscInt snaps_in;
25: PetscInt stepsleft;
26: PetscInt check;
27: PetscInt oldcapo;
28: PetscInt capo;
29: PetscInt fine;
30: PetscInt info;
31: } RevolveCTX;
32: #endif
34: typedef struct _Stack {
35: PetscInt stacksize;
36: PetscInt top;
37: StackElement *container;
38: PetscInt numY;
39: PetscBool solution_only;
40: } Stack;
42: typedef struct _DiskStack {
43: PetscInt stacksize;
44: PetscInt top;
45: PetscInt *container;
46: } DiskStack;
48: typedef struct _TJScheduler {
49: SchedulerType stype;
50: #ifdef PETSC_HAVE_REVOLVE
51: RevolveCTX *rctx,*rctx2;
52: PetscBool use_online;
53: PetscInt store_stride;
54: #endif
55: PetscBool recompute;
56: PetscBool skip_trajectory;
57: PetscBool save_stack;
58: MPI_Comm comm;
59: PetscInt max_cps_ram; /* maximum checkpoints in RAM */
60: PetscInt max_cps_disk; /* maximum checkpoints on disk */
61: PetscInt stride;
62: PetscInt total_steps; /* total number of steps */
63: Stack stack;
64: DiskStack diskstack;
65: } TJScheduler;
69: static PetscErrorCode TurnForwardWithStepsize(TS ts,PetscReal nextstepsize)
70: {
71: PetscReal stepsize;
75: /* reverse the direction */
76: TSGetTimeStep(ts,&stepsize);
77: stepsize = nextstepsize;
78: TSSetTimeStep(ts,stepsize);
79: return(0);
80: }
84: static PetscErrorCode TurnForward(TS ts)
85: {
86: PetscReal stepsize;
90: /* reverse the direction */
91: TSGetTimeStep(ts,&stepsize);
92: TSSetTimeStep(ts,-stepsize);
93: return(0);
94: }
98: static PetscErrorCode TurnBackward(TS ts)
99: {
100: PetscReal stepsize;
104: /* reverse the direction */
105: stepsize = ts->ptime_prev-ts->ptime;
106: TSSetTimeStep(ts,stepsize);
107: return(0);
108: }
112: static PetscErrorCode StackCreate(Stack *stack,PetscInt size,PetscInt ny)
113: {
117: stack->top = -1;
118: stack->numY = ny;
120: PetscMalloc1(size*sizeof(StackElement),&stack->container);
121: return(0);
122: }
126: static PetscErrorCode StackDestroy(Stack *stack)
127: {
128: PetscInt i;
132: if (stack->top > -1) {
133: for (i=0;i<=stack->top;i++) {
134: VecDestroy(&stack->container[i]->X);
135: if (!stack->solution_only) {
136: VecDestroyVecs(stack->numY,&stack->container[i]->Y);
137: }
138: PetscFree(stack->container[i]);
139: }
140: }
141: PetscFree(stack->container);
142: return(0);
143: }
147: static PetscErrorCode StackResize(Stack *stack,PetscInt newsize)
148: {
149: StackElement *newcontainer;
150: PetscInt i;
154: PetscMalloc1(newsize*sizeof(StackElement),&newcontainer);
155: for (i=0;i<stack->stacksize;i++) {
156: newcontainer[i] = stack->container[i];
157: }
158: PetscFree(stack->container);
159: stack->container = newcontainer;
160: stack->stacksize = newsize;
161: return(0);
162: }
166: static PetscErrorCode StackPush(Stack *stack,StackElement e)
167: {
169: if (stack->top+1 >= stack->stacksize) SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_MEMC,"Maximum stack size (%D) exceeded",stack->stacksize);
170: stack->container[++stack->top] = e;
171: return(0);
172: }
176: static PetscErrorCode StackPop(Stack *stack,StackElement *e)
177: {
179: if (stack->top == -1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_MEMC,"Empty stack");
180: *e = stack->container[stack->top--];
181: return(0);
182: }
186: static PetscErrorCode StackTop(Stack *stack,StackElement *e)
187: {
189: *e = stack->container[stack->top];
190: return(0);
191: }
193: #ifdef PETSC_HAVE_REVOLVE
196: static PetscErrorCode StackFind(Stack *stack,StackElement *e,PetscInt index)
197: {
199: *e = stack->container[index];
200: return(0);
201: }
202: #endif
206: static PetscErrorCode OutputBIN(const char *filename,PetscViewer *viewer)
207: {
211: PetscViewerCreate(PETSC_COMM_WORLD,viewer);
212: PetscViewerSetType(*viewer,PETSCVIEWERBINARY);
213: PetscViewerFileSetMode(*viewer,FILE_MODE_WRITE);
214: PetscViewerFileSetName(*viewer,filename);
215: return(0);
216: }
220: static PetscErrorCode WriteToDisk(PetscInt stepnum,PetscReal time,PetscReal timeprev,Vec X,Vec *Y,PetscInt numY,PetscBool solution_only,PetscViewer viewer)
221: {
222: PetscInt i;
226: PetscViewerBinaryWrite(viewer,&stepnum,1,PETSC_INT,PETSC_FALSE);
227: VecView(X,viewer);
228: for (i=0;!solution_only && i<numY;i++) {
229: VecView(Y[i],viewer);
230: }
231: PetscViewerBinaryWrite(viewer,&time,1,PETSC_REAL,PETSC_FALSE);
232: PetscViewerBinaryWrite(viewer,&timeprev,1,PETSC_REAL,PETSC_FALSE);
233: return(0);
234: }
238: static PetscErrorCode ReadFromDisk(PetscInt *stepnum,PetscReal *time,PetscReal *timeprev,Vec X,Vec *Y,PetscInt numY,PetscBool solution_only,PetscViewer viewer)
239: {
240: PetscInt i;
244: PetscViewerBinaryRead(viewer,stepnum,1,NULL,PETSC_INT);
245: VecLoad(X,viewer);
246: for (i=0;!solution_only && i<numY;i++) {
247: VecLoad(Y[i],viewer);
248: }
249: PetscViewerBinaryRead(viewer,time,1,NULL,PETSC_REAL);
250: PetscViewerBinaryRead(viewer,timeprev,1,NULL,PETSC_REAL);
251: return(0);
252: }
256: static PetscErrorCode StackDumpAll(TS ts,Stack *stack,PetscInt id)
257: {
258: Vec *Y;
259: PetscInt i;
260: StackElement e;
261: PetscViewer viewer;
262: char filename[PETSC_MAX_PATH_LEN];
266: if (id == 1) {
267: PetscMPIInt rank;
268: MPI_Comm_rank(PetscObjectComm((PetscObject)ts),&rank);
269: if (!rank) {
270: PetscRMTree("SA-data");
271: PetscMkdir("SA-data");
272: }
273: }
274: PetscSNPrintf(filename,sizeof(filename),"SA-data/SA-STACK%06d.bin",id);
275: OutputBIN(filename,&viewer);
276: for (i=0;i<stack->stacksize;i++) {
277: e = stack->container[i];
278: PetscLogEventBegin(TSTrajectory_DiskWrite,ts,0,0,0);
279: WriteToDisk(e->stepnum,e->time,e->timeprev,e->X,e->Y,stack->numY,stack->solution_only,viewer);
280: PetscLogEventEnd(TSTrajectory_DiskWrite,ts,0,0,0);
281: ts->trajectory->diskwrites++;
282: }
283: /* save the last step for restart, the last step is in memory when using single level schemes, but not necessarily the case for multi level schemes */
284: TSGetStages(ts,&stack->numY,&Y);
285: PetscLogEventBegin(TSTrajectory_DiskWrite,ts,0,0,0);
286: WriteToDisk(ts->total_steps,ts->ptime,ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,viewer);
287: PetscLogEventEnd(TSTrajectory_DiskWrite,ts,0,0,0);
288: ts->trajectory->diskwrites++;
289: for (i=0;i<stack->stacksize;i++) {
290: StackPop(stack,&e);
291: VecDestroy(&e->X);
292: if (!stack->solution_only) {
293: VecDestroyVecs(stack->numY,&e->Y);
294: }
295: PetscFree(e);
296: }
297: PetscViewerDestroy(&viewer);
298: return(0);
299: }
303: static PetscErrorCode StackLoadAll(TS ts,Stack *stack,PetscInt id)
304: {
305: Vec *Y;
306: PetscInt i;
307: StackElement e;
308: PetscViewer viewer;
309: char filename[PETSC_MAX_PATH_LEN];
313: PetscPrintf(PETSC_COMM_WORLD,"\x1B[33mLoad stack from file\033[0m\n");
314: PetscSNPrintf(filename,sizeof filename,"SA-data/SA-STACK%06d.bin",id);
315: PetscViewerBinaryOpen(PETSC_COMM_WORLD,filename,FILE_MODE_READ,&viewer);
316: for (i=0;i<stack->stacksize;i++) {
317: PetscCalloc1(1,&e);
318: TSGetStages(ts,&stack->numY,&Y);
319: VecDuplicate(Y[0],&e->X);
320: if (!stack->solution_only && stack->numY>0) {
321: VecDuplicateVecs(Y[0],stack->numY,&e->Y);
322: }
323: StackPush(stack,e);
324: }
325: for (i=0;i<stack->stacksize;i++) {
326: e = stack->container[i];
327: PetscLogEventBegin(TSTrajectory_DiskRead,ts,0,0,0);
328: ReadFromDisk(&e->stepnum,&e->time,&e->timeprev,e->X,e->Y,stack->numY,stack->solution_only,viewer);
329: PetscLogEventEnd(TSTrajectory_DiskRead,ts,0,0,0);
330: ts->trajectory->diskreads++;
331: }
332: /* load the last step into TS */
333: TSGetStages(ts,&stack->numY,&Y);
334: PetscLogEventBegin(TSTrajectory_DiskRead,ts,0,0,0);
335: ReadFromDisk(&ts->total_steps,&ts->ptime,&ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,viewer);
336: PetscLogEventEnd(TSTrajectory_DiskRead,ts,0,0,0);
337: ts->trajectory->diskreads++;
338: TurnBackward(ts);
339: PetscViewerDestroy(&viewer);
340: return(0);
341: }
343: #ifdef PETSC_HAVE_REVOLVE
346: static PetscErrorCode StackLoadLast(TS ts,Stack *stack,PetscInt id)
347: {
348: Vec *Y;
349: PetscInt size;
350: PetscViewer viewer;
351: char filename[PETSC_MAX_PATH_LEN];
352: #if defined(PETSC_HAVE_MPIIO)
353: PetscBool usempiio;
354: #endif
355: int fd;
356: off_t off,offset;
360: PetscPrintf(PETSC_COMM_WORLD,"\x1B[33mLoad last stack element from file\033[0m\n");
361: TSGetStages(ts,&stack->numY,&Y);
362: VecGetSize(Y[0],&size);
363: /* VecView writes to file two extra int's for class id and number of rows */
364: off = -((stack->solution_only?0:stack->numY)+1)*(size*PETSC_BINARY_SCALAR_SIZE+2*PETSC_BINARY_INT_SIZE)-PETSC_BINARY_INT_SIZE-2*PETSC_BINARY_SCALAR_SIZE;
366: PetscSNPrintf(filename,sizeof filename,"SA-data/SA-STACK%06d.bin",id);
367: PetscViewerBinaryOpen(PETSC_COMM_WORLD,filename,FILE_MODE_READ,&viewer);
368: #if defined(PETSC_HAVE_MPIIO)
369: PetscViewerBinaryGetUseMPIIO(viewer,&usempiio);
370: if (usempiio) {
371: PetscViewerBinaryGetMPIIODescriptor(viewer,(MPI_File*)&fd);
372: PetscBinarySynchronizedSeek(PETSC_COMM_WORLD,fd,off,PETSC_BINARY_SEEK_END,&offset);
373: } else {
374: #endif
375: PetscViewerBinaryGetDescriptor(viewer,&fd);
376: PetscBinarySeek(fd,off,PETSC_BINARY_SEEK_END,&offset);
377: #if defined(PETSC_HAVE_MPIIO)
378: }
379: #endif
380: /* load the last step into TS */
381: PetscLogEventBegin(TSTrajectory_DiskRead,ts,0,0,0);
382: ReadFromDisk(&ts->total_steps,&ts->ptime,&ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,viewer);
383: PetscLogEventEnd(TSTrajectory_DiskRead,ts,0,0,0);
384: ts->trajectory->diskreads++;
385: TurnBackward(ts);
386: PetscViewerDestroy(&viewer);
387: return(0);
388: }
389: #endif
393: static PetscErrorCode DumpSingle(TS ts,Stack *stack,PetscInt id)
394: {
395: Vec *Y;
396: PetscInt stepnum;
397: PetscViewer viewer;
398: char filename[PETSC_MAX_PATH_LEN];
402: TSGetTotalSteps(ts,&stepnum);
403: if (id == 1) {
404: PetscMPIInt rank;
405: MPI_Comm_rank(PetscObjectComm((PetscObject)ts),&rank);
406: if (!rank) {
407: PetscRMTree("SA-data");
408: PetscMkdir("SA-data");
409: }
410: }
411: PetscSNPrintf(filename,sizeof(filename),"SA-data/SA-CPS%06d.bin",id);
412: OutputBIN(filename,&viewer);
414: TSGetStages(ts,&stack->numY,&Y);
415: PetscLogEventBegin(TSTrajectory_DiskWrite,ts,0,0,0);
416: WriteToDisk(stepnum,ts->ptime,ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,viewer);
417: PetscLogEventEnd(TSTrajectory_DiskWrite,ts,0,0,0);
418: ts->trajectory->diskwrites++;
420: PetscViewerDestroy(&viewer);
421: return(0);
422: }
426: static PetscErrorCode LoadSingle(TS ts,Stack *stack,PetscInt id)
427: {
428: Vec *Y;
429: PetscViewer viewer;
430: char filename[PETSC_MAX_PATH_LEN];
434: PetscPrintf(PETSC_COMM_WORLD,"\x1B[33mLoad a single point from file\033[0m\n");
435: PetscSNPrintf(filename,sizeof filename,"SA-data/SA-CPS%06d.bin",id);
436: PetscViewerBinaryOpen(PETSC_COMM_WORLD,filename,FILE_MODE_READ,&viewer);
438: TSGetStages(ts,&stack->numY,&Y);
439: PetscLogEventBegin(TSTrajectory_DiskRead,ts,0,0,0);
440: ReadFromDisk(&ts->total_steps,&ts->ptime,&ts->ptime_prev,ts->vec_sol,Y,stack->numY,stack->solution_only,viewer);
441: PetscLogEventEnd(TSTrajectory_DiskRead,ts,0,0,0);
442: ts->trajectory->diskreads++;
444: PetscViewerDestroy(&viewer);
445: return(0);
446: }
450: static PetscErrorCode ElementCreate(TS ts,Stack *stack,StackElement *e,PetscInt stepnum,PetscReal time,Vec X)
451: {
452: Vec *Y;
453: PetscInt i;
454: PetscReal timeprev;
458: PetscCalloc1(1,e);
459: VecDuplicate(X,&(*e)->X);
460: VecCopy(X,(*e)->X);
461: if (stack->numY > 0 && !stack->solution_only) {
462: TSGetStages(ts,&stack->numY,&Y);
463: VecDuplicateVecs(Y[0],stack->numY,&(*e)->Y);
464: for (i=0;i<stack->numY;i++) {
465: VecCopy(Y[i],(*e)->Y[i]);
466: }
467: }
468: (*e)->stepnum = stepnum;
469: (*e)->time = time;
470: /* for consistency */
471: if (stepnum == 0) {
472: (*e)->timeprev = (*e)->time - ts->time_step;
473: } else {
474: TSGetPrevTime(ts,&timeprev);
475: (*e)->timeprev = timeprev;
476: }
477: return(0);
478: }
482: static PetscErrorCode ElementDestroy(Stack *stack,StackElement e)
483: {
487: VecDestroy(&e->X);
488: if (!stack->solution_only) {
489: VecDestroyVecs(stack->numY,&e->Y);
490: }
491: PetscFree(e);
492: return(0);
493: }
497: static PetscErrorCode UpdateTS(TS ts,Stack *stack,StackElement e)
498: {
499: Vec *Y;
500: PetscInt i;
504: VecCopy(e->X,ts->vec_sol);
505: if (!stack->solution_only) {
506: TSGetStages(ts,&stack->numY,&Y);
507: for (i=0;i<stack->numY;i++) {
508: VecCopy(e->Y[i],Y[i]);
509: }
510: }
511: TSSetTimeStep(ts,e->timeprev-e->time); /* stepsize will be negative */
512: ts->ptime = e->time;
513: ts->ptime_prev = e->timeprev;
514: return(0);
515: }
519: static PetscErrorCode ReCompute(TS ts,TJScheduler *tjsch,PetscInt stepnumbegin,PetscInt stepnumend)
520: {
521: Stack *stack = &tjsch->stack;
522: PetscInt i,adjsteps;
526: adjsteps = ts->steps;
527: ts->steps = stepnumbegin; /* global step number */
528: for (i=stepnumbegin;i<stepnumend;i++) { /* assume fixed step size */
529: if (stack->solution_only && !tjsch->skip_trajectory) { /* revolve online need this */
530: TSTrajectorySet(ts->trajectory,ts,ts->steps,ts->ptime,ts->vec_sol);
531: }
532: TSMonitor(ts,ts->steps,ts->ptime,ts->vec_sol);
533: TSStep(ts);
534: if (!stack->solution_only && !tjsch->skip_trajectory) {
535: TSTrajectorySet(ts->trajectory,ts,ts->steps,ts->ptime,ts->vec_sol);
536: }
537: TSEventHandler(ts);
538: if (!ts->steprollback) {
539: TSPostStep(ts);
540: }
541: }
542: TurnBackward(ts);
543: ts->trajectory->recomps += stepnumend-stepnumbegin; /* recomputation counter */
544: ts->steps = adjsteps;
545: ts->total_steps = stepnumend;
546: return(0);
547: }
551: static PetscErrorCode TopLevelStore(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscInt localstepnum,PetscInt laststridesize,PetscBool *done)
552: {
553: Stack *stack = &tjsch->stack;
554: DiskStack *diskstack = &tjsch->diskstack;
555: PetscInt stridenum;
559: *done = PETSC_FALSE;
560: stridenum = stepnum/tjsch->stride;
561: /* make sure saved checkpoint id starts from 1
562: skip last stride when using stridenum+1
563: skip first stride when using stridenum */
564: if (stack->solution_only) {
565: if (tjsch->save_stack) {
566: if (localstepnum == tjsch->stride-1 && stepnum < tjsch->total_steps-laststridesize) { /* current step will be saved without going through stack */
567: StackDumpAll(ts,stack,stridenum+1);
568: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum+1;
569: PetscPrintf(PETSC_COMM_WORLD,"\x1B[33mDump stack to file\033[0m\n");
570: *done = PETSC_TRUE;
571: }
572: } else {
573: if (localstepnum == 0 && stepnum < tjsch->total_steps-laststridesize) {
574: DumpSingle(ts,stack,stridenum+1);
575: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum+1;
576: PetscPrintf(PETSC_COMM_WORLD,"\x1B[33mDump a single point (solution) to file\033[0m\n");
577: *done = PETSC_TRUE;
578: }
579: }
580: } else {
581: if (tjsch->save_stack) {
582: if (localstepnum == 0 && stepnum < tjsch->total_steps && stepnum != 0) { /* skip the first stride */
583: StackDumpAll(ts,stack,stridenum);
584: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum;
585: PetscPrintf(PETSC_COMM_WORLD,"\x1B[33mDump stack to file\033[0m\n");
586: *done = PETSC_TRUE;
587: }
588: } else {
589: if (localstepnum == 1 && stepnum < tjsch->total_steps-laststridesize) {
590: DumpSingle(ts,stack,stridenum+1);
591: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) diskstack->container[++diskstack->top] = stridenum+1;
592: PetscPrintf(PETSC_COMM_WORLD,"\x1B[33mDump a single point (solution+stages) to file\033[0m\n");
593: *done = PETSC_TRUE;
594: }
595: }
596: }
597: return(0);
598: }
602: static PetscErrorCode SetTrajN(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
603: {
604: Stack *stack = &tjsch->stack;
605: StackElement e;
609: /* skip the last step */
610: if (ts->reason) { /* only affect the forward run */
611: /* update total_steps in the end of forward run */
612: if (stepnum != tjsch->total_steps) tjsch->total_steps = stepnum;
613: if (stack->solution_only) {
614: /* get rid of the solution at second last step */
615: StackPop(stack,&e);
616: ElementDestroy(stack,e);
617: }
618: return(0);
619: }
620: /* do not save trajectory at the recompute stage for solution_only mode */
621: if (stack->solution_only && tjsch->recompute) return(0);
622: /* skip the first step for no_solution_only mode */
623: if (!stack->solution_only && stepnum == 0) return(0);
625: /* resize the stack */
626: if (stack->top+1 == stack->stacksize) {
627: StackResize(stack,2*stack->stacksize);
628: }
629: /* update timenext for the previous step; necessary for step adaptivity */
630: if (stack->top > -1) {
631: StackTop(stack,&e);
632: e->timenext = ts->ptime;
633: }
634: if (stepnum < stack->top) SETERRQ(tjsch->comm,PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
635: ElementCreate(ts,stack,&e,stepnum,time,X);
636: StackPush(stack,e);
637: return(0);
638: }
642: static PetscErrorCode GetTrajN(TS ts,TJScheduler *tjsch,PetscInt stepnum)
643: {
644: Stack *stack = &tjsch->stack;
645: StackElement e;
649: if (stepnum == tjsch->total_steps) {
650: TurnBackward(ts);
651: return(0);
652: }
653: /* restore a checkpoint */
654: StackTop(stack,&e);
655: UpdateTS(ts,stack,e);
656: if (stack->solution_only) {/* recompute one step */
657: tjsch->recompute = PETSC_TRUE;
658: TurnForwardWithStepsize(ts,e->timenext-e->time);
659: ReCompute(ts,tjsch,e->stepnum,stepnum);
660: }
661: StackPop(stack,&e);
662: ElementDestroy(stack,e);
663: return(0);
664: }
668: static PetscErrorCode SetTrajTLNR(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
669: {
670: Stack *stack = &tjsch->stack;
671: PetscInt localstepnum,laststridesize;
672: StackElement e;
673: PetscBool done;
677: if (!stack->solution_only && stepnum == 0) return(0);
678: if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
679: if (tjsch->save_stack && tjsch->recompute) return(0);
681: localstepnum = stepnum%tjsch->stride;
682: /* (stride size-1) checkpoints are saved in each stride; an extra point is added by StackDumpAll() */
683: laststridesize = tjsch->total_steps%tjsch->stride;
684: if (!laststridesize) laststridesize = tjsch->stride;
686: if (!tjsch->recompute) {
687: TopLevelStore(ts,tjsch,stepnum,localstepnum,laststridesize,&done);
688: if (!tjsch->save_stack && stepnum < tjsch->total_steps-laststridesize) return(0);
689: }
690: if (!stack->solution_only && localstepnum == 0) return(0); /* skip last point in each stride at recompute stage or last stride */
691: if (stack->solution_only && localstepnum == tjsch->stride-1) return(0); /* skip last step in each stride at recompute stage or last stride */
693: ElementCreate(ts,stack,&e,stepnum,time,X);
694: StackPush(stack,e);
695: return(0);
696: }
700: static PetscErrorCode GetTrajTLNR(TS ts,TJScheduler *tjsch,PetscInt stepnum)
701: {
702: Stack *stack = &tjsch->stack;
703: PetscInt id,localstepnum,laststridesize;
704: StackElement e;
708: if (stepnum == tjsch->total_steps) {
709: TurnBackward(ts);
710: return(0);
711: }
713: localstepnum = stepnum%tjsch->stride;
714: laststridesize = tjsch->total_steps%tjsch->stride;
715: if (!laststridesize) laststridesize = tjsch->stride;
716: if (stack->solution_only) {
717: /* fill stack with info */
718: if (localstepnum == 0 && tjsch->total_steps-stepnum >= laststridesize) {
719: id = stepnum/tjsch->stride;
720: if (tjsch->save_stack) {
721: StackLoadAll(ts,stack,id);
722: tjsch->recompute = PETSC_TRUE;
723: tjsch->skip_trajectory = PETSC_TRUE;
724: TurnForward(ts);
725: ReCompute(ts,tjsch,id*tjsch->stride-1,id*tjsch->stride);
726: tjsch->skip_trajectory = PETSC_FALSE;
727: } else {
728: LoadSingle(ts,stack,id);
729: tjsch->recompute = PETSC_TRUE;
730: TurnForward(ts);
731: ReCompute(ts,tjsch,(id-1)*tjsch->stride,id*tjsch->stride);
732: }
733: return(0);
734: }
735: /* restore a checkpoint */
736: StackPop(stack,&e);
737: UpdateTS(ts,stack,e);
738: tjsch->recompute = PETSC_TRUE;
739: tjsch->skip_trajectory = PETSC_TRUE;
740: TurnForward(ts);
741: ReCompute(ts,tjsch,e->stepnum,stepnum);
742: tjsch->skip_trajectory = PETSC_FALSE;
743: ElementDestroy(stack,e);
744: } else {
745: /* fill stack with info */
746: if (localstepnum == 0 && tjsch->total_steps-stepnum >= laststridesize) {
747: id = stepnum/tjsch->stride;
748: if (tjsch->save_stack) {
749: StackLoadAll(ts,stack,id);
750: } else {
751: LoadSingle(ts,stack,id);
752: ElementCreate(ts,stack,&e,(id-1)*tjsch->stride+1,ts->ptime,ts->vec_sol);
753: StackPush(stack,e);
754: tjsch->recompute = PETSC_TRUE;
755: TurnForward(ts);
756: ReCompute(ts,tjsch,e->stepnum,id*tjsch->stride);
757: }
758: return(0);
759: }
760: /* restore a checkpoint */
761: StackPop(stack,&e);
762: UpdateTS(ts,stack,e);
763: ElementDestroy(stack,e);
764: }
765: return(0);
766: }
768: #ifdef PETSC_HAVE_REVOLVE
769: static void printwhattodo(PetscInt whattodo,RevolveCTX *rctx,PetscInt shift)
770: {
771: switch(whattodo) {
772: case 1:
773: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mAdvance from %D to %D\033[0m\n",rctx->oldcapo+shift,rctx->capo+shift);
774: break;
775: case 2:
776: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mStore in checkpoint number %D (located in RAM)\033[0m\n",rctx->check);
777: break;
778: case 3:
779: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mFirst turn: Initialize adjoints and reverse first step\033[0m\n");
780: break;
781: case 4:
782: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mForward and reverse one step\033[0m\n");
783: break;
784: case 5:
785: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mRestore in checkpoint number %D (located in RAM)\033[0m\n",rctx->check);
786: break;
787: case 7:
788: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mStore in checkpoint number %D (located on disk)\033[0m\n",rctx->check);
789: break;
790: case 8:
791: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mRestore in checkpoint number %D (located on disk)\033[0m\n",rctx->check);
792: break;
793: case -1:
794: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mError!");
795: break;
796: }
797: }
799: static void printwhattodo2(PetscInt whattodo,RevolveCTX *rctx,PetscInt shift)
800: {
801: switch(whattodo) {
802: case 1:
803: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35m[Top Level] Advance from stride %D to stride %D\033[0m\n",rctx->oldcapo+shift,rctx->capo+shift);
804: break;
805: case 2:
806: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35m[Top Level] Store in checkpoint number %D\033[0m\n",rctx->check);
807: break;
808: case 3:
809: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35m[Top Level] First turn: Initialize adjoints and reverse first stride\033[0m\n");
810: break;
811: case 4:
812: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35m[Top Level] Forward and reverse one stride\033[0m\n");
813: break;
814: case 5:
815: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35m[Top Level] Restore in checkpoint number %D\033[0m\n",rctx->check);
816: break;
817: case 7:
818: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35m[Top Level] Store in top-level checkpoint number %D\033[0m\n",rctx->check);
819: break;
820: case 8:
821: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35m[Top Level] Restore in top-level checkpoint number %D\033[0m\n",rctx->check);
822: break;
823: case -1:
824: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35m[Top Level] Error!");
825: break;
826: }
827: }
831: static PetscErrorCode InitRevolve(PetscInt fine,PetscInt snaps,RevolveCTX *rctx)
832: {
834: revolve_reset();
835: revolve_create_offline(fine,snaps);
836: rctx->snaps_in = snaps;
837: rctx->fine = fine;
838: rctx->check = 0;
839: rctx->capo = 0;
840: rctx->reverseonestep = PETSC_FALSE;
841: /* check stepsleft? */
842: return(0);
843: }
847: static PetscErrorCode FastForwardRevolve(RevolveCTX *rctx)
848: {
849: PetscInt whattodo;
852: whattodo = 0;
853: while(whattodo!=3) { /* we have to fast forward revolve to the beginning of the backward sweep due to unfriendly revolve interface */
854: whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
855: }
856: return(0);
857: }
861: static PetscErrorCode ApplyRevolve(SchedulerType stype,RevolveCTX *rctx,PetscInt total_steps,PetscInt stepnum,PetscInt localstepnum,PetscBool toplevel,PetscInt *store)
862: {
863: PetscInt shift,whattodo;
866: *store = 0;
867: if (rctx->stepsleft > 0) { /* advance the solution without checkpointing anything as Revolve requires */
868: rctx->stepsleft--;
869: return(0);
870: }
871: /* let Revolve determine what to do next */
872: shift = stepnum-localstepnum;
873: rctx->oldcapo = rctx->capo;
874: rctx->capo = localstepnum;
876: if (!toplevel) whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
877: else whattodo = revolve2_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
878: if (stype == REVOLVE_ONLINE && whattodo == 8) whattodo = 5;
879: if (stype == REVOLVE_ONLINE && whattodo == 7) whattodo = 2;
880: if (!toplevel) printwhattodo(whattodo,rctx,shift);
881: else printwhattodo2(whattodo,rctx,shift);
882: if (whattodo == -1) SETERRQ(PETSC_COMM_WORLD,PETSC_ERR_LIB,"Error in the Revolve library");
883: if (whattodo == 1) { /* advance some time steps */
884: if (stype == REVOLVE_ONLINE && rctx->capo >= total_steps-1) {
885: revolve_turn(total_steps,&rctx->capo,&rctx->fine);
886: if (!toplevel) printwhattodo(whattodo,rctx,shift);
887: else printwhattodo2(whattodo,rctx,shift);
888: }
889: rctx->stepsleft = rctx->capo-rctx->oldcapo-1;
890: }
891: if (whattodo == 3 || whattodo == 4) { /* ready for a reverse step */
892: rctx->reverseonestep = PETSC_TRUE;
893: }
894: if (whattodo == 5) { /* restore a checkpoint and ask Revolve what to do next */
895: rctx->oldcapo = rctx->capo;
896: if (!toplevel) whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where); /* must return 1 or 3 or 4*/
897: else whattodo = revolve2_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
898: if (!toplevel) printwhattodo(whattodo,rctx,shift);
899: else printwhattodo2(whattodo,rctx,shift);
900: if (whattodo == 3 || whattodo == 4) rctx->reverseonestep = PETSC_TRUE;
901: if (whattodo == 1) rctx->stepsleft = rctx->capo-rctx->oldcapo;
902: }
903: if (whattodo == 7) { /* save the checkpoint to disk */
904: *store = 2;
905: rctx->oldcapo = rctx->capo;
906: whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where); /* must return 1 */
907: printwhattodo(whattodo,rctx,shift);
908: rctx->stepsleft = rctx->capo-rctx->oldcapo-1;
909: }
910: if (whattodo == 2) { /* store a checkpoint to RAM and ask Revolve how many time steps to advance next */
911: *store = 1;
912: rctx->oldcapo = rctx->capo;
913: if (!toplevel) whattodo = revolve_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where); /* must return 1 */
914: else whattodo = revolve2_action(&rctx->check,&rctx->capo,&rctx->fine,rctx->snaps_in,&rctx->info,&rctx->where);
915: if (!toplevel) printwhattodo(whattodo,rctx,shift);
916: else printwhattodo2(whattodo,rctx,shift);
917: if (stype == REVOLVE_ONLINE && rctx->capo >= total_steps-1) {
918: revolve_turn(total_steps,&rctx->capo,&rctx->fine);
919: printwhattodo(whattodo,rctx,shift);
920: }
921: rctx->stepsleft = rctx->capo-rctx->oldcapo-1;
922: }
923: return(0);
924: }
928: static PetscErrorCode SetTrajROF(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
929: {
930: Stack *stack = &tjsch->stack;
931: PetscInt store;
932: StackElement e;
936: if (!stack->solution_only && stepnum == 0) return(0);
937: if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
938: ApplyRevolve(tjsch->stype,tjsch->rctx,tjsch->total_steps,stepnum,stepnum,PETSC_FALSE,&store);
939: if (store == 1) {
940: if (stepnum < stack->top) SETERRQ(tjsch->comm,PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
941: ElementCreate(ts,stack,&e,stepnum,time,X);
942: StackPush(stack,e);
943: }
944: return(0);
945: }
949: static PetscErrorCode GetTrajROF(TS ts,TJScheduler *tjsch,PetscInt stepnum)
950: {
951: Stack *stack = &tjsch->stack;
952: PetscInt whattodo,shift,store;
953: StackElement e;
957: if (stepnum == 0 || stepnum == tjsch->total_steps) {
958: TurnBackward(ts);
959: tjsch->rctx->reverseonestep = PETSC_FALSE;
960: return(0);
961: }
962: /* restore a checkpoint */
963: StackTop(stack,&e);
964: UpdateTS(ts,stack,e);
965: if (stack->solution_only) { /* start with restoring a checkpoint */
966: tjsch->rctx->capo = stepnum;
967: tjsch->rctx->oldcapo = tjsch->rctx->capo;
968: shift = 0;
969: whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where);
970: printwhattodo(whattodo,tjsch->rctx,shift);
971: } else { /* 2 revolve actions: restore a checkpoint and then advance */
972: ApplyRevolve(tjsch->stype,tjsch->rctx,tjsch->total_steps,stepnum,stepnum,PETSC_FALSE,&store);
973: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mSkip the step from %D to %D (stage values already checkpointed)\033[0m\n",tjsch->rctx->oldcapo,tjsch->rctx->oldcapo+1);
974: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
975: }
976: if (stack->solution_only || (!stack->solution_only && e->stepnum < stepnum)) {
977: tjsch->recompute = PETSC_TRUE;
978: TurnForward(ts);
979: ReCompute(ts,tjsch,e->stepnum,stepnum);
980: }
981: if ((stack->solution_only && e->stepnum+1 == stepnum) || (!stack->solution_only && e->stepnum == stepnum)) {
982: StackPop(stack,&e);
983: ElementDestroy(stack,e);
984: }
985: tjsch->rctx->reverseonestep = PETSC_FALSE;
986: return(0);
987: }
991: static PetscErrorCode SetTrajRON(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
992: {
993: Stack *stack = &tjsch->stack;
994: Vec *Y;
995: PetscInt i,store;
996: PetscReal timeprev;
997: StackElement e;
998: RevolveCTX *rctx = tjsch->rctx;
1002: if (!stack->solution_only && stepnum == 0) return(0);
1003: if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
1004: ApplyRevolve(tjsch->stype,rctx,tjsch->total_steps,stepnum,stepnum,PETSC_FALSE,&store);
1005: if (store == 1) {
1006: if (rctx->check != stack->top+1) { /* overwrite some non-top checkpoint in the stack */
1007: StackFind(stack,&e,rctx->check);
1008: VecCopy(X,e->X);
1009: if (stack->numY > 0 && !stack->solution_only) {
1010: TSGetStages(ts,&stack->numY,&Y);
1011: for (i=0;i<stack->numY;i++) {
1012: VecCopy(Y[i],e->Y[i]);
1013: }
1014: }
1015: e->stepnum = stepnum;
1016: e->time = time;
1017: TSGetPrevTime(ts,&timeprev);
1018: e->timeprev = timeprev;
1019: } else {
1020: if (stepnum < stack->top) SETERRQ(tjsch->comm,PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
1021: ElementCreate(ts,stack,&e,stepnum,time,X);
1022: StackPush(stack,e);
1023: }
1024: }
1025: return(0);
1026: }
1030: static PetscErrorCode GetTrajRON(TS ts,TJScheduler *tjsch,PetscInt stepnum)
1031: {
1032: Stack *stack = &tjsch->stack;
1033: PetscInt whattodo,shift;
1034: StackElement e;
1038: if (stepnum == 0 || stepnum == tjsch->total_steps) {
1039: TurnBackward(ts);
1040: tjsch->rctx->reverseonestep = PETSC_FALSE;
1041: return(0);
1042: }
1043: tjsch->rctx->capo = stepnum;
1044: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1045: shift = 0;
1046: whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where); /* whattodo=restore */
1047: if (whattodo == 8) whattodo = 5;
1048: printwhattodo(whattodo,tjsch->rctx,shift);
1049: /* restore a checkpoint */
1050: StackFind(stack,&e,tjsch->rctx->check);
1051: UpdateTS(ts,stack,e);
1052: if (!stack->solution_only) { /* whattodo must be 5 */
1053: /* ask Revolve what to do next */
1054: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1055: whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where); /* must return 1 or 3 or 4*/
1056: printwhattodo(whattodo,tjsch->rctx,shift);
1057: if (whattodo == 3 || whattodo == 4) tjsch->rctx->reverseonestep = PETSC_TRUE;
1058: if (whattodo == 1) tjsch->rctx->stepsleft = tjsch->rctx->capo-tjsch->rctx->oldcapo;
1059: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mSkip the step from %D to %D (stage values already checkpointed)\033[0m\n",tjsch->rctx->oldcapo,tjsch->rctx->oldcapo+1);
1060: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
1061: }
1062: if (stack->solution_only || (!stack->solution_only && e->stepnum < stepnum)) {
1063: tjsch->recompute = PETSC_TRUE;
1064: TurnForward(ts);
1065: ReCompute(ts,tjsch,e->stepnum,stepnum);
1066: }
1067: tjsch->rctx->reverseonestep = PETSC_FALSE;
1068: return(0);
1069: }
1073: static PetscErrorCode SetTrajTLR(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
1074: {
1075: Stack *stack = &tjsch->stack;
1076: PetscInt store,localstepnum,laststridesize;
1077: StackElement e;
1078: PetscBool done = PETSC_FALSE;
1082: if (!stack->solution_only && stepnum == 0) return(0);
1083: if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
1085: localstepnum = stepnum%tjsch->stride;
1086: laststridesize = tjsch->total_steps%tjsch->stride;
1087: if (!laststridesize) laststridesize = tjsch->stride;
1089: if (!tjsch->recompute) {
1090: TopLevelStore(ts,tjsch,stepnum,localstepnum,laststridesize,&done);
1091: /* revolve is needed for the last stride; different starting points for last stride between solutin_only and !solutin_only */
1092: if (!stack->solution_only && !tjsch->save_stack && stepnum <= tjsch->total_steps-laststridesize) return(0);
1093: if (stack->solution_only && !tjsch->save_stack && stepnum < tjsch->total_steps-laststridesize) return(0);
1094: }
1095: if (tjsch->save_stack && done) {
1096: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1097: return(0);
1098: }
1099: if (laststridesize < tjsch->stride) {
1100: if (stack->solution_only && stepnum == tjsch->total_steps-laststridesize && !tjsch->recompute) { /* step tjsch->total_steps-laststridesize-1 is skipped, but the next step is not */
1101: InitRevolve(laststridesize,tjsch->max_cps_ram,tjsch->rctx);
1102: }
1103: if (!stack->solution_only && stepnum == tjsch->total_steps-laststridesize+1 && !tjsch->recompute) { /* step tjsch->total_steps-laststridesize is skipped, but the next step is not */
1104: InitRevolve(laststridesize,tjsch->max_cps_ram,tjsch->rctx);
1105: }
1106: }
1107: ApplyRevolve(tjsch->stype,tjsch->rctx,tjsch->total_steps,stepnum,localstepnum,PETSC_FALSE,&store);
1108: if (store == 1) {
1109: if (localstepnum < stack->top) SETERRQ(tjsch->comm,PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
1110: ElementCreate(ts,stack,&e,stepnum,time,X);
1111: StackPush(stack,e);
1112: }
1113: return(0);
1114: }
1118: static PetscErrorCode GetTrajTLR(TS ts,TJScheduler *tjsch,PetscInt stepnum)
1119: {
1120: Stack *stack = &tjsch->stack;
1121: PetscInt whattodo,shift;
1122: PetscInt localstepnum,stridenum,laststridesize,store;
1123: StackElement e;
1127: localstepnum = stepnum%tjsch->stride;
1128: stridenum = stepnum/tjsch->stride;
1129: if (stepnum == tjsch->total_steps) {
1130: TurnBackward(ts);
1131: tjsch->rctx->reverseonestep = PETSC_FALSE;
1132: return(0);
1133: }
1134: laststridesize = tjsch->total_steps%tjsch->stride;
1135: if (!laststridesize) laststridesize = tjsch->stride;
1136: if (stack->solution_only) {
1137: /* fill stack */
1138: if (localstepnum == 0 && stepnum <= tjsch->total_steps-laststridesize) {
1139: if (tjsch->save_stack) {
1140: StackLoadAll(ts,stack,stridenum);
1141: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1142: FastForwardRevolve(tjsch->rctx);
1143: tjsch->recompute = PETSC_TRUE;
1144: tjsch->skip_trajectory = PETSC_TRUE;
1145: TurnForward(ts);
1146: ReCompute(ts,tjsch,stridenum*tjsch->stride-1,stridenum*tjsch->stride);
1147: tjsch->skip_trajectory = PETSC_FALSE;
1148: } else {
1149: LoadSingle(ts,stack,stridenum);
1150: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1151: tjsch->recompute = PETSC_TRUE;
1152: TurnForward(ts);
1153: ReCompute(ts,tjsch,(stridenum-1)*tjsch->stride,stridenum*tjsch->stride);
1154: }
1155: return(0);
1156: }
1157: /* restore a checkpoint */
1158: StackTop(stack,&e);
1159: UpdateTS(ts,stack,e);
1160: /* start with restoring a checkpoint */
1161: tjsch->rctx->capo = stepnum;
1162: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1163: shift = stepnum-localstepnum;
1164: whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where);
1165: printwhattodo(whattodo,tjsch->rctx,shift);
1166: tjsch->recompute = PETSC_TRUE;
1167: TurnForward(ts);
1168: ReCompute(ts,tjsch,e->stepnum,stepnum);
1169: if (e->stepnum+1 == stepnum) {
1170: StackPop(stack,&e);
1171: ElementDestroy(stack,e);
1172: }
1173: } else {
1174: /* fill stack with info */
1175: if (localstepnum == 0 && tjsch->total_steps-stepnum >= laststridesize) {
1176: if (tjsch->save_stack) {
1177: StackLoadAll(ts,stack,stridenum);
1178: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1179: FastForwardRevolve(tjsch->rctx);
1180: } else {
1181: LoadSingle(ts,stack,stridenum);
1182: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1183: ApplyRevolve(tjsch->stype,tjsch->rctx,tjsch->total_steps,(stridenum-1)*tjsch->stride+1,1,PETSC_FALSE,&store);
1184: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mSkip the step from %D to %D (stage values already checkpointed)\033[0m\n",(stridenum-1)*tjsch->stride+tjsch->rctx->oldcapo,(stridenum-1)*tjsch->stride+tjsch->rctx->oldcapo+1);
1185: ElementCreate(ts,stack,&e,(stridenum-1)*tjsch->stride+1,ts->ptime,ts->vec_sol);
1186: StackPush(stack,e);
1187: tjsch->recompute = PETSC_TRUE;
1188: TurnForward(ts);
1189: ReCompute(ts,tjsch,e->stepnum,stridenum*tjsch->stride);
1190: }
1191: return(0);
1192: }
1193: /* restore a checkpoint */
1194: StackTop(stack,&e);
1195: UpdateTS(ts,stack,e);
1196: /* 2 revolve actions: restore a checkpoint and then advance */
1197: ApplyRevolve(tjsch->stype,tjsch->rctx,tjsch->total_steps,stepnum,localstepnum,PETSC_FALSE,&store);
1198: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mSkip the step from %D to %D (stage values already checkpointed)\033[0m\n",stepnum-localstepnum+tjsch->rctx->oldcapo,stepnum-localstepnum+tjsch->rctx->oldcapo+1);
1199: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
1200: if (e->stepnum < stepnum) {
1201: tjsch->recompute = PETSC_TRUE;
1202: TurnForward(ts);
1203: ReCompute(ts,tjsch,e->stepnum,stepnum);
1204: }
1205: if (e->stepnum == stepnum) {
1206: StackPop(stack,&e);
1207: ElementDestroy(stack,e);
1208: }
1209: }
1210: tjsch->rctx->reverseonestep = PETSC_FALSE;
1211: return(0);
1212: }
1216: static PetscErrorCode SetTrajTLTR(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
1217: {
1218: Stack *stack = &tjsch->stack;
1219: PetscInt store,localstepnum,stridenum,laststridesize;
1220: StackElement e;
1221: PetscBool done = PETSC_FALSE;
1225: if (!stack->solution_only && stepnum == 0) return(0);
1226: if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
1228: localstepnum = stepnum%tjsch->stride; /* index at the bottom level (inside a stride) */
1229: stridenum = stepnum/tjsch->stride; /* index at the top level */
1230: laststridesize = tjsch->total_steps%tjsch->stride;
1231: if (!laststridesize) laststridesize = tjsch->stride;
1232: if (stack->solution_only && localstepnum == 0 && !tjsch->rctx2->reverseonestep) {
1233: ApplyRevolve(tjsch->stype,tjsch->rctx2,(tjsch->total_steps+tjsch->stride-1)/tjsch->stride,stridenum,stridenum,PETSC_TRUE,&tjsch->store_stride);
1234: if (laststridesize < tjsch->stride && stepnum == tjsch->total_steps-laststridesize) {
1235: InitRevolve(laststridesize,tjsch->max_cps_ram,tjsch->rctx);
1236: }
1237: }
1238: if (!stack->solution_only && localstepnum == 1 && !tjsch->rctx2->reverseonestep) {
1239: ApplyRevolve(tjsch->stype,tjsch->rctx2,(tjsch->total_steps+tjsch->stride-1)/tjsch->stride,stridenum,stridenum,PETSC_TRUE,&tjsch->store_stride);
1240: if (laststridesize < tjsch->stride && stepnum == tjsch->total_steps-laststridesize+1) {
1241: InitRevolve(laststridesize,tjsch->max_cps_ram,tjsch->rctx);
1242: }
1243: }
1244: if (tjsch->store_stride) {
1245: TopLevelStore(ts,tjsch,stepnum,localstepnum,laststridesize,&done);
1246: if (done) {
1247: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1248: return(0);
1249: }
1250: }
1251: if (stepnum < tjsch->total_steps-laststridesize) {
1252: if (tjsch->save_stack && !tjsch->store_stride && !tjsch->rctx2->reverseonestep) return(0); /* store or forward-and-reverse at top level trigger revolve at bottom level */
1253: if (!tjsch->save_stack && !tjsch->rctx2->reverseonestep) return(0); /* store operation does not require revolve be called at bottom level */
1254: }
1255: /* Skipping stepnum=0 for !stack->only is enough for TLR, but not for TLTR. Here we skip the first step for each stride so that the top-level revolve is applied (always at localstepnum=1) ahead of the bottom-level revolve */
1256: if (!stack->solution_only && localstepnum == 0 && stepnum != tjsch->total_steps && !tjsch->recompute) return(0);
1257: ApplyRevolve(tjsch->stype,tjsch->rctx,tjsch->total_steps,stepnum,localstepnum,PETSC_FALSE,&store);
1258: if (store == 1) {
1259: if (localstepnum < stack->top) SETERRQ(tjsch->comm,PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
1260: ElementCreate(ts,stack,&e,stepnum,time,X);
1261: StackPush(stack,e);
1262: }
1263: return(0);
1264: }
1268: static PetscErrorCode GetTrajTLTR(TS ts,TJScheduler *tjsch,PetscInt stepnum)
1269: {
1270: Stack *stack = &tjsch->stack;
1271: DiskStack *diskstack = &tjsch->diskstack;
1272: PetscInt whattodo,shift;
1273: PetscInt localstepnum,stridenum,restoredstridenum,laststridesize,store;
1274: StackElement e;
1278: localstepnum = stepnum%tjsch->stride;
1279: stridenum = stepnum/tjsch->stride;
1280: if (stepnum == tjsch->total_steps) {
1281: TurnBackward(ts);
1282: tjsch->rctx->reverseonestep = PETSC_FALSE;
1283: return(0);
1284: }
1285: laststridesize = tjsch->total_steps%tjsch->stride;
1286: if (!laststridesize) laststridesize = tjsch->stride;
1287: /*
1288: Last stride can be adjoined directly. All the other strides require that the stack in memory be ready before an adjoint step is taken (at the end of each stride). The following two cases need to be addressed differently:
1289: Case 1 (save_stack)
1290: Restore a disk checkpoint; update TS with the last element in the restored data; recompute to the current point.
1291: Case 2 (!save_stack)
1292: Restore a disk checkpoint; update TS with the restored point; recompute to the current point.
1293: */
1294: if (localstepnum == 0 && stepnum <= tjsch->total_steps-laststridesize) {
1295: /* restore the top element in the stack for disk checkpoints */
1296: restoredstridenum = diskstack->container[diskstack->top];
1297: tjsch->rctx2->reverseonestep = PETSC_FALSE;
1298: /* top-level revolve must be applied before current step, just like the solution_only mode for single-level revolve */
1299: if (!tjsch->save_stack && stack->solution_only) { /* start with restoring a checkpoint */
1300: tjsch->rctx2->capo = stridenum;
1301: tjsch->rctx2->oldcapo = tjsch->rctx2->capo;
1302: shift = 0;
1303: whattodo = revolve2_action(&tjsch->rctx2->check,&tjsch->rctx2->capo,&tjsch->rctx2->fine,tjsch->rctx2->snaps_in,&tjsch->rctx2->info,&tjsch->rctx2->where);
1304: printwhattodo2(whattodo,tjsch->rctx2,shift);
1305: } else { /* 2 revolve actions: restore a checkpoint and then advance */
1306: ApplyRevolve(tjsch->stype,tjsch->rctx2,(tjsch->total_steps+tjsch->stride-1)/tjsch->stride,stridenum,stridenum,PETSC_TRUE,&tjsch->store_stride);
1307: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35m[Top Level] Skip the stride from %D to %D (stage values already checkpointed)\033[0m\n",tjsch->rctx2->oldcapo,tjsch->rctx2->oldcapo+1);
1308: if (!tjsch->rctx2->reverseonestep && tjsch->rctx2->stepsleft > 0) tjsch->rctx2->stepsleft--;
1309: }
1310: /* fill stack */
1311: if (stack->solution_only) {
1312: if (tjsch->save_stack) {
1313: if (restoredstridenum < stridenum) {
1314: StackLoadLast(ts,stack,restoredstridenum);
1315: } else {
1316: StackLoadAll(ts,stack,restoredstridenum);
1317: }
1318: /* recompute one step ahead */
1319: tjsch->recompute = PETSC_TRUE;
1320: tjsch->skip_trajectory = PETSC_TRUE;
1321: TurnForward(ts);
1322: ReCompute(ts,tjsch,stridenum*tjsch->stride-1,stridenum*tjsch->stride);
1323: tjsch->skip_trajectory = PETSC_FALSE;
1324: if (restoredstridenum < stridenum) {
1325: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1326: tjsch->recompute = PETSC_TRUE;
1327: TurnForward(ts);
1328: ReCompute(ts,tjsch,restoredstridenum*tjsch->stride,stepnum);
1329: } else { /* stack ready, fast forward revolve status */
1330: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1331: FastForwardRevolve(tjsch->rctx);
1332: }
1333: } else {
1334: LoadSingle(ts,stack,restoredstridenum);
1335: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1336: tjsch->recompute = PETSC_TRUE;
1337: TurnForward(ts);
1338: ReCompute(ts,tjsch,(restoredstridenum-1)*tjsch->stride,stepnum);
1339: }
1340: } else {
1341: if (tjsch->save_stack) {
1342: if (restoredstridenum < stridenum) {
1343: StackLoadLast(ts,stack,restoredstridenum);
1344: /* reset revolve */
1345: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1346: tjsch->recompute = PETSC_TRUE;
1347: TurnForward(ts);
1348: ReCompute(ts,tjsch,restoredstridenum*tjsch->stride,stepnum);
1349: } else { /* stack ready, fast forward revolve status */
1350: StackLoadAll(ts,stack,restoredstridenum);
1351: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1352: FastForwardRevolve(tjsch->rctx);
1353: }
1354: } else {
1355: LoadSingle(ts,stack,restoredstridenum);
1356: InitRevolve(tjsch->stride,tjsch->max_cps_ram,tjsch->rctx);
1357: /* push first element to stack */
1358: if (tjsch->store_stride || tjsch->rctx2->reverseonestep) {
1359: shift = (restoredstridenum-1)*tjsch->stride-localstepnum;
1360: ApplyRevolve(tjsch->stype,tjsch->rctx,tjsch->total_steps,(restoredstridenum-1)*tjsch->stride+1,1,PETSC_FALSE,&store);
1361: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mSkip the step from %D to %D (stage values already checkpointed)\033[0m\n",(restoredstridenum-1)*tjsch->stride,(restoredstridenum-1)*tjsch->stride+1);
1362: ElementCreate(ts,stack,&e,(restoredstridenum-1)*tjsch->stride+1,ts->ptime,ts->vec_sol);
1363: StackPush(stack,e);
1364: }
1365: tjsch->recompute = PETSC_TRUE;
1366: TurnForward(ts);
1367: ReCompute(ts,tjsch,(restoredstridenum-1)*tjsch->stride+1,stepnum);
1368: }
1369: }
1370: if (restoredstridenum == stridenum) diskstack->top--;
1371: tjsch->rctx->reverseonestep = PETSC_FALSE;
1372: return(0);
1373: }
1375: if (stack->solution_only) {
1376: /* restore a checkpoint */
1377: StackTop(stack,&e);
1378: UpdateTS(ts,stack,e);
1379: /* start with restoring a checkpoint */
1380: tjsch->rctx->capo = stepnum;
1381: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1382: shift = stepnum-localstepnum;
1383: whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where);
1384: printwhattodo(whattodo,tjsch->rctx,shift);
1385: tjsch->recompute = PETSC_TRUE;
1386: TurnForward(ts);
1387: ReCompute(ts,tjsch,e->stepnum,stepnum);
1388: if (e->stepnum+1 == stepnum) {
1389: StackPop(stack,&e);
1390: ElementDestroy(stack,e);
1391: }
1392: } else {
1393: /* restore a checkpoint */
1394: StackTop(stack,&e);
1395: UpdateTS(ts,stack,e);
1396: /* 2 revolve actions: restore a checkpoint and then advance */
1397: ApplyRevolve(tjsch->stype,tjsch->rctx,tjsch->total_steps,stepnum,localstepnum,PETSC_FALSE,&store);
1398: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mSkip the step from %D to %D (stage values already checkpointed)\033[0m\n",stepnum-localstepnum+tjsch->rctx->oldcapo,stepnum-localstepnum+tjsch->rctx->oldcapo+1);
1399: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
1400: if (e->stepnum < stepnum) {
1401: tjsch->recompute = PETSC_TRUE;
1402: TurnForward(ts);
1403: ReCompute(ts,tjsch,e->stepnum,stepnum);
1404: }
1405: if (e->stepnum == stepnum) {
1406: StackPop(stack,&e);
1407: ElementDestroy(stack,e);
1408: }
1409: }
1410: tjsch->rctx->reverseonestep = PETSC_FALSE;
1411: return(0);
1412: }
1416: static PetscErrorCode SetTrajRMS(TS ts,TJScheduler *tjsch,PetscInt stepnum,PetscReal time,Vec X)
1417: {
1418: Stack *stack = &tjsch->stack;
1419: PetscInt store;
1420: StackElement e;
1424: if (!stack->solution_only && stepnum == 0) return(0);
1425: if (stack->solution_only && stepnum == tjsch->total_steps) return(0);
1426: ApplyRevolve(tjsch->stype,tjsch->rctx,tjsch->total_steps,stepnum,stepnum,PETSC_FALSE,&store);
1427: if (store == 1){
1428: if (stepnum < stack->top) SETERRQ(tjsch->comm,PETSC_ERR_MEMC,"Illegal modification of a non-top stack element");
1429: ElementCreate(ts,stack,&e,stepnum,time,X);
1430: StackPush(stack,e);
1431: } else if (store == 2) {
1432: DumpSingle(ts,stack,tjsch->rctx->check+1);
1433: }
1434: return(0);
1435: }
1439: static PetscErrorCode GetTrajRMS(TS ts,TJScheduler *tjsch,PetscInt stepnum)
1440: {
1441: Stack *stack = &tjsch->stack;
1442: PetscInt whattodo,shift;
1443: PetscInt restart;
1444: PetscBool ondisk;
1445: StackElement e;
1449: if (stepnum == 0 || stepnum == tjsch->total_steps) {
1450: TurnBackward(ts);
1451: tjsch->rctx->reverseonestep = PETSC_FALSE;
1452: return(0);
1453: }
1454: tjsch->rctx->capo = stepnum;
1455: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1456: shift = 0;
1457: whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where); /* whattodo=restore */
1458: printwhattodo(whattodo,tjsch->rctx,shift);
1459: /* restore a checkpoint */
1460: restart = tjsch->rctx->capo;
1461: if (!tjsch->rctx->where) {
1462: ondisk = PETSC_TRUE;
1463: LoadSingle(ts,stack,tjsch->rctx->check+1);
1464: TurnBackward(ts);
1465: } else {
1466: ondisk = PETSC_FALSE;
1467: StackTop(stack,&e);
1468: UpdateTS(ts,stack,e);
1469: }
1470: if (!stack->solution_only) { /* whattodo must be 5 or 8 */
1471: /* ask Revolve what to do next */
1472: tjsch->rctx->oldcapo = tjsch->rctx->capo;
1473: whattodo = revolve_action(&tjsch->rctx->check,&tjsch->rctx->capo,&tjsch->rctx->fine,tjsch->rctx->snaps_in,&tjsch->rctx->info,&tjsch->rctx->where); /* must return 1 or 3 or 4*/
1474: printwhattodo(whattodo,tjsch->rctx,shift);
1475: if (whattodo == 3 || whattodo == 4) tjsch->rctx->reverseonestep = PETSC_TRUE;
1476: if (whattodo == 1) tjsch->rctx->stepsleft = tjsch->rctx->capo-tjsch->rctx->oldcapo;
1477: PetscPrintf(PETSC_COMM_WORLD,"\x1B[35mSkip the step from %D to %D (stage values already checkpointed)\033[0m\n",tjsch->rctx->oldcapo,tjsch->rctx->oldcapo+1);
1478: if (!tjsch->rctx->reverseonestep && tjsch->rctx->stepsleft > 0) tjsch->rctx->stepsleft--;
1479: restart++; /* skip one step */
1480: }
1481: if (stack->solution_only || (!stack->solution_only && restart < stepnum)) {
1482: tjsch->recompute = PETSC_TRUE;
1483: TurnForward(ts);
1484: ReCompute(ts,tjsch,restart,stepnum);
1485: }
1486: if (!ondisk && ( (stack->solution_only && e->stepnum+1 == stepnum) || (!stack->solution_only && e->stepnum == stepnum) )) {
1487: StackPop(stack,&e);
1488: ElementDestroy(stack,e);
1489: }
1490: tjsch->rctx->reverseonestep = PETSC_FALSE;
1491: return(0);
1492: }
1493: #endif
1497: static PetscErrorCode TSTrajectorySet_Memory(TSTrajectory tj,TS ts,PetscInt stepnum,PetscReal time,Vec X)
1498: {
1499: TJScheduler *tjsch = (TJScheduler*)tj->data;
1503: if (!tjsch->recompute) { /* use global stepnum in the forward sweep */
1504: TSGetTotalSteps(ts,&stepnum);
1505: }
1506: /* for consistency */
1507: if (!tjsch->recompute && stepnum == 0) ts->ptime_prev = ts->ptime-ts->time_step;
1508: switch (tjsch->stype) {
1509: case NONE:
1510: SetTrajN(ts,tjsch,stepnum,time,X);
1511: break;
1512: case TWO_LEVEL_NOREVOLVE:
1513: SetTrajTLNR(ts,tjsch,stepnum,time,X);
1514: break;
1515: #ifdef PETSC_HAVE_REVOLVE
1516: case TWO_LEVEL_REVOLVE:
1517: SetTrajTLR(ts,tjsch,stepnum,time,X);
1518: break;
1519: case TWO_LEVEL_TWO_REVOLVE:
1520: SetTrajTLTR(ts,tjsch,stepnum,time,X);
1521: break;
1522: case REVOLVE_OFFLINE:
1523: SetTrajROF(ts,tjsch,stepnum,time,X);
1524: break;
1525: case REVOLVE_ONLINE:
1526: SetTrajRON(ts,tjsch,stepnum,time,X);
1527: break;
1528: case REVOLVE_MULTISTAGE:
1529: SetTrajRMS(ts,tjsch,stepnum,time,X);
1530: break;
1531: #endif
1532: default:
1533: break;
1534: }
1535: return(0);
1536: }
1540: static PetscErrorCode TSTrajectoryGet_Memory(TSTrajectory tj,TS ts,PetscInt stepnum,PetscReal *t)
1541: {
1542: TJScheduler *tjsch = (TJScheduler*)tj->data;
1546: TSGetTotalSteps(ts,&stepnum);
1547: if (stepnum == 0) return(0);
1548: switch (tjsch->stype) {
1549: case NONE:
1550: GetTrajN(ts,tjsch,stepnum);
1551: break;
1552: case TWO_LEVEL_NOREVOLVE:
1553: GetTrajTLNR(ts,tjsch,stepnum);
1554: break;
1555: #ifdef PETSC_HAVE_REVOLVE
1556: case TWO_LEVEL_REVOLVE:
1557: GetTrajTLR(ts,tjsch,stepnum);
1558: break;
1559: case TWO_LEVEL_TWO_REVOLVE:
1560: GetTrajTLTR(ts,tjsch,stepnum);
1561: break;
1562: case REVOLVE_OFFLINE:
1563: GetTrajROF(ts,tjsch,stepnum);
1564: break;
1565: case REVOLVE_ONLINE:
1566: GetTrajRON(ts,tjsch,stepnum);
1567: break;
1568: case REVOLVE_MULTISTAGE:
1569: GetTrajRMS(ts,tjsch,stepnum);
1570: break;
1571: #endif
1572: default:
1573: break;
1574: }
1575: return(0);
1576: }
1580: PETSC_UNUSED static PetscErrorCode TSTrajectorySetStride_Memory(TSTrajectory tj,TS ts,PetscInt stride)
1581: {
1582: TJScheduler *tjsch = (TJScheduler*)tj->data;
1585: tjsch->stride = stride;
1586: return(0);
1587: }
1591: PETSC_UNUSED static PetscErrorCode TSTrajectorySetMaxCpsRAM_Memory(TSTrajectory tj,TS ts,PetscInt max_cps_ram)
1592: {
1593: TJScheduler *tjsch = (TJScheduler*)tj->data;
1596: tjsch->max_cps_ram = max_cps_ram;
1597: return(0);
1598: }
1602: PETSC_UNUSED static PetscErrorCode TSTrajectorySetMaxCpsDisk_Memory(TSTrajectory tj,TS ts,PetscInt max_cps_disk)
1603: {
1604: TJScheduler *tjsch = (TJScheduler*)tj->data;
1607: tjsch->max_cps_disk = max_cps_disk;
1608: return(0);
1609: }
1611: #ifdef PETSC_HAVE_REVOLVE
1614: PETSC_UNUSED static PetscErrorCode TSTrajectorySetRevolveOnline(TSTrajectory tj,PetscBool use_online)
1615: {
1616: TJScheduler *tjsch = (TJScheduler*)tj->data;
1619: tjsch->use_online = use_online;
1620: return(0);
1621: }
1622: #endif
1626: PETSC_UNUSED static PetscErrorCode TSTrajectorySetSaveStack(TSTrajectory tj,PetscBool save_stack)
1627: {
1628: TJScheduler *tjsch = (TJScheduler*)tj->data;
1631: tjsch->save_stack = save_stack;
1632: return(0);
1633: }
1637: PETSC_UNUSED static PetscErrorCode TSTrajectorySetSolutionOnly(TSTrajectory tj,PetscBool solution_only)
1638: {
1639: TJScheduler *tjsch = (TJScheduler*)tj->data;
1640: Stack *stack = &tjsch->stack;
1643: stack->solution_only = solution_only;
1644: return(0);
1645: }
1649: static PetscErrorCode TSTrajectorySetFromOptions_Memory(PetscOptionItems *PetscOptionsObject,TSTrajectory tj)
1650: {
1651: TJScheduler *tjsch = (TJScheduler*)tj->data;
1655: PetscOptionsHead(PetscOptionsObject,"Memory based TS trajectory options");
1656: {
1657: PetscOptionsInt("-ts_trajectory_max_cps_ram","Maximum number of checkpoints in RAM","TSTrajectorySetMaxCpsRAM_Memory",tjsch->max_cps_ram,&tjsch->max_cps_ram,NULL);
1658: PetscOptionsInt("-ts_trajectory_max_cps_disk","Maximum number of checkpoints on disk","TSTrajectorySetMaxCpsDisk_Memory",tjsch->max_cps_disk,&tjsch->max_cps_disk,NULL);
1659: PetscOptionsInt("-ts_trajectory_stride","Stride to save checkpoints to file","TSTrajectorySetStride_Memory",tjsch->stride,&tjsch->stride,NULL);
1660: #ifdef PETSC_HAVE_REVOLVE
1661: PetscOptionsBool("-ts_trajectory_revolve_online","Trick TS trajectory into using online mode of revolve","TSTrajectorySetRevolveOnline",tjsch->use_online,&tjsch->use_online,NULL);
1662: #endif
1663: PetscOptionsBool("-ts_trajectory_save_stack","Save all stack to disk","TSTrajectorySetSaveStack",tjsch->save_stack,&tjsch->save_stack,NULL);
1664: PetscOptionsBool("-ts_trajectory_solution_only","Checkpoint solution only","TSTrajectorySetSolutionOnly",tjsch->stack.solution_only,&tjsch->stack.solution_only,NULL);
1665: }
1666: PetscOptionsTail();
1667: return(0);
1668: }
1672: static PetscErrorCode TSTrajectorySetUp_Memory(TSTrajectory tj,TS ts)
1673: {
1674: TJScheduler *tjsch = (TJScheduler*)tj->data;
1675: Stack *stack = &tjsch->stack;
1676: #ifdef PETSC_HAVE_REVOLVE
1677: RevolveCTX *rctx,*rctx2;
1678: DiskStack *diskstack = &tjsch->diskstack;
1679: PetscInt diskblocks;
1680: #endif
1681: PetscInt numY;
1682: PetscBool flg;
1686: PetscStrcmp(((PetscObject)ts->adapt)->type_name,TSADAPTNONE,&flg);
1687: if (flg) tjsch->total_steps = PetscMin(ts->max_steps,(PetscInt)(ceil(ts->max_time/ts->time_step))); /* fixed time step */
1688: if (tjsch->max_cps_ram > 0) stack->stacksize = tjsch->max_cps_ram;
1690: if (tjsch->stride > 1) { /* two level mode */
1691: if (tjsch->save_stack && tjsch->max_cps_disk > 1 && tjsch->max_cps_disk <= tjsch->max_cps_ram) SETERRQ(tjsch->comm,PETSC_ERR_ARG_INCOMP,"The specified disk capacity is not enough to store a full stack of RAM checkpoints. You might want to change the disk capacity or use single level checkpointing instead.");
1692: if (tjsch->max_cps_disk <= 1 && tjsch->max_cps_ram > 1 && tjsch->max_cps_ram <= tjsch->stride-1) tjsch->stype = TWO_LEVEL_REVOLVE; /* use revolve_offline for each stride */
1693: if (tjsch->max_cps_disk > 1 && tjsch->max_cps_ram > 1 && tjsch->max_cps_ram <= tjsch->stride-1) tjsch->stype = TWO_LEVEL_TWO_REVOLVE; /* use revolve_offline for each stride */
1694: if (tjsch->max_cps_disk <= 1 && (tjsch->max_cps_ram >= tjsch->stride || tjsch->max_cps_ram == -1)) tjsch->stype = TWO_LEVEL_NOREVOLVE; /* can also be handled by TWO_LEVEL_REVOLVE */
1695: } else { /* single level mode */
1696: if (flg) { /* fixed time step */
1697: if (tjsch->max_cps_ram >= tjsch->total_steps-1 || tjsch->max_cps_ram < 1) tjsch->stype = NONE; /* checkpoint all */
1698: else tjsch->stype = (tjsch->max_cps_disk>1) ? REVOLVE_MULTISTAGE : REVOLVE_OFFLINE;
1699: } else tjsch->stype = NONE; /* checkpoint all for adaptive time step */
1700: #ifdef PETSC_HAVE_REVOLVE
1701: if (tjsch->use_online) tjsch->stype = REVOLVE_ONLINE; /* trick into online (for testing purpose only) */
1702: #endif
1703: }
1705: if (tjsch->stype > TWO_LEVEL_NOREVOLVE) {
1706: #ifndef PETSC_HAVE_REVOLVE
1707: SETERRQ(tjsch->comm,PETSC_ERR_SUP,"revolve is needed when there is not enough memory to checkpoint all time steps according to the user's settings, please reconfigure with the additional option --download-revolve.");
1708: #else
1709: switch (tjsch->stype) {
1710: case TWO_LEVEL_REVOLVE:
1711: revolve_create_offline(tjsch->stride,tjsch->max_cps_ram);
1712: break;
1713: case TWO_LEVEL_TWO_REVOLVE:
1714: diskblocks = tjsch->save_stack ? tjsch->max_cps_disk/(tjsch->max_cps_ram+1) : tjsch->max_cps_disk; /* The block size depends on whether the stack is saved. */
1715: diskstack->stacksize = diskblocks;
1716: revolve_create_offline(tjsch->stride,tjsch->max_cps_ram);
1717: revolve2_create_offline((tjsch->total_steps+tjsch->stride-1)/tjsch->stride,diskblocks);
1718: PetscCalloc1(1,&rctx2);
1719: rctx2->snaps_in = diskblocks;
1720: rctx2->reverseonestep = PETSC_FALSE;
1721: rctx2->check = 0;
1722: rctx2->oldcapo = 0;
1723: rctx2->capo = 0;
1724: rctx2->info = 2;
1725: rctx2->fine = (tjsch->total_steps+tjsch->stride-1)/tjsch->stride;
1726: tjsch->rctx2 = rctx2;
1727: diskstack->top = -1;
1728: PetscMalloc1(diskstack->stacksize*sizeof(PetscInt),&diskstack->container);
1729: break;
1730: case REVOLVE_OFFLINE:
1731: revolve_create_offline(tjsch->total_steps,tjsch->max_cps_ram);
1732: break;
1733: case REVOLVE_ONLINE:
1734: stack->stacksize = tjsch->max_cps_ram;
1735: revolve_create_online(tjsch->max_cps_ram);
1736: break;
1737: case REVOLVE_MULTISTAGE:
1738: revolve_create_multistage(tjsch->total_steps,tjsch->max_cps_ram+tjsch->max_cps_disk,tjsch->max_cps_ram);
1739: break;
1740: default:
1741: break;
1742: }
1743: PetscCalloc1(1,&rctx);
1744: rctx->snaps_in = tjsch->max_cps_ram; /* for theta methods snaps_in=2*max_cps_ram */
1745: rctx->reverseonestep = PETSC_FALSE;
1746: rctx->check = 0;
1747: rctx->oldcapo = 0;
1748: rctx->capo = 0;
1749: rctx->info = 2;
1750: rctx->fine = (tjsch->stride > 1) ? tjsch->stride : tjsch->total_steps;
1751: tjsch->rctx = rctx;
1752: if (tjsch->stype == REVOLVE_ONLINE) rctx->fine = -1;
1753: #endif
1754: } else {
1755: if (tjsch->stype == TWO_LEVEL_NOREVOLVE) stack->stacksize = tjsch->stride-1; /* need tjsch->stride-1 at most */
1756: if (tjsch->stype == NONE) {
1757: if (flg) stack->stacksize = stack->solution_only ? tjsch->total_steps : tjsch->total_steps-1; /* fix time step */
1758: else { /* adaptive time step */
1759: if(tjsch->max_cps_ram == -1) stack->stacksize = ts->max_steps; /* if max_cps_ram is not specified, use maximal allowed number of steps for stack size */
1760: tjsch->total_steps = stack->solution_only ? stack->stacksize:stack->stacksize+1; /* will be updated as time integration advances */
1761: }
1762: }
1763: }
1765: tjsch->recompute = PETSC_FALSE;
1766: tjsch->comm = PetscObjectComm((PetscObject)ts);
1767: TSGetStages(ts,&numY,PETSC_IGNORE);
1768: StackCreate(stack,stack->stacksize,numY);
1769: return(0);
1770: }
1774: static PetscErrorCode TSTrajectoryDestroy_Memory(TSTrajectory tj)
1775: {
1776: TJScheduler *tjsch = (TJScheduler*)tj->data;
1780: if (tjsch->stype > TWO_LEVEL_NOREVOLVE) {
1781: #ifdef PETSC_HAVE_REVOLVE
1782: revolve_reset();
1783: if (tjsch->stype == TWO_LEVEL_TWO_REVOLVE) {
1784: revolve2_reset();
1785: PetscFree(tjsch->diskstack.container);
1786: }
1787: #endif
1788: }
1789: StackDestroy(&tjsch->stack);
1790: #ifdef PETSC_HAVE_REVOLVE
1791: if (tjsch->stype > TWO_LEVEL_NOREVOLVE) {
1792: PetscFree(tjsch->rctx);
1793: PetscFree(tjsch->rctx2);
1794: }
1795: #endif
1796: PetscFree(tjsch);
1797: return(0);
1798: }
1800: /*MC
1801: TSTRAJECTORYMEMORY - Stores each solution of the ODE/ADE in memory
1803: Level: intermediate
1805: .seealso: TSTrajectoryCreate(), TS, TSTrajectorySetType()
1807: M*/
1810: PETSC_EXTERN PetscErrorCode TSTrajectoryCreate_Memory(TSTrajectory tj,TS ts)
1811: {
1812: TJScheduler *tjsch;
1816: tj->ops->set = TSTrajectorySet_Memory;
1817: tj->ops->get = TSTrajectoryGet_Memory;
1818: tj->ops->setup = TSTrajectorySetUp_Memory;
1819: tj->ops->destroy = TSTrajectoryDestroy_Memory;
1820: tj->ops->setfromoptions = TSTrajectorySetFromOptions_Memory;
1822: PetscCalloc1(1,&tjsch);
1823: tjsch->stype = NONE;
1824: tjsch->max_cps_ram = -1; /* -1 indicates that it is not set */
1825: tjsch->max_cps_disk = -1; /* -1 indicates that it is not set */
1826: tjsch->stride = 0; /* if not zero, two-level checkpointing will be used */
1827: #ifdef PETSC_HAVE_REVOLVE
1828: tjsch->use_online = PETSC_FALSE;
1829: #endif
1830: tjsch->save_stack = PETSC_TRUE;
1832: tjsch->stack.solution_only = PETSC_TRUE;
1834: tj->data = tjsch;
1836: return(0);
1837: }