Actual source code: fretrieve.c
2: /*
3: Code for opening and closing files.
4: */
5: #include <petscsys.h>
6: #if defined(PETSC_HAVE_PWD_H)
7: #include <pwd.h>
8: #endif
9: #include <ctype.h>
10: #include <sys/stat.h>
11: #if defined(PETSC_HAVE_UNISTD_H)
12: #include <unistd.h>
13: #endif
14: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
15: #include <sys/utsname.h>
16: #endif
17: #include <fcntl.h>
18: #include <time.h>
19: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
20: #include <sys/systeminfo.h>
21: #endif
23: /*
24: Private routine to delete tmp/shared storage
26: This is called by MPI, not by users.
28: Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
30: */
31: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
32: {
33: PetscInfo(NULL,"Deleting tmp/shared data in an MPI_Comm %ld\n",(long)comm);
34: PetscFree(count_val);
35: return MPI_SUCCESS;
36: }
38: /*@C
39: PetscGetTmp - Gets the name of the tmp directory
41: Collective
43: Input Parameters:
44: + comm - MPI_Communicator that may share /tmp
45: - len - length of string to hold name
47: Output Parameter:
48: . dir - directory name
50: Options Database Keys:
51: + -shared_tmp - indicates the directory is shared among the MPI ranks
52: . -not_shared_tmp - indicates the directory is not shared among the MPI ranks
53: - -tmp tmpdir - name of the directory you wish to use as /tmp
55: Environmental Variables:
56: + PETSC_SHARED_TMP - indicates the directory is shared among the MPI ranks
57: . PETSC_NOT_SHARED_TMP - indicates the directory is not shared among the MPI ranks
58: - PETSC_TMP - name of the directory you wish to use as /tmp
60: Level: developer
62: @*/
63: PetscErrorCode PetscGetTmp(MPI_Comm comm,char dir[],size_t len)
64: {
65: PetscBool flg;
67: PetscOptionsGetenv(comm,"PETSC_TMP",dir,len,&flg);
68: if (!flg) {
69: PetscStrncpy(dir,"/tmp",len);
70: }
71: return 0;
72: }
74: /*@C
75: PetscSharedTmp - Determines if all processors in a communicator share a
76: /tmp or have different ones.
78: Collective
80: Input Parameters:
81: . comm - MPI_Communicator that may share /tmp
83: Output Parameters:
84: . shared - PETSC_TRUE or PETSC_FALSE
86: Options Database Keys:
87: + -shared_tmp - indicates the directory is shared among the MPI ranks
88: . -not_shared_tmp - indicates the directory is not shared among the MPI ranks
89: - -tmp tmpdir - name of the directory you wish to use as /tmp
91: Environmental Variables:
92: + PETSC_SHARED_TMP - indicates the directory is shared among the MPI ranks
93: . PETSC_NOT_SHARED_TMP - indicates the directory is not shared among the MPI ranks
94: - PETSC_TMP - name of the directory you wish to use as /tmp
96: Level: developer
98: Notes:
99: Stores the status as a MPI attribute so it does not have
100: to be redetermined each time.
102: Assumes that all processors in a communicator either
103: 1) have a common /tmp or
104: 2) each has a separate /tmp
105: eventually we can write a fancier one that determines which processors
106: share a common /tmp.
108: This will be very slow on runs with a large number of processors since
109: it requires O(p*p) file opens.
111: If the environmental variable PETSC_TMP is set it will use this directory
112: as the "/tmp" directory.
114: @*/
115: PetscErrorCode PetscSharedTmp(MPI_Comm comm,PetscBool *shared)
116: {
117: PetscMPIInt size,rank,*tagvalp,sum,cnt,i;
118: PetscBool flg,iflg;
119: FILE *fd;
120: static PetscMPIInt Petsc_Tmp_keyval = MPI_KEYVAL_INVALID;
121: int err;
123: MPI_Comm_size(comm,&size);
124: if (size == 1) {
125: *shared = PETSC_TRUE;
126: return 0;
127: }
129: PetscOptionsGetenv(comm,"PETSC_SHARED_TMP",NULL,0,&flg);
130: if (flg) {
131: *shared = PETSC_TRUE;
132: return 0;
133: }
135: PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_TMP",NULL,0,&flg);
136: if (flg) {
137: *shared = PETSC_FALSE;
138: return 0;
139: }
141: if (Petsc_Tmp_keyval == MPI_KEYVAL_INVALID) {
142: MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_Tmp_keyval,NULL);
143: }
145: MPI_Comm_get_attr(comm,Petsc_Tmp_keyval,(void**)&tagvalp,(int*)&iflg);
146: if (!iflg) {
147: char filename[PETSC_MAX_PATH_LEN],tmpname[PETSC_MAX_PATH_LEN];
149: /* This communicator does not yet have a shared tmp attribute */
150: PetscMalloc1(1,&tagvalp);
151: MPI_Comm_set_attr(comm,Petsc_Tmp_keyval,tagvalp);
153: PetscOptionsGetenv(comm,"PETSC_TMP",tmpname,238,&iflg);
154: if (!iflg) {
155: PetscStrcpy(filename,"/tmp");
156: } else {
157: PetscStrcpy(filename,tmpname);
158: }
160: PetscStrcat(filename,"/petsctestshared");
161: MPI_Comm_rank(comm,&rank);
163: /* each processor creates a /tmp file and all the later ones check */
164: /* this makes sure no subset of processors is shared */
165: *shared = PETSC_FALSE;
166: for (i=0; i<size-1; i++) {
167: if (rank == i) {
168: fd = fopen(filename,"w");
170: err = fclose(fd);
172: }
173: MPI_Barrier(comm);
174: if (rank >= i) {
175: fd = fopen(filename,"r");
176: if (fd) cnt = 1;
177: else cnt = 0;
178: if (fd) {
179: err = fclose(fd);
181: }
182: } else cnt = 0;
184: MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
185: if (rank == i) unlink(filename);
187: if (sum == size) {
188: *shared = PETSC_TRUE;
189: break;
191: }
192: *tagvalp = (int)*shared;
193: PetscInfo(NULL,"processors %s %s\n",(*shared) ? "share":"do NOT share",(iflg ? tmpname:"/tmp"));
194: } else *shared = (PetscBool) *tagvalp;
195: return 0;
196: }
198: /*@C
199: PetscSharedWorkingDirectory - Determines if all processors in a communicator share a working directory or have different ones.
201: Collective
203: Input Parameter:
204: . comm - MPI_Communicator that may share working directory
206: Output Parameter:
207: . shared - PETSC_TRUE or PETSC_FALSE
209: Options Database Keys:
210: + -shared_working_directory - indicates the directory is shared among the MPI ranks
211: - -not_shared_working_directory - indicates the directory is shared among the MPI ranks
213: Environmental Variables:
214: + PETSC_SHARED_WORKING_DIRECTORY - indicates the directory is shared among the MPI ranks
215: - PETSC_NOT_SHARED_WORKING_DIRECTORY - indicates the directory is shared among the MPI ranks
217: Level: developer
219: Notes:
220: Stores the status as a MPI attribute so it does not have to be redetermined each time.
222: Assumes that all processors in a communicator either
223: $ 1) have a common working directory or
224: $ 2) each has a separate working directory
225: eventually we can write a fancier one that determines which processors share a common working directory.
227: This will be very slow on runs with a large number of processors since it requires O(p*p) file opens.
228: @*/
229: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm, PetscBool *shared)
230: {
231: PetscMPIInt size,rank,*tagvalp,sum,cnt,i;
232: PetscBool flg,iflg;
233: FILE *fd;
234: static PetscMPIInt Petsc_WD_keyval = MPI_KEYVAL_INVALID;
235: int err;
237: MPI_Comm_size(comm,&size);
238: if (size == 1) {
239: *shared = PETSC_TRUE;
240: return 0;
241: }
243: PetscOptionsGetenv(comm,"PETSC_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
244: if (flg) {
245: *shared = PETSC_TRUE;
246: return 0;
247: }
249: PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
250: if (flg) {
251: *shared = PETSC_FALSE;
252: return 0;
253: }
255: if (Petsc_WD_keyval == MPI_KEYVAL_INVALID) {
256: MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_WD_keyval,NULL);
257: }
259: MPI_Comm_get_attr(comm,Petsc_WD_keyval,(void**)&tagvalp,(int*)&iflg);
260: if (!iflg) {
261: char filename[PETSC_MAX_PATH_LEN];
263: /* This communicator does not yet have a shared attribute */
264: PetscMalloc1(1,&tagvalp);
265: MPI_Comm_set_attr(comm,Petsc_WD_keyval,tagvalp);
267: PetscGetWorkingDirectory(filename,240);
268: PetscStrcat(filename,"/petsctestshared");
269: MPI_Comm_rank(comm,&rank);
271: /* each processor creates a file and all the later ones check */
272: /* this makes sure no subset of processors is shared */
273: *shared = PETSC_FALSE;
274: for (i=0; i<size-1; i++) {
275: if (rank == i) {
276: fd = fopen(filename,"w");
278: err = fclose(fd);
280: }
281: MPI_Barrier(comm);
282: if (rank >= i) {
283: fd = fopen(filename,"r");
284: if (fd) cnt = 1;
285: else cnt = 0;
286: if (fd) {
287: err = fclose(fd);
289: }
290: } else cnt = 0;
292: MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
293: if (rank == i) unlink(filename);
295: if (sum == size) {
296: *shared = PETSC_TRUE;
297: break;
299: }
300: *tagvalp = (int)*shared;
301: } else *shared = (PetscBool) *tagvalp;
302: PetscInfo(NULL,"processors %s working directory\n",(*shared) ? "shared" : "do NOT share");
303: return 0;
304: }
306: /*@C
307: PetscFileRetrieve - Obtains a file from a URL or compressed
308: and copies into local disk space as uncompressed.
310: Collective
312: Input Parameters:
313: + comm - processors accessing the file
314: . url - name of file, including entire URL (with or without .gz)
315: - llen - length of localname
317: Output Parameters:
318: + localname - name of local copy of file - valid on only process zero
319: - found - if found or retrieved the file - valid on all processes
321: Notes:
322: if the file already exists local this function just returns without downloading it.
324: Level: intermediate
325: @*/
326: PetscErrorCode PetscFileRetrieve(MPI_Comm comm,const char url[],char localname[],size_t llen,PetscBool *found)
327: {
328: char buffer[PETSC_MAX_PATH_LEN],*par,*tlocalname,name[PETSC_MAX_PATH_LEN];
329: FILE *fp;
330: PetscMPIInt rank;
331: size_t len = 0;
332: PetscBool flg1,flg2,flg3,flg4,download,compressed = PETSC_FALSE;
334: MPI_Comm_rank(comm,&rank);
335: if (rank == 0) {
336: *found = PETSC_FALSE;
338: PetscStrstr(url,".gz",&par);
339: if (par) {
340: PetscStrlen(par,&len);
341: if (len == 3) compressed = PETSC_TRUE;
342: }
344: PetscStrncmp(url,"ftp://",6,&flg1);
345: PetscStrncmp(url,"http://",7,&flg2);
346: PetscStrncmp(url,"file://",7,&flg3);
347: PetscStrncmp(url,"https://",8,&flg4);
348: download = (PetscBool) (flg1 || flg2 || flg3 || flg4);
350: if (!download && !compressed) {
351: PetscStrncpy(localname,url,llen);
352: PetscTestFile(url,'r',found);
353: if (*found) {
354: PetscInfo(NULL,"Found file %s\n",url);
355: } else {
356: PetscInfo(NULL,"Did not find file %s\n",url);
357: }
358: goto done;
359: }
361: /* look for uncompressed file in requested directory */
362: if (compressed) {
363: PetscStrncpy(localname,url,llen);
364: PetscStrstr(localname,".gz",&par);
365: *par = 0; /* remove .gz extension */
366: PetscTestFile(localname,'r',found);
367: if (*found) goto done;
368: }
370: /* look for file in current directory */
371: PetscStrrchr(url,'/',&tlocalname);
372: PetscStrncpy(localname,tlocalname,llen);
373: if (compressed) {
374: PetscStrstr(localname,".gz",&par);
375: *par = 0; /* remove .gz extension */
376: }
377: PetscTestFile(localname,'r',found);
378: if (*found) goto done;
380: if (download) {
381: /* local file is not already here so use curl to get it */
382: PetscStrncpy(localname,tlocalname,llen);
383: PetscStrcpy(buffer,"curl --fail --silent --show-error ");
384: PetscStrcat(buffer,url);
385: PetscStrcat(buffer," > ");
386: PetscStrcat(buffer,localname);
387: #if defined(PETSC_HAVE_POPEN)
388: PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
389: PetscPClose(PETSC_COMM_SELF,fp);
390: #else
391: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
392: #endif
393: PetscTestFile(localname,'r',found);
394: if (*found) {
395: FILE *fd;
396: char buf[1024],*str,*substring;
398: /* check if the file didn't exist so it downloaded an HTML message instead */
399: fd = fopen(localname,"r");
401: str = fgets(buf,sizeof(buf)-1,fd);
402: while (str) {
403: PetscStrstr(buf,"<!DOCTYPE html>",&substring);
405: PetscStrstr(buf,"Not Found",&substring);
407: str = fgets(buf,sizeof(buf)-1,fd);
408: }
409: fclose(fd);
410: }
411: } else if (compressed) {
412: PetscTestFile(url,'r',found);
413: if (!*found) goto done;
414: PetscStrncpy(localname,url,llen);
415: }
416: if (compressed) {
417: PetscStrrchr(localname,'/',&tlocalname);
418: PetscStrncpy(name,tlocalname,PETSC_MAX_PATH_LEN);
419: PetscStrstr(name,".gz",&par);
420: *par = 0; /* remove .gz extension */
421: /* uncompress file */
422: PetscStrcpy(buffer,"gzip -c -d ");
423: PetscStrcat(buffer,localname);
424: PetscStrcat(buffer," > ");
425: PetscStrcat(buffer,name);
426: #if defined(PETSC_HAVE_POPEN)
427: PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
428: PetscPClose(PETSC_COMM_SELF,fp);
429: #else
430: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
431: #endif
432: PetscStrncpy(localname,name,llen);
433: PetscTestFile(localname,'r',found);
434: }
435: }
436: done:
437: MPI_Bcast(found,1,MPIU_BOOL,0,comm);
438: MPI_Bcast(localname, llen, MPI_CHAR, 0, comm);
439: return 0;
440: }