Actual source code: fretrieve.c
2: /*
3: Code for opening and closing files.
4: */
5: #include <petscsys.h>
6: #if defined(PETSC_HAVE_PWD_H)
7: #include <pwd.h>
8: #endif
9: #include <ctype.h>
10: #include <sys/stat.h>
11: #if defined(PETSC_HAVE_UNISTD_H)
12: #include <unistd.h>
13: #endif
14: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
15: #include <sys/utsname.h>
16: #endif
17: #include <fcntl.h>
18: #include <time.h>
19: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
20: #include <sys/systeminfo.h>
21: #endif
23: /*
24: Private routine to delete tmp/shared storage
26: This is called by MPI, not by users.
28: Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
30: */
31: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
32: {
36: PetscInfo1(NULL,"Deleting tmp/shared data in an MPI_Comm %ld\n",(long)comm);
37: PetscFree(count_val);
38: PetscFunctionReturn(MPI_SUCCESS);
39: }
41: /*@C
42: PetscGetTmp - Gets the name of the tmp directory
44: Collective
46: Input Parameters:
47: + comm - MPI_Communicator that may share /tmp
48: - len - length of string to hold name
50: Output Parameter:
51: . dir - directory name
53: Options Database Keys:
54: + -shared_tmp - indicates the directory is shared among the MPI ranks
55: . -not_shared_tmp - indicates the directory is not shared among the MPI ranks
56: - -tmp tmpdir - name of the directory you wish to use as /tmp
58: Environmental Variables:
59: + PETSC_SHARED_TMP - indicates the directory is shared among the MPI ranks
60: . PETSC_NOT_SHARED_TMP - indicates the directory is not shared among the MPI ranks
61: - PETSC_TMP - name of the directory you wish to use as /tmp
63: Level: developer
65: @*/
66: PetscErrorCode PetscGetTmp(MPI_Comm comm,char dir[],size_t len)
67: {
69: PetscBool flg;
72: PetscOptionsGetenv(comm,"PETSC_TMP",dir,len,&flg);
73: if (!flg) {
74: PetscStrncpy(dir,"/tmp",len);
75: }
76: return(0);
77: }
79: /*@C
80: PetscSharedTmp - Determines if all processors in a communicator share a
81: /tmp or have different ones.
83: Collective
85: Input Parameters:
86: . comm - MPI_Communicator that may share /tmp
88: Output Parameters:
89: . shared - PETSC_TRUE or PETSC_FALSE
91: Options Database Keys:
92: + -shared_tmp - indicates the directory is shared among the MPI ranks
93: . -not_shared_tmp - indicates the directory is not shared among the MPI ranks
94: - -tmp tmpdir - name of the directory you wish to use as /tmp
96: Environmental Variables:
97: + PETSC_SHARED_TMP - indicates the directory is shared among the MPI ranks
98: . PETSC_NOT_SHARED_TMP - indicates the directory is not shared among the MPI ranks
99: - PETSC_TMP - name of the directory you wish to use as /tmp
101: Level: developer
103: Notes:
104: Stores the status as a MPI attribute so it does not have
105: to be redetermined each time.
107: Assumes that all processors in a communicator either
108: 1) have a common /tmp or
109: 2) each has a separate /tmp
110: eventually we can write a fancier one that determines which processors
111: share a common /tmp.
113: This will be very slow on runs with a large number of processors since
114: it requires O(p*p) file opens.
116: If the environmental variable PETSC_TMP is set it will use this directory
117: as the "/tmp" directory.
119: @*/
120: PetscErrorCode PetscSharedTmp(MPI_Comm comm,PetscBool *shared)
121: {
122: PetscErrorCode ierr;
123: PetscMPIInt size,rank,*tagvalp,sum,cnt,i;
124: PetscBool flg,iflg;
125: FILE *fd;
126: static PetscMPIInt Petsc_Tmp_keyval = MPI_KEYVAL_INVALID;
127: int err;
130: MPI_Comm_size(comm,&size);
131: if (size == 1) {
132: *shared = PETSC_TRUE;
133: return(0);
134: }
136: PetscOptionsGetenv(comm,"PETSC_SHARED_TMP",NULL,0,&flg);
137: if (flg) {
138: *shared = PETSC_TRUE;
139: return(0);
140: }
142: PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_TMP",NULL,0,&flg);
143: if (flg) {
144: *shared = PETSC_FALSE;
145: return(0);
146: }
148: if (Petsc_Tmp_keyval == MPI_KEYVAL_INVALID) {
149: MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_Tmp_keyval,NULL);
150: }
152: MPI_Comm_get_attr(comm,Petsc_Tmp_keyval,(void**)&tagvalp,(int*)&iflg);
153: if (!iflg) {
154: char filename[PETSC_MAX_PATH_LEN],tmpname[PETSC_MAX_PATH_LEN];
156: /* This communicator does not yet have a shared tmp attribute */
157: PetscMalloc1(1,&tagvalp);
158: MPI_Comm_set_attr(comm,Petsc_Tmp_keyval,tagvalp);
160: PetscOptionsGetenv(comm,"PETSC_TMP",tmpname,238,&iflg);
161: if (!iflg) {
162: PetscStrcpy(filename,"/tmp");
163: } else {
164: PetscStrcpy(filename,tmpname);
165: }
167: PetscStrcat(filename,"/petsctestshared");
168: MPI_Comm_rank(comm,&rank);
170: /* each processor creates a /tmp file and all the later ones check */
171: /* this makes sure no subset of processors is shared */
172: *shared = PETSC_FALSE;
173: for (i=0; i<size-1; i++) {
174: if (rank == i) {
175: fd = fopen(filename,"w");
176: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
177: err = fclose(fd);
178: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
179: }
180: MPI_Barrier(comm);
181: if (rank >= i) {
182: fd = fopen(filename,"r");
183: if (fd) cnt = 1;
184: else cnt = 0;
185: if (fd) {
186: err = fclose(fd);
187: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
188: }
189: } else cnt = 0;
191: MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
192: if (rank == i) unlink(filename);
194: if (sum == size) {
195: *shared = PETSC_TRUE;
196: break;
197: } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share /tmp ");
198: }
199: *tagvalp = (int)*shared;
200: PetscInfo2(NULL,"processors %s %s\n",(*shared) ? "share":"do NOT share",(iflg ? tmpname:"/tmp"));
201: } else *shared = (PetscBool) *tagvalp;
202: return(0);
203: }
205: /*@C
206: PetscSharedWorkingDirectory - Determines if all processors in a communicator share a
207: working directory or have different ones.
209: Collective
211: Input Parameter:
212: . comm - MPI_Communicator that may share working directory
214: Output Parameter:
215: . shared - PETSC_TRUE or PETSC_FALSE
217: Options Database Keys:
218: + -shared_working_directory - indicates the directory is shared among the MPI ranks
219: - -not_shared_working_directory - indicates the directory is shared among the MPI ranks
221: Environmental Variables:
222: + PETSC_SHARED_WORKING_DIRECTORY - indicates the directory is shared among the MPI ranks
223: - PETSC_NOT_SHARED_WORKING_DIRECTORY - indicates the directory is shared among the MPI ranks
225: Level: developer
227: Notes:
228: Stores the status as a MPI attribute so it does not have
229: to be redetermined each time.
231: Assumes that all processors in a communicator either
232: 1) have a common working directory or
233: 2) each has a separate working directory
234: eventually we can write a fancier one that determines which processors
235: share a common working directory.
237: This will be very slow on runs with a large number of processors since
238: it requires O(p*p) file opens.
240: @*/
241: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm,PetscBool *shared)
242: {
243: PetscErrorCode ierr;
244: PetscMPIInt size,rank,*tagvalp,sum,cnt,i;
245: PetscBool flg,iflg;
246: FILE *fd;
247: static PetscMPIInt Petsc_WD_keyval = MPI_KEYVAL_INVALID;
248: int err;
251: MPI_Comm_size(comm,&size);
252: if (size == 1) {
253: *shared = PETSC_TRUE;
254: return(0);
255: }
257: PetscOptionsGetenv(comm,"PETSC_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
258: if (flg) {
259: *shared = PETSC_TRUE;
260: return(0);
261: }
263: PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
264: if (flg) {
265: *shared = PETSC_FALSE;
266: return(0);
267: }
269: if (Petsc_WD_keyval == MPI_KEYVAL_INVALID) {
270: MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_WD_keyval,NULL);
271: }
273: MPI_Comm_get_attr(comm,Petsc_WD_keyval,(void**)&tagvalp,(int*)&iflg);
274: if (!iflg) {
275: char filename[PETSC_MAX_PATH_LEN];
277: /* This communicator does not yet have a shared attribute */
278: PetscMalloc1(1,&tagvalp);
279: MPI_Comm_set_attr(comm,Petsc_WD_keyval,tagvalp);
281: PetscGetWorkingDirectory(filename,240);
282: PetscStrcat(filename,"/petsctestshared");
283: MPI_Comm_rank(comm,&rank);
285: /* each processor creates a file and all the later ones check */
286: /* this makes sure no subset of processors is shared */
287: *shared = PETSC_FALSE;
288: for (i=0; i<size-1; i++) {
289: if (rank == i) {
290: fd = fopen(filename,"w");
291: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
292: err = fclose(fd);
293: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
294: }
295: MPI_Barrier(comm);
296: if (rank >= i) {
297: fd = fopen(filename,"r");
298: if (fd) cnt = 1;
299: else cnt = 0;
300: if (fd) {
301: err = fclose(fd);
302: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
303: }
304: } else cnt = 0;
306: MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
307: if (rank == i) unlink(filename);
309: if (sum == size) {
310: *shared = PETSC_TRUE;
311: break;
312: } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share working directory");
313: }
314: *tagvalp = (int)*shared;
315: } else *shared = (PetscBool) *tagvalp;
316: PetscInfo1(NULL,"processors %s working directory\n",(*shared) ? "shared" : "do NOT share");
317: return(0);
318: }
320: /*@C
321: PetscFileRetrieve - Obtains a file from a URL or compressed
322: and copies into local disk space as uncompressed.
324: Collective
326: Input Parameters:
327: + comm - processors accessing the file
328: . url - name of file, including entire URL (with or without .gz)
329: - llen - length of localname
331: Output Parameters:
332: + localname - name of local copy of file - valid on only process zero
333: - found - if found or retrieved the file - valid on all processes
335: Notes:
336: if the file already exists local this function just returns without downloading it.
338: Level: intermediate
339: @*/
340: PetscErrorCode PetscFileRetrieve(MPI_Comm comm,const char url[],char localname[],size_t llen,PetscBool *found)
341: {
342: char buffer[PETSC_MAX_PATH_LEN],*par,*tlocalname,name[PETSC_MAX_PATH_LEN];
343: FILE *fp;
345: PetscMPIInt rank;
346: size_t len = 0;
347: PetscBool flg1,flg2,flg3,flg4,download,compressed = PETSC_FALSE;
350: MPI_Comm_rank(comm,&rank);
351: if (rank == 0) {
352: *found = PETSC_FALSE;
354: PetscStrstr(url,".gz",&par);
355: if (par) {
356: PetscStrlen(par,&len);
357: if (len == 3) compressed = PETSC_TRUE;
358: }
360: PetscStrncmp(url,"ftp://",6,&flg1);
361: PetscStrncmp(url,"http://",7,&flg2);
362: PetscStrncmp(url,"file://",7,&flg3);
363: PetscStrncmp(url,"https://",8,&flg4);
364: download = (PetscBool) (flg1 || flg2 || flg3 || flg4);
366: if (!download && !compressed) {
367: PetscStrncpy(localname,url,llen);
368: PetscTestFile(url,'r',found);
369: if (*found) {
370: PetscInfo1(NULL,"Found file %s\n",url);
371: } else {
372: PetscInfo1(NULL,"Did not find file %s\n",url);
373: }
374: goto done;
375: }
377: /* look for uncompressed file in requested directory */
378: if (compressed) {
379: PetscStrncpy(localname,url,llen);
380: PetscStrstr(localname,".gz",&par);
381: *par = 0; /* remove .gz extension */
382: PetscTestFile(localname,'r',found);
383: if (*found) goto done;
384: }
386: /* look for file in current directory */
387: PetscStrrchr(url,'/',&tlocalname);
388: PetscStrncpy(localname,tlocalname,llen);
389: if (compressed) {
390: PetscStrstr(localname,".gz",&par);
391: *par = 0; /* remove .gz extension */
392: }
393: PetscTestFile(localname,'r',found);
394: if (*found) goto done;
396: if (download) {
397: /* local file is not already here so use curl to get it */
398: PetscStrncpy(localname,tlocalname,llen);
399: PetscStrcpy(buffer,"curl --fail --silent --show-error ");
400: PetscStrcat(buffer,url);
401: PetscStrcat(buffer," > ");
402: PetscStrcat(buffer,localname);
403: #if defined(PETSC_HAVE_POPEN)
404: PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
405: PetscPClose(PETSC_COMM_SELF,fp);
406: #else
407: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
408: #endif
409: PetscTestFile(localname,'r',found);
410: if (*found) {
411: FILE *fd;
412: char buf[1024],*str,*substring;
414: /* check if the file didn't exist so it downloaded an HTML message instead */
415: fd = fopen(localname,"r");
416: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"PetscTestFile() indicates %s exists but fopen() cannot open it",localname);
417: str = fgets(buf,sizeof(buf)-1,fd);
418: while (str) {
419: PetscStrstr(buf,"<!DOCTYPE html>",&substring);
420: if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
421: PetscStrstr(buf,"Not Found",&substring);
422: if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
423: str = fgets(buf,sizeof(buf)-1,fd);
424: }
425: fclose(fd);
426: }
427: } else if (compressed) {
428: PetscTestFile(url,'r',found);
429: if (!*found) goto done;
430: PetscStrncpy(localname,url,llen);
431: }
432: if (compressed) {
433: PetscStrrchr(localname,'/',&tlocalname);
434: PetscStrncpy(name,tlocalname,PETSC_MAX_PATH_LEN);
435: PetscStrstr(name,".gz",&par);
436: *par = 0; /* remove .gz extension */
437: /* uncompress file */
438: PetscStrcpy(buffer,"gzip -c -d ");
439: PetscStrcat(buffer,localname);
440: PetscStrcat(buffer," > ");
441: PetscStrcat(buffer,name);
442: #if defined(PETSC_HAVE_POPEN)
443: PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
444: PetscPClose(PETSC_COMM_SELF,fp);
445: #else
446: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
447: #endif
448: PetscStrncpy(localname,name,llen);
449: PetscTestFile(localname,'r',found);
450: }
451: }
452: done:
453: MPI_Bcast(found,1,MPIU_BOOL,0,comm);
454: MPI_Bcast(localname, llen, MPI_CHAR, 0, comm);
455: return(0);
456: }