Actual source code: fretrieve.c
petsc-3.14.6 2021-03-30
2: /*
3: Code for opening and closing files.
4: */
5: #include <petscsys.h>
6: #if defined(PETSC_HAVE_PWD_H)
7: #include <pwd.h>
8: #endif
9: #include <ctype.h>
10: #include <sys/stat.h>
11: #if defined(PETSC_HAVE_UNISTD_H)
12: #include <unistd.h>
13: #endif
14: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
15: #include <sys/utsname.h>
16: #endif
17: #include <fcntl.h>
18: #include <time.h>
19: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
20: #include <sys/systeminfo.h>
21: #endif
23: /*
24: Private routine to delete tmp/shared storage
26: This is called by MPI, not by users.
28: Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
30: */
31: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
32: {
36: PetscInfo1(NULL,"Deleting tmp/shared data in an MPI_Comm %ld\n",(long)comm);CHKERRMPI(ierr);
37: PetscFree(count_val);CHKERRMPI(ierr);
38: PetscFunctionReturn(MPI_SUCCESS);
39: }
41: /*@C
42: PetscGetTmp - Gets the name of the tmp directory
44: Collective
46: Input Parameters:
47: + comm - MPI_Communicator that may share /tmp
48: - len - length of string to hold name
50: Output Parameters:
51: . dir - directory name
53: Options Database Keys:
54: + -shared_tmp
55: . -not_shared_tmp
56: - -tmp tmpdir
58: Environmental Variables:
59: + PETSC_SHARED_TMP
60: . PETSC_NOT_SHARED_TMP
61: - PETSC_TMP
63: Level: developer
66: If the environmental variable PETSC_TMP is set it will use this directory
67: as the "/tmp" directory.
69: @*/
70: PetscErrorCode PetscGetTmp(MPI_Comm comm,char dir[],size_t len)
71: {
73: PetscBool flg;
76: PetscOptionsGetenv(comm,"PETSC_TMP",dir,len,&flg);
77: if (!flg) {
78: PetscStrncpy(dir,"/tmp",len);
79: }
80: return(0);
81: }
83: /*@C
84: PetscSharedTmp - Determines if all processors in a communicator share a
85: /tmp or have different ones.
87: Collective
89: Input Parameters:
90: . comm - MPI_Communicator that may share /tmp
92: Output Parameters:
93: . shared - PETSC_TRUE or PETSC_FALSE
95: Options Database Keys:
96: + -shared_tmp
97: . -not_shared_tmp
98: - -tmp tmpdir
100: Environmental Variables:
101: + PETSC_SHARED_TMP
102: . PETSC_NOT_SHARED_TMP
103: - PETSC_TMP
105: Level: developer
107: Notes:
108: Stores the status as a MPI attribute so it does not have
109: to be redetermined each time.
111: Assumes that all processors in a communicator either
112: 1) have a common /tmp or
113: 2) each has a separate /tmp
114: eventually we can write a fancier one that determines which processors
115: share a common /tmp.
117: This will be very slow on runs with a large number of processors since
118: it requires O(p*p) file opens.
120: If the environmental variable PETSC_TMP is set it will use this directory
121: as the "/tmp" directory.
123: @*/
124: PetscErrorCode PetscSharedTmp(MPI_Comm comm,PetscBool *shared)
125: {
126: PetscErrorCode ierr;
127: PetscMPIInt size,rank,*tagvalp,sum,cnt,i;
128: PetscBool flg,iflg;
129: FILE *fd;
130: static PetscMPIInt Petsc_Tmp_keyval = MPI_KEYVAL_INVALID;
131: int err;
134: MPI_Comm_size(comm,&size);
135: if (size == 1) {
136: *shared = PETSC_TRUE;
137: return(0);
138: }
140: PetscOptionsGetenv(comm,"PETSC_SHARED_TMP",NULL,0,&flg);
141: if (flg) {
142: *shared = PETSC_TRUE;
143: return(0);
144: }
146: PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_TMP",NULL,0,&flg);
147: if (flg) {
148: *shared = PETSC_FALSE;
149: return(0);
150: }
152: if (Petsc_Tmp_keyval == MPI_KEYVAL_INVALID) {
153: MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_Tmp_keyval,NULL);
154: }
156: MPI_Comm_get_attr(comm,Petsc_Tmp_keyval,(void**)&tagvalp,(int*)&iflg);
157: if (!iflg) {
158: char filename[PETSC_MAX_PATH_LEN],tmpname[PETSC_MAX_PATH_LEN];
160: /* This communicator does not yet have a shared tmp attribute */
161: PetscMalloc1(1,&tagvalp);
162: MPI_Comm_set_attr(comm,Petsc_Tmp_keyval,tagvalp);
164: PetscOptionsGetenv(comm,"PETSC_TMP",tmpname,238,&iflg);
165: if (!iflg) {
166: PetscStrcpy(filename,"/tmp");
167: } else {
168: PetscStrcpy(filename,tmpname);
169: }
171: PetscStrcat(filename,"/petsctestshared");
172: MPI_Comm_rank(comm,&rank);
174: /* each processor creates a /tmp file and all the later ones check */
175: /* this makes sure no subset of processors is shared */
176: *shared = PETSC_FALSE;
177: for (i=0; i<size-1; i++) {
178: if (rank == i) {
179: fd = fopen(filename,"w");
180: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
181: err = fclose(fd);
182: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
183: }
184: MPI_Barrier(comm);
185: if (rank >= i) {
186: fd = fopen(filename,"r");
187: if (fd) cnt = 1;
188: else cnt = 0;
189: if (fd) {
190: err = fclose(fd);
191: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
192: }
193: } else cnt = 0;
195: MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
196: if (rank == i) unlink(filename);
198: if (sum == size) {
199: *shared = PETSC_TRUE;
200: break;
201: } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share /tmp ");
202: }
203: *tagvalp = (int)*shared;
204: PetscInfo2(NULL,"processors %s %s\n",(*shared) ? "share":"do NOT share",(iflg ? tmpname:"/tmp"));
205: } else *shared = (PetscBool) *tagvalp;
206: return(0);
207: }
209: /*@C
210: PetscSharedWorkingDirectory - Determines if all processors in a communicator share a
211: working directory or have different ones.
213: Collective
215: Input Parameters:
216: . comm - MPI_Communicator that may share working directory
218: Output Parameters:
219: . shared - PETSC_TRUE or PETSC_FALSE
221: Options Database Keys:
222: + -shared_working_directory
223: - -not_shared_working_directory
225: Environmental Variables:
226: + PETSC_SHARED_WORKING_DIRECTORY
227: . PETSC_NOT_SHARED_WORKING_DIRECTORY
229: Level: developer
231: Notes:
232: Stores the status as a MPI attribute so it does not have
233: to be redetermined each time.
235: Assumes that all processors in a communicator either
236: 1) have a common working directory or
237: 2) each has a separate working directory
238: eventually we can write a fancier one that determines which processors
239: share a common working directory.
241: This will be very slow on runs with a large number of processors since
242: it requires O(p*p) file opens.
244: @*/
245: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm,PetscBool *shared)
246: {
247: PetscErrorCode ierr;
248: PetscMPIInt size,rank,*tagvalp,sum,cnt,i;
249: PetscBool flg,iflg;
250: FILE *fd;
251: static PetscMPIInt Petsc_WD_keyval = MPI_KEYVAL_INVALID;
252: int err;
255: MPI_Comm_size(comm,&size);
256: if (size == 1) {
257: *shared = PETSC_TRUE;
258: return(0);
259: }
261: PetscOptionsGetenv(comm,"PETSC_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
262: if (flg) {
263: *shared = PETSC_TRUE;
264: return(0);
265: }
267: PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
268: if (flg) {
269: *shared = PETSC_FALSE;
270: return(0);
271: }
273: if (Petsc_WD_keyval == MPI_KEYVAL_INVALID) {
274: MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_WD_keyval,NULL);
275: }
277: MPI_Comm_get_attr(comm,Petsc_WD_keyval,(void**)&tagvalp,(int*)&iflg);
278: if (!iflg) {
279: char filename[PETSC_MAX_PATH_LEN];
281: /* This communicator does not yet have a shared attribute */
282: PetscMalloc1(1,&tagvalp);
283: MPI_Comm_set_attr(comm,Petsc_WD_keyval,tagvalp);
285: PetscGetWorkingDirectory(filename,240);
286: PetscStrcat(filename,"/petsctestshared");
287: MPI_Comm_rank(comm,&rank);
289: /* each processor creates a file and all the later ones check */
290: /* this makes sure no subset of processors is shared */
291: *shared = PETSC_FALSE;
292: for (i=0; i<size-1; i++) {
293: if (rank == i) {
294: fd = fopen(filename,"w");
295: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
296: err = fclose(fd);
297: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
298: }
299: MPI_Barrier(comm);
300: if (rank >= i) {
301: fd = fopen(filename,"r");
302: if (fd) cnt = 1;
303: else cnt = 0;
304: if (fd) {
305: err = fclose(fd);
306: if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
307: }
308: } else cnt = 0;
310: MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
311: if (rank == i) unlink(filename);
313: if (sum == size) {
314: *shared = PETSC_TRUE;
315: break;
316: } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share working directory");
317: }
318: *tagvalp = (int)*shared;
319: } else *shared = (PetscBool) *tagvalp;
320: PetscInfo1(NULL,"processors %s working directory\n",(*shared) ? "shared" : "do NOT share");
321: return(0);
322: }
325: /*@C
326: PetscFileRetrieve - Obtains a file from a URL or compressed
327: and copies into local disk space as uncompressed.
329: Collective
331: Input Parameter:
332: + comm - processors accessing the file
333: . url - name of file, including entire URL (with or without .gz)
334: - llen - length of localname
336: Output Parameter:
337: + localname - name of local copy of file - valid on only process zero
338: - found - if found or retrieved the file - valid on all processes
340: Notes:
341: if the file already exists local this function just returns without downloading it.
343: Level: intermediate
344: @*/
345: PetscErrorCode PetscFileRetrieve(MPI_Comm comm,const char url[],char localname[],size_t llen,PetscBool *found)
346: {
347: char buffer[PETSC_MAX_PATH_LEN],*par,*tlocalname,name[PETSC_MAX_PATH_LEN];
348: FILE *fp;
350: PetscMPIInt rank;
351: size_t len = 0;
352: PetscBool flg1,flg2,flg3,flg4,download,compressed = PETSC_FALSE;
355: MPI_Comm_rank(comm,&rank);
356: if (!rank) {
357: *found = PETSC_FALSE;
359: PetscStrstr(url,".gz",&par);
360: if (par) {
361: PetscStrlen(par,&len);
362: if (len == 3) compressed = PETSC_TRUE;
363: }
365: PetscStrncmp(url,"ftp://",6,&flg1);
366: PetscStrncmp(url,"http://",7,&flg2);
367: PetscStrncmp(url,"file://",7,&flg3);
368: PetscStrncmp(url,"https://",8,&flg4);
369: download = (PetscBool) (flg1 || flg2 || flg3 || flg4);
371: if (!download && !compressed) {
372: PetscStrncpy(localname,url,llen);
373: PetscTestFile(url,'r',found);
374: if (*found) {
375: PetscInfo1(NULL,"Found file %s\n",url);
376: } else {
377: PetscInfo1(NULL,"Did not find file %s\n",url);
378: }
379: goto done;
380: }
382: /* look for uncompressed file in requested directory */
383: if (compressed) {
384: PetscStrncpy(localname,url,llen);
385: PetscStrstr(localname,".gz",&par);
386: *par = 0; /* remove .gz extension */
387: PetscTestFile(localname,'r',found);
388: if (*found) goto done;
389: }
391: /* look for file in current directory */
392: PetscStrrchr(url,'/',&tlocalname);
393: PetscStrncpy(localname,tlocalname,llen);
394: if (compressed) {
395: PetscStrstr(localname,".gz",&par);
396: *par = 0; /* remove .gz extension */
397: }
398: PetscTestFile(localname,'r',found);
399: if (*found) goto done;
401: if (download) {
402: /* local file is not already here so use curl to get it */
403: PetscStrncpy(localname,tlocalname,llen);
404: PetscStrcpy(buffer,"curl --fail --silent --show-error ");
405: PetscStrcat(buffer,url);
406: PetscStrcat(buffer," > ");
407: PetscStrcat(buffer,localname);
408: #if defined(PETSC_HAVE_POPEN)
409: PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
410: PetscPClose(PETSC_COMM_SELF,fp);
411: #else
412: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
413: #endif
414: PetscTestFile(localname,'r',found);
415: if (*found) {
416: FILE *fd;
417: char buf[1024],*str,*substring;
419: /* check if the file didn't exist so it downloaded an HTML message instead */
420: fd = fopen(localname,"r");
421: if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"PetscTestFile() indicates %s exists but fopen() cannot open it",localname);
422: str = fgets(buf,sizeof(buf)-1,fd);
423: while (str) {
424: PetscStrstr(buf,"<!DOCTYPE html>",&substring);
425: if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
426: PetscStrstr(buf,"Not Found",&substring);
427: if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
428: str = fgets(buf,sizeof(buf)-1,fd);
429: }
430: fclose(fd);
431: }
432: } else if (compressed) {
433: PetscTestFile(url,'r',found);
434: if (!*found) goto done;
435: PetscStrncpy(localname,url,llen);
436: }
437: if (compressed) {
438: PetscStrrchr(localname,'/',&tlocalname);
439: PetscStrncpy(name,tlocalname,PETSC_MAX_PATH_LEN);
440: PetscStrstr(name,".gz",&par);
441: *par = 0; /* remove .gz extension */
442: /* uncompress file */
443: PetscStrcpy(buffer,"gzip -c -d ");
444: PetscStrcat(buffer,localname);
445: PetscStrcat(buffer," > ");
446: PetscStrcat(buffer,name);
447: #if defined(PETSC_HAVE_POPEN)
448: PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
449: PetscPClose(PETSC_COMM_SELF,fp);
450: #else
451: SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
452: #endif
453: PetscStrncpy(localname,name,llen);
454: PetscTestFile(localname,'r',found);
455: }
456: }
457: done:
458: MPI_Bcast(found,1,MPIU_BOOL,0,comm);
459: MPI_Bcast(localname, llen, MPI_CHAR, 0, comm);
460: return(0);
461: }