# -*- mode: makefile-gmake -*-
#
#    Contains rules for building PETSc application codes (not PETSc libraries).
#      * for example, make ex1
#      * included only in makefiles in directories that can build executables such as /tutorials, /tests, and their subdirectores
#      * requires GNUmake
#
# It may be included in users makefiles (with include ${PETSC_DIR}/lib/petsc/conf/rules)
# The makefile will then compile executables that link against the PETSc libraries
# See https://petsc.org/release/docs/manual/getting_started/#writing-c-c-or-fortran-applications

.SUFFIXES: .F .F90 .f90 ${SUFFIXES} .PETSc .C .cc .cpp .cxx .r .rm .so .html .ad .m .F95 .f95 .fiat .cu .hip.cpp .kokkos.cxx .raja.cxx *.sycl.cxx

clean-legacy:
	@-${RM} ${CLEANFILES} ${TESTS} *.o *.lo *~ \
               ex[0-9] ex[0-9][0-9] ex[0-9][0-9][0-9] \
               ex[0-9]f ex[0-9][0-9]f ex[0-9][0-9][0-9]f \
               ex[0-9]k ex[0-9][0-9]k ex[0-9][0-9][0-9]k \
               ex[0-9]f90 ex[0-9][0-9]f90 ex[0-9][0-9][0-9]f90 \
               ex[0-9]cu ex[0-9][0-9]cu ex[0-9][0-9][0-9]cu \
               ex[0-9].exe ex[0-9][0-9].exe ex[0-9][0-9][0-9].exe \
               ex[0-9]f.exe ex[0-9][0-9]f.exe ex[0-9][0-9][0-9]f.exe \
               ex[0-9]f90.exe ex[0-9][0-9]f90.exe ex[0-9][0-9][0-9]f90.exe \
               ex[0-9]cu.exe ex[0-9][0-9]cu.exe ex[0-9][0-9][0-9]cu.exe \
               ex[0-9]hip ex[0-9][0-9]hip ex[0-9][0-9][0-9]hip \
               ex[0-9]hip.exe ex[0-9][0-9]hip.exe ex[0-9][0-9][0-9]hip.exe \
               ex[0-9]sycl ex[0-9][0-9]sycl ex[0-9][0-9][0-9]sycl \
               ex[0-9]sycl.exe ex[0-9][0-9]sycl.exe ex[0-9][0-9][0-9]sycl.exe \
              PI* *.ln l.outa* mputil.mp_* core core.* *.tmp *.map gmon.out *.gcov.html \
              trashz \#*\# *.mex* *.stolen *.trace Log.* *.stolen \
              output/*~ .mpirtmp mon.out *.aus *.mon.* p4pg ins10*.c \
               *.cp_ *.cp__ *.c*.c \
               *.dep *.proj ctoatmp PETScArena* *.L *.anl *.mod .mpi* *.d \
              *.class *.ouit *.ad.* g_* silly.cmp *.tmp.* *.ilk *.pdb *.inst.c *.rej *.gcda *.gcno
	@-${RM} -rf ${CLEANDIRS} *.dSYM AD_cache SunWS_cache

clean:: clean-legacy

#
#  These rules are for compiling the test examples.
#
.cpp.rm .cxx.rm .cc.rm .C.rm .F.rm .F90.rm .f.rm .c.rm .cu.rm:
	-@${RM} $* *.o $*.mon.* gmon.out mon.out
	-@${RM} *.exe *.ilk *.pdb *.tds
.cu.PETSc .hip.cpp.PETSc .kokkos.cxx.PETSc .c.PETSc .cxx.PETSc:
	-+@${OMAKE}  PETSC_ARCH=${PETSC_ARCH}  $* > trashz 2>&1
	-@grep -v clog trashz | grep -v "information sections" | \
          grep -v "warning C4003: not enough actual parameters for macro 'PETSC_PASTE3_" | \
          grep -v "(aka 'long \*') doesn't match specified 'MPI' type tag that requires 'long long \*'" | \
          grep -v "note: expanded from macro" |\
          grep -v "MPI_" | \
          grep -v "warnings generated" | \
          grep -v "WARNING: TOC" | \
          grep -v "D4024 : unrecognized" | \
          grep -v "tentative definition of size" | \
          grep -v "Extra instructions" | \
          grep -v "Unused external reference" | \
          grep -v "Warning: attribute unused is unsupported and will be skipped" | \
          grep -v "f90 continuing despite warning messages" | \
          grep -v "symbol if the" | \
          grep -v "ignoring symbol version info" | \
          grep -v "warning: initializer element is not computable at load time" | \
          grep -v "warning: ISO C90 forbids mixed declarations and code" | \
          grep -v "warning: ISO C90 does not support 'static' or type qualifiers in parameter array declarators" | \
          grep -v "warning, duplicate dylib" | \
          grep -v "warning: duplicate dylib" | \
          grep -v "preempts that definition" | \
          grep -v "is an array from" | \
         grep -v "At least one PA 2.0" | \
          grep -v "Cannot cast" | \
          grep -v "WARNING 134: weak definition of" | \
          grep -v "Warning(s) detected" | \
          grep -v "object file assumed" | \
          grep -v "consider using mkstemp"  |\
          grep -v EXTERNAL  |\
          grep -v "warning prebinding disabled"  |\
          grep -v volatile  |\
          grep -v -i inconsistent |\
          grep -v Anachronism | \
          grep -v "/opt/ibmcmp/xlsmp/1.3/lib" | \
          grep -v "add line info to anonymous symbol" | \
          grep -v "/opt/ibmcmp/xlsmp/1.3/../.." | \
          grep -v "IPO Error: unresolved" | \
         grep -v "is being replaced by a real definition" | \
          grep -v "may result in errors or" | \
          grep -v "is deprecated" | \
          grep -v " -Werror" | \
          grep -v " was built for newer macOS version " | \
          grep -v "duplicate -rpath" | \
          grep -v "only the last is used because nvcc can only accept a single optimization setting" | \
         grep -E -i '(Error|warning|Can|Unresolved)' >> /dev/null;\
         if [ "$$?" != 1 ]; then \
          printf ${PETSC_TEXT_HILIGHT}"*******************Error detected during compile or link!*******************\n";\
          echo "See https://petsc.org/release/faq/";\
          echo ${PWD} $* ;\
          printf "*********************************************************************************"${PETSC_TEXT_NORMAL}"\n" ;\
         cat trashz ; fi; ${RM} trashz

.F.PETSc .F90.PETSc:
	-+@${OMAKE}  PETSC_ARCH=${PETSC_ARCH}  $* > trashz 2>&1
	-@grep -v EXTERNAL trashz | grep -v Wall | \
          grep -v "warning: In-place macro substitution leaves line truncated" | \
          grep -v "Warning: Same actual argument associated with INTENT(IN) argument 'errorcode' and INTENT(OUT) argument 'ierror' at (1)" | \
          grep -v "Unused external reference" | \
          grep -v "D4024 : unrecognized" | \
          grep -v "WARNING: TOC overflow." | \
          grep -v "Extra instructions are being" | \
          grep -v "tentative definition of size" | \
          grep -v "symbol if the symbol" | \
          grep -v -i inconsistent | \
          grep -v -i "unused dummy" | \
          grep -v "alignment lost in merging tentative definition" | \
         grep -v "WARNING:  -cpp is ignored" | \
          grep -v "ignoring symbol version info" | \
         grep -v "At least one PA 2.0" | \
         grep -v "Inconsistent structure" | \
          grep -v "object file assumed" | \
         grep -v "ex20.F:30:" | \
         grep -v "ex20f.F:31: warning" | \
         grep -v "f90 continuing despite warning messages" | \
          grep -v "is an array from" | \
          grep -v "warning, duplicate dylib" | \
          grep -v "warning: duplicate dylib" | \
          grep -v "consider using mkstemp"  |\
          grep -v "Nonconforming tab character"  |\
         grep -v "Unused external reference" | \
          grep -v "WARNING 134: weak definition of" | \
          grep -v 'continuing despite warning messages' | \
          grep -v "add line info to anonymous symbol" | \
          grep -v "warning prebinding disabled"  |\
          grep -v "ex20f.F:34: warning:" | \
         grep -v "Unused dummy argument" | \
         grep -v "is being replaced by a real definition" | \
          grep -v "IPO Error: unresolved" | \
          grep -v "warning multiple definitions of symbol _matdensegetarray_" | \
          grep -v " -Werror" | \
          grep -v " was built for newer macOS version " | \
          grep -v "duplicate -rpath" | \
         grep -E -i '(Error|warning|Can|Unresolved)'  >> /dev/null ; \
         if [ "$$?" != 1 ]; then \
          printf ${PETSC_TEXT_HILIGHT}"*******************Error detected during compile or link!*******************\n";\
          echo "See https://petsc.org/release/faq/";\
          echo ${PWD} $* ;\
          printf "*********************************************************"${PETSC_TEXT_NORMAL}"\n" ;\
         cat trashz ; fi; ${RM} trashz;

.c.o:
	${PETSC_COMPILE_SINGLE} $<

.cpp.o .cxx.o .cc.o .C.o:
	${PETSC_CXXCOMPILE_SINGLE} $<

.cu.o:
	${PETSC_CUCOMPILE_SINGLE} $<

.hip.cpp.o:
	${PETSC_HIPCOMPILE_SINGLE} $<

.kokkos.cxx.o:
	${PETSC_KOKKOSCOMPILE_SINGLE} $<

.sycl.cxx.o:
	${PETSC_SYCLCOMPILE_SINGLE} $<

.raja.cxx.o:
	${PETSC_RAJACOMPILE_SINGLE} $<

.F.o .F90.o .F95.o:
	${PETSC_FCOMPILE_SINGLE} $<

.f.o .f90.o .f95.o:
	${FC} -c ${FC_FLAGS} ${FFLAGS} -o $@ $<

ifeq ($(RM),true)
.SECONDARY: $(%.o) $(%f.o) $(%f90.o)
endif

LINK.c = $(CLINKER) $(CCPPFLAGS) $(LDFLAGS) $(EXEFLAGS)
LINK.o = $(CLINKER) $(LDFLAGS) $(EXEFLAGS)
LINK.F = $(FLINKER) $(FCPPFLAGS) $(LDFLAGS)
LINK.cc = $(CXXLINKER) $(CXXCPPFLAGS) $(LDFLAGS)

# make macros KOKKOS_USE_CUDA_COMPILER etc are defined in `${PETSC_DIR}/${PETSC_ARCH}/lib/petsc/conf/petscvariables`
ifneq ($(KOKKOS_USE_CUDA_COMPILER),)
  # Kokkos requires nvcc to be in PATH and the C++ compiler to be given in an environmental variable
  # We prefer to use the Kokkos nvcc_wrapper compiler wrapper for compiling Kokkos files. The command is adapted from PETSC_COMPILE.kokkos.cxx
  # in gmakefile.test, with additional "-o $*.o" to generate intended foo.o instead of foo.kokkos.o
  KOKKOS_COMPILE_SINGLE = PATH=`dirname $(CUDAC)`:$(PATH) NVCC_WRAPPER_DEFAULT_COMPILER="$(CUDA_CXX)" $(KOKKOS_BIN)/nvcc_wrapper --expt-extended-lambda -c $(CUDAC_FLAGS) ${PETSC_CCPPFLAGS} $(CUDACPPFLAGS) $(CUDA_CXXFLAGS) $(MPICXX_INCLUDES) -o $*.o
  KOKKOS_LINK = PATH=`dirname $(CUDAC)`:$(PATH) NVCC_WRAPPER_DEFAULT_COMPILER=$(CUDA_CXX) $(KOKKOS_BIN)/nvcc_wrapper --expt-extended-lambda $(CUDAC_FLAGS) $(MPICXX_INCLUDES) ${PETSC_CCPPFLAGS} $(CUDA_CXXFLAGS) $(CUDACPPFLAGS) $(CUDA_CXXLIBS) $(MPICXX_LIBS) $(LDFLAGS)
else ifneq ($(KOKKOS_USE_HIP_COMPILER),)
  KOKKOS_COMPILE_SINGLE = ${PETSC_HIPCOMPILE_SINGLE}
  KOKKOS_LINK = $(HIPC) $(HIPC_FLAGS) $(HIPPP_FLAGS) $(HIPOPTFLAGS) $(CXXFLAGS) $(CXXCPPFLAGS) $(LDFLAGS) $(MPICXX_INCLUDES) $(MPICXX_LIBS)
else ifneq ($(KOKKOS_USE_SYCL_COMPILER),)
  KOKKOS_COMPILE_SINGLE = ${PETSC_SYCLCOMPILE_SINGLE}
  KOKKOS_LINK = $(SYCLC) $(SYCLC_FLAGS) $(SYCLC_LINKER_FLAGS) $(SYCLOPTFLAGS) $(CXXFLAGS) $(CXXCPPFLAGS) $(SYCLPP_FLAGS) $(LDFLAGS) $(MPICXX_INCLUDES) $(MPICXX_LIBS)
else
  KOKKOS_COMPILE_SINGLE = ${PETSC_CXXCOMPILE_SINGLE}
  KOKKOS_LINK = $(CXXLINKER) $(CXX_FLAGS) $(CXXFLAGS) $(CXXCPPFLAGS) $(LDFLAGS) $(MPICXX_INCLUDES) $(MPICXX_LIBS)
endif

# Why do we filter out -fvisibility=hidden? See comments in gmakefile.test on PETSC_COMPILE.kokkos.cxx
PETSC_KOKKOSCOMPILE_SINGLE = $(filter-out -fvisibility=hidden,$(subst -Xcompiler -fvisibility=hidden ,,$(strip $(KOKKOS_COMPILE_SINGLE))))
LINK.kokkos.cxx = $(filter-out -fvisibility=hidden,$(subst -Xcompiler -fvisibility=hidden ,,$(strip $(KOKKOS_LINK))))

LINK.hip.cpp = $(HIPC) $(HIPC_FLAGS) $(HIPPP_FLAGS) $(HIPOPTFLAGS) $(CXXFLAGS) $(CXXCPPFLAGS) $(LDFLAGS) $(MPICXX_INCLUDES) $(MPICXX_LIBS)

ifneq ($(RAJA_USE_CUDA_COMPILER),)
  PETSC_RAJACOMPILE_SINGLE  = ${CUDAC} -o $*.o $(MPICXX_INCLUDES) ${CUDAC_FLAGS} -x cu -Xcudafe "--display_error_number" -c --compiler-options="${PETSC_CCPPFLAGS} ${PETSC_CC_INCLUDES} ${CUDA_CXXFLAGS}" --expt-extended-lambda --expt-relaxed-constexpr
else
    PETSC_RAJACOMPILE_SINGLE = ${CXX} -o $*.o -c ${CXX_FLAGS} ${CXXFLAGS} ${CXXCPPFLAGS}
endif

LDLIBS += $(PRELIBS) $(PETSC_LIB) $(LIBS)

% : %.c
	$(LINK.c) $^ $(LOADLIBES) $(LDLIBS) -o $@
% : %.F90
	$(LINK.F) $^ $(LOADLIBES) $(LDLIBS) -o $@
% : %.kokkos.cxx
	$(LINK.kokkos.cxx) $^ $(LOADLIBES) $(LDLIBS) -o $@
% : %.cxx
	$(LINK.cc) $^ $(LOADLIBES) $(LDLIBS) -o $@
% : %.hip.cpp
	$(LINK.hip.cpp) $^ $(LOADLIBES) $(LDLIBS) -o $@

# Shall we compile *.c to *.o and then link *.o to executables in two separate steps?
SEPARATE_COMPILE_LINK =

# See MR !2750
ifneq (,$(findstring -TP,$(PCC_FLAGS)))
  SEPARATE_COMPILE_LINK = 1
endif

# To avoid warnings when compiling *.c with sycl compilers
#   icx -fsycl -o ex1 ex1.c ..
#   icx: warning: treating 'c' input as 'c++' when -fsycl is used [-Wexpected-file-type]
ifneq (,$(findstring -fsycl,$(PCC_LINKER_FLAGS)))
  SEPARATE_COMPILE_LINK = 1
endif

ifdef SEPARATE_COMPILE_LINK
% : %.c
	$(subst $@, $@.o, ${PETSC_COMPILE_SINGLE}) $<
	${CLINKER} -o $@ $@.o $(filter-out $@.o $<,$^) $(LOADLIBES) $(LDLIBS)
endif

# We don't have % : %.cu target as we can't use nvcc as linker - due to difference in linker options [wrt CLINKER etc.]. For example, nvcc does not accept -Wl,-rpath
# % : %.cu

#  makes an Emscripten https://emscripten.org/ executable from a PETSc C main program
#  See config/examples/arch-ci-linux-emcc.py for an example of building the PETSc libraries with Emscripten
#  node drops a variety of silly crums that are not all trivially blockable
#
ifneq (,$(findstring emcc,$(CC)))
% : %.c
	$(subst $@, $@.o, ${PETSC_COMPILE_SINGLE}) $<
	${CLINKER} -s MAIN_MODULE -s ASSERTIONS=2 -s SAFE_HEAP=1 -s STACK_OVERFLOW_CHECK=1 -s ALLOW_MEMORY_GROWTH $(EXEFLAGS) -o $@.js $@.o $(PETSC_LIB)
	-@printf '#!/usr/bin/env sh\nnode --redirect-warnings=/dev/null $@.js $$* | grep -v "Heap resize call from" ' > $@
endif

help-applications:
	-@echo ""
	-@echo "make applicationname - compiles applicationname.[c,cxx,F,F90] file into an executable"
	-@echo "make clean - delete any object files and executables in the current directory"
	-@echo "make libs - rebuilds any needed changes to the PETSc libraries"
	-@echo ""

include ${PETSC_DIR}/lib/petsc/conf/rules_doc.mk
