#===============================================================================
# Copyright 2020-2022 Intel Corporation.
#
# This software and the related documents are Intel copyrighted  materials,  and
# your use of  them is  governed by the  express license  under which  they were
# provided to you (License).  Unless the License provides otherwise, you may not
# use, modify, copy, publish, distribute,  disclose or transmit this software or
# the related documents without Intel's prior written permission.
#
# This software and the related documents  are provided as  is,  with no express
# or implied  warranties,  other  than those  that are  expressly stated  in the
# License.
#===============================================================================

# This file shows how to enable cluster libraries in the MKL::MKL target with ENABLE_CDFT, ENABLE_SCALAPACK, and ENABLE_BLACS options.
# See c_mpi/CMakeLists.txt for how to use specific cluster targets.

cmake_minimum_required(VERSION 3.13)
enable_testing()

# Add cmake scripts and modules to CMake search path
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../cmake")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")

# Generate domainList and ${domain}_funcList
include(generate_examples_list)

# Define what cluster libraries to include to MKL::MKL
if("cdft" IN_LIST domainList OR "fftw3x_cdft" IN_LIST domainList)
  set(ENABLE_CDFT ON)
endif()
if("scalapack" IN_LIST domainList)
  set(ENABLE_SCALAPACK ON)
endif()
set(ENABLE_BLACS ON)

# Define language and compiler
set(TEST_LANG Fortran)
set(TEST_EXT f)
set(DATA_EXT dat)
include(setup_examples)

project(MKL_Examples LANGUAGES ${TEST_LANG} C)

find_package(MKL CONFIG REQUIRED)

# W/A for known problem in Intel(R) MPI Library 2021.1
if(MKL_MPI STREQUAL "intelmpi" AND DEFINED ENV{I_MPI_ROOT})
  if(UNIX AND $ENV{I_MPI_ROOT} MATCHES "2021.1")
    set(MPI_C_ADDITIONAL_INCLUDE_DIRS $ENV{I_MPI_ROOT}/include)
    set(MPI_Fortran_ADDITIONAL_INCLUDE_DIRS $ENV{I_MPI_ROOT}/include)
  endif()
endif()

# Force mshpc to be the first in mpi find package
if(MKL_MPI STREQUAL "msmpi" OR MKL_MPI STREQUAL "mshpc")
  set(MPI_GUESS_LIBRARY_NAME MSMPI)
  # temporary workaround to skip internal fortran mpi testing
  set(MPI_Fortran_WORKS 1)
endif()

find_package(MPI REQUIRED)

# Override default compile/link lines for mpi script
if(USE_MPI_SCRIPT AND WIN32)
  set(CMAKE_C_COMPILE_OBJECT "<CMAKE_C_COMPILER> <DEFINES> <INCLUDES> <FLAGS> /Fo<OBJECT> -c <SOURCE>")
  set(CMAKE_C_LINK_EXECUTABLE "<CMAKE_C_COMPILER> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
  set(CMAKE_Fortran_COMPILE_OBJECT "<CMAKE_Fortran_COMPILER> <DEFINES> <INCLUDES> <FLAGS> /Fo<OBJECT> -c <SOURCE>")
  set(CMAKE_Fortran_LINK_EXECUTABLE "<CMAKE_Fortran_COMPILER> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
endif()

# Try to identify MPI for correct BLACS if MKL_MPI is not defined
if(MPI_${TEST_LANG}_FOUND AND NOT MKL_MPI)
  if(UNIX AND MPI_${TEST_LANG}_COMPILER MATCHES "openmpi")
    set(CURRENT_MPI "openmpi")
  elseif(WIN32 AND MPI_${TEST_LANG}_COMPILER MATCHES "msmpi" OR MPI_${TEST_LANG}_COMPILER MATCHES "Microsoft MPI")
    set(CURRENT_MPI "msmpi")
  elseif(MPI_${TEST_LANG}_COMPILER MATCHES "mpich")
    set(CURRENT_MPI "mpich")
  elseif(MPI_${TEST_LANG}_COMPILER MATCHES ".*intel.*mpi.*")
    set(CURRENT_MPI "intelmpi")
    if(WIN32 AND MPI_LOCALONLY)
      set(MPI_LOCAL_OPT -localonly)
    endif()
  endif()
  set(MKL_MPI "${CURRENT_MPI}")
endif()

# Build MPI mod files for MS HPC
set(MSMPI_LOPT "")
if(MKL_MPI STREQUAL "msmpi" OR MKL_MPI STREQUAL "mshpc")
  find_file(mpi_f90_file "mpi.f90" PATHS ${MPI_${TEST_LANG}_INCLUDE_DIRS})
  add_library(msmpi_mod_files OBJECT "${mpi_f90_file}")
  target_compile_options(msmpi_mod_files PRIVATE "${MPI_${TEST_LANG}_INCLUDE_DIRS}")
  list(APPEND MSMPI_LOPT msmpi_mod_files)
endif()

# fftw3-mpi.f03 uses C_INT
if(MKL_INTERFACE STREQUAL "ilp64")
  list(REMOVE_ITEM domainList "fftw3x_cdft")
endif()

# Define target for each function from each domain
if(domainList)
foreach(domain IN LISTS domainList)
  set(TEST_INCLUDE "${MPI_${TEST_LANG}_MODULE_DIR}" "${MPI_${TEST_LANG}_INCLUDE_DIRS}")
  set(TEST_LOPT "${MSMPI_LOPT}")

  set(TEST_COPT "${MPI_${TEST_LANG}_COMPILE_OPTIONS}")
  set(TEST_COPT77 "")

  file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${domain}_mod")
  set(CMAKE_Fortran_MODULE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${domain}_mod")
  list(APPEND TEST_INCLUDE ${CMAKE_CURRENT_BINARY_DIR}/${domain}_mod)
  list(APPEND TEST_INCLUDE "${PROJECT_SOURCE_DIR}/${domain}/source")

  if(EXISTS ${MKL_INCLUDE}/mkl_${domain}.f90)
    add_library(${domain}_mod_files OBJECT ${MKL_INCLUDE}/mkl_${domain}.f90)
    target_compile_options(${domain}_mod_files PRIVATE ${TEST_COPT} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    list(APPEND TEST_LOPT ${domain}_mod_files)
  endif()

  if(domain STREQUAL "cdft")
    if(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM")
      list(APPEND TEST_COPT -fpp -fixed)
    elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
      list(APPEND TEST_COPT -x f95-cpp-input -ffixed-form)
    endif()
    # Intel(R) MPI Library provides ILP64 MPI Interfaces
    if(MKL_INTERFACE STREQUAL "ilp64" AND MKL_MPI STREQUAL "intelmpi" AND USE_MPI_SCRIPT)
      list(APPEND TEST_COPT -DMPI_KIND_=8)
      list(APPEND TEST_COPT -ilp64)
      list(APPEND TEST_LOPT -ilp64)
    else()
      list(APPEND TEST_COPT -DMPI_KIND_=4)
    endif()

    add_library(dp_cdft_support OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/cdft/source/dp_cdft_example_support.f90)
    add_library(sp_cdft_support OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/cdft/source/sp_cdft_example_support.f90)
    target_include_directories(dp_cdft_support PRIVATE ${TEST_INCLUDE} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    target_include_directories(sp_cdft_support PRIVATE ${TEST_INCLUDE} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    target_compile_options(dp_cdft_support PRIVATE ${TEST_COPT} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    target_compile_options(sp_cdft_support PRIVATE ${TEST_COPT} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    target_link_libraries(dp_cdft_support PRIVATE cdft_mod_files ${MSMPI_LOPT})
    target_link_libraries(sp_cdft_support PRIVATE cdft_mod_files ${MSMPI_LOPT})
    list(APPEND TEST_LOPT dp_cdft_support)
    list(APPEND TEST_LOPT sp_cdft_support)
  endif()
  if(domain STREQUAL "cluster_sparse_solver")
    if(CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM")
      if(UNIX)
        list(APPEND TEST_COPT77 -132)
      else()
        list(APPEND TEST_COPT77 /4L132)
      endif()
    elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
      list(APPEND TEST_COPT77 -ffixed-form -ffixed-line-length-123)
    endif()

    add_library(sparse_blas_mod_files OBJECT ${MKL_INCLUDE}/mkl_spblas.f90)
    if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
      # Suppress C/Fortran interoperability (i.e., C-binding) warning of possible
      # integer size difference between Fortran and C APIs. For correct
      # compile/link line, please use MKLConfig.cmake or oneMKL Link Line Advisor.
      target_compile_options(sparse_blas_mod_files PRIVATE ${TEST_COPT} -Wno-c-binding-type $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    else()
      target_compile_options(sparse_blas_mod_files PRIVATE ${TEST_COPT} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    endif()
    list(APPEND TEST_LOPT sparse_blas_mod_files)
  endif()
  if(domain STREQUAL "fftw3x_cdft")
    if(CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
      list(APPEND TEST_COPT -fno-range-check)
    endif()
    set(objname ${domain}_${MKL_INTERFACE}_obj)
    file(GLOB WRAPPERS_SRC ${MKL_ROOT}/share/mkl/interfaces/${domain}/wrappers/*)
    add_library(${objname}_single OBJECT ${WRAPPERS_SRC})
    add_library(${objname}_double OBJECT ${WRAPPERS_SRC})
    set_property(TARGET ${objname}_single PROPERTY C_STANDARD 99)
    set_property(TARGET ${objname}_double PROPERTY C_STANDARD 99)
    target_compile_options(${objname}_single PRIVATE -DMKL_SINGLE
      $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    target_compile_options(${objname}_double PRIVATE -DMKL_DOUBLE
      $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    target_include_directories(${objname}_single PRIVATE ${MPI_C_INCLUDE_DIRS} ${MKL_INCLUDE}/fftw
      $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    target_include_directories(${objname}_double PRIVATE ${MPI_C_INCLUDE_DIRS} ${MKL_INCLUDE}/fftw
      $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    list(APPEND TEST_LOPT ${objname}_single ${objname}_double)
    list(APPEND TEST_INCLUDE ${MKL_INCLUDE}/fftw)
  endif()
  if(domain STREQUAL "scalapack")
    if(UNIX AND CMAKE_Fortran_COMPILER_ID STREQUAL "Intel" OR CMAKE_Fortran_COMPILER_ID STREQUAL "IntelLLVM")
      list(APPEND TEST_COPT -nogen-interfaces)
    endif()

    if(MKL_INTERFACE STREQUAL "ilp64" AND MKL_MPI STREQUAL "intelmpi" AND USE_MPI_SCRIPT)
      list(APPEND TEST_COPT -DMPI_KIND_=8)
      list(APPEND TEST_COPT -ilp64)
      list(APPEND TEST_LOPT -ilp64)
    else()
      list(APPEND TEST_COPT -DMPI_KIND_=4)
    endif()

    file(GLOB COMMON_SRC ${PROJECT_SOURCE_DIR}/${domain}/*/common/*.f)
    add_library(${domain}_common OBJECT ${COMMON_SRC})
    target_compile_options(${domain}_common PRIVATE ${TEST_COPT} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    list(APPEND TEST_LOPT ${domain}_common)
  endif()

# Build target for each example
  message(STATUS "Functions list ${domain}: ${${domain}_funcList}")
  foreach(func IN LISTS ${domain}_funcList)
    string(STRIP ${func} func_name)
    set(func "${domain}-${func_name}")

    file(GLOB_RECURSE ${domain}_${func}_SRC ${PROJECT_SOURCE_DIR}/${domain}/*/${func_name}.${TEST_EXT}*)
    if(NOT ${domain}_${func}_SRC)
      message(FATAL_ERROR "${domain} source file ${func_name}.${TEST_EXT}* was not found")
    endif()

    add_executable(${func} ${${domain}_${func}_SRC})
    target_include_directories(${func} PUBLIC ${TEST_INCLUDE} ${MKL_INCLUDE})
    if(${${domain}_${func}_SRC} MATCHES ".f$")
      target_compile_options(${func} PUBLIC ${TEST_COPT} ${TEST_COPT77} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    else()
      target_compile_options(${func} PUBLIC ${TEST_COPT} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    endif()
    target_link_libraries(${func} PUBLIC ${TEST_LOPT} $<LINK_ONLY:MKL::MKL> MPI::MPI_C MPI::MPI_Fortran)

    # Register example as ctest
    if(domain STREQUAL "pblas")
      configure_file(${PROJECT_SOURCE_DIR}/${domain}/data/${func_name}.in ${CMAKE_CURRENT_BINARY_DIR} COPYONLY)
    endif()
    file(GLOB_RECURSE ${domain}_${func_name}_DATA ${PROJECT_SOURCE_DIR}/${domain}/*/${func_name}.${DATA_EXT})
    if(EXISTS ${${domain}_${func_name}_DATA})
      if(domain STREQUAL "scalapack")
        add_test(NAME ${func} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPI_LOCAL_OPT} ${PROJECT_BINARY_DIR}/${func} ${${domain}_${func_name}_DATA} ${PROJECT_BINARY_DIR}/${func}.res)
      else()
        add_test(NAME ${func} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPI_LOCAL_OPT} ${PROJECT_BINARY_DIR}/${func} ${${domain}_${func_name}_DATA} )
      endif()
    else()
      add_test(NAME ${func} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPI_LOCAL_OPT} ${PROJECT_BINARY_DIR}/${func})
    endif()

    # Add Environment variables
    if(MKL_ENV)
      set_tests_properties(${func} PROPERTIES ENVIRONMENT "${MKL_ENV}")
    endif()
  endforeach() #${domain}_funcList
endforeach() #domainList
endif() #not empty domainList
