cloudsc.cmake

Go to the documentation of this file.

# Copyright (c) 2020 - 2025 David Guibert
# Copyright (c) 2024 - 2025 Antoine Morvan
# Copyright (c) 2024 - 2025 Niclas Schroeter
# All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
 
option(ENABLE_cloudsc_single_precision "Build CLOUDSC in single precision" OFF)
set(cloudsc_block_size "" CACHE STRING "The block size argument for CLOUDSC")
 
if(cloudsc_block_size STREQUAL "")
  if(ENABLE_cloudsc_single_precision)
    set(cloudsc_block_size "32")
  else()
    set(cloudsc_block_size "16")
  endif()
endif()
 
option(ENABLE_cloudsc_gpu "Enable cloudsc gpu prototypes" OFF)
cmake_dependent_option(ENABLE_cloudsc_acc "Enable cloudsc acc components" ON "ENABLE_cloudsc_gpu" OFF)
option(ENABLE_cloudsc_omp "Enable cloudsc openMP components" ON)
cmake_dependent_option(ENABLE_cloudsc_omp_gpu "Enable cloudsc openMP components (gpu)" ON "ENABLE_cloudsc_gpu" OFF)
set(cloudsc_omp OFF)
if(ENABLE_cloudsc_omp)
  set(cloudsc_omp ON)
endif()
if(ENABLE_cloudsc_omp_gpu)
  set(cloudsc_omp ON)
endif()
cmake_dependent_option(ENABLE_cloudsc_cuda "Enable cloudsc cuda components" ON "ENABLE_cloudsc_gpu" OFF)
cmake_dependent_option(ENABLE_cloudsc_hip "Enable cloudsc hip components" ON "ENABLE_cloudsc_gpu" OFF)
 
set_property(GLOBAL PROPERTY cloudsc_can_use_system TRUE)
set_property(GLOBAL PROPERTY cloudsc_depends ecbuild hdf5)
set_property(GLOBAL PROPERTY cloudsc_depends_optional mpi)
if(ENABLE_cloudsc_gpu)
  set_property(GLOBAL APPEND PROPERTY cloudsc_depends cuda)
endif()
 
set(cloudsc_cmake_args "" CACHE STRING "Custom CMake arguments for CloudSC")
set(cloudsc_build_args "" CACHE STRING "Custom Make arguments for CloudSC")
 
if(cloudsc_enabled)
  if(USE_SYSTEM_cloudsc)
 
    message(STATUS "cloudsc: enabled (system installed)")
 
    function(find_cloudsc_program BIN)
      find_program(CLOUDSC_EXE ${BIN} REQUIRED)
      get_filename_component(CLOUDSC_EXE_DIR "${CLOUDSC_EXE}" DIRECTORY)
      set_if_not_present(ENV{PATH} "$ENV{PATH}" "${CLOUDSC_EXE_DIR}")
    endfunction()
 
    find_cloudsc_program("dwarf-cloudsc-fortran")
 
    if(ENABLE_cloudsc_acc)
      find_cloudsc_program("dwarf-cloudsc-gpu-scc-k-caching")
    endif()
    if(ENABLE_cloudsc_omp_gpu)
      find_cloudsc_program("dwarf-cloudsc-gpu-omp-scc-hoist")
    endif()
    if(ENABLE_cloudsc_cuda)
      find_cloudsc_program("dwarf-cloudsc-c-cuda-k-caching")
    endif()
    if(ENABLE_cloudsc_hip)
      find_cloudsc_program("dwarf-cloudsc-hip-k-caching")
    endif()
 
    # dummy cloudsc external project to get tests
    ExternalProject_Data_Add(
      cloudsc
      ${cloudsc_revision}
      CONFIGURE_COMMAND ""
      BUILD_COMMAND ""
      INSTALL_COMMAND
        ${CMAKE_COMMAND} -E copy_directory <SOURCE_DIR>/config-files
        <INSTALL_DIR>/share/dwarf-p-cloudsc
      COMMAND ${CMAKE_COMMAND} -E make_directory
              <BINARY_DIR>/test-dwarf-p-cloudsc)
 
    ExternalProject_Get_Property(cloudsc INSTALL_DIR)
    ExternalProject_Get_Property(cloudsc BINARY_DIR)
    set(cloudsc_DIR ${INSTALL_DIR})
    set(cloudsc_INPUT_DIR ${cloudsc_DIR}/share/dwarf-p-cloudsc)
    set(cloudsc_BIN_DIR ${BINARY_DIR})
 
  else()
    message(STATUS "cloudsc: enabled (internally built)")
    set(cloudsc_depends_ep ecbuild)
    if(USE_SYSTEM_hdf5)
      list(APPEND cloudsc_depends_ep hdf5::hdf5)
    else()
      list(APPEND cloudsc_depends_ep hdf5)
    endif()
    if(mpi_enabled)
      list(APPEND cloudsc_depends_ep MPI::MPI_C MPI::MPI_Fortran)
    endif()
 
    ExternalProject_Data_Add(
      cloudsc
      ${cloudsc_revision}
      DEPENDS ${cloudsc_depends_ep}
 
      # execute 'true' instead of the patch command when condition is true
      PATCH_COMMAND $<IF:$<NOT:$<BOOL:${ENABLE_cloudsc_omp}>>,true,${Patch_EXECUTABLE} -p1 -i ${CMAKE_SOURCE_DIR}/projects/cloudsc_omp_link.patch>
      COMMAND       $<IF:$<BOOL:${ENABLE_cloudsc_omp_gpu}>,true,${Patch_EXECUTABLE} -p1 -i ${CMAKE_SOURCE_DIR}/projects/cloudsc_no_omp_target.patch>
 
      CONFIGURE_COMMAND env
        PATH=$ENV{PATH} CPATH=$ENV{CPATH} C_INCLUDE_PATH=$ENV{C_INCLUDE_PATH}
        LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} LIBRARY_PATH=$ENV{LIBRARY_PATH}
        PKG_CONFIG_PATH=<INSTALL_DIR>/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}
        CC=${MPI_C_COMPILER} CXX=${MPI_CXX_COMPILER} FC=${MPI_Fortran_COMPILER}
        ecbuild -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER}
        -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER}
        -DCMAKE_Fortran_COMPILER:FILEPATH=${CMAKE_Fortran_COMPILER}
        -DCMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS}
        -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS}
        -DCMAKE_INSTALL_PREFIX:FILEPATH=<INSTALL_DIR>
 
        -DENABLE_CLOUDSC_C=OFF
        -DENABLE_CLOUDSC_FORTRAN=ON
        # OpenACC
        -DENABLE_ACC:BOOL=${ENABLE_cloudsc_acc}
        -DENABLE_CLOUDSC_GPU_SCC_K_CACHING=${ENABLE_cloudsc_acc}
        # OpenMP
        -DENABLE_OMP:BOOL=${cloudsc_omp}
        -DENABLE_CLOUDSC_GPU_OMP_SCC_HOIST=${ENABLE_cloudsc_omp_gpu}
        # Cuda
        -DENABLE_CUDA:BOOL=${ENABLE_cloudsc_cuda}
        -DENABLE_CLOUDSC_C_CUDA=${ENABLE_cloudsc_cuda}
        # HIP
        -DENABLE_HIP:BOOL=${ENABLE_cloudsc_hip}
        -DENABLE_CLOUDSC_HIP:BOOL=${ENABLE_cloudsc_hip}
 
        -DENABLE_MPI:BOOL=${mpi_enabled}
        -DENABLE_SINGLE_PRECISION=${ENABLE_cloudsc_single_precision}
 
        -DCMAKE_POLICY_DEFAULT_CMP0104=NEW # initialize CMAKE_CUDA_ARCHITECTURES https://cmake.org/cmake/help/latest/policy/CMP0104.html
        ${default_cmake_args} ${cloudsc_cmake_args}
        <SOURCE_DIR>
      BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} -j${BUILD_PARALLEL_LEVEL} ${default_build_args} ${cloudsc_build_args}
      INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} -j${BUILD_PARALLEL_LEVEL} install
      COMMAND ${CMAKE_COMMAND} -E copy_directory <SOURCE_DIR>/config-files
              <INSTALL_DIR>/share/dwarf-p-cloudsc
      COMMAND ${CMAKE_COMMAND} -E make_directory
              <BINARY_DIR>/test-dwarf-p-cloudsc)
 
    ExternalProject_Get_Property(cloudsc INSTALL_DIR)
    set(cloudsc_DIR ${INSTALL_DIR})
    set(cloudsc_INPUT_DIR ${cloudsc_DIR}/share/dwarf-p-cloudsc)
 
    set_if_not_present(ENV{PATH} "$ENV{PATH}" "${cloudsc_DIR}/bin")
    set_if_not_present(
      ENV{LD_LIBRARY_PATH} "$ENV{LD_LIBRARY_PATH}" "${cloudsc_DIR}/lib"
      "${cloudsc_DIR}/lib64")
 
    ExternalProject_Get_Property(cloudsc BINARY_DIR)
    set(cloudsc_BIN_DIR ${BINARY_DIR})
  endif()
 
  # common for all tests
  add_test(
    NAME cloudsc-stage-in
    COMMAND $<TARGET_FILE:job_launcher> --name=cloudsc-stage-in
      --log-dir=${LOG_DIR}
      --
      cp ${cloudsc_INPUT_DIR}/{reference.h5,input.h5} .
    WORKING_DIRECTORY ${cloudsc_BIN_DIR}/test-dwarf-p-cloudsc)
 
  set_tests_properties(cloudsc-stage-in PROPERTIES FIXTURES_SETUP _setup-cloudsc)
 
  function(link_cloudsc_tests testname validation)
    set_tests_properties(${testname} PROPERTIES FIXTURES_REQUIRED _setup-cloudsc)
    if(TEST ${validation})
      set_tests_properties(${testname} PROPERTIES FIXTURES_SETUP _run-${testname})
      set_tests_properties(${validation} PROPERTIES FIXTURES_REQUIRED _run-${testname})
    endif()
  endfunction()
 
  function(add_cloudsc_test testname executable args)
    add_test(
      NAME cloudsc-${testname}
      COMMAND
        env CPATH=$ENV{CPATH} C_INCLUDE_PATH=$ENV{C_INCLUDE_PATH}
        PATH=$ENV{PATH} LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}
        LIBRARY_PATH=$ENV{LIBRARY_PATH} $<TARGET_FILE:job_launcher>
        --name=cloudsc-${testname}
        --log-dir=${LOG_DIR}
        --
        ${executable} ${args}
      WORKING_DIRECTORY ${cloudsc_BIN_DIR}/test-dwarf-p-cloudsc)
 
    # The validation is expected to fail for single precision due to a lack
    # of references in CLOUDSC itself.
    if(NOT ENABLE_cloudsc_single_precision)
      add_test(
        NAME cloudsc-${testname}-validation
        COMMAND
        $<TARGET_FILE:job_launcher> --name=cloudsc-${testname}-validation
        --log-dir=${LOG_DIR}
        --
        ${CMAKE_SOURCE_DIR}/validation/cloudsc.sh
        ${LOG_DIR}/cloudsc-${testname}.log
        WORKING_DIRECTORY ${cloudsc_BIN_DIR}/test-dwarf-p-cloudsc)
    endif()
 
    link_cloudsc_tests(cloudsc-${testname}
      cloudsc-${testname}-validation)
  endfunction()
 
  # the first parameter is the number of threads (need to be adapted in the job-launcher)
  add_cloudsc_test(fortran-small dwarf-cloudsc-fortran "\${OMP_NUM_THREADS-1} 16384 ${cloudsc_block_size}")
  add_cloudsc_test(fortran-medium dwarf-cloudsc-fortran "\${OMP_NUM_THREADS-1} 131072 ${cloudsc_block_size}")
  add_cloudsc_test(fortran-big dwarf-cloudsc-fortran "\${OMP_NUM_THREADS-1} 524288 ${cloudsc_block_size}")
 
  # OpenACC
  if(ENABLE_cloudsc_acc)
    add_cloudsc_test(gpu-acc dwarf-cloudsc-gpu-scc-k-caching "1 163840 128")
  endif()
  # OpenMP
  if(ENABLE_cloudsc_omp_gpu)
    add_cloudsc_test(gpu-omp dwarf-cloudsc-gpu-omp-scc-hoist "1 163840 128")
  endif()
  # Cuda
  if(ENABLE_cloudsc_cuda)
    add_cloudsc_test(gpu-cuda dwarf-cloudsc-c-cuda-k-caching "1 163840 128")
  endif()
  # HIP
  if(ENABLE_cloudsc_hip)
    add_cloudsc_test(gpu-hip dwarf-cloudsc-hip-k-caching "1 163840 128")
  endif()
 
  if(mpi_enabled)
    add_cloudsc_test(mpi-nproma32 dwarf-cloudsc-fortran "\${OMP_NUM_THREADS-1} 2097152 32")
    add_cloudsc_test(mpi-nproma64 dwarf-cloudsc-fortran "\${OMP_NUM_THREADS-1} 2097152 64")
    add_cloudsc_test(mpi-nproma128 dwarf-cloudsc-fortran "\${OMP_NUM_THREADS-1} 2097152 128")
    add_cloudsc_test(mpi-nproma256 dwarf-cloudsc-fortran "\${OMP_NUM_THREADS-1} 2097152 256")
  endif()
 
endif(cloudsc_enabled)