ectrans.cmake

Go to the documentation of this file.

# Copyright (c) 2020 - 2024 David Guibert
# Copyright (c) 2024 Sam Hatfield
# Copyright (c) 2024 Antoine Morvan
# Copyright (c) 2024 - 2025 Niclas Schroeter
# All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
 
set_property(GLOBAL PROPERTY ectrans_can_use_system TRUE)
set_property(GLOBAL PROPERTY ectrans_depends mpi ecbuild fiat)
 
option(ENABLE_ectrans_gpu "Enable ectrans GPU benchmarks" OFF)
option(ENABLE_ectrans_cpu "Enable ectrans CPU benchmarks" ON)
set(ectrans_cmake_args "" CACHE STRING "Custom CMake arguments for ectrans")
set(ectrans_build_args "" CACHE STRING "Custom Make arguments for ectrans")
 
if(ENABLE_ectrans_cpu)
  set_property(GLOBAL APPEND PROPERTY ectrans_depends openblas fftw)
endif()
 
if(ENABLE_ectrans_gpu)
  set_property(GLOBAL APPEND PROPERTY ectrans_depends cuda)
endif()
 
if(ectrans_enabled)
  if(USE_SYSTEM_ectrans)
    message(STATUS "ectrans: enabled (system installed)")
 
    if(ENABLE_ectrans_cpu)
      find_program(ECTRANS_EXE ectrans-benchmark-cpu-sp  REQUIRED)
      set(ectrans_BIN_DIR ${CMAKE_BINARY_DIR})
      get_filename_component(ECTRANS_EXE_DIR "${ECTRANS_EXE}" DIRECTORY)
      set_if_not_present(ENV{PATH} "$ENV{PATH}" "${ECTRANS_EXE_DIR}")
    endif()
 
    if(ENABLE_ectrans_gpu)
      find_program(ECTRANS_EXE ectrans-benchmark-gpu-sp  REQUIRED)
      set(ectrans_BIN_DIR ${CMAKE_BINARY_DIR})
      get_filename_component(ECTRANS_EXE_DIR "${ECTRANS_EXE}" DIRECTORY)
      set_if_not_present(ENV{PATH} "$ENV{PATH}" "${ECTRANS_EXE_DIR}")
    endif()
  else()
    message(STATUS "ectrans: enabled (internally built)")
 
    set(ectrans_depends_ep ecbuild fiat MPI::MPI_C MPI::MPI_Fortran)
 
    if(ENABLE_ectrans_cpu)
      list(APPEND ectrans_depends_ep openblas)
      list(APPEND ectrans_depends_ep fftw fftw_single)
 
      #see FindFFTW.cmake
      if(FFTW_ROOT)
        list(APPEND ectrans_cmake_args "-DFFTW_ROOT=${FFTW_ROOT}")
      endif()
    endif()
 
    ExternalProject_Data_Add(
      ectrans
      ${ectrans_revision}
      DEPENDS ${ectrans_depends_ep}
      CMAKE_ARGS ${cmake_params}
                 -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS}
                 -DCMAKE_INSTALL_PREFIX:FILEPATH=<INSTALL_DIR>
                 -DENABLE_MPI=ON -DENABLE_OMP=ON
                 -DENABLE_DOUBLE_PRECISION=ON
                 -DENABLE_SINGLE_PRECISION=ON
                 -DENABLE_CPU=${ENABLE_ectrans_cpu}
                 -DENABLE_TESTS=${ENABLE_ectrans_cpu}
                 -DENABLE_TRANSI=${ENABLE_ectrans_cpu}
                 -DENABLE_GPU=${ENABLE_ectrans_gpu}
                 -DENABLE_ACC=${ENABLE_ectrans_gpu}
                 -DENABLE_GPU_GRAPHS_GEMM=OFF
                 -DENABLE_GPU_GRAPHS_FFT=OFF
                 ${default_cmake_args}
                 ${ectrans_cmake_args}
      BUILD_COMMAND ${CMAKE_MAKE_PROGRAM} -j${BUILD_PARALLEL_LEVEL} ${default_build_args} ${ectrans_build_args})
 
    ExternalProject_Get_Property(ectrans INSTALL_DIR)
    set(ectrans_BIN_DIR ${INSTALL_DIR})
    set_if_not_present(ENV{PATH} "$ENV{PATH}" "${ectrans_BIN_DIR}/bin")
  endif()
 
  # Options shared by all ecTrans cases
  # --norms: output spectral error norms to verify numerical correctness
  # --meminfo: output memory usage and core binding information
  # -n 100: run benchmark for 100 iterations
  # -l 137: use 137 vertical levels (as in standard configurations of the IFS)
  # --vordiv: enable computation of grid-point vorticity and divergence during the inverse transform
  # --uvders: enable computation of derivatives of U and V during the inverse transform
  # --scders: enable computation of derivatives of scalar fields during the inverse transform
  # higher
  # --check: The multiplier of the machine epsilon used as a tolerance for correctness checking
  set(ectrans_core_options "--norms --meminfo -n 100 -l 137 --vordiv --uvders --scders --check 1000")
 
  # Define cases small, medium, and big, based on truncation (i.e. spectral resolution)
  # We choose truncations of 319, 639 and 1279 because these are common resolutions for the IFS, and
  # it's relatively easy to construct a weak-scaling benchmark suite with a reasonable number of
  # nodes based on these three by assuming the work scales with the cube of truncation
  # E.g., for ECMWF Atos, we use 1, 8, and 64 nodes for these three cases, respectively
  set(cases "small" "medium" "big")
  set(truncs 319 639 1279)
 
  # Define test cases for ecTrans
  # Note that all cases use the single-precision binary of ecTrans, as ECMWF uses single precision
  # for all forecast configurations of the IFS by default
  if(ENABLE_ectrans_cpu)
    foreach(case trunc IN ZIP_LISTS cases truncs)
      add_test(
        NAME ectrans-cpu-${case}
        COMMAND
          env LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} LIBRARY_PATH=$ENV{LIBRARY_PATH} PATH=$ENV{PATH}
          $<TARGET_FILE:job_launcher> --name=ectrans-cpu-${case}
          --log-dir=${LOG_DIR}
          --
          ectrans-benchmark-cpu-sp ${ectrans_core_options} -t ${trunc}
        WORKING_DIRECTORY ${ectrans_BIN_DIR})
    endforeach()
  endif()
 
  if(ENABLE_ectrans_gpu)
    foreach(case trunc IN ZIP_LISTS cases truncs)
      add_test(
        NAME ectrans-gpu-${case}
        COMMAND
          env LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} LIBRARY_PATH=$ENV{LIBRARY_PATH} PATH=$ENV{PATH}
          $<TARGET_FILE:job_launcher> --name=ectrans-gpu-${case}
          --log-dir=${LOG_DIR}
          --
          ectrans-benchmark-gpu-sp ${ectrans_core_options} -t ${trunc}
        WORKING_DIRECTORY ${ectrans_BIN_DIR})
    endforeach()
  endif()
endif(ectrans_enabled)