From 916edf5463082616ca0115528ede1ed4b05240d1 Mon Sep 17 00:00:00 2001
From: Jim Wittig <jwittig@ucar.edu>
Date: Tue, 12 May 2026 14:15:57 -0600
Subject: [PATCH 1/2] add updated files from the JCSDA-internal mpas-bundle
 repo.

---
 CMakeLists.txt                | 114 ++++++++++++++++++----------------
 README.md                     |   6 ++
 env-setup/crontab.sh          |  58 +++++++++++------
 env-setup/gnu-derecho.csh     |  29 +++++++--
 env-setup/gnu-derecho.sh      |  30 ++++++---
 env-setup/intel-derecho.csh   |  26 +++++---
 env-setup/intel-derecho.sh    |  26 +++++---
 env-setup/mpas-bundle-cron.sh |  87 +++++++++++++++++++-------
 env-setup/nvhpc-derecho.csh   |  21 +++++++
 env-setup/nvhpc-derecho.sh    |  21 +++++++
 env-setup/run_make.bundle.sh  |  14 +++--
 11 files changed, 305 insertions(+), 127 deletions(-)
 create mode 100644 env-setup/nvhpc-derecho.csh
 create mode 100644 env-setup/nvhpc-derecho.sh

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78ae10f..6d3e3bf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,4 +1,4 @@
-# (C) Copyright 2017-2024 UCAR
+# (C) Copyright 2017-2026 UCAR
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@@ -25,59 +25,67 @@ include( ecbuild_bundle )
 ecbuild_bundle_initialize()
 # Use external jedi-cmake or build in bundle
 
-if(DEFINED ENV{jedi_cmake_ROOT})
-  include( $ENV{jedi_cmake_ROOT}/share/jedicmake/Functions/git_functions.cmake )
-  list( APPEND CMAKE_MODULE_PATH $ENV{jedi_cmake_ROOT}/share/jedicmake/Modules )
-  message (INFO "CMAKE_MODULE_PATH")
+# If the gpu toolchain is loaded, only configure MPAS-Model
+if(CMAKE_Fortran_COMPILER_ID MATCHES NVHPC)
+    ecbuild_bundle( PROJECT MPAS      GIT "https://github.com/MPAS-Dev/MPAS-Model.git" BRANCH develop UPDATE )
 else()
-  ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA/jedi-cmake.git" BRANCH develop UPDATE RECURSIVE )
-  include( jedicmake/cmake/Functions/git_functions.cmake )
-  list( APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/jedicmake/CMakeModules/Modules )
-endif()
-
-option(BUNDLE_SKIP_ECKIT "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
-option(BUNDLE_SKIP_FCKIT "Don't build fckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF
-option(BUNDLE_SKIP_ATLAS "Don't build atlas" "ON" ) # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF
-ecbuild_bundle( PROJECT eckit     GIT "https://github.com/ecmwf/eckit.git" TAG 1.24.4 )
-ecbuild_bundle( PROJECT fckit     GIT "https://github.com/ecmwf/fckit.git" TAG 0.11.0 )
-ecbuild_bundle( PROJECT atlas     GIT "https://github.com/ecmwf/atlas.git" TAG 0.34.0 )
-
-#TODO: When mpas-bundle becomes a public repo, consider changing the default value of BUNDLE_SKIP_ROPP-UFO to "ON"
-option(BUNDLE_SKIP_ROPP-UFO "Don't build ROPP-UFO"  "ON") # Build ropp-ufo unless user passes -DBUNDLE_SKIP_ROPP-UFO=ON
-ecbuild_bundle( PROJECT ropp-ufo  GIT "https://github.com/JCSDA-internal/ropp-test.git"   BRANCH develop UPDATE )
-option(BUNDLE_SKIP_RTTOV "Don't build rttov"  "ON") # Skip rttov build unless user passes -DBUNDLE_SKIP_RTTOV=OFF
-ecbuild_bundle( PROJECT rttov     GIT "https://github.com/JCSDA-internal/rttov.git"       BRANCH develop UPDATE )
-
-ecbuild_bundle( PROJECT oops      GIT "https://github.com/JCSDA/oops.git"        BRANCH develop UPDATE )
-ecbuild_bundle( PROJECT vader     GIT "https://github.com/JCSDA/vader.git"       BRANCH develop UPDATE )
-ecbuild_bundle( PROJECT saber     GIT "https://github.com/JCSDA/saber.git"       BRANCH develop UPDATE)
-ecbuild_bundle( PROJECT crtm      GIT "https://github.com/JCSDA/CRTMv3.git"      BRANCH develop UPDATE )
-
-option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON)
-ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git"   BRANCH develop UPDATE )
-ecbuild_bundle( PROJECT ioda      GIT "https://github.com/JCSDA/ioda.git"                 BRANCH develop UPDATE )
-option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
-ecbuild_bundle( PROJECT ufo-data  GIT "https://github.com/JCSDA-internal/ufo-data.git"    BRANCH develop UPDATE )
-ecbuild_bundle( PROJECT ufo       GIT "https://github.com/JCSDA/ufo.git"                  BRANCH develop UPDATE )
-
-
-# Find external ESMF for mpas-model (optional)
-find_package(ESMF 8.3.0 MODULE)
-
-set(MPAS_DOUBLE_PRECISION "ON" CACHE STRING "MPAS-Model: Use double precision 64-bit Floating point.")
-set(MPAS_CORES init_atmosphere atmosphere CACHE STRING "MPAS-Model: cores to build.")
-
-ecbuild_bundle( PROJECT MPAS      GIT "https://github.com/MPAS-Dev/MPAS-Model" BRANCH develop UPDATE )
-option(ENABLE_MPAS_JEDI_DATA "Obtain mpas-jedi test data from mpas-jedi-data repository (vs tarball)" ON)
-ecbuild_bundle( PROJECT mpas-jedi-data  GIT "https://github.com/JCSDA-internal/mpas-jedi-data.git"  BRANCH develop UPDATE )
-ecbuild_bundle( PROJECT mpas-jedi GIT "https://github.com/JCSDA/mpas-jedi"     BRANCH develop UPDATE )
-
-# Set GIT_BRANCH_FUNC to MPAS-JEDI's current branch so that it can be used for mpas-jedi-data
-find_branch_name(REPO_DIR_NAME mpas-jedi)
-
-# If mpas-jedi's current branch is available in mpas-jedi-data repo, that branch will be checked out
-branch_checkout (REPO_DIR_NAME mpas-jedi-data
+    if(DEFINED ENV{jedi_cmake_ROOT})
+      include( $ENV{jedi_cmake_ROOT}/share/jedicmake/Functions/git_functions.cmake )
+      list( APPEND CMAKE_MODULE_PATH $ENV{jedi_cmake_ROOT}/share/jedicmake/Modules )
+      message (INFO "CMAKE_MODULE_PATH")
+    else()
+      ecbuild_bundle( PROJECT jedicmake GIT "https://github.com/JCSDA/jedi-cmake.git" BRANCH develop UPDATE RECURSIVE )
+      include( jedicmake/cmake/Functions/git_functions.cmake )
+      list( APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/jedicmake/CMakeModules/Modules )
+    endif()
+
+    option(BUNDLE_SKIP_ECKIT "Don't build eckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_ECKIT=OFF
+    option(BUNDLE_SKIP_FCKIT "Don't build fckit" "ON" ) # Skip eckit build unless user passes -DBUNDLE_SKIP_FCKIT=OFF
+    option(BUNDLE_SKIP_ATLAS "Don't build atlas" "ON" ) # Skip atlas build unless user passes -DBUNDLE_SKIP_ATLAS=OFF
+    ecbuild_bundle( PROJECT eckit     GIT "https://github.com/ecmwf/eckit.git" TAG 1.24.4 )
+    ecbuild_bundle( PROJECT fckit     GIT "https://github.com/ecmwf/fckit.git" TAG 0.11.0 )
+    ecbuild_bundle( PROJECT atlas     GIT "https://github.com/ecmwf/atlas.git" TAG 0.34.0 )
+
+    #TODO: When mpas-bundle becomes a public repo, consider changing the default value of BUNDLE_SKIP_ROPP-UFO to "ON"
+    option(BUNDLE_SKIP_ROPP-UFO "Don't build ROPP-UFO"  "ON") # Build ropp-ufo unless user passes -DBUNDLE_SKIP_ROPP-UFO=ON
+    ecbuild_bundle( PROJECT ropp-ufo  GIT "https://github.com/JCSDA-internal/ropp-test.git"   BRANCH develop UPDATE )
+    option(BUNDLE_SKIP_RTTOV "Don't build rttov"  "ON") # Skip rttov build unless user passes -DBUNDLE_SKIP_RTTOV=OFF
+    ecbuild_bundle( PROJECT rttov     GIT "https://github.com/JCSDA-internal/rttov.git"       BRANCH develop UPDATE )
+
+    ecbuild_bundle( PROJECT oops      GIT "https://github.com/JCSDA/oops.git"        BRANCH develop UPDATE )
+    ecbuild_bundle( PROJECT vader     GIT "https://github.com/JCSDA/vader.git"       BRANCH develop UPDATE )
+    ecbuild_bundle( PROJECT saber     GIT "https://github.com/JCSDA/saber.git"       BRANCH develop UPDATE )
+    ecbuild_bundle( PROJECT crtm      GIT "https://github.com/JCSDA/CRTMv3.git"               BRANCH develop UPDATE )
+
+    option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON)
+    ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git"   BRANCH develop UPDATE )
+    ecbuild_bundle( PROJECT ioda      GIT "https://github.com/JCSDA/ioda.git"        BRANCH develop UPDATE )
+    option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
+    ecbuild_bundle( PROJECT ufo-data  GIT "https://github.com/JCSDA-internal/ufo-data.git"    BRANCH develop UPDATE )
+
+    # Build IODA converters if requested
+    option(BUILD_IODA_CONVERTERS "Build IODA Converters" OFF)
+    if(BUILD_IODA_CONVERTERS)
+      ecbuild_bundle( PROJECT iodaconv GIT "https://github.com/jcsda-internal/ioda-converters.git" BRANCH develop UPDATE )
+    endif()
+
+    # Find external ESMF for mpas-model (optional)
+    find_package(ESMF 8.3.0 MODULE)
+
+    set(MPAS_DOUBLE_PRECISION "ON" CACHE STRING "MPAS-Model: Use double precision 64-bit Floating point.")
+    set(MPAS_CORES init_atmosphere atmosphere CACHE STRING "MPAS-Model: cores to build.")
+
+    ecbuild_bundle( PROJECT MPAS      GIT "https://github.com/MPAS-Dev/MPAS-Model" BRANCH develop UPDATE )
+    option(ENABLE_MPAS_JEDI_DATA "Obtain mpas-jedi test data from mpas-jedi-data repository (vs tarball)" ON)
+    ecbuild_bundle( PROJECT mpas-jedi-data  GIT "https://github.com/JCSDA-internal/mpas-jedi-data.git"  BRANCH develop UPDATE )
+    ecbuild_bundle( PROJECT mpas-jedi GIT "https://github.com/JCSDA/mpas-jedi"      BRANCH develop UPDATE )
+
+    # Set GIT_BRANCH_FUNC to MPAS-JEDI's current branch so that it can be used for mpas-jedi-data
+    find_branch_name(REPO_DIR_NAME mpas-jedi)
+
+    # If mpas-jedi's current branch is available in mpas-jedi-data repo, that branch will be checked out
+    branch_checkout (REPO_DIR_NAME mpas-jedi-data
                  BRANCH ${GIT_BRANCH_FUNC} )
-
+endif()
 
 ecbuild_bundle_finalize()
diff --git a/README.md b/README.md
index 1503994..10b9d68 100644
--- a/README.md
+++ b/README.md
@@ -45,6 +45,8 @@ _**For performance and memory reasons, it is recommended to compile ```mpas-bund
 
     ```bash
     git clone https://github.com/JCSDA/mpas-bundle.git
+  ```
+    ```bash
     cd mpas-bundle
     ```
 
@@ -60,8 +62,12 @@ _**For performance and memory reasons, it is recommended to compile ```mpas-bund
   | __csh/tcsh__ | `source <mpas_bundle_dir>/env-setup/gnu-derecho.csh` | `source <mpas_bundle_dir>/env-setup/intel-derecho.csh` |
 * Create and navigate into the build directory.
 
+  ```bash
     mkdir -p <mpas-bundle_build_dir> 
+  ```
+  ```bash
     cd <mpas-bundle_build_dir> 
+    ```
 * To configure the build using CMake, set the `MPAS_DOUBLE_PRECISION` flag according to your usage needs: 
   enable it (`-DMPAS_DOUBLE_PRECISION=ON`) for running the `mpas-jedi` test suite, or 
   disable it for `MPAS-Workflow` calculations when using the `mpas-bundle` build. 
diff --git a/env-setup/crontab.sh b/env-setup/crontab.sh
index 9dcfe56..9aadd8b 100644
--- a/env-setup/crontab.sh
+++ b/env-setup/crontab.sh
@@ -2,21 +2,27 @@
 # run scripts to build mpas-bundle and run ctest
 # to use this, 
 # 1. log on to cron.hpc.ucar.edu
-# 2. run: crontab bundle-crontab
+# 2. run: crontab crontab.sh
 
 # set this to the directory the mpas-bundle repository has been cloned to
-bundle_dir=/glade/work/jwittig/repos1/mpas-bundle-cron/mpas-bundle/
+# don't put it on scratch or it will get corrupted due to scratch retention policy
+bundle_dir=/glade/work/jwittig/repos1/mpas-bundle-cron-src/mpas-bundle
+# set this to where the builds are - we want them on scratch so they get cleaned up
+builds_dir=/glade/derecho/scratch/jwittig/repos-s/mpas-bundle-cron/
 # the directory where the single precision bundle build is
-bundle_build_dir=/glade/work/jwittig/repos1/mpas-bundle-cron/build-gnu-1p/
+bundle_build_dir="'/glade/derecho/scratch/jwittig/repos-s/mpas-bundle-cron/build-gnu-1p_[01]'"
 # this is the build script to execute, relative to the mpas-bundle source directory
 build_script=env-setup/mpas-bundle-cron.sh
 
+# the workflow directory with the develop branch, used when running the cylc workflow
+workflow_dev_dir=/glade/derecho/scratch/jwittig/repos-s/MPAS-Workflow-develop
 # the workflow directory with the script to run a workflow and make graphs
 workflow_dir=/glade/derecho/scratch/jwittig/repos-s/MPAS-Workflow-cron
 # the script to run a cylc job and to create graphs, relative to the workflow directory
 workflow_script=env-setup/run_cylc.sh
 # the workflow scenario to run, relative to the workflow directory
 workflow_scenario=scenarios/3denvar_OIE120km_WarmStart_VarBC_cron.yaml
+workflow_scenario2=scenarios/3dhybrid_OIE120km_WarmStart_cron.yaml
 
 # the directory with the graphics scripts
 graphs_dir=/glade/derecho/scratch/jwittig/repos-s/mpas-jedi-cron/graphics
@@ -26,36 +32,48 @@ graphs_out_dir=/glade/derecho/scratch/jwittig/graphs/data/
 # derecho hpc
 derecho=derecho.hpc.ucar.edu
 # mmm web server
-webserver=whitedwarf.mmm.ucar.edu
+webserver=eris.mmm.ucar.edu
 # destination for graphs
 web_graphs_dir=/web/htdocs/projects/mpas-jedi/weekly-cycling/cylc_graphs
 
 # start at 11:05 PM and clean up log files
-05 23 * * * ssh $derecho "cd ~/my_cron_logs && (gunzip mpas-bundle-cron.log.tar.gz ; tar --remove-files -uf mpas-bundle-cron.log.tar mpas-bundle-cron.log.2* ; tar --remove-files -uf mpas-bundle-cron.log.tar git_shas* ; gzip mpas-bundle-cron.log.tar)"
+05 23 * * 7 ssh $derecho "cd ~/my_cron_logs && (gunzip mpas-bundle-cron.log.tar.gz ; tar --remove-files -uf mpas-bundle-cron.log.tar mpas-bundle-cron.log.2* ; tar --remove-files -uf mpas-bundle-cron.log.tar git_shas* ; gzip mpas-bundle-cron.log.tar)"
 
-# start at 12:05 AM Sun through Fri 
-# change 'gnu' to 'intel' to use the intel build toolchain, or 'both' to build both
-# double precision (-p 2), to be used for ctests.
-# don't do anything if no source code changed from previous run (don't provide -f)
-#05 00 * * 0-5 $bundle_dir/$build_script -d $bundle_dir -q main@desched1 -c gnu  -p 2
-#05 00 * * 0-5 $bundle_dir/$build_script -d $bundle_dir -q develop@desched1 -c gnu  -p 2 -l $bundle_dir/$build_script.lock
+# at 7:30 AM on 7th day of the month clean up last months cylc log files
+lastmon=$(date --date='-1 month' +%Y-%m)
+30 7 7 * * ssh $derecho "cd ~/my_cron_logs/cylc/logs && tar --remove-files -czf run_cylc.cron.$lastmon.tgz *.$lastmon*.log"
+
+bld_suffix=date +%0m_%0d_%y
+
+# start at 12:05 AM on Mon-Fri
+# make a double precision build of mpas-bundle and run ctest
+# the build script won't do anything if there have been no changes to the
+# modules used to create mpas-bundle (unless the '-f' parameter is provided).
+05 00 * * 1-5 ssh $derecho "echo ${bld_suffix} ${bundle_dir} 'make 2p'  >> ~/my_cron_logs/cron.log && cd $bundle_dir && git fetch -p >> ~/my_cron_logs/cron.log ; git status >> ~/my_cron_logs/cron.log ; git pull >> ~/my_cron_logs/cron.log" ;$bundle_dir/$build_script -d $bundle_dir -b $builds_dir -q develop@desched1 -c gnu -p 2 -a nmmm0015
 
 # start at 12:05 AM on Sat 
 # always build, even if no source change from previous run (-f)
-05 00 * * 6 $bundle_dir/$build_script -d $bundle_dir -q develop@desched1 -c gnu -p 2 -f
+05 00 * * 6 $bundle_dir/$build_script -d $bundle_dir -b $builds_dir -q develop@desched1 -c gnu -p 2 -f -a nmmm0015
 
 # start at 1:05 AM on Sat 
 # always build, even if no source change from previous run (-f)
 # single precision (-p 1), to be used for cylc experiment.
-05 01 * * 6 $bundle_dir/$build_script -d $bundle_dir -q develop@desched1 -c gnu -p 1 -f -l $bundle_dir/$build_script.lock
+# use the date as part of the build directory name, so each week's build is unique.
+05 01 * * 6 $bundle_dir/$build_script -d $bundle_dir -b $builds_dir -q develop@desched1 -c gnu -p 1 -f -l $bundle_dir/$build_script.lock -x $(${bld_suffix}) -a nmmm0015
+# build gpu enabled MPAS-Model
+#20 12 * * 1 $bundle_dir/$build_script -d $bundle_dir -b $builds_dir -q develop@desched1 -c nvhpc -p 1 -f -l $bundle_dir/$build_script.lock -x $(${bld_suffix}) -a nmmm0015
 
-# at 12:05 am on Sun run the workflow
+# at 12:05 am on Sun update the develop branch of  MPAS-Workflow repo and run the workflow
 suffix=$(date +%F)
-05 00 * * 7 ssh $derecho $workflow_dir/$workflow_script -w $workflow_dir -d $bundle_build_dir -k $bundle_dir/$build_script.lock -s $workflow_scenario -x $suffix
-
-# at 12:05 am Mon-Fri try to graph results from the completed workflow runs
-05 00 * * 1-5 ssh $derecho $workflow_dir/$workflow_script -w $workflow_dir -g $graphs_dir -o $graphs_out_dir
+05 00 * * 7 ssh $derecho "echo $suffix 'run workflow in ' $workflow_dir >> ~/my_cron_logs/cron.log && cd $workflow_dir && git fetch -p >> ~/my_cron_logs/cron.log 2>&1 && git co develop >> ~/my_cron_logs/cron.log 2>&1 && git pull >> ~/my_cron_logs/cron.log 2>&1 && $workflow_dir/$workflow_script -w $workflow_dir -d $bundle_build_dir -k $bundle_dir/$build_script.lock -s $workflow_scenario -x $suffix"
+# at 1:05 am on Sun run the 3dhybrid workflow
+05 01 * * 7 ssh $derecho "echo $suffix 'run workflow in ' $workflow_dir >> ~/my_cron_logs/cron.log && cd $workflow_dir && $workflow_dir/$workflow_script -w $workflow_dir -d $bundle_build_dir -k $bundle_dir/$build_script.lock -s $workflow_scenario2 -x $suffix"
 
-# at 6:05 am weekdays tar any graphics files and copy them to an mmm web server
-05 06 * * 1-5 echo "`date` checking $graphs_out_dir" >> ~/my_cron_logs/cp_graphs.log && for dir in $(/bin/ls $graphs_out_dir); do echo "`date` dir to copy:$dir" >> ~/my_cron_logs/cp_graphs.log; echo "ssh $derecho cd $graphs_out_dir && tar --remove-files -czf $dir.tgz $dir" >> ~/my_cron_logs/cp_graphs.log; ssh $derecho "cd $graphs_out_dir && tar --remove-files -czf $dir.tgz $dir"; echo "scp $graphs_out_dir/$dir.tgz $webserver:$web_graphs_dir/" >> ~/my_cron_logs/cp_graphs.log; scp $graphs_out_dir/$dir.tgz $webserver:$web_graphs_dir/; echo "ssh $webserver cd $web_graphs_dir/ && tar -xzf $dir.tgz" >> ~/my_cron_logs/cp_graphs.log; ssh $webserver "cd $web_graphs_dir/ && tar -xzf $dir.tgz"; mv $graphs_out_dir/$dir.tgz $graphs_out_dir/../copied; echo "finished $dir" >> ~/my_cron_logs/cp_graphs.log; done
+# run the weekly cylc scenario using a gpu build of MPAS-Model
+suffix_gpu=2025-05-30_new
+workflow_scenario_gpu=scenarios/3denvar_OIE120km_WarmStart_VarBC_gpu_cron.yaml
+#55 15 * * * ssh $derecho "echo $suffix >> ~/my_cron_logs/cron.log 2>&1 && cd $workflow_dev_dir && git fetch -p >> ~/my_cron_logs/cron.log 2>&1 && git co develop >> ~/my_cron_logs/cron.log 2>&1 && git pull >> ~/my_cron_logs/cron.log && $workflow_dir/$workflow_script -w $workflow_dir -d $bundle_build_dir -k $bundle_dir/$build_script.lock -s $workflow_scenario_gpu -x ${suffix}_gpu"
 
+# at 2:05 am Mon-Fri try to graph results from the completed workflow runs
+casper=casper.hpc.ucar.edu
+05 02 * * 1-5 ssh $casper "$workflow_dir/$workflow_script -w $workflow_dir -g $graphs_dir -o $graphs_out_dir -m $webserver -c $web_graphs_dir/"
diff --git a/env-setup/gnu-derecho.csh b/env-setup/gnu-derecho.csh
index 1d210e0..c378b1d 100644
--- a/env-setup/gnu-derecho.csh
+++ b/env-setup/gnu-derecho.csh
@@ -10,16 +10,33 @@ endif
 
 module purge
 # ignore that the sticky module ncarenv/... is not unloaded
-module load ncarenv/23.09
+module load ncarenv/24.12
 module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles
 module load ecflow/5.8.4
 module load mysql/8.0.33
 
-module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core
-module load stack-gcc/12.2.0
-module load stack-cray-mpich/8.1.25
-module load stack-python/3.10.13
-module load jedi-mpas-env
+module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core
+
+module load stack-gcc/12.4.0
+module load stack-cray-mpich/8.1.29
+module load stack-python/3.11.7
+module load py-pycodestyle/2.11.0
+# module load jedi-mpas-env is commented out because it loads many modules not needed for
+# building and running mpas-bundle on derecho.
+# To load all of the modules which were loaded prior to 01/15/2026,
+# uncomment out "module load jedi-mpas-env"
+# module load jedi-mpas-env
+module load atlas/0.40.0
+module load ecbuild/3.7.2
+module load netcdf-cxx4/4.3.1
+module load parallelio/2.6.2
+module load gsl-lite/0.37.0
+module load nccmp/1.9.0.1
+module load udunits/2.2.28
+# Following modules are required for ioda-converters build
+module load bufr/12.1.0
+module load py-pybind11/2.13.5
+module list
 
 limit stacksize unlimited
 setenv F_UFMTENDIAN 'big_endian:101-200'
diff --git a/env-setup/gnu-derecho.sh b/env-setup/gnu-derecho.sh
index 8eae3fc..55c3036 100644
--- a/env-setup/gnu-derecho.sh
+++ b/env-setup/gnu-derecho.sh
@@ -11,19 +11,35 @@ fi
 
 module purge
 # ignore that the sticky module ncarenv/... is not unloaded
-module load ncarenv/23.09
+module load ncarenv/24.12
 module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles
 module load ecflow/5.8.4
 module load mysql/8.0.33
 
-module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core
-module load stack-gcc/12.2.0
-module load stack-cray-mpich/8.1.25
-module load stack-python/3.10.13
-module load jedi-mpas-env
+module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core
+
+module load stack-gcc/12.4.0
+module load stack-cray-mpich/8.1.29
+module load stack-python/3.11.7
+# module load jedi-mpas-env is commented out because it loads many modules not needed for
+# building and running mpas-bundle on derecho.
+# To load all of the modules which were loaded prior to 01/15/2026,
+# uncomment out "module load jedi-mpas-env"
+# module load jedi-mpas-env
+module load py-pycodestyle/2.11.0
+module load atlas/0.40.0
+module load ecbuild/3.7.2
+module load netcdf-cxx4/4.3.1
+module load parallelio/2.6.2
+module load gsl-lite/0.37.0
+module load nccmp/1.9.0.1
+module load udunits/2.2.28
+# Following modules are required for ioda-converters build
+module load bufr/12.1.0
+module load py-pybind11/2.13.5
 module list
 
 ulimit -s unlimited
-export F_UFMTENDIAN='big_endian:101-200'
+export GFORTRAN_CONVERT_UNIT='big_endian:101-200'
 export LD_LIBRARY_PATH=`pwd`/lib:$LD_LIBRARY_PATH
 
diff --git a/env-setup/intel-derecho.csh b/env-setup/intel-derecho.csh
index 286167e..fd09f6d 100644
--- a/env-setup/intel-derecho.csh
+++ b/env-setup/intel-derecho.csh
@@ -13,17 +13,29 @@ endif
 #
 
 module purge
-module use /lustre/desc1/scratch/epicufsrt/contrib/modulefiles
-module use /lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra
+module load ncarenv/24.12
+module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles
 module load ecflow/5.8.4
 module load mysql/8.0.33
 
-module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core
+module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-oneapi-2024.2.1/install/modulefiles/Core
 
-module load stack-intel/2021.10.0
-module load stack-cray-mpich/8.1.25
-module load stack-python/3.10.13
-module load jedi-mpas-env
+module load stack-oneapi/2024.2.1
+module load stack-cray-mpich/8.1.29
+module load stack-python/3.11.7
+# module load jedi-mpas-env is commented out because it loads many modules not needed for
+# building and running mpas-bundle on derecho.
+# To load all of the modules which were loaded prior to 01/15/2026,
+# uncomment out "module load jedi-mpas-env"
+#module load jedi-mpas-env
+module load py-pycodestyle/2.11.0
+module load atlas/0.40.0
+module load ecbuild/3.7.2
+module load netcdf-cxx4/4.3.1
+module load parallelio/2.6.2
+module load gsl-lite/0.37.0
+module load nccmp/1.9.0.1
+module load udunits/2.2.28
 module list
 
 limit stacksize unlimited
diff --git a/env-setup/intel-derecho.sh b/env-setup/intel-derecho.sh
index aecca3b..554b659 100644
--- a/env-setup/intel-derecho.sh
+++ b/env-setup/intel-derecho.sh
@@ -10,17 +10,29 @@ if command -v conda >/dev/null 2>&1; then
 fi
 
 module purge
-module use /lustre/desc1/scratch/epicufsrt/contrib/modulefiles
-module use /lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra
+module load ncarenv/24.12
+module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles
 module load ecflow/5.8.4
 module load mysql/8.0.33
 
-module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core
+module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-oneapi-2024.2.1/install/modulefiles/Core
 
-module load stack-intel/2021.10.0
-module load stack-cray-mpich/8.1.25
-module load stack-python/3.10.13
-module load jedi-mpas-env
+module load stack-oneapi/2024.2.1
+module load stack-cray-mpich/8.1.29
+module load stack-python/3.11.7
+# module load jedi-mpas-env is commented out because it loads many modules not needed for
+# building and running mpas-bundle on derecho.
+# To load all of the modules which were loaded prior to 01/15/2026,
+# uncomment out "module load jedi-mpas-env"
+#module load jedi-mpas-env
+module load py-pycodestyle/2.11.0
+module load atlas/0.40.0
+module load ecbuild/3.7.2
+module load netcdf-cxx4/4.3.1
+module load parallelio/2.6.2
+module load gsl-lite/0.37.0
+module load nccmp/1.9.0.1
+module load udunits/2.2.28
 module list
 
 ulimit -s unlimited
diff --git a/env-setup/mpas-bundle-cron.sh b/env-setup/mpas-bundle-cron.sh
index 8f564b6..bdcd7f3 100755
--- a/env-setup/mpas-bundle-cron.sh
+++ b/env-setup/mpas-bundle-cron.sh
@@ -4,7 +4,7 @@
 # Script to build mpas-bundle and run mpas-jedi ctests.
 # mpas-bundle may be built with both the gnu and intel tool chains.
 # After running mpas-jedi ctests the results will be put into an html document,
-# which will be copied to the web pages directory on whitedwarf.mmm.ucar.edu
+# which will be copied to the web pages directory on an mmm server.
 #
 # This script assumes the mpas-bundle repository has been cloned into
 #  <some_dir>/<mpas-bundle_dir>    # directory mpas-bundle has been cloned to
@@ -32,7 +32,7 @@ init_logs()
   HTML_BODY_FILE="${cron_logdir}/body.html"
 }
 
-# where to copy html output files on whitedwarf.mmm.ucar.edu
+# where to copy html output files on mmm web pages server
 html_dir="/web/htdocs/projects/mpas-jedi/weekly-ctests"
 # available queues to use
 # normal cpu hours are charged
@@ -48,7 +48,7 @@ usage()
   log "usage:"
   log "  mpas-bundle-cron.sh ${args}"
   log "    -d <bundle_dir> is where the mpas-bundle repo has been cloned to"
-  log "    -c <compiler> is one of gnu, intel or both"
+  log "    -c <compiler> is one of gnu, intel, nvhpc, or all"
   log "    -x <suffix> is a suffix to add to the build directory"
   log "    -q <queue> is ${MAIN_Q}, ${DEV_Q} or ${PREEMPT_Q}, defaults to ${MAIN_Q}"
   log "    -a <account> is the account to use when submitting PBS jbs (e.g. 'nmmm0015')"
@@ -87,7 +87,8 @@ another_instance()
 check_pbs_return()
 {
   local jobno=$1
-  local pbs_status=$(qstat -x -f $jobno | grep 'Exit_status')
+  local pbs_status=$(QSCACHE_BYPASS=1 qstat -x -f $jobno | grep 'Exit_status')
+  log "jobno=$jobno pbs_status=$pbs_status"
 
   if [ ! -z "${pbs_status}" ]; then
     retcode=$(echo $pbs_status|awk '{print $3}')
@@ -107,11 +108,21 @@ queue_wait()
 
   qstat ${job} &> /dev/null
   local rc=$?
-  while [ ${rc} == "${pcode}" ]; do
-    sleep ${secs}
-    qstat ${job} &> /dev/null
-    rc=$?
-  done
+  log "queue_wait started job=$job pcode=$pcode rc=$rc"
+  if [ $pcode -gt 0 ]; then
+    while [ ${rc} -gt 0 ]; do
+      sleep ${secs}
+      qstat ${job} &> /dev/null
+      rc=$?
+    done
+  else
+    while [ ${rc} -eq 0 ]; do
+      sleep ${secs}
+      qstat ${job} &> /dev/null
+      rc=$?
+    done
+  fi
+  log "queue_wait finished job=$job pcode=$pcode rc=$rc"
 }
 
 # write html header, including the table directive
@@ -146,10 +157,11 @@ run_cmake()
 cat > $cmake_script << CMAKE_EOF
 #!/bin/bash
 #
-cd $1 && source ../mpas-bundle/env-setup/$2-derecho.sh && if [ -f Makefile ]; then echo "make update" && make update |& tee make.update.log; fi && cmake -DCMAKE_VERBOSE_MAKEFILE=ON -DBUNDLE_SKIP_RTTOV=ON -DMPAS_DOUBLE_PRECISION=$3 ctest_update  ../mpas-bundle;
+cd $1 && source $2/env-setup/$3-derecho.sh && if [ -f Makefile ]; then echo "make update" && make update |& tee make.update.log; fi && cmake -DCMAKE_VERBOSE_MAKEFILE=ON -DBUNDLE_SKIP_RTTOV=ON -DMPAS_DOUBLE_PRECISION=$4 ctest_update  $2;
 CMAKE_EOF
 
   chmod 755 ${cmake_script}
+  log "ssh derecho.hpc.ucar.edu ${cmake_script} |& tee ${cmake_script}.log"
   ssh derecho.hpc.ucar.edu ${cmake_script} |& tee ${cmake_script}.log
 }
 
@@ -304,16 +316,18 @@ make_html()
   log "            chref=$href sha_file=$sha_file sha_file:t=${sha_file##*/}"
 
   # get tool versions to put into summary table
-  local spack_stack=$(grep spack-stack- ../mpas-bundle/env-setup/${cc}-derecho.sh | awk -F/ '{print $8}')
+  local spack_stack=$(grep spack-stack- ${BUNDLE_DIR}/env-setup/${cc}-derecho.sh | awk -F/ '{print $8}')
   log "spack-stack: ${spack_stack}"
   if [ ${cc} == "gnu" ]; then
     local comp="gcc"
-  else
+  elif [ ${cc} == "intel" ]; then
     local comp="intel"
+  elif [ ${cc} == "nvhpc" ]; then
+    local comp="nvhpc"
   fi
-  local compiler=$(grep "load *stack-${comp}" ../mpas-bundle/env-setup/${cc}-derecho.sh | awk '{print $3}')
+  local compiler=$(grep "load .*${comp}" ${BUNDLE_DIR}/env-setup/${cc}-derecho.sh | awk '{print $3}')
   log "compiler: ${compiler}"
-  local mpich=$(grep "load *stack-cray" ../mpas-bundle/env-setup/${cc}-derecho.sh | awk '{print $3}')
+  local mpich=$(grep "load .*cray-mpich" ${BUNDLE_DIR}/env-setup/${cc}-derecho.sh | awk '{print $3}')
   log "mpich: ${mpich}"
 
 
@@ -334,7 +348,7 @@ make_html()
   print_footer index.html
   rm body.html
 
-  local host="whitedwarf.mmm.ucar.edu"
+  local host="eris.mmm.ucar.edu"
   local dest="${host}:${dest_dir}"
   local index_tarfile="index.tar"
 
@@ -385,7 +399,7 @@ build_and_test()
 
   #--------------------------------------------------------------
   # go to build directory, exit on failure:
-  BUILD_DIR="${BUNDLE_DIR}/../build-${cc}-${build_dir_suffix}"
+  BUILD_DIR="${BUILD_DIR_ROOT}/build-${cc}-${build_dir_suffix}"
   if [ "$suffix" != "" ]; then
     BUILD_DIR="${BUILD_DIR}_${suffix}"
   fi
@@ -399,7 +413,7 @@ build_and_test()
   # run cmake on a login node (compute nodes have poor internet transmission)
   # block until cmake finishes
   if [[ "$run_cmake" == "yes" ]]; then
-    run_cmake ${BUILD_DIR} ${cc} $dbl_p
+    run_cmake ${BUILD_DIR} ${BUNDLE_DIR} ${cc} $dbl_p
   fi
 
   # create script to run gnu make and run it
@@ -425,6 +439,21 @@ build_and_test()
     queue_wait ${make_job} 0 60
     summary="Single precision build - no ctests run"
     ctest_time="NA"
+
+    # make a symlink to the latest single precision build on success
+    check_pbs_return $make_job
+    local make_rc=$?
+    if [ "$make_rc" -eq 0 ]; then
+      build_dir_root=${BUILD_DIR%%_*}
+      latest_dir="${build_dir_root}_latest"
+      log "rm $latest_dir"
+      rm $latest_dir
+      log "ln -s $(pwd) $latest_dir"
+      ln -s $(pwd) $latest_dir
+    else
+      summary="single precision make job failed return code $make_rc"
+      ctest_time="NA"
+    fi
   fi
 
   #
@@ -437,6 +466,7 @@ build_and_test()
 CTEST_LOGFILE="ctest.pbs.sh.log"
 QUEUE=$MAIN_Q
 BUNDLE_DIR=""
+BUILD_DIR_ROOT=""
 tools=""
 precision="2"
 force_build=0
@@ -447,10 +477,11 @@ suffix=""
 ACCOUNT="nmmm0015"
 
 # get comamnd line args
-while getopts d:q:a:p:c:l:o:x:fhn flag
+while getopts d:b:q:a:p:c:l:o:x:fhn flag
 do
   case "${flag}" in
     d) BUNDLE_DIR="${OPTARG}";;
+    b) BUILD_DIR_ROOT="${OPTARG}";;
     q) QUEUE="${OPTARG}";;
     a) ACCOUNT="${OPTARG}";;
     p) precision=${OPTARG};;
@@ -476,11 +507,21 @@ if [ "$BUNDLE_DIR" = "" ]; then
   log "source dir <-d source_dir> is required"
   usage
 fi
+
 if [ ! -d ${BUNDLE_DIR} ]; then
   log "source dir ${BUNDLE_DIR} does not exist"
   usage
 fi
 
+if [ "$BUILD_DIR_ROOT" == "" ]; then
+  BUILD_DIR_ROOT="${BUNDLE_DIR}/.."
+else
+  if [ ! -d ${BUILD_DIR_ROOT} ]; then
+    log "output dir ${BUILD_DIR_ROOT} does not exist"
+    usage
+  fi
+fi
+
 if [[ "$QUEUE" != "$MAIN_Q" && "$QUEUE" != "$PREEMPT_Q" && "$QUEUE" != "$DEV_Q" ]]; then
   log "queue \"${QUEUE}\" is invalid"
   usage
@@ -535,14 +576,18 @@ if [ "$force_build" -eq 0 ]; then
 else
   log "building with tools ${tools}"
   # build gnu version and run ctest
-  if [[ "$tools" == "gnu" || "$tools" == "both" ]]; then
-    log "build_and_test gnu $dbl_precision $html_dir $run_cmake $sha_file $suffix"
+  if [[ "$tools" == "gnu" || "$tools" == "all" ]]; then
     build_and_test "gnu" $dbl_precision $html_dir $run_cmake $sha_file $suffix
   fi
 
   # build intel version and run ctest
-  if [[ "$tools" == "intel" || "$tools" == "both" ]]; then
+  if [[ "$tools" == "intel" || "$tools" == "all" ]]; then
     build_and_test "intel" $dbl_precision $html_dir $run_cmake $sha_file $suffix
   fi
+
+  # build nvhpc version and run ctest
+  if [[ "$tools" == "nvhpc" || "$tools" == "all" ]]; then
+    build_and_test "nvhpc" $dbl_precision $html_dir $run_cmake $sha_file $suffix
+  fi
 fi
 
diff --git a/env-setup/nvhpc-derecho.csh b/env-setup/nvhpc-derecho.csh
new file mode 100644
index 0000000..5fd4fde
--- /dev/null
+++ b/env-setup/nvhpc-derecho.csh
@@ -0,0 +1,21 @@
+#!/bin/csh
+#
+
+source /etc/profile.d/z00_modules.csh
+
+module --force purge
+module load ncarenv/24.12
+
+module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core
+module load stack-gcc/12.4.0
+module load ecbuild/3.7.2
+module load cmake/3.27.9
+
+module load craype/2.7.31
+module load nvhpc/24.11 
+module load ncarcompilers/1.0.0
+module load cray-mpich/8.1.29
+module load cuda/12.3.2
+module load parallel-netcdf/1.14.0
+module list
+
diff --git a/env-setup/nvhpc-derecho.sh b/env-setup/nvhpc-derecho.sh
new file mode 100644
index 0000000..8f83fc0
--- /dev/null
+++ b/env-setup/nvhpc-derecho.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+
+source /etc/profile.d/z00_modules.sh
+
+module --force purge
+module load ncarenv/24.12
+
+module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core
+module load stack-gcc/12.4.0
+module load ecbuild/3.7.2
+module load cmake/3.27.9
+
+module load craype/2.7.31
+module load nvhpc/24.11 
+module load ncarcompilers/1.0.0
+module load cray-mpich/8.1.29
+module load cuda/12.3.2
+module load parallel-netcdf/1.14.0
+module list
+
diff --git a/env-setup/run_make.bundle.sh b/env-setup/run_make.bundle.sh
index 3e1be7b..a52d3c0 100755
--- a/env-setup/run_make.bundle.sh
+++ b/env-setup/run_make.bundle.sh
@@ -36,7 +36,7 @@ LOG=""
 VERBOSE=""
 
 # derecho params
-DERECHO_CC="gnu intel"
+DERECHO_CC="gnu intel nvhpc"
 DERECHO_CC_INTEL="intel"
 DERECHO_Q=("main" "develop" )
 QUEUE_OPTS=${DERECHO_Q[@]}
@@ -47,7 +47,7 @@ export GFORTRAN_CONVERT_UNIT='big_endian:101-200'
 
 usage()
 {
-	echo "usage: $0 -A account -c gnu|intel [-x make|ctest|both|echo] [-l] "
+	echo "usage: $0 -A account -c gnu|intel|nvhpc [-x make|ctest|both|echo] [-l] "
 	echo "  [-q queue] [-p priority][-t threads] [-N name] [-f job-file] [-m] [-n] [-e env-dir] [-h] "
 	echo
 	echo "  account is the HPC account number"
@@ -79,7 +79,7 @@ elif [ "cron" == "$HOST" ]; then
 fi
 
 if [ "$HPC" = "derecho" ]; then
-	QUEUE="-q ${DERECHO_Q[0]}"
+	QUEUE="-q ${DERECHO_Q[1]}"
 	QUEUE_OPTS=${DERECHO_Q[@]}
 else
 	echo "unsupported HPC, must run on HPC login node"
@@ -113,14 +113,14 @@ done
 
 if [ "$ACCOUNT" = "" ]; then
 	echo "account (-A) is required"
-	echo "   something like nmmm0004"
+	echo "   something like nmmm0015"
 	echo
 	usage
 fi
 
 if [ "$COMPILER" = "" ]; then
 	echo "compiler (-c) is required"
-	echo "gnu or intel"
+	echo "  one of $DERECHO_CC"
 	echo
 	usage
 fi
@@ -159,8 +159,10 @@ if [ "$HPC" == "derecho" ]; then
     MODFILE="${ENV_DIR}/gnu-derecho.sh"
 	elif [ "$COMPILER" == "intel" ]; then
     MODFILE="${ENV_DIR}/intel-derecho.sh"
+	elif [ "$COMPILER" == "nvhpc" ]; then
+    MODFILE="${ENV_DIR}/nvhpc-derecho.sh"
 	else
-		echo unknown compiler: $COMPILER, must be either "gnu" or "intel"
+		echo "unknown compiler: $COMPILER, must be one of $DERECHO_CC"
 		echo
 		usage
 	fi

From f7b8173cbaec2107444c2910946558e2b470fc31 Mon Sep 17 00:00:00 2001
From: Jim Wittig <jwittig@ucar.edu>
Date: Fri, 15 May 2026 12:19:34 -0600
Subject: [PATCH 2/2] Add the ufo (which was mistakenly removed).

---
 CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6d3e3bf..9f59d83 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -62,6 +62,7 @@ else()
     ecbuild_bundle( PROJECT ioda      GIT "https://github.com/JCSDA/ioda.git"        BRANCH develop UPDATE )
     option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
     ecbuild_bundle( PROJECT ufo-data  GIT "https://github.com/JCSDA-internal/ufo-data.git"    BRANCH develop UPDATE )
+    ecbuild_bundle( PROJECT ufo       GIT "https://github.com/JCSDA/ufo.git"                  BRANCH develop UPDATE )
 
     # Build IODA converters if requested
     option(BUILD_IODA_CONVERTERS "Build IODA Converters" OFF)