V2.1.00 (#223)

* fix Planet::RegisterInDump (#229)

* fix segFault with gridCoarsening+MHD when DIMENSIONS==2 (#230)

* add a comment on how to disable OMPiO on openmpi in the documentation

* Update faq.rst with known MacOs linker bug

* Fix parabolic MHD+coarsening (#233)

* adding a new test for grid Coarsening

* fix currents used with grid Coarsening

* reconstruct BX2s only on the axis (#234)

* update version & changelog

* dumpimage that can work on distributed data

* use correct bound for userdef slices

* vtk slice automatically handles all of the variables

* fix potential segfault in dumpimage

* produce slices when emergency outputs

* add compile time info

* force disable Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC in tests
as it is now enabled by default in Kokkos > 4.1
kokkos/kokkos@ac23f36

* Bump up to Kokkos 4.3.1

---------

Co-authored-by: volodia99 <gaylor.wafflard@univ-grenoble-alpes.fr>
Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
Co-authored-by: Clément Robert <cr52@protonmail.com>

Author: 4 people

CHANGELOG.md

@@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.1.0] 2024-05-10
+### Changed
+- VTK slices are automatically produced along with standard VTK when an emergency abort is triggered.
+- Bump up Kokkos version to 4.3.01
+- Add compile time info on startup
+
+### Added
+- DumpImage can load dump and distribute dump in memory, following the domain decomposition used in the current simulation
+- Vtk slices now show all of the fluids (not only hydro)
+
+
 ## [2.0.5] 2024-03-29
 ### Added
 - implement per-fluid flux boundary conditions (experimental) following a suggestion from Paul Segretain (#235)

CMakeLists.txt

@@ -3,10 +3,10 @@ set(CMAKE_BUILD_TYPE Release)
 set (CMAKE_CXX_STANDARD 17)
 
 set(Idefix_VERSION_MAJOR 2)
-set(Idefix_VERSION_MINOR 0)
-set(Idefix_VERSION_PATCH 05)
+set(Idefix_VERSION_MINOR 1)
+set(Idefix_VERSION_PATCH 00)
 
-project (idefix VERSION 2.0.05)
+project (idefix VERSION 2.1.00)
 option(Idefix_MHD "enable MHD" OFF)
 option(Idefix_MPI "enable Message Passing Interface parallelisation" OFF)
 option(Idefix_HIGH_ORDER_FARGO "Force Fargo to use a PPM reconstruction scheme" OFF)

doc/python_requirements.txt

@@ -3,7 +3,7 @@
 # Doxygen https://www.doxygen.nl/download.html
 #
 # Install Sphinx and required plugins with the following command
-# python -m pip install -r python-requirements.txt
+# python -m pip install -r python_requirements.txt
 
 wheel>=0.38.4 # help forward compatibility for pip with old sphinx plugins
 sphinx==5.3.0

doc/source/conf.py

@@ -23,7 +23,7 @@
 author = 'Geoffroy Lesur'
 
 # The full version, including alpha/beta/rc tags
-release = '2.0.05'
+release = '2.1.00'
 
 
 

doc/source/faq.rst

@@ -61,9 +61,6 @@ How can I stop the code without loosing the current calculation?
 I'm doing performance measures. How do I disable all outputs in *Idefix*?
   Add ``-nowrite`` when you call *Idefix* executable.
 
-I sometimes get incoherent values in my VTK files when running Idefix with OpenMPI>4, but the code seems to be running fine.
-  This is probably a bug ot the MPI library for some of the MPI I/O calls used by the VTK class. We recommend passing ``--mca io ^ompio`` to mpirun to avoid
-  this problem which seems to be a bug of the OMPIO layer of OpenMPI.
 
 Developement
 ------------

pytools/idfx_test.py

@@ -116,6 +116,8 @@ def configure(self,definitionFile=""):
       comm.append("-DKokkos_ENABLE_CUDA=ON")
       # disable fmad operations on Cuda to make it compatible with CPU arithmetics
       comm.append("-DIdefix_CXX_FLAGS=--fmad=false")
+      # disable Async cuda malloc for tests performed on old UCX implementations
+      comm.append("-DKokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC=OFF")
 
     if self.hip:
       comm.append("-DKokkos_ENABLE_HIP=ON")

src/input.cpp

@@ -420,4 +420,5 @@ void Input::PrintOptions() {
 void Input::PrintVersion() {
   idfx::cout << "              Idefix version " << IDEFIX_VERSION << std::endl;
   idfx::cout << "              Built against Kokkos " << KOKKOS_VERSION << std::endl;
+  idfx::cout << "              Compiled on " << __DATE__ <<  " at " << __TIME__ << std::endl;
 }

src/kokkos

@@ -9,7 +9,7 @@
 //@HEADER
 // ************************************************************************
 //
-//                        IDEFIX v 2.0.05
+//                        IDEFIX v 2.1.00
 //
 // ************************************************************************
 //@HEADER

src/main.cpp

@@ -14,7 +14,7 @@
   #include <experimental/filesystem>
   namespace fs = std::experimental::filesystem;
 #else
-  error "Missing the <filesystem> header."
+  #error "Missing the <filesystem> header."
 #endif
 #include <iomanip>
 #include "dump.hpp"
@@ -62,6 +62,97 @@ void  Dump::RegisterVariable(IdefixHostArray4D<real>& in,
 }
 
 
+
+void Dump::CreateMPIDataType(GridBox gb, bool read) {
+  #ifdef WITH_MPI
+    int start[3];
+    int size[3];
+    int subsize[3];
+
+    // the grid is required to now the current MPÏ domain decomposition
+    Grid *grid = data->mygrid;
+
+    // Dimensions for cell-centered fields
+    for(int dir = 0; dir < 3 ; dir++) {
+      size[2-dir] = gb.sizeGlob[dir];
+      start[2-dir] = gb.start[dir];
+      subsize[2-dir] = gb.size[dir];
+    }
+    if(read) {
+      MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
+                                            MPI_ORDER_C, realMPI, &this->descCR));
+      MPI_SAFE_CALL(MPI_Type_commit(&this->descCR));
+    } else {
+      MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
+                                            MPI_ORDER_C, realMPI, &this->descCW));
+      MPI_SAFE_CALL(MPI_Type_commit(&this->descCW));
+    }
+
+    // Dimensions for face-centered field
+    for(int face = 0; face < 3 ; face++) {
+      for(int dir = 0; dir < 3 ; dir++) {
+        size[2-dir] = gb.sizeGlob[dir];
+        start[2-dir] = gb.start[dir];
+        subsize[2-dir] = gb.size[dir];
+      }
+      if(read) {
+        // Add the extra guy in the face direction
+        size[2-face]++;
+        subsize[2-face]++; // valid only for reading
+                          //since it involves an overlap of data between procs
+
+        MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
+                                              MPI_ORDER_C, realMPI, &this->descSR[face]));
+        MPI_SAFE_CALL(MPI_Type_commit(&this->descSR[face]));
+      } else {
+        // Now for writing, it is only the last proc which keeps one additional cell
+        size[2-face]++;
+        if(grid->xproc[face] == grid->nproc[face] - 1  ) subsize[2-face]++;
+        MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
+                                              MPI_ORDER_C, realMPI, &this->descSW[face]));
+        MPI_SAFE_CALL(MPI_Type_commit(&this->descSW[face]));
+      }
+    }
+    // Dimensions for edge-centered field
+    for(int nv = 0; nv < 3 ; nv++) {
+      // load the array size
+      for(int dir = 0; dir < 3 ; dir++) {
+        size[2-dir] = gb.sizeGlob[dir];
+        start[2-dir] = gb.start[dir];
+        subsize[2-dir] = gb.size[dir];
+      }
+
+      if(read) {
+        // Extra cell in the dirs perp to field
+        for(int i = 0 ; i < DIMENSIONS ; i++) {
+          if(i!=nv) {
+            size[2-i]++;
+            subsize[2-i]++; // valid only for reading
+                            //since it involves an overlap of data between procs
+          }
+        }
+        MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
+                                              MPI_ORDER_C, realMPI, &this->descER[nv]));
+        MPI_SAFE_CALL(MPI_Type_commit(&this->descER[nv]));
+      } else {
+        // Now for writing, it is only the last proc which keeps one additional cell,
+        // so we remove what we added for reads
+        for(int i = 0 ; i < DIMENSIONS ; i++) {
+          if(i!=nv) {
+            size[2-i]++;
+            if(grid->xproc[i] == grid->nproc[i] - 1  ) {
+              subsize[2-i]++;
+            }
+          }
+        }
+        MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
+                                              MPI_ORDER_C, realMPI, &this->descEW[nv]));
+        MPI_SAFE_CALL(MPI_Type_commit(&this->descEW[nv]));
+      }
+    }
+  #endif
+}
+
 void Dump::Init(DataBlock *datain) {
   idfx::pushRegion("Dump::Init");
   this->data = datain;
@@ -102,79 +193,15 @@ void Dump::Init(DataBlock *datain) {
 
   #ifdef WITH_MPI
     Grid *grid = data->mygrid;
-
-    int start[3];
-    int size[3];
-    int subsize[3];
-
-    // Dimensions for cell-centered fields
+    GridBox gb;
     for(int dir = 0; dir < 3 ; dir++) {
-      size[2-dir] = grid->np_int[dir];
-      start[2-dir] = data->gbeg[dir]-data->nghost[dir];
-      subsize[2-dir] = data->np_int[dir];
+      gb.start[dir] = data->gbeg[dir]-data->nghost[dir];
+      gb.size[dir] = data->np_int[dir];
+      gb.sizeGlob[dir] = grid->np_int[dir];
     }
-
-    MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
-                                           MPI_ORDER_C, realMPI, &this->descC));
-    MPI_SAFE_CALL(MPI_Type_commit(&this->descC));
-
-    // Dimensions for face-centered field
-    for(int face = 0; face < 3 ; face++) {
-      for(int dir = 0; dir < 3 ; dir++) {
-        size[2-dir] = grid->np_int[dir];
-        start[2-dir] = data->gbeg[dir]-data->nghost[dir];
-        subsize[2-dir] = data->np_int[dir];
-      }
-      // Add the extra guy in the face direction
-      size[2-face]++;
-      subsize[2-face]++; // valid only for reading
-                         //since it involves an overlap of data between procs
-
-      MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
-                                             MPI_ORDER_C, realMPI, &this->descSR[face]));
-      MPI_SAFE_CALL(MPI_Type_commit(&this->descSR[face]));
-
-      // Now for writing, it is only the last proc which keeps one additional cell
-      if(grid->xproc[face] != grid->nproc[face] - 1  ) subsize[2-face]--;
-      MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
-                                             MPI_ORDER_C, realMPI, &this->descSW[face]));
-      MPI_SAFE_CALL(MPI_Type_commit(&this->descSW[face]));
-    }
-    // Dimensions for edge-centered field
-    for(int nv = 0; nv < 3 ; nv++) {
-      // load the array size
-      for(int dir = 0; dir < 3 ; dir++) {
-        size[2-dir] = grid->np_int[dir];
-        start[2-dir] = data->gbeg[dir]-data->nghost[dir];
-        subsize[2-dir] = data->np_int[dir];
-      }
-
-      // Extra cell in the dirs perp to field
-      for(int i = 0 ; i < DIMENSIONS ; i++) {
-        if(i!=nv) {
-          size[2-i]++;
-          subsize[2-i]++; // valid only for reading
-                          //since it involves an overlap of data between procs
-        }
-      }
-      MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
-                                            MPI_ORDER_C, realMPI, &this->descER[nv]));
-      MPI_SAFE_CALL(MPI_Type_commit(&this->descER[nv]));
-
-      // Now for writing, it is only the last proc which keeps one additional cell,
-      // so we remove what we added for reads
-      for(int i = 0 ; i < DIMENSIONS ; i++) {
-        if(i!=nv) {
-          if(grid->xproc[i] != grid->nproc[i] - 1  ) {
-            subsize[2-i]--;
-          }
-        }
-      }
-      MPI_SAFE_CALL(MPI_Type_create_subarray(3, size, subsize, start,
-                                            MPI_ORDER_C, realMPI, &this->descEW[nv]));
-      MPI_SAFE_CALL(MPI_Type_commit(&this->descEW[nv]));
-    }
-
+    // Create MPI datatypes for read/write
+    CreateMPIDataType(gb, false);
+    CreateMPIDataType(gb, true);
   #endif
 
   // Register variables that are needed in restart dumps
@@ -661,6 +688,7 @@ bool Dump::Read(Output& output, int readNumber ) {
     }
     // Todo: check that coordinates are identical
   }
+
   std::unordered_set<std::string> notFound {};
   for(auto it = dumpFieldMap.begin(); it != dumpFieldMap.end(); it++) {
     notFound.insert(it->first);
@@ -701,7 +729,7 @@ bool Dump::Read(Output& output, int readNumber ) {
             }
           }
           if(scalar.GetLocation() == DumpField::ArrayLocation::Center) {
-            ReadDistributed(fileHdl, ndim, nx, nxglob, descC, scrch);
+            ReadDistributed(fileHdl, ndim, nx, nxglob, descCR, scrch);
           } else if(scalar.GetLocation() == DumpField::ArrayLocation::Face) {
             ReadDistributed(fileHdl, ndim, nx, nxglob, descSR[direction], scrch);
           } else if(scalar.GetLocation() == DumpField::ArrayLocation::Edge) {
@@ -887,7 +915,7 @@ int Dump::Write(Output& output) {
       }
 
       if(scalar.GetLocation() == DumpField::ArrayLocation::Center) {
-        WriteDistributed(fileHdl, 3, nx, nxtot, fieldName, this->descC, scrch);
+        WriteDistributed(fileHdl, 3, nx, nxtot, fieldName, this->descCW, scrch);
       } else if(scalar.GetLocation() == DumpField::ArrayLocation::Face) {
         WriteDistributed(fileHdl, 3, nx, nxtot, fieldName, this->descSW[dir], scrch);
       } else if(scalar.GetLocation() == DumpField::ArrayLocation::Edge) {