Merge pull request #2685 from devitocodes/data-singlempi

data: prevent mpi comm with single rank

Author: mloubout

devito/data/data.py

@@ -154,7 +154,7 @@ def _local(self):
 
     def _global(self, glb_idx, decomposition):
         """A "global" view of ``self`` over a given Decomposition."""
-        if self._is_distributed:
+        if self._is_decomposed:
             raise ValueError("Cannot derive a decomposed view from a decomposed Data")
         if len(decomposition) != self.ndim:
             raise ValueError("`decomposition` should have ndim=%d entries" % self.ndim)
@@ -176,13 +176,12 @@ def _check_idx(func):
         @wraps(func)
         def wrapper(data, *args, **kwargs):
             glb_idx = args[0]
-            is_gather = isinstance(kwargs.get('gather_rank', None), int)
+            is_gather = isinstance(kwargs.get('gather_rank'), int)
             if is_gather and all(i == slice(None, None, 1) for i in glb_idx):
                 comm_type = gather
-            elif len(args) > 1 and isinstance(args[1], Data) \
-                    and args[1]._is_mpi_distributed:
+            elif len(args) > 1 and isinstance(args[1], Data) and args[1]._is_decomposed:
                 comm_type = index_by_index
-            elif data._is_mpi_distributed:
+            elif data._is_decomposed:
                 for i in as_tuple(glb_idx):
                     if isinstance(i, slice) and i.step is not None and i.step < 0:
                         comm_type = index_by_index
@@ -196,8 +195,9 @@ def wrapper(data, *args, **kwargs):
         return wrapper
 
     @property
-    def _is_mpi_distributed(self):
-        return self._is_distributed and configuration['mpi']
+    def _is_decomposed(self):
+        return self._is_distributed and configuration['mpi'] and \
+            self._distributor.comm.size > 1
 
     def __repr__(self):
         return super(Data, self._local).__repr__()
@@ -341,7 +341,7 @@ def __setitem__(self, glb_idx, val, comm_type):
                 super().__setitem__(loc_idx, val)
             else:
                 super().__setitem__(glb_idx, val)
-        elif isinstance(val, Data) and val._is_distributed:
+        elif isinstance(val, Data) and val._is_decomposed:
             if comm_type is index_by_index:
                 glb_idx, val = self._process_args(glb_idx, val)
                 val_idx = as_tuple([slice(i.glb_min, i.glb_max+1, 1) for
@@ -361,14 +361,14 @@ def __setitem__(self, glb_idx, val, comm_type):
                         or data_global[j].size == 0
                     if not skip:
                         self.__setitem__(idx_global[j], data_global[j])
-            elif self._is_distributed:
+            elif self._is_decomposed:
                 # `val` is decomposed, `self` is decomposed -> local set
                 super().__setitem__(glb_idx, val)
             else:
                 # `val` is decomposed, `self` is replicated -> gatherall-like
                 raise NotImplementedError
         elif isinstance(val, np.ndarray):
-            if self._is_distributed:
+            if self._is_decomposed:
                 # `val` is replicated, `self` is decomposed -> `val` gets decomposed
                 glb_idx = self._normalize_index(glb_idx)
                 glb_idx, val = self._process_args(glb_idx, val)
@@ -401,7 +401,7 @@ def __setitem__(self, glb_idx, val, comm_type):
                 pass
             super().__setitem__(glb_idx, val)
         elif isinstance(val, Iterable):
-            if self._is_mpi_distributed:
+            if self._is_decomposed:
                 raise NotImplementedError("With MPI, data can only be set "
                                           "via scalars, numpy arrays or "
                                           "other data ")
@@ -478,7 +478,7 @@ def _index_glb_to_loc(self, glb_idx):
         if len(glb_idx) > self.ndim:
             # Maybe user code is trying to add a new axis (see np.newaxis),
             # so the resulting array will be higher dimensional than `self`
-            if self._is_mpi_distributed:
+            if self._is_decomposed:
                 raise ValueError("Cannot increase dimensionality of MPI-distributed Data")
             else:
                 # As by specification, we are forced to ignore modulo indexing
@@ -494,7 +494,7 @@ def _index_glb_to_loc(self, glb_idx):
                 try:
                     v = convert_index(i, dec, mode='glb_to_loc')
                 except TypeError:
-                    if self._is_mpi_distributed:
+                    if self._is_decomposed:
                         raise NotImplementedError("Unsupported advanced indexing with "
                                                   "MPI-distributed Data")
                     v = i

examples/seismic/self_adjoint/sa_03_iso_correctness.ipynb

@@ -128,12 +128,12 @@ def test_linearization_F(self, shape, dtype, so):
         ns = 2 * size + 1
         if len(shape) == 2:
             nx2, nz2 = shape[0]//2, shape[1]//2
-            dm.data[nx2-size:nx2+size, nz2-size:nz2+size] = \
+            dm.data[nx2-size:nx2+size+1, nz2-size:nz2+size+1] = \
                 -1 + 2 * np.random.rand(ns, ns)
         else:
             nx2, ny2, nz2 = shape[0]//2, shape[1]//2, shape[2]//2
             nx, ny, nz = shape
-            dm.data[nx2-size:nx2+size, ny2-size:ny2+size, nz2-size:nz2+size] = \
+            dm.data[nx2-size:nx2+size+1, ny2-size:ny2+size+1, nz2-size:nz2+size+1] = \
                 -1 + 2 * np.random.rand(ns, ns, ns)
 
         # Compute F(m + dm)
@@ -194,12 +194,12 @@ def test_linearity_forward_J(self, shape, dtype, so):
         ns = 2 * size + 1
         if len(shape) == 2:
             nx2, nz2 = shape[0]//2, shape[1]//2
-            m1.data[nx2-size:nx2+size, nz2-size:nz2+size] = \
+            m1.data[nx2-size:nx2+size+1, nz2-size:nz2+size+1] = \
                 -1 + 2 * np.random.rand(ns, ns)
         else:
             nx2, ny2, nz2 = shape[0]//2, shape[1]//2, shape[2]//2
             nx, ny, nz = shape
-            m1.data[nx2-size:nx2+size, ny2-size:ny2+size, nz2-size:nz2+size] = \
+            m1.data[nx2-size:nx2+size+1, ny2-size:ny2+size+1, nz2-size:nz2+size+1] = \
                 -1 + 2 * np.random.rand(ns, ns, ns)
 
         a = np.random.rand()
@@ -242,12 +242,12 @@ def test_linearity_adjoint_J(self, shape, dtype, so):
         ns = 2 * size + 1
         if len(shape) == 2:
             nx2, nz2 = shape[0]//2, shape[1]//2
-            m1.data[nx2-size:nx2+size, nz2-size:nz2+size] = \
+            m1.data[nx2-size:nx2+size+1, nz2-size:nz2+size+1] = \
                 -1 + 2 * np.random.rand(ns, ns)
         else:
             nx2, ny2, nz2 = shape[0]//2, shape[1]//2, shape[2]//2
             nx, ny, nz = shape
-            m1.data[nx2-size:nx2+size, ny2-size:ny2+size, nz2-size:nz2+size] = \
+            m1.data[nx2-size:nx2+size+1, ny2-size:ny2+size+1, nz2-size:nz2+size+1] = \
                 -1 + 2 * np.random.rand(ns, ns, ns)
 
         a = np.random.rand()
@@ -289,12 +289,12 @@ def test_adjoint_J(self, shape, dtype, so):
         ns = 2 * size + 1
         if len(shape) == 2:
             nx2, nz2 = shape[0]//2, shape[1]//2
-            dm1.data[nx2-size:nx2+size, nz2-size:nz2+size] = \
+            dm1.data[nx2-size:nx2+size+1, nz2-size:nz2+size+1] = \
                 -1 + 2 * np.random.rand(ns, ns)
         else:
             nx2, ny2, nz2 = shape[0]//2, shape[1]//2, shape[2]//2
             nx, ny, nz = shape
-            dm1.data[nx2-size:nx2+size, ny2-size:ny2+size, nz2-size:nz2+size] = \
+            dm1.data[nx2-size:nx2+size+1, ny2-size:ny2+size+1, nz2-size:nz2+size+1] = \
                 -1 + 2 * np.random.rand(ns, ns, ns)
 
         # Data perturbation