Merge pull request #280 from stan-dev/feature/22-better-error-handling

Feature/22 better error handling

Author: mitzimorris

cmdstanpy/model.py

@@ -313,7 +313,6 @@ def optimize(
         seed: int = None,
         inits: Union[Dict, float, str] = None,
         output_dir: str = None,
-        save_diagnostics: bool = True,
         algorithm: str = None,
         init_alpha: float = None,
         iter: int = None,
@@ -367,11 +366,6 @@ def optimize(
             files are written. If unspecified, output files will be written
             to a temporary directory which is deleted upon session exit.
 
-        :param save_diagnostics: Whether or not to save diagnostics. If True,
-            csv output files are written to an output file with filename
-            template '<model_name>-<YYYYMMDDHHMM>-diagnostic-<chain_id>',
-            e.g. 'bernoulli-201912081451-diagnostic-1.csv'.
-
         :param algorithm: Algorithm to use. One of: 'BFGS', 'LBFGS', 'Newton'
 
         :param init_alpha: Line search step size for first iteration
@@ -393,7 +387,7 @@ def optimize(
                 seed=seed,
                 inits=_inits,
                 output_dir=output_dir,
-                save_diagnostics=save_diagnostics,
+                save_diagnostics=False,
                 method_args=optimize_args,
             )
 
@@ -402,12 +396,8 @@ def optimize(
             self._run_cmdstan(runset, dummy_chain_id)
 
         if not runset._check_retcodes():
-            msg = 'Error during optimizing'
-            if runset._retcode(dummy_chain_id) != 0:
-                msg = '{}, error code {}'.format(
-                    msg, runset._retcode(dummy_chain_id)
-                )
-                raise RuntimeError(msg)
+            msg = 'Error during optimization.\n{}'.format(runset.get_err_msgs())
+            raise RuntimeError(msg)
         mle = CmdStanMLE(runset)
         return mle
 
@@ -750,17 +740,9 @@ def sample(
                 # re-enable logger for console
                 self._logger.propagate = True
 
-            err_msg = 'Error during sampling.\n'
             if not runset._check_retcodes():
-                for i in range(chains):
-                    if runset._retcode(i) != 0:
-                        err_msg = '{}chain {} returned error code {}\n'.format(
-                            err_msg, i + 1, runset._retcode(i)
-                        )
-                console_errs = runset._get_err_msgs()
-                if len(console_errs) > 0:
-                    err_msg = '{}{}'.format(err_msg, ''.join(console_errs))
-                raise RuntimeError(err_msg)
+                msg = 'Error during sampling.\n{}'.format(runset.get_err_msgs())
+                raise RuntimeError(msg)
 
             mcmc = CmdStanMCMC(runset, validate_csv, logger=self._logger)
         return mcmc
@@ -899,12 +881,9 @@ def generate_quantities(
                     executor.submit(self._run_cmdstan, runset, i)
 
             if not runset._check_retcodes():
-                msg = 'Error during generate_quantities'
-                for i in range(chains):
-                    if runset._retcode(i) != 0:
-                        msg = '{}, chain {} returned error code {}'.format(
-                            msg, i, runset._retcode(i)
-                        )
+                msg = 'Error during generate_quantities.\n{}'.format(
+                    runset.get_err_msgs()
+                )
                 raise RuntimeError(msg)
             quantities = CmdStanGQ(runset=runset, mcmc_sample=sample_drawset)
         return quantities
@@ -1034,12 +1013,10 @@ def variational(
         if not valid:
             raise RuntimeError('The algorithm may not have converged.')
         if not runset._check_retcodes():
-            msg = 'Error during variational inference'
-            if runset._retcode(dummy_chain_id) != 0:
-                msg = '{}, error code {}'.format(
-                    msg, runset._retcode(dummy_chain_id)
-                )
-                raise RuntimeError(msg)
+            msg = 'Error during variational inference.\n{}'.format(
+                runset.get_err_msgs()
+            )
+            raise RuntimeError(msg)
         # pylint: disable=invalid-name
         vb = CmdStanVB(runset)
         return vb

cmdstanpy/stanfit.py

@@ -126,9 +126,25 @@ def __init__(
     def __repr__(self) -> str:
         repr = 'RunSet: chains={}'.format(self._chains)
         repr = '{}\n cmd:\n\t{}'.format(repr, self._cmds[0])
-        repr = '{}\n csv_files:\n\t{}\n output_files:\n\t{}'.format(
-            repr, '\n\t'.join(self._csv_files), '\n\t'.join(self._stdout_files)
-        )
+        repr = '{}\n retcodes={}'.format(repr, self._retcodes)
+        if os.path.exists(self._csv_files[0]):
+            repr = '{}\n csv_files:\n\t{}'.format(
+                repr, '\n\t'.join(self._csv_files)
+            )
+        if self._args.save_diagnostics and os.path.exists(
+            self._diagnostic_files[0]
+        ):
+            repr = '{}\n diagnostics_files:\n\t{}'.format(
+                repr, '\n\t'.join(self._diagnostic_files)
+            )
+        if os.path.exists(self._stdout_files[0]):
+            repr = '{}\n console_msgs:\n\t{}'.format(
+                repr, '\n\t'.join(self._stdout_files)
+            )
+        if os.path.exists(self._stderr_files[0]):
+            repr = '{}\n error_msgs:\n\t{}'.format(
+                repr, '\n\t'.join(self._stderr_files)
+            )
         return repr
 
     @property
@@ -199,27 +215,37 @@ def _set_retcode(self, idx: int, val: int) -> None:
         """Set retcode for chain[idx] to val."""
         self._retcodes[idx] = val
 
-    def _get_err_msgs(self) -> List[str]:
+    def get_err_msgs(self) -> List[str]:
         """Checks console messages for each chain."""
         msgs = []
+        msgs.append(self.__repr__())
         for i in range(self._chains):
             if (
                 os.path.exists(self._stderr_files[i])
                 and os.stat(self._stderr_files[i]).st_size > 0
             ):
                 with open(self._stderr_files[i], 'r') as fd:
-                    msgs.append('chain {}:\n{}\n'.format(i + 1, fd.read()))
+                    msgs.append(
+                        'chain_id {}:\n{}\n'.format(
+                            self._chain_ids[i], fd.read()
+                        )
+                    )
             if (
                 os.path.exists(self._stdout_files[i])
                 and os.stat(self._stdout_files[i]).st_size > 0
             ):
                 with open(self._stdout_files[i], 'r') as fd:
                     contents = fd.read()
-                    pat = re.compile(r'^Exception.*$', re.M)
+                    # pattern matches initial "Exception" or "Error" msg
+                    pat = re.compile(r'^E[rx].*$', re.M)
                 errors = re.findall(pat, contents)
                 if len(errors) > 0:
-                    msgs.append('chain {}: {}\n'.format(i + 1, errors))
-        return msgs
+                    msgs.append(
+                        'chain_id {}:\n\t{}\n'.format(
+                            self._chain_ids[i], '\n\t'.join(errors)
+                        )
+                    )
+        return '\n'.join(msgs)
 
     def save_csvfiles(self, dir: str = None) -> None:
         """

test/test_optimize.py

@@ -124,9 +124,7 @@ def test_optimize_bad(self):
         )
         exp_bound_model = CmdStanModel(stan_file=stan)
         no_data = {}
-        with self.assertRaisesRegex(
-            Exception, 'Error during optimizing, error code 70'
-        ):
+        with self.assertRaisesRegex(RuntimeError, 'Error during optimization'):
             exp_bound_model.optimize(
                 data=no_data, seed=1239812093, inits=None, algorithm='BFGS'
             )

test/test_runset.py

@@ -13,7 +13,7 @@
 
 
 class RunSetTest(unittest.TestCase):
-    def test_check_retcodes(self):
+    def test_check_repr(self):
         exe = os.path.join(DATAFILES_PATH, 'bernoulli' + EXTENSION)
         jdata = os.path.join(DATAFILES_PATH, 'bernoulli.data.json')
         sampler_args = SamplerArgs()
@@ -26,8 +26,26 @@ def test_check_retcodes(self):
             method_args=sampler_args,
         )
         runset = RunSet(args=cmdstan_args)
+
         self.assertIn('RunSet: chains=4', runset.__repr__())
         self.assertIn('method=sample', runset.__repr__())
+        self.assertIn('retcodes=[-1, -1, -1, -1]', runset.__repr__())
+        self.assertIn('csv_files:', runset.__repr__())
+        self.assertNotIn('diagnostics_files:', runset.__repr__())
+
+    def test_check_retcodes(self):
+        exe = os.path.join(DATAFILES_PATH, 'bernoulli' + EXTENSION)
+        jdata = os.path.join(DATAFILES_PATH, 'bernoulli.data.json')
+        sampler_args = SamplerArgs()
+        chain_ids = [1, 2, 3, 4]  # default
+        cmdstan_args = CmdStanArgs(
+            model_name='bernoulli',
+            model_exe=exe,
+            chain_ids=chain_ids,
+            data=jdata,
+            method_args=sampler_args,
+        )
+        runset = RunSet(args=cmdstan_args)
 
         retcodes = runset._retcodes
         self.assertEqual(4, len(retcodes))
@@ -60,7 +78,7 @@ def test_get_err_msgs(self):
             stdout_file = 'chain-' + str(i + 1) + '-missing-data-stdout.txt'
             path = os.path.join(DATAFILES_PATH, stdout_file)
             runset._stdout_files[i] = path
-        errs = '\n\t'.join(runset._get_err_msgs())
+        errs = runset.get_err_msgs()
         self.assertIn('Exception', errs)
 
     def test_output_filenames(self):

test/test_sample.py

@@ -720,7 +720,7 @@ def test_validate_bad_run(self):
                 DATAFILES_PATH, 'runset-bad', 'bad-transcript-bern-4.txt'
             ),
         ]
-        self.assertEqual(len(runset._get_err_msgs()), 4)
+        self.assertIn('Exception', runset.get_err_msgs())
 
         # csv file headers inconsistent
         runset._csv_files = [