Further restructuring to ensure app_profile/config is sourced ASAP

Author: Nathan Lee

pyrunner/cli.py

@@ -30,9 +30,8 @@ def main():
     setup()
   else:
     try:
-      pyrunner = PyRunner()
-      pyrunner.parse_args()
-      exit_status = pyrunner.execute()
+      app = PyRunner()
+      exit_status = app.execute()
     except ValueError as value_error:
       exit_status = 2
       print(str(value_error))

pyrunner/core/constants.py

@@ -42,16 +42,15 @@
 from pyrunner import PyRunner
 from pathlib import Path
 
+# Determine absolute path of this file's parent directory at runtime
 abs_dir_path = os.path.dirname(os.path.realpath(__file__))
 
-app = PyRunner()
+# Store path to default config and .lst file
+config_file = '{{}}/config/app_profile'.format(abs_dir_path)
+proc_file = '{{}}/config/{app_name}.lst'.format(abs_dir_path)
 
-# Assign default config and .lst file
-app.config_file = '{{}}/config/app_profile'.format(abs_dir_path)
-app.proc_file = '{{}}/config/{app_name}.lst'.format(abs_dir_path)
-
-# Parse command line args
-app.parse_args()
+# Init PyRunner and assign default config and .lst file
+app = PyRunner(config_file=config_file, proc_file=proc_file)
 
 if __name__ == '__main__':
   # Initiate job and exit driver with return code

pyrunner/core/pyrunner.py

@@ -47,35 +47,31 @@ def __init__(self, **kwargs):
     self.engine = ExecutionEngine()
 
     self._init_params = {
-      'restart' : False,
-      'config_file' : None,
-      'proc_file' : None,
-      'exec_proc_name' : None,
-      'exec_only_list' : [],
+      'config_file'       : kwargs.get('config_file'),
+      'proc_file'         : kwargs.get('proc_file'),
+      'restart'           : False,
+      'cvar_list'         : [],
+      'exec_proc_name'    : None,
+      'exec_only_list'    : [],
       'exec_disable_list' : [],
-      'exec_from_id' : None,
-      'exec_to_id' : None
+      'exec_from_id'      : None,
+      'exec_to_id'        : None
     }
 
+    self._on_fresh_start_func = None
     self._on_start_func = None
     self._on_restart_func = None
     self._on_success_func = None
     self._on_fail_func = None
     self._on_exit_func = None
 
-    # Backwards compatability
-    self.source_config_file = self.config.source_config_file
-    self.load_proc_list_file = self.load_from_file
-    self.load_last_failed = self.load_state
-    
-    if kwargs.get('parse_args', False) == True:
-      self.parse_args()
+    self.parse_args()
 
   def reset_env(self):
     os.environ.clear()
     os.environ.update(self._environ)
 
-  def load_from_file(self, proc_file, restart=False):
+  def load_proc_file(self, proc_file, restart=False):
     if not proc_file or not os.path.isfile(proc_file):
       return False
 
@@ -140,16 +136,15 @@ def on_exit(self, func):
     self._on_exit_func = func
 
   def prepare(self):
-    # Source config
-    self.source_config_file(self._init_params['config_file'])
-    
     # Initialize NodeRegister
     if self._init_params['restart']:
-      if not self.load_last_failed():
-        self._init_params['restart'] = False
-        self.load_proc_list_file(self._init_params['proc_file'])
-    else:
-      self.load_proc_list_file(self._init_params['proc_file'])
+      self.load_state()
+    elif self._init_params.get('proc_file'):
+      self.load_proc_file(self._init_params['proc_file'])
+    
+    # Inject Context var overrides
+    for k,v in self._init_params['cvar_list']:
+      self.engine.context.set(k, v)
 
     # Modify NodeRegister
     if self._init_params['exec_proc_name']:
@@ -323,7 +318,7 @@ def save_state(self, suppress_output=False):
     return
 
   def load_state(self):
-    if not self.load_from_file(self.config.ctllog_file, True):
+    if not self.load_proc_file(self.config.ctllog_file, True):
       return False
 
     if not os.path.isfile(self.config.ctx_file):
@@ -346,6 +341,13 @@ def delete_state(self):
     if os.path.isfile(self.config.ctx_file):
       os.remove(self.config.ctx_file)
 
+  def is_restartable(self):
+    if not os.path.isfile(self.config.ctllog_file):
+      return False
+    if not os.path.isfile(self.config.ctx_file):
+      return False
+    return True
+  
   # NodeRegister wiring
   def add_node(self, **kwargs)    : return self.register.add_node(**kwargs)
   def exec_only(self, id_list)    : return self.register.exec_only(id_list)
@@ -402,13 +404,11 @@ def parse_args(self):
         os.environ[parts[0]] = parts[1]
       elif opt == '--cvar':
         parts = arg.split('=')
-        self.engine.context.set(parts[0], parts[1])
+        self._init_params['cvar_list'].append((parts[0], parts[1]))
       elif opt == '--nozip':
         self.config['nozip'] = True
       elif opt == '--dump-logs':
         self.config['dump_logs'] = True
-      #elif opt == '--context':
-      #  ctx_file = arg
       elif opt == '--dryrun':
         self.config['dryrun'] = True
       elif opt in ['-i', '--interactive']:
@@ -434,6 +434,16 @@ def parse_args(self):
         sys.exit(0)
       else:
         raise ValueError("Error during parsing of opts")
+    
+    # We need to check for and source the app_profile/config file ASAP,
+    # but only after --env vars are processed
+    if not self._init_params.get('config_file'):
+      raise RuntimeError('Config file (app_profile) has not been provided')
+    self.config.source_config_file(self._init_params['config_file'])
+    
+    # Check if restart is possible (ctllog/ctx files exist)
+    if self._init_params['restart'] and not self.is_restartable():
+      self._init_params['restart'] = False
 
   def show_help(self):
     print("Required:")

pyrunner/version.py

@@ -1 +1 @@
-__version__ = '4.1.3'
+__version__ = '4.2.0'