Skip to content

Commit 0e36d31

Browse files
author
Nathan Lee
committed
Merge branch 'develop'
2 parents 8c7dee3 + 59e2f7a commit 0e36d31

4 files changed

Lines changed: 63 additions & 42 deletions

File tree

‎pyrunner/core/config.py‎

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -81,27 +81,28 @@ def __init__(self):
8181
'exec_to_id' : { 'type': int , 'preserve': False, 'env': None, 'value': None, 'default': None },
8282

8383
# Preservable runtime params
84-
'app_version' : { 'type': str , 'preserve': True, 'env': 'APP_VERSION' , 'value': None, 'default': '0.0.0' },
85-
'app_name' : { 'type': str , 'preserve': True, 'env': 'APP_NAME' , 'value': None, 'default': "PyrunnerApp_{}".format(uuid.uuid4()) },
86-
'app_start_time' : { 'type': str , 'preserve': True, 'env': None , 'value': None, 'default': None },
87-
'app_root_dir' : { 'type': str , 'preserve': True, 'env': 'APP_ROOT_DIR' , 'value': None, 'default': None },
88-
'config_dir' : { 'type': str , 'preserve': True, 'env': 'APP_CONFIG_DIR' , 'value': None, 'default': None },
89-
'temp_dir' : { 'type': str , 'preserve': True, 'env': 'APP_TEMP_DIR' , 'value': None, 'default': None },
90-
'log_dir' : { 'type': str , 'preserve': True, 'env': 'APP_LOG_DIR' , 'value': None, 'default': None },
91-
'root_log_dir' : { 'type': str , 'preserve': True, 'env': 'APP_ROOT_LOG_DIR' , 'value': None, 'default': None },
92-
'worker_dir' : { 'type': str , 'preserve': True, 'env': 'APP_WORKER_DIR' , 'value': None, 'default': None },
93-
'nozip' : { 'type': bool, 'preserve': False, 'env': 'APP_NOZIP' , 'value': None, 'default': False },
94-
'dump_logs' : { 'type': bool, 'preserve': False, 'env': 'APP_DUMP_LOGS' , 'value': None, 'default': False },
95-
'email' : { 'type': str , 'preserve': False, 'env': 'APP_EMAIL' , 'value': None, 'default': None },
96-
'debug' : { 'type': bool, 'preserve': False, 'env': 'APP_DEBUG' , 'value': None, 'default': False },
97-
'tickrate' : { 'type': int , 'preserve': False, 'env': 'APP_TICKRATE' , 'value': None, 'default': 1 },
98-
'save_interval' : { 'type': int , 'preserve': False, 'env': 'APP_SAVE_INTERVAL' , 'value': None, 'default': 10 },
99-
'max_procs' : { 'type': int , 'preserve': False, 'env': 'APP_MAX_PROCS' , 'value': None, 'default': -1 },
100-
'log_retention' : { 'type': int , 'preserve': True, 'env': 'APP_LOG_RETENTION' , 'value': None, 'default': 30 },
101-
'dryrun' : { 'type': bool, 'preserve': False, 'env': 'APP_DRYRUN' , 'value': None, 'default': False },
102-
'email_on_fail' : { 'type': bool, 'preserve': False, 'env': 'APP_EMAIL_ON_FAIL' , 'value': None, 'default': True },
103-
'email_on_success' : { 'type': bool, 'preserve': False, 'env': 'APP_EMAIL_ON_SUCCESS', 'value': None, 'default': True },
104-
'test_mode' : { 'type': bool, 'preserve': True, 'env': 'APP_TEST_MODE' , 'value': None, 'default': False }
84+
'app_version' : { 'type': str , 'preserve': True, 'env': 'APP_VERSION' , 'value': None, 'default': '0.0.0' },
85+
'app_name' : { 'type': str , 'preserve': True, 'env': 'APP_NAME' , 'value': None, 'default': "PyrunnerApp_{}".format(uuid.uuid4()) },
86+
'app_start_time' : { 'type': str , 'preserve': True, 'env': None , 'value': None, 'default': None },
87+
'app_root_dir' : { 'type': str , 'preserve': True, 'env': 'APP_ROOT_DIR' , 'value': None, 'default': None },
88+
'config_dir' : { 'type': str , 'preserve': True, 'env': 'APP_CONFIG_DIR' , 'value': None, 'default': None },
89+
'temp_dir' : { 'type': str , 'preserve': True, 'env': 'APP_TEMP_DIR' , 'value': None, 'default': None },
90+
'log_dir' : { 'type': str , 'preserve': True, 'env': 'APP_LOG_DIR' , 'value': None, 'default': None },
91+
'root_log_dir' : { 'type': str , 'preserve': True, 'env': 'APP_ROOT_LOG_DIR' , 'value': None, 'default': None },
92+
'worker_dir' : { 'type': str , 'preserve': True, 'env': 'APP_WORKER_DIR' , 'value': None, 'default': None },
93+
'nozip' : { 'type': bool, 'preserve': False, 'env': 'APP_NOZIP' , 'value': None, 'default': False },
94+
'dump_logs' : { 'type': bool, 'preserve': False, 'env': 'APP_DUMP_LOGS' , 'value': None, 'default': False },
95+
'email' : { 'type': str , 'preserve': False, 'env': 'APP_EMAIL' , 'value': None, 'default': None },
96+
'debug' : { 'type': bool, 'preserve': False, 'env': 'APP_DEBUG' , 'value': None, 'default': False },
97+
'tickrate' : { 'type': int , 'preserve': False, 'env': 'APP_TICKRATE' , 'value': None, 'default': 1 },
98+
'time_between_tasks' : { 'type': int , 'preserve': True, 'env': 'APP_TIME_BETWEEN_TASKS', 'value': None, 'default': 0 },
99+
'save_interval' : { 'type': int , 'preserve': False, 'env': 'APP_SAVE_INTERVAL' , 'value': None, 'default': 10 },
100+
'max_procs' : { 'type': int , 'preserve': False, 'env': 'APP_MAX_PROCS' , 'value': None, 'default': -1 },
101+
'log_retention' : { 'type': int , 'preserve': True, 'env': 'APP_LOG_RETENTION' , 'value': None, 'default': 30 },
102+
'dryrun' : { 'type': bool, 'preserve': False, 'env': 'APP_DRYRUN' , 'value': None, 'default': False },
103+
'email_on_fail' : { 'type': bool, 'preserve': False, 'env': 'APP_EMAIL_ON_FAIL' , 'value': None, 'default': True },
104+
'email_on_success' : { 'type': bool, 'preserve': False, 'env': 'APP_EMAIL_ON_SUCCESS' , 'value': None, 'default': True },
105+
'test_mode' : { 'type': bool, 'preserve': True, 'env': 'APP_TEST_MODE' , 'value': None, 'default': False }
105106
}
106107
self._iter_keys = None
107108

‎pyrunner/core/engine.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ def __init__(self):
3636
self.register = None
3737
self.start_time = None
3838
self.save_state_func = lambda *args: None
39+
self._wait_until = 0
3940

4041
# Initialization of Manager proxy objects and Context
4142
self._manager = Manager()
@@ -114,6 +115,10 @@ def initiate(self, **kwargs):
114115
if self.config['max_procs'] > 0 and len(self.register.running_nodes) >= self.config['max_procs']:
115116
break
116117

118+
if not time.time() >= self._wait_until:
119+
break
120+
121+
self._wait_until = time.time() + self.config['time_between_tasks']
117122
runnable = True
118123
for p in node.parent_nodes:
119124
if p.id >= 0 and p not in self.register.completed_nodes.union(self.register.norun_nodes):

‎pyrunner/core/pyrunner.py‎

Lines changed: 35 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -344,16 +344,17 @@ def parse_args(self):
344344
'setup', 'help', 'nozip', 'interactive', 'abort',
345345
'restart', 'version', 'dryrun', 'debug',
346346
'preserve-context', 'dump-logs', 'allow-duplicate-jobs',
347-
'email=', 'email-on-fail=', 'email-on-success=', 'ef=', 'es=',
348-
'env=', 'cvar=', 'context=',
347+
'email=', 'email-on-fail=', 'email-on-success=',
348+
'env=', 'cvar=', 'context=', 'time-between-tasks=',
349349
'to=', 'from=', 'descendants=', 'ancestors=',
350350
'norun=', 'exec-only=', 'exec-proc-name=',
351351
'max-procs=', 'serde=', 'exec-loop-interval='
352352
]
353353

354354
try:
355355
opts, _ = getopt.getopt(sys.argv[1:], opt_list, longopt_list)
356-
except getopt.GetoptError:
356+
except getopt.GetoptError as e:
357+
print(str(e))
357358
self.show_help()
358359
sys.exit(1)
359360

@@ -378,9 +379,9 @@ def parse_args(self):
378379
self.config['exec_to_id'] = int(arg)
379380
elif opt in ['-e', '--email']:
380381
self.config['email'] = arg
381-
elif opt in ['--ef', '--email-on-fail']:
382+
elif opt in ['--email-on-fail']:
382383
self.config['email_on_fail'] = arg
383-
elif opt in ['--es', '--email-on-success']:
384+
elif opt in ['--email-on-success']:
384385
self.config['email_on_success'] = arg
385386
elif opt == '--env':
386387
parts = arg.split('=')
@@ -398,6 +399,8 @@ def parse_args(self):
398399
self.engine.context.interactive = True
399400
elif opt in ['-t', '--tickrate']:
400401
self.config['tickrate'] = int(arg)
402+
elif opt in ['--time-between-tasks']:
403+
self.config['time_between_tasks'] = int(arg)
401404
elif opt in ['--preserve-context']:
402405
self.preserve_context = True
403406
elif opt in ['--allow-duplicate-jobs']:
@@ -436,20 +439,32 @@ def parse_args(self):
436439
self.config['restart'] = False
437440

438441
def show_help(self):
439-
print("Required:")
440-
print(" -c [CFG_FILENAME] : Provide full path to config file.")
441-
print(" -l [LST_FILENAME] : Provide full path to process list filename.")
442-
print("")
443442
print("Options:")
444-
print(" -r : Use this instead of -l option to execute from last point of failure.")
445-
print(" -n : Maximum number of concurrent processes (Default 10).")
446-
print(" -x : Comma separated list of processes ID's to execute. All other processes will be set to NORUN")
447-
print(" -h --help : Show help (you're reading it right now).")
448-
print(" -e --email : Email to send job notification email upon completion or failure.")
449-
print(" -ef --email-on-fail : Email to send job notification email upon failure.")
450-
print(" -es --email-on-success : Email to send job notification email upon completion.")
451-
print(" -d --debug : Prints list of Pending, Running, Failed, and Defaulted tasks instead of summary counts.")
452-
print(" -i --interactive : Interactive mode. This will force the execution engine to request user input for each non-existent Context variable.")
453-
print(" --env : Allows user to provide key/value pair to export to the environment prior to execution. Can provide this option multiple times.")
454-
print(" --cvar : Allows user to provide key/value pair to initialize the Context object with prior to execution. Can provide this option multiple times.")
443+
print(" -c <path> Path to config file.")
444+
print(" -l <path> Path to process list filename.")
445+
print(" -r, --restart Start from last known point-of-failure, if any.")
446+
print(" -n, --max_procs <num> Maximum number of concurrent processes.")
447+
print(" --exec-proc-name <proc name> Execute only a single process/task with the given name.")
448+
print(" -x, --exec-only <comma seperated nums> Comma separated list of process ID's to execute. All other processes will be set to NORUN.")
449+
print(" -N, --norun <comma separated nums> Comma separated list of process ID's to NOT execute (set to NORUN).")
450+
print(" -D, --descendents <comma separated nums> Comma separated list of process ID's to execute, along with their descendent processes (child procs and beyond).")
451+
print(" -A, --ancestors <comma separated nums> Comma separated list of process ID's to execute, along with their ancestors processes (parent procs and beyond).")
452+
print(" -e, --email <email> Email to send job notification email upon completion or failure.")
453+
print(" --email-on-fail <true|false> Enable/disable job notification email upon failure.")
454+
print(" --email-on-success <true|false> Enable/disable job notification email upon success.")
455+
print(" -d, --debug Prints list of Pending, Running, Failed, and Defaulted tasks instead of summary counts.")
456+
print(" -i, --interactive Interactive mode. This will force the execution engine to request user input for each non-existent Context variable.")
457+
print(" --env <VAR_NAME=var_value> Provide key/value pair to export to the environment prior to execution. Can provide this option multiple times.")
458+
print(" --cvar <VAR_NAME=var_value> Provide key/value pair to initialize the Context object with prior to execution. Can provide this option multiple times.")
459+
print(" --nozip Disable behavior which zips up all log files after job exit.")
460+
print(" --dump-logs Enable behavior which prints all failure logs, if any, to STDOUT after job exit.")
461+
print(" -t, --tickrate <num> Number of times per second that the executon engine should poll child processes/launch new processes. Default is 1.")
462+
print(" --time-between-tasks <seconds> Number of seconds, at minimum, that the execution engine should wait after launching a process before launching another.")
463+
print(" --serde <serializer/deserializer> Specify the process list serializer/deserializer. Default is LST.")
464+
print(" --preserve-context Disables behavior which deletes the job's context file after successful job exit.")
465+
print(" --allow-duplicate-jobs Enables running more than 1 instance of a unique job (based on APP_NAME).")
466+
print(" --abort Aborts running instance of a job (based on APP_NAME), if any.")
467+
print(" --setup Run the PyRunner basic project setup.")
468+
print(" -v, --version Print PyRunner version.")
469+
print(" -h, --help Show help (you're reading it right now).")
455470
return

‎pyrunner/version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = '5.0.0'
1+
__version__ = '5.1.0'

0 commit comments

Comments
 (0)