Address general issues and no longer wait 45 for starting the profile

Author: alexanderbock

runner/main.py

@@ -47,8 +47,10 @@ def submit_image(result: TestResult, hardware: str, timestamp: str, file: str,
   Submits a new candidate image to the provided URL. This function logs a method
   indicating whether the image submission succeeded
   """
+  f = open(file, "rb")
   res = requests.post(
     url,
+    timeout=30,
     data = {
       "group": result.group,
       "name": result.name,
@@ -59,7 +61,7 @@ def submit_image(result: TestResult, hardware: str, timestamp: str, file: str,
       "commitHash": result.commit
     },
     files = {
-      "file": open(file, "rb"),
+      "file": f,
       "log": result.error
     }
   )
@@ -68,6 +70,7 @@ def submit_image(result: TestResult, hardware: str, timestamp: str, file: str,
     print(res.text)
   else:
     print("Image submitted successfully")
+  f.close()
 
 
 
@@ -132,80 +135,88 @@ def setup_argparse():
   return args
 
 
+if __name__ == "__main__":
+  global_start = time.perf_counter()
+  if os.path.exists("config.json"):
+    submit_images = True
+    with open("config.json") as f:
+      config = json.load(f)
+      url = config["url"]
+      submit_url = f"{url}/api/submit-test"
+      hardware = config["hardware"]
+      runner_id = config["id"]
+  else:
+    print("No 'config.json' provided. Test results will be stored locally instead")
+    submit_images = False
+
+
+  args = setup_argparse()
 
-global_start = time.perf_counter()
-if os.path.exists("config.json"):
-  submit_images = True
-  with open("config.json") as f:
-    config = json.load(f)
-    url = config["url"]
-    submit_url = f"{url}/api/submit-test"
-    hardware = config["hardware"]
-    runner_id = config["id"]
-else:
-  print("No 'config.json' provided. Test results will be stored locally instead")
-  submit_images = False
-
-
-args = setup_argparse()
-
-# Find the executable location and its name
-if os.name == "nt":
-  # Windows
-  executable = f"{args.dir}/bin/RelWithDebInfo/OpenSpace.exe"
-else:
-  # Linux/Mac
-  executable = f"{args.dir}/bin/OpenSpace"
-
-if not os.path.exists(executable):
-  raise Exception(f"Could not find executable '{executable}'")
-
-
-if args.overwrite_path != None:
-  write_configuration_overwrite(args.dir, args.overwrite_path)
-
-
-
-# Running the tests
-if args.test is None:
-  print("Running all tests in OpenSpace folder")
-  files = glob.glob(f"{args.dir}/{test_base_dir}/**/*.ostest", recursive=True)
-  for file in files:
-    # Normalize the path endings to always do forward slashes
-    file = file.replace(os.sep, "/")
-    timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
-    if args.dry_run:
-      print(f"Test: '{file}' run against executable '{executable}'")
-      continue
-
-    result = run_single_test(file, executable)
-    for file in result.files:
-      if submit_images:
-        submit_image(result, hardware, timestamp, file, runner_id, submit_url)
-      else:
-        store_image(result, file)
-    time.sleep(5.0)
-else:
-  tests = args.test.split(",")
-  print(f"Running tests: {tests}")
-  for test in tests:
-    path = f"{args.dir}/{test_base_dir}/{test}.ostest"
-    if not os.path.isfile(path):
-      raise Exception(f"Could not find test '{path}'")
-
-    timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
-
-    if args.dry_run:
-      print(f"Test: '{path}' run against executable '{executable}'")
-      continue
-
-    result = run_single_test(path, executable)
-    for file in result.files:
-      if submit_images:
-        submit_image(result, hardware, timestamp, file, runner_id, submit_url)
-      else:
-        store_image(result, file)
-    time.sleep(5.0)
-
-global_end = time.perf_counter()
-print(f"Total time for all tests: {global_end - global_start}")
+  # Find the executable location and its name
+  if os.name == "nt":
+    # Windows
+    executable = f"{args.dir}/bin/RelWithDebInfo/OpenSpace.exe"
+  else:
+    # Linux
+    executable = f"{args.dir}/bin/OpenSpace"
+
+  if not os.path.exists(executable):
+    raise Exception(f"Could not find executable '{executable}'")
+
+
+  if args.overwrite_path != None:
+    write_configuration_overwrite(args.dir, args.overwrite_path)
+
+
+
+  # Running the tests
+  if args.test is None:
+    print("Running all tests in OpenSpace folder")
+    files = glob.glob(f"{args.dir}/{test_base_dir}/**/*.ostest", recursive=True)
+    for file in files:
+      # Normalize the path endings to always do forward slashes
+      file = file.replace(os.sep, "/")
+      timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
+      if args.dry_run:
+        print(f"Test: '{file}' run against executable '{executable}'")
+        continue
+
+      try:
+        result = run_single_test(file, executable)
+      except Exception as e:
+        print(f"Test '{file}' failed with error: {e}")
+        continue
+      for img in result.files:
+        if submit_images:
+          submit_image(result, hardware, timestamp, img, runner_id, submit_url)
+        else:
+          store_image(result, img)
+      time.sleep(5.0)
+  else:
+    tests = args.test.split(",")
+    print(f"Running tests: {tests}")
+    for test in tests:
+      path = f"{args.dir}/{test_base_dir}/{test}.ostest"
+      if not os.path.isfile(path):
+        raise Exception(f"Could not find test '{path}'")
+
+      timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+      if args.dry_run:
+        print(f"Test: '{path}' run against executable '{executable}'")
+        continue
+
+      try:
+        result = run_single_test(path, executable)
+      except Exception as e:
+        print(f"Test '{path}' failed with error: {e}")
+        continue
+      for file in result.files:
+        if submit_images:
+          submit_image(result, hardware, timestamp, file, runner_id, submit_url)
+        else:
+          store_image(result, file)
+      time.sleep(5.0)
+
+  global_end = time.perf_counter()
+  print(f"Total time for all tests: {global_end - global_start}")

runner/testsuite/instruction.py

@@ -22,7 +22,7 @@
 # OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                                          #
 ##########################################################################################
 
-import time
+import asyncio
 
 
 
@@ -57,7 +57,7 @@ def __init__(self, obj):
 
     if not obj["type"] in Allowed_Types:
       type = obj["type"]
-      raise Exception(f"Invalid type '{type}")
+      raise Exception(f"Invalid type '{type}'")
 
     self.type = obj["type"]
 
@@ -142,7 +142,7 @@ async def run(self, openspace):
         print("    Take Screenshot")
         # We'll wait an extra 5 seconds before taking a screenshot just to be sure that
         # everything is finished
-        time.sleep(5)
+        await asyncio.sleep(5)
 
         # Take the screenshot
         await openspace.takeScreenshot()
@@ -152,7 +152,7 @@ async def run(self, openspace):
         # screenshot. The writing should be on the order of 100 ms + about 35 ms for two
         # frames get us to 135 ms. Lets be on the safe side with a 35x margin and go for
         # a wait of 5 seconds
-        time.sleep(5)
+        await asyncio.sleep(5)
 
       case "script":
         print(f"    Script: {self.value}")
@@ -164,7 +164,7 @@ async def run(self, openspace):
 
       case "wait":
         print(f"    Wait: {self.value}")
-        time.sleep(int(self.value))
+        await asyncio.sleep(int(self.value))
 
       case _:
-        raise Exception(f"Unrecognized instruction type '{type}'")
+        raise Exception(f"Unrecognized instruction type '{self.type}'")

runner/testsuite/openspace.py

@@ -51,7 +51,7 @@ def write_configuration_overwrite(base_path, data_path):
     # Remove the version checking URL as it would otherwise count as a "user"
     f.write("VersionCheckUrl = [[]]\n")
 
-    # Checking the OpenGL state will cause OpenGL errors to appear in the log
+    # Setting this to true will cause OpenGL errors to appear in the log
     f.write("CheckOpenGLState = true\n")
 
     # We can reduce the amount of time that we have to wait for OpenSpace to shut down
@@ -91,7 +91,7 @@ async def internal_run(openspace, test):
   This function assumes that the `openspace` library object is already authenticated and
   connected to the OpenSpace instance and is ready to take commands.
   """
-  print(f"  Starting test")
+  print("  Starting test")
   await setup_test_run(openspace)
   await test.run(openspace)
   print("  Finished test")
@@ -138,11 +138,6 @@ def run_single_test(test_path, executable) -> TestResult:
     stderr=subprocess.PIPE
   )
 
-  # Add a sleeping time instead of repeatedly trying to reconnect. Starting up OpenSpace
-  # in general takes longer than this, so we don't actually lose any time
-  time.sleep(45)
-
-
   async def mainLoop():
     """
     The main loop of an async event loop that is needed for the OpenSpace Python API to

runner/testsuite/test.py

@@ -22,6 +22,7 @@
 # OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                                          #
 ##########################################################################################
 
+import asyncio
 import json
 import os
 import time
@@ -41,7 +42,7 @@ class TestResult:
   """
   group: str
   name: str
-  files = list[str]
+  files: list[str]
   timing: float
   commit: str
   error: str
@@ -55,13 +56,13 @@ class Test:
   """
   def __init__(self, path: str):
     assert(os.path.isfile(path))
-    self.test_path = path
+    self.test_path = path.replace(os.sep, "/")
 
     with open(path) as f:
       content = json.load(f)
 
     if content["profile"] is None:
-      raise f"Missing 'profile' in test {path}'"
+      raise Exception(f"Missing 'profile' in test {path}'")
     self.profile = content["profile"]
 
     if content["commands"] is None:
@@ -103,4 +104,4 @@ async def run(self, openspace):
     """
     for instruction in self.instructions:
       await instruction.run(openspace)
-      time.sleep(1.0)
+      await asyncio.sleep(1.0)