document process

Author: Borda

.actions/helpers.py .actions/assistant.py.actions/helpers.py renamed to .actions/assistant.py

@@ -130,7 +130,7 @@ def get_running_torch_version():
 )
 
 
-class HelperCLI:
+class AssistantCLI:
 
     DIR_NOTEBOOKS = ".notebooks"
     META_REQUIRED_FIELDS = ("title", "author", "license", "description")
@@ -158,7 +158,7 @@ class HelperCLI:
 
     @staticmethod
     def _meta_file(folder: str) -> str:
-        files = glob.glob(os.path.join(folder, HelperCLI.META_FILE_REGEX), flags=glob.BRACE)
+        files = glob.glob(os.path.join(folder, AssistantCLI.META_FILE_REGEX), flags=glob.BRACE)
         if len(files) == 1:
             return files[0]
 
@@ -171,9 +171,9 @@ def augment_script(fpath: str):
         """
         with open(fpath) as fp:
             py_file = fp.readlines()
-        fpath_meta = HelperCLI._meta_file(os.path.dirname(fpath))
+        fpath_meta = AssistantCLI._meta_file(os.path.dirname(fpath))
         meta = yaml.safe_load(open(fpath_meta))
-        meta_miss = [fl for fl in HelperCLI.META_REQUIRED_FIELDS if fl not in meta]
+        meta_miss = [fl for fl in AssistantCLI.META_REQUIRED_FIELDS if fl not in meta]
         if meta_miss:
             raise ValueError(f"Meta file '{fpath_meta}' is missing the following fields: {meta_miss}")
         meta.update(
@@ -188,7 +188,7 @@ def augment_script(fpath: str):
         setup = TEMPLATE_SETUP % dict(requirements=" ".join([f'"{req}"' for req in requires]))
         py_file = [header + setup] + py_file + [TEMPLATE_FOOTER]
 
-        py_file = HelperCLI._replace_images(py_file, os.path.dirname(fpath))
+        py_file = AssistantCLI._replace_images(py_file, os.path.dirname(fpath))
 
         with open(fpath, "w") as fp:
             fp.writelines(py_file)
@@ -226,10 +226,10 @@ def _replace_images(lines: list, local_dir: str) -> list:
 
     @staticmethod
     def _is_ipynb_parent_dir(dir_path: str) -> bool:
-        if HelperCLI._meta_file(dir_path):
+        if AssistantCLI._meta_file(dir_path):
             return True
         sub_dirs = [d for d in glob.glob(os.path.join(dir_path, "*")) if os.path.isdir(d)]
-        return any(HelperCLI._is_ipynb_parent_dir(d) for d in sub_dirs)
+        return any(AssistantCLI._is_ipynb_parent_dir(d) for d in sub_dirs)
 
     @staticmethod
     def group_folders(
@@ -255,7 +255,7 @@ def group_folders(
             root_path: path to the root tobe added for all local folder paths in files
 
         Example:
-            >> python helpers.py group-folders ../target-diff.txt --fpaths_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']"
+            >> python assistant.py group-folders ../target-diff.txt --fpaths_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']"
         """
         with open(fpath_gitdiff) as fp:
             changed = [ln.strip() for ln in fp.readlines()]
@@ -275,19 +275,19 @@ def group_folders(
         # unique folders
         dirs = set(dirs)
         # drop folder with skip folder
-        dirs = [pd for pd in dirs if not any(nd in HelperCLI.SKIP_DIRS for nd in pd.split(os.path.sep))]
+        dirs = [pd for pd in dirs if not any(nd in AssistantCLI.SKIP_DIRS for nd in pd.split(os.path.sep))]
         # valid folder has meta
         dirs_exist = [d for d in dirs if os.path.isdir(d)]
-        dirs_invalid = [d for d in dirs_exist if not HelperCLI._meta_file(d)]
+        dirs_invalid = [d for d in dirs_exist if not AssistantCLI._meta_file(d)]
         if strict and dirs_invalid:
-            msg = f"Following folders do not have valid `{HelperCLI.META_FILE_REGEX}`"
+            msg = f"Following folders do not have valid `{AssistantCLI.META_FILE_REGEX}`"
             warn(f"{msg}: \n {os.linesep.join(dirs_invalid)}")
             # check if there is other valid folder in its tree
-            dirs_invalid = [pd for pd in dirs_invalid if not HelperCLI._is_ipynb_parent_dir(pd)]
+            dirs_invalid = [pd for pd in dirs_invalid if not AssistantCLI._is_ipynb_parent_dir(pd)]
             if dirs_invalid:
                 raise FileNotFoundError(f"{msg} nor sub-folder: \n {os.linesep.join(dirs_invalid)}")
 
-        dirs_change = [d for d in dirs_exist if HelperCLI._meta_file(d)]
+        dirs_change = [d for d in dirs_exist if AssistantCLI._meta_file(d)]
         with open(fpath_change_folders, "w") as fp:
             fp.write(os.linesep.join(sorted(dirs_change)))
 
@@ -301,19 +301,21 @@ def parse_requirements(dir_path: str):
         Args:
             dir_path: path to the folder
         """
-        fpath = HelperCLI._meta_file(dir_path)
+        fpath = AssistantCLI._meta_file(dir_path)
         assert fpath, f"Missing Meta file in {dir_path}"
         meta = yaml.safe_load(open(fpath))
         pprint(meta)
 
         req = meta.get("requirements", [])
-        fname = os.path.join(dir_path, HelperCLI.REQUIREMENTS_FILE)
+        fname = os.path.join(dir_path, AssistantCLI.REQUIREMENTS_FILE)
         print(f"File for requirements: {fname}")
         with open(fname, "w") as fp:
             fp.write(os.linesep.join(req))
 
         pip_args = {
-            k.replace(HelperCLI.META_PIP_KEY, ""): v for k, v in meta.items() if k.startswith(HelperCLI.META_PIP_KEY)
+            k.replace(AssistantCLI.META_PIP_KEY, ""): v
+            for k, v in meta.items()
+            if k.startswith(AssistantCLI.META_PIP_KEY)
         }
         cmd_args = []
         for pip_key in pip_args:
@@ -323,7 +325,7 @@ def parse_requirements(dir_path: str):
                 arg = arg % RUNTIME_VERSIONS
                 cmd_args.append(f"--{pip_key} {arg}")
 
-        fname = os.path.join(dir_path, HelperCLI.PIP_ARGS_FILE)
+        fname = os.path.join(dir_path, AssistantCLI.PIP_ARGS_FILE)
         print(f"File for PIP arguments: {fname}")
         with open(fname, "w") as fp:
             fp.write(" ".join(cmd_args))
@@ -349,7 +351,7 @@ def _get_card_item_cell(path_ipynb: str, path_meta: str, path_thumb: Optional[st
 
         dirname = os.path.basename(os.path.dirname(path_ipynb))
         if dirname != ".notebooks":
-            meta["tags"].append(HelperCLI.DIR_TO_TAG.get(dirname, dirname))
+            meta["tags"].append(AssistantCLI.DIR_TO_TAG.get(dirname, dirname))
 
         meta["tags"] = [tag.replace(" ", "-") for tag in meta["tags"]]
         meta["tags"] = ",".join(meta["tags"])
@@ -379,7 +381,7 @@ def _resolve_path_thumb(path_ipynb: str, path_meta: str) -> Optional[str]:
         assert len(paths) == 1, f"Found multiple possible thumbnail paths for notebook: {path_ipynb}."
         path_thumb = paths[0]
         path_thumb = path_thumb.split(os.path.sep)
-        path_thumb = os.path.sep.join(path_thumb[path_thumb.index(HelperCLI.DIR_NOTEBOOKS) + 1 :])
+        path_thumb = os.path.sep.join(path_thumb[path_thumb.index(AssistantCLI.DIR_NOTEBOOKS) + 1 :])
         return path_thumb
 
     @staticmethod
@@ -401,22 +403,22 @@ def copy_notebooks(
         """
         ls_ipynb = []
         for sub in patterns:
-            ls_ipynb += glob.glob(os.path.join(path_root, HelperCLI.DIR_NOTEBOOKS, sub, "*.ipynb"))
+            ls_ipynb += glob.glob(os.path.join(path_root, AssistantCLI.DIR_NOTEBOOKS, sub, "*.ipynb"))
 
         os.makedirs(os.path.join(docs_root, path_docs_ipynb), exist_ok=True)
         ipynb_content = []
         for path_ipynb in tqdm.tqdm(ls_ipynb):
             ipynb = path_ipynb.split(os.path.sep)
-            sub_ipynb = os.path.sep.join(ipynb[ipynb.index(HelperCLI.DIR_NOTEBOOKS) + 1 :])
+            sub_ipynb = os.path.sep.join(ipynb[ipynb.index(AssistantCLI.DIR_NOTEBOOKS) + 1 :])
             new_ipynb = os.path.join(docs_root, path_docs_ipynb, sub_ipynb)
             os.makedirs(os.path.dirname(new_ipynb), exist_ok=True)
 
             path_meta = path_ipynb.replace(".ipynb", ".yaml")
-            path_thumb = HelperCLI._resolve_path_thumb(path_ipynb, path_meta)
+            path_thumb = AssistantCLI._resolve_path_thumb(path_ipynb, path_meta)
 
             if path_thumb is not None:
                 new_thumb = os.path.join(docs_root, path_docs_images, path_thumb)
-                old_path_thumb = os.path.join(path_root, HelperCLI.DIR_NOTEBOOKS, path_thumb)
+                old_path_thumb = os.path.join(path_root, AssistantCLI.DIR_NOTEBOOKS, path_thumb)
                 os.makedirs(os.path.dirname(new_thumb), exist_ok=True)
                 copyfile(old_path_thumb, new_thumb)
                 path_thumb = os.path.join(path_docs_images, path_thumb)
@@ -426,7 +428,7 @@ def copy_notebooks(
             with open(path_ipynb) as f:
                 ipynb = json.load(f)
 
-            ipynb["cells"].append(HelperCLI._get_card_item_cell(path_ipynb, path_meta, path_thumb))
+            ipynb["cells"].append(AssistantCLI._get_card_item_cell(path_ipynb, path_meta, path_thumb))
 
             with open(new_ipynb, "w") as f:
                 json.dump(ipynb, f)
@@ -439,7 +441,7 @@ def valid_accelerator(dir_path: str):
         Args:
             dir_path: path to the folder
         """
-        fpath = HelperCLI._meta_file(dir_path)
+        fpath = AssistantCLI._meta_file(dir_path)
         assert fpath, f"Missing Meta file in {dir_path}"
         meta = yaml.safe_load(open(fpath))
         # default is CPU runtime
@@ -453,7 +455,7 @@ def update_env_details(dir_path: str):
         Args:
              dir_path: path to the folder
         """
-        fpath = HelperCLI._meta_file(dir_path)
+        fpath = AssistantCLI._meta_file(dir_path)
         assert fpath, f"Missing Meta file in {dir_path}"
         meta = yaml.safe_load(open(fpath))
         # default is COU runtime
@@ -474,9 +476,9 @@ def _parse(pkg: str, keys: str = " <=>[]") -> str:
         meta["environment"] = [env[r] for r in require]
         meta["published"] = datetime.now().isoformat()
 
-        fmeta = os.path.join(HelperCLI.DIR_NOTEBOOKS, dir_path) + ".yaml"
+        fmeta = os.path.join(AssistantCLI.DIR_NOTEBOOKS, dir_path) + ".yaml"
         yaml.safe_dump(meta, stream=open(fmeta, "w"), sort_keys=False)
 
 
 if __name__ == "__main__":
-    fire.Fire(HelperCLI)
+    fire.Fire(AssistantCLI)

.actions/git-diff-sync.sh

@@ -30,7 +30,7 @@ cat "dirs-$b2.txt"
 printf "\n\n"
 git merge --ff -s resolve origin/$1
 
-python .actions/helpers.py group-folders target-diff.txt --fpaths_actual_dirs "['dirs-$b1.txt', 'dirs-$b2.txt']"
+python .actions/assistant.py group-folders target-diff.txt --fpaths_actual_dirs "['dirs-$b1.txt', 'dirs-$b2.txt']"
 printf "\n\nChanged folders:\n"
 cat changed-folders.txt
 printf "\n\nDropped folders:\n"

.actions/ipynb-generate.sh

@@ -5,11 +5,13 @@ printf "Converting: $1\n\n"
 
 # check that there is only one python script
 python -c "import os, glob ; assert(len(glob.glob(os.path.join('$1', '*.py'))) == 1)"
-# check that there is meta file
-python -c "import os ; assert any(os.path.isfile(os.path.join('$1', f'.meta{ext}')) for ext in ['.yml', '.yaml'])"
+# check that there is exactly one meta recipe
+python -c "import os ; assert sum(os.path.isfile(os.path.join('$1', f'.meta{ext}')) for ext in ['.yml', '.yaml']) == 1"
 py_file=( $(ls "$1"/*.py) )
 printf $py_file
 
-python .actions/helpers.py augment-script $py_file
+# add header and footer to the python script
+python .actions/assistant.py augment-script $py_file
 
+# generate notebook from given script
 python -m jupytext --set-formats "ipynb,py:percent" $py_file

.actions/ipynb-render.sh

@@ -3,14 +3,17 @@
 set -e
 printf "Rendering: $1\n\n"
 
+# check that there is no ipython file in the folder
 python -c "import os, glob ; assert(len(glob.glob(os.path.join('$1', '*.ipynb'))) == 1)"
 ipynb_file=( $(ls "$1"/*.ipynb) )
 printf $ipynb_file
 
-python -c "import os ; assert any(os.path.isfile(os.path.join('$1', f'.meta{ext}')) for ext in ['.yml', '.yaml'])"
+# check that there i exactly one meta recipe
+python -c "import os ; assert sum(os.path.isfile(os.path.join('$1', f'.meta{ext}')) for ext in ['.yml', '.yaml']) == 1"
 meta_file=( $(ls "$1"/.meta.*) )
 printf $meta_file
 
+# check that folder has at most one thumb file
 python -c "import os, glob ; assert(len(glob.glob(os.path.join('$1', '.thumb.*'))) <= 1)"
 thumb_file=( $(ls "$1"/.thumb.* 2>/dev/null || echo "") ) || true
 [ ! -z $thumb_file ] && printf $thumb_file
@@ -21,6 +24,7 @@ printf $pub_file
 pub_meta_file=".notebooks/$1.yaml"
 printf $pub_meta_file
 
+# if a notebook has thumb image gets its path
 if [ ! -z $thumb_file ]; then
   pub_thumb_file=".notebooks/"${thumb_file/"$1"\/.thumb/$1}
   printf $pub_thumb_file
@@ -29,18 +33,22 @@ fi
 pub_dir=$(dirname "$pub_file")
 mkdir -p $pub_dir
 
-python .actions/helpers.py parse-requirements $1
+# just in case reinstall the global
 pip install --quiet --requirement requirements.txt --upgrade-strategy only-if-needed
+# prepare the requirements specific for the particular notebook
+python .actions/assistant.py parse-requirements $1
 cat "$1/requirements.txt"
 pip_args=$(cat "$1/pip_arguments.txt")
 printf "pip arguments:\n $pip_args\n\n"
+# and install specific packages
 pip install --requirement "$1/requirements.txt" $pip_args
 
+# dry run does not execute the notebooks just takes them as they are
 if [ ! -z "${DRY_RUN}" ] && [ "${DRY_RUN}" == "1" ]; then
   cp $ipynb_file $pub_file
 else
   printf "available: $ACCELERATOR\n"
-  accel=$(python .actions/helpers.py valid-accelerator $1 2>&1)
+  accel=$(python .actions/assistant.py valid-accelerator $1 2>&1)
   if [ $accel == 1 ]
   then
     printf "Processing: $ipynb_file\n"
@@ -51,14 +59,18 @@ else
   fi
 fi
 
-python .actions/helpers.py update-env-details $1
+# Export the actual packages used in runtime
+python .actions/assistant.py update-env-details $1
 
+# copy and add to version the enriched meta config
 cp $meta_file $pub_meta_file
 git add $pub_meta_file
 
+# if thumb image is linked to to the notebook, copy and version it too
 if [ ! -z $thumb_file ]; then
   cp $thumb_file $pub_thumb_file
   git add $pub_thumb_file
 fi
 
+# add the generated notebook to version
 git add $pub_file

.actions/ipynb-test.sh

@@ -3,27 +3,32 @@
 set -e
 printf "Testing: $1\n\n"
 
+# check that there is no ipython file in the folder
 python -c "import glob ; assert(len(glob.glob('$1/*.ipynb')) == 1)"
 ipynb_file=( $(ls "$1"/*.ipynb) )
 py_file=( $(ls "$1"/*.py) )
 printf $ipynb_file
 
 pip install --quiet --requirement requirements.txt --upgrade-strategy only-if-needed
 
-python .actions/helpers.py parse-requirements $1
+# prepare the requirements specific for the particular notebook
+python .actions/assistant.py parse-requirements $1
 cat "$1/requirements.txt"
 
+# prepare isolated environment with inheriting the global packages
 python -m virtualenv --system-site-packages "$1/venv"
 source "$1/venv/bin/activate"
 pip --version
+# just in case reinstall the global
 pip install --quiet --requirement requirements.txt --upgrade-strategy only-if-needed
 pip_args=$(cat "$1/pip_arguments.txt")
 printf "pip arguments:\n $pip_args\n\n"
+# and install specific packages
 pip install --requirement "$1/requirements.txt" $pip_args
 pip list
 
 printf "available: $ACCELERATOR\n"
-accel=$(python .actions/helpers.py valid-accelerator $1 2>&1)
+accel=$(python .actions/assistant.py valid-accelerator $1 2>&1)
 if [ $accel == 1 ]
 then
   #python $py_file
@@ -33,4 +38,6 @@ else
   printf "WARNING: not valid accelerator so no tests will be run.\n"
 fi
 
+# deactivate and clean local environment
 deactivate
+rm -rf "$1/venv"

.azure-pipelines/ipynb-tests.yml

@@ -48,7 +48,7 @@ jobs:
         head=$(git rev-parse origin/main)
         printf "Head: $head\n"
         git diff --name-only $head --output=target-diff.txt
-        python .actions/helpers.py group-folders target-diff.txt
+        python .actions/assistant.py group-folders target-diff.txt
         printf "Changed folders:\n"
         cat changed-folders.txt
       displayName: 'Process folders'

.github/workflows/ci_docs.yml

@@ -40,7 +40,7 @@ jobs:
         run: |
           head=$(git rev-parse origin/"${{ github.base_ref }}")
           git diff --name-only $head --output=master-diff.txt
-          python .actions/helpers.py group-folders master-diff.txt
+          python .actions/assistant.py group-folders master-diff.txt
           printf "Changed folders:\n"
           cat changed-folders.txt
         shell: bash

.github/workflows/ci_testing.yml

@@ -57,7 +57,7 @@ jobs:
         run: |
           head=$(git rev-parse origin/${{ github.base_ref }})
           git diff --name-only $head --output=target-diff.txt
-          python .actions/helpers.py group-folders target-diff.txt
+          python .actions/assistant.py group-folders target-diff.txt
           printf "Changed folders:\n"
           cat changed-folders.txt
         shell: bash