Merge pull request #1803 from xlt208/lingtaox

Updating update_items.py for updating sample notebooks online

Author: jyaistMap

update_items.py

@@ -12,19 +12,21 @@
 
 """
 
-import os
-import sys
 import argparse
-import traceback
 import json
-import re
 import logging
-log = logging.getLogger(__name__)
+import os
+import re
+import sys
+import traceback
 
-import yaml
 import nbformat
-from nbconvert import HTMLExporter
+import requests
+import yaml
 from arcgis.gis import GIS
+from nbconvert import HTMLExporter
+
+log = logging.getLogger(__name__)
 
 ITEMS_METADATA_YAML_PATH = os.path.join(".", "items_metadata.yaml")
 THUMBNAILS_DIR = os.path.join(".", "static", "thumbnails")
@@ -35,21 +37,27 @@
 NB_PORTAL_TYPE_KEYWORDS = "Notebook, Python"
 NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED = \
     {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Advanced',
-     'notebookRuntimeVersion': '5.0'}
+     'notebookRuntimeVersion': ''}
 NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED_GPU = \
-     {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Advanced with GPU support',
-      'notebookRuntimeVersion': '5.0'}
+    {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Advanced with GPU support',
+     'notebookRuntimeVersion': ''}
 NB_ITEM_PROPERTIES_RUNTIME_STAMP_STANDARD = \
     {'notebookRuntimeName': 'ArcGIS Notebook Python 3 Standard',
-     'notebookRuntimeVersion': '5.0'}
+     'notebookRuntimeVersion': ''}
 NB_ITEM_FOLDER = "Notebook Samples"
 
+
 def _main():
     """Parses arguments, connects to GIS, reads YAML, uploads NBs"""
     args = _parse_cmd_line_args()
     _setup_logging(args)
     gis = GIS(args.portal_url, args.username,
               args.password, verify_cert=False)
+    if args.version:
+        NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED["notebookRuntimeVersion"] = args.version
+        NB_ITEM_PROPERTIES_RUNTIME_STAMP_STANDARD["notebookRuntimeVersion"] = args.version
+    else:
+        _get_current_runtime(gis)
     items_metadata_yaml = _read_items_metadata_yaml()
     if args.replace_profiles:
         _replace_profiles()
@@ -58,100 +66,120 @@ def _main():
     if s.failed_uploads:
         raise Exception(f"Some uploads failed: {s.failed_uploads}")
 
+
 def _parse_cmd_line_args():
     """Parse CMD args, returns an object instance of all user passed in args"""
-    parser = argparse.ArgumentParser(description = "Takes all notebooks "\
-        "this in `gallery` directory, and will upload it to the specified "\
-        "portal/org in the right group with the right categories. "\
-        "(default is geosaurus.maps.arcgis.com, 'Esri Sample Notebooks' group)",
-        formatter_class=argparse.RawTextHelpFormatter)
+    parser = argparse.ArgumentParser(description="Takes all notebooks "
+                                     "this in `gallery` directory, and will upload it to the specified "
+                                     "portal/org in the right group with the right categories. "
+                                     "(default is geosaurus.maps.arcgis.com, 'Esri Sample Notebooks' group)",
+                                     formatter_class=argparse.RawTextHelpFormatter)
     parser.add_argument("--username", "-u", type=str,
-        help="Required username for the portal/org")
+                        help="Required username for the portal/org")
     parser.add_argument("--password", "-p", type=str,
-        help="Required password for the portal/org")
+                        help="Required password for the portal/org")
     parser.add_argument("--portal-url", "-r", type=str,
-        help="The portal to connect to (Default:geosaurus.maps.arcgis.com)",
-        default="https://geosaurus.maps.arcgis.com/")
+                        help="The portal to connect to (default: geosaurus.maps.arcgis.com)",
+                        default="https://geosaurus.maps.arcgis.com/")
+    parser.add_argument("--version", "-ver", type=str,
+                        help="The version of notebook runtime to set on sample notebooks "
+                        "(default: the latest version)",
+                        default="")
     parser.add_argument("--verbose", "-v", action="store_true",
-       help="Print all DEBUG log messages instead of just INFO")
+                        help="Print all DEBUG log messages instead of just INFO")
     parser.add_argument("--replace-profiles", "-c", action="store_true",
-       help="Replace all profiles in notebooks with their appropriate username "\
-            "and passwords. Does this by running misc/tools/replace_profiles.py")
-    args = parser.parse_args(sys.argv[1:]) #don't use filename as 1st arg
+                        help="Replace all profiles in notebooks with their appropriate username "
+                        "and passwords. Does this by running misc/tools/replace_profiles.py")
+    args = parser.parse_args(sys.argv[1:])  # don't use filename as 1st arg
     return args
 
+
 def _setup_logging(args):
     """Sets up the logging based on args"""
     if args.verbose:
         log.setLevel(logging.DEBUG)
     else:
-         log.setLevel(logging.INFO)
+        log.setLevel(logging.INFO)
     stdout_handler = logging.StreamHandler(stream=sys.stdout)
     stdout_handler.setLevel(logging.DEBUG)
     stdout_handler.setFormatter(logging.Formatter(
-        '-----    %(levelname)s    |    '\
-        '%(asctime)s    |    '\
-        '%(filename)s line %(lineno)d'\
-        '     -----\n'\
+        '-----    %(levelname)s    |    '
+        '%(asctime)s    |    '
+        '%(filename)s line %(lineno)d'
+        '     -----\n'
         '"%(message)s"'))
     log.addHandler(stdout_handler)
     log.info("Logging at level {}.".format(logging.getLevelName(log.level)))
     log.debug("args passed in => {}".format(args))
 
+
+def _get_current_runtime(gis):
+    ntbk_svr = gis.notebook_server[0]
+    rest = ntbk_svr._url.replace("/admin", "/rest")
+    rest_info = requests.get(f"{rest}/info?f=json", timeout=5, verify=False)
+    version = rest_info.json()["currentRuntimeVersion"]
+    NB_ITEM_PROPERTIES_RUNTIME_STAMP_ADVANCED["notebookRuntimeVersion"] = version
+    NB_ITEM_PROPERTIES_RUNTIME_STAMP_STANDARD["notebookRuntimeVersion"] = version
+    return version
+
+
 def _read_items_metadata_yaml():
     """Returns the items_metadata.yaml file as a dict"""
-    with open(ITEMS_METADATA_YAML_PATH) as f:
+    with open(ITEMS_METADATA_YAML_PATH, encoding="utf-8") as f:
         return yaml.safe_load(f)
 
+
 def _replace_profiles():
     """Runs misc/tools/replace_profiles.py to go through each notebook in the
     repo and replace profiles with usernames/passwords
     """
     cmd = f"{sys.executable} {REPLACE_PROFILES_SCRIPT}"
     os.system(cmd)
 
+
 class ItemsUploader:
     def __init__(self, gis, items_metadata_yaml):
         self._gis = gis
         self._items_metadata_yaml = items_metadata_yaml
         self.failed_uploads = []
 
-    def upload_items(self, share_after_upload = True):
+    def upload_items(self, share_after_upload=True):
         for entry in self._items_metadata_yaml["samples"] + \
-                     self._items_metadata_yaml["guides"] + \
-                     self._items_metadata_yaml["labs"]:
+                self._items_metadata_yaml["guides"] + \
+                self._items_metadata_yaml["labs"]:
             self._stage_and_upload_item(entry, share_after_upload)
 
-    def _stage_and_upload_item(self, entry, share_after_upload = True):
+    def _stage_and_upload_item(self, entry, share_after_upload=True):
         log.info(f"Uploading {entry['title']}")
         log.debug(f"    sample: {entry}")
         try:
             nb_path = entry["path"]
             self._preupload_check(entry['title'], nb_path)
-            runtime_stamp = self._infer_runtime_stamp(entry.get("runtime", "standard"))
+            runtime_stamp = self._infer_runtime_stamp(
+                entry.get("runtime", "standard"))
             categories = entry.get("categories", None)
             self._stamp_file_with_runtime(nb_path, runtime_stamp)
             item_id = self._infer_item_id(entry["url"])
             item = self.update_item(
-                                item_id = item_id,
-                                item_type = NB_PORTAL_TYPE,
-                                item_type_keywords = NB_PORTAL_TYPE_KEYWORDS,
-                                title = entry['title'],
-                                categories = categories,
-                                snippet = entry['snippet'],
-                                description = entry['description'],
-                                license_info = entry['licenseInfo'],
-                                tags = entry['tags'],
-                                nb_path = nb_path,
-                                runtime_stamp = runtime_stamp,
-                                thumbnail = entry['thumbnail'])
+                item_id=item_id,
+                item_type=NB_PORTAL_TYPE,
+                item_type_keywords=NB_PORTAL_TYPE_KEYWORDS,
+                title=entry['title'],
+                categories=categories,
+                snippet=entry['snippet'],
+                description=entry['description'],
+                license_info=entry['licenseInfo'],
+                tags=entry['tags'],
+                nb_path=nb_path,
+                runtime_stamp=runtime_stamp,
+                thumbnail=entry['thumbnail'])
             if share_after_upload:
-                item.share(everyone = True)
+                item.sharing.sharing_level = "EVERYONE"
             item.protect()
             if categories:
                 self._assign_categories_to_item(item, categories)
             self._apply_html_preview_to_item(item, nb_path)
-            log.info(f"    Uploaded succeded -> {item.homepage}")
+            log.info(f"    Uploaded succeeded -> {item.homepage}")
         except Exception as e:
             self.failed_uploads.append(entry['title'])
             log.warn(f"    Couldn't upload {entry['title']}: {e}")
@@ -170,11 +198,11 @@ def update_item(self, item_id, item_type, item_type_keywords, title, categories,
                     snippet, description, license_info, tags, nb_path,
                     runtime_stamp, thumbnail):
         """Actually uploads the notebook item to the portal"""
-        item_properties = {"title" : title,
-                           "snippet" : snippet,
-                           "description" : description,
-                           "licenseInfo" : license_info,
-                           "tags" : tags,
+        item_properties = {"title": title,
+                           "snippet": snippet,
+                           "description": description,
+                           "licenseInfo": license_info,
+                           "tags": tags,
                            "properties": runtime_stamp}
         if categories:
             item_properties["categories"] = categories
@@ -188,11 +216,12 @@ def update_item(self, item_id, item_type, item_type_keywords, title, categories,
             log.debug(f'item {existing_item.homepage} exists, updating...')
             item_properties["url"] = existing_item.homepage
             existing_item.update(item_properties,
-                                 data = nb_path,
-                                 thumbnail = thumbnail)
+                                 data=nb_path,
+                                 thumbnail=thumbnail)
             resp = existing_item
         else:
-            raise Exception(f"Could not find item {item_id} to update. Failing!")
+            raise Exception(
+                f"Could not find item {item_id} to update. Failing!")
         return resp
 
     def _assign_categories_to_item(self, item, categories):
@@ -207,8 +236,8 @@ def _apply_html_preview_to_item(self, item, nb_path):
 
         json_file_name = "notebook_preview.json"
         json_file_path = os.path.join(".", json_file_name)
-        with open(json_file_path, 'w') as f:
-            json.dump({"html" : html_str}, f)
+        with open(json_file_path, 'w', encoding="utf-8") as f:
+            json.dump({"html": html_str}, f)
 
         if item.resources.list():
             item.resources.remove()
@@ -239,11 +268,12 @@ def _stamp_file_with_runtime(self, notebook_file_path, runtime_stamp):
         nb['metadata']['esriNotebookRuntime'] = runtime_stamp
         nbformat.write(nb, notebook_file_path, nbformat.NO_CONVERT)
 
+
 if __name__ == "__main__":
     try:
         _main()
         sys.exit(0)
     except Exception as e:
         log.exception(e)
-        log.info("Program did not succesfully complete (unhandled exception)")
+        log.info("Program did not successfully complete (unhandled exception)")
         sys.exit(1)