Add github_archiver and tests

Author: fungss

funcs/github_archiver.py

@@ -0,0 +1,117 @@
+from abc import ABC, abstractmethod
+import os
+from github import Github, Auth, InputGitTreeElement
+import base64
+
+
+class GithubArchiverInterface(ABC):
+    @abstractmethod
+    def identify_target_files():
+        """
+        Read list of target files to be pushed to github, excluding replit's system files.
+        Raise error if target folder does not exist or is empty.
+        """
+        pass
+
+    @abstractmethod
+    def commit_to_github():
+        """
+        Commit target files to github.
+        """
+        pass
+
+
+class GithubArchiver(GithubArchiverInterface):
+
+    def __init__(self, project_name, github_access_token, commit_message="Auto-archive") -> None:
+        self._project_name = project_name
+        self._file_paths = dict()
+        self._file_list = list()
+        self._commit_sha = ""
+        self.__github_access_token = github_access_token
+        self._commit_message = commit_message
+
+    def get_project_name(self) -> str:
+        return self._project_name
+
+    def identify_target_files(self) -> None:
+        download_folder_path = "./screen-shots"
+        extracted_folder_path = os.path.join(download_folder_path, self.get_project_name())
+        assert os.path.isdir(extracted_folder_path) is True, "Target folder does not exist"
+        assert len(os.listdir(extracted_folder_path)) != 0, "Target folder is empty"
+
+        replit_junk = [
+            '.cache',
+            '.upm',
+            '.replit',
+            'poetry.lock',
+            'pyproject.toml',
+            'replit_zip_error_log.txt',
+            'replit.nix',
+        ]
+
+        # Walk through the directory and its subdirectories
+        for root, dirs, files in os.walk(extracted_folder_path):
+            for file in files:
+                file_full_path = os.path.join(root, file)
+                file_relative_path = file_full_path.replace(extracted_folder_path, self.get_project_name())
+                if not any(excluded in file_relative_path for excluded in replit_junk):
+                    self._file_paths[file_relative_path] = file_full_path
+                    self._file_list.append(file_relative_path)
+
+    def get_target_files(self) -> list:
+        return self._file_list
+
+    def commit_to_github(self) -> None:
+        assert len(self._file_list) != 0, "Target files are not identified"
+        auth = Auth.Token(self.__github_access_token)
+        g = Github(auth=auth)
+        repo = g.get_user().get_repo('The-Archive')
+        main_branch = repo.get_branch("main")
+        main_tree = repo.get_git_tree(sha=main_branch.commit.sha)
+
+        tree = list()
+        for file_relative_path, file_full_path in self._file_paths.items():
+
+            with open(file_full_path, "rb") as file:
+                file_content = file.read()
+
+            file_content_based64 = base64.b64encode(file_content)
+
+            blob = repo.create_git_blob(
+                    content=file_content_based64.decode('utf-8'),
+                    encoding="base64"
+                )
+
+            tree.append(
+                InputGitTreeElement(
+                    path=file_relative_path,
+                    mode="100644",
+                    type="blob",
+                    sha=blob.sha,
+                )
+            )
+
+        new_tree = repo.create_git_tree(
+            tree=tree,
+            base_tree=main_tree
+        )
+
+        commit = repo.create_git_commit(
+            message=self._commit_message,
+            tree=repo.get_git_tree(sha=new_tree.sha),
+            parents=[repo.get_git_commit(main_branch.commit.sha)],
+        )
+
+        archive_ref = repo.get_git_ref(ref='heads/main')
+        print(f"Archive_ref: {archive_ref}")
+        self._commit_sha = commit.sha
+
+        # Commit to Github
+        archive_ref.edit(sha=commit.sha)
+        print("Upload complete")
+
+        g.close()
+
+    def get_commit_sha(self) -> str:
+        return self._commit_sha

funcs/replit_playwright.py

@@ -1,23 +1,59 @@
+appnope==0.1.3
+asttokens==2.4.1
 certifi==2023.11.17
+cffi==1.16.0
 charset-normalizer==3.3.2
+comm==0.2.0
+cryptography==41.0.7
+debugpy==1.8.0
+decorator==5.1.1
+Deprecated==1.2.14
+executing==2.0.1
 flake8==6.1.0
 greenlet==3.0.1
 idna==3.6
 iniconfig==2.0.0
+ipykernel==6.27.1
+ipython==8.19.0
+jedi==0.19.1
+jupyter_client==8.6.0
+jupyter_core==5.5.1
+matplotlib-inline==0.1.6
 mccabe==0.7.0
+nest-asyncio==1.5.8
 packaging==23.2
+parso==0.8.3
+pexpect==4.9.0
+platformdirs==4.1.0
 playwright==1.40.0
 playwright-stealth==1.0.6
 pluggy==1.3.0
+prompt-toolkit==3.0.43
+psutil==5.9.7
+ptyprocess==0.7.0
+pure-eval==0.2.2
 pycodestyle==2.11.0
+pycparser==2.21
 pyee==11.0.1
 pyflakes==3.1.0
+PyGithub==2.1.1
+Pygments==2.17.2
+PyJWT==2.8.0
+PyNaCl==1.5.0
 pytest==7.4.3
 pytest-base-url==2.0.0
 pytest-playwright==0.4.3
+python-dateutil==2.8.2
 python-dotenv==1.0.0
 python-slugify==8.0.1
+pyzmq==25.1.2
 requests==2.31.0
+six==1.16.0
+stack-data==0.6.3
 text-unidecode==1.3
+tornado==6.4
+traitlets==5.14.0
 typing_extensions==4.9.0
 urllib3==2.1.0
+wcwidth==0.2.12
+wrapt==1.16.0

requirements.txt

@@ -0,0 +1,78 @@
+from github import Github, Auth
+import unittest
+from funcs.github_archiver import GithubArchiver
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+
+class Test(unittest.TestCase):
+
+    def test_archiver_raise_error_if_target_folder_does_not_exist(self):
+        archiver = GithubArchiver(
+            project_name="directory_not_exist",
+            github_access_token=os.environ['GITHUB_ACCESS_TOKEN']
+        )
+        with self.assertRaises(AssertionError) as ctx_manager:
+            archiver.identify_target_files()
+        self.assertEqual(str(ctx_manager.exception), "Target folder does not exist")
+
+    def test_archiver_raise_error_if_target_folder_is_empty(self):
+        archiver = GithubArchiver(
+            project_name="empty_folder",
+            github_access_token=os.environ['GITHUB_ACCESS_TOKEN']
+        )
+        with self.assertRaises(AssertionError) as ctx_manager:
+            archiver.identify_target_files()
+        self.assertEqual(str(ctx_manager.exception), "Target folder is empty")
+
+    def test_archiver_return_list_of_target_files(self):
+        target_list = [
+            "SlipperyGargantuanDebuggers/test-README.md",
+            "SlipperyGargantuanDebuggers/road.jpg",
+            "SlipperyGargantuanDebuggers/test.py",
+            "SlipperyGargantuanDebuggers/main.py",
+            "SlipperyGargantuanDebuggers/Group-1/test-1.txt",
+            "SlipperyGargantuanDebuggers/Group-2/test-2.txt",
+        ]
+
+        archiver = GithubArchiver(
+            project_name="SlipperyGargantuanDebuggers",
+            xxwgithub_access_token=os.environ['GITHUB_ACCESS_TOKEN']
+        )
+        archiver.identify_target_files()
+        # https://stackoverflow.com/questions/12813633/how-to-assert-two-list-contain-the-same-elements-in-python
+        self.assertCountEqual(archiver.get_target_files(), target_list)
+
+    def test_archiver_raise_error_if_target_files_not_set(self):
+        archiver = GithubArchiver(
+            project_name="SlipperyGargantuanDebuggers",
+            github_access_token=os.environ['GITHUB_ACCESS_TOKEN']
+        )
+        with self.assertRaises(AssertionError) as ctx_manager:
+            archiver.commit_to_github()
+
+        self.assertEqual(str(ctx_manager.exception), "Target files are not identified")
+
+    def test_archiver_upload_target_files_to_github(self):
+        archiver = GithubArchiver(
+            project_name="SlipperyGargantuanDebuggers",
+            github_access_token=os.environ['GITHUB_ACCESS_TOKEN']
+        )
+        archiver.identify_target_files()
+        archiver.commit_to_github()
+
+        auth = Auth.Token(os.environ['GITHUB_ACCESS_TOKEN'])
+        g = Github(auth=auth)
+        repo = g.get_user().get_repo('The-Archive')
+        commit = repo.get_commit(archiver.get_commit_sha())
+        target_list = list()
+        for file in commit.files:
+            target_list.append(file.filename)
+        g.close()
+
+        self.assertCountEqual(archiver.get_target_files(), target_list)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_github_archiver.py

@@ -1,6 +1,6 @@
 import unittest
 from funcs.replit_scrapper import ReplitScrapper
-import os
+# import os
 from dotenv import load_dotenv
 load_dotenv()
 
@@ -20,7 +20,7 @@ def test_scrapper_return_replit_url(self):
         scrapper.set_replit_url(test_url)
         self.assertEqual(scrapper.get_replit_url(), test_url)
 
-    # Commented out to avoid acount freezes
+    # Commented out to avoid replit acount freezes
     # def test_scrapper_login_with_invalid_credentials(self):
     #     scrapper = ReplitScrapper(login_name = os.environ['EMAIL'], login_password = "ThisIsNotTheCorrectPassword")
     #     with self.assertRaises(ValueError) as ctx_manager: