Add class ReplitScrapper and corresponding tests

Author: fungss

funcs/replit_scrapper.py

@@ -0,0 +1,125 @@
+from playwright.sync_api import sync_playwright
+from playwright_stealth import stealth_sync
+
+
+class ReplitScrapper():
+    ua = (
+        """
+        Mozilla/5.0 (Windows NT 10.0; Win64; x64)
+        AppleWebKit/537.36 (KHTML, like Gecko)
+        Chrome/116.0.0.0
+        Safari/537.36
+        Edg/116.0.1938.81
+        """
+    )
+
+    def __init__(self, login_name, login_password):
+        self.__login_name = login_name
+        self.__login_password = login_password
+        self._replit_url = None
+        self._downloaded_filename = None
+
+    def set_replit_url(self, replit_url) -> None:
+        if replit_url is None:
+            raise ValueError
+        self._replit_url = replit_url
+
+    def get_replit_url(self) -> str:
+        if self._replit_url is None:
+            raise ValueError("Missing replit_url")
+        return self._replit_url
+
+    def _set_downloaded_filename(self, filename) -> None:
+        if filename is None:
+            raise ValueError("ReplitScrapper._set_downloaded_filename() argument is None")
+        self._downloaded_filename = filename
+
+    def get_downloaded_filename(self) -> str:
+        if self._downloaded_filename is None:
+            raise ValueError("Missing downloaded_filename")
+        return self._downloaded_filename
+
+    def _visit_replit_repo(self, page) -> None:
+        response = page.goto(self.get_replit_url(), wait_until="domcontentloaded")
+        if response.status != 200:
+            if response.status == 404:
+                print(f"response.status = {response.status}")
+                raise ValueError("Invalid replit_url")
+            else:
+                print(f"response.status = {response.status}")
+                raise ValueError("ReplitScrapper._visit_replit_repo() something other than 404 happened")
+
+    def _login_replit(self, page) -> None:
+        # Login
+        page.goto('https://replit.com/login', wait_until="domcontentloaded")
+        page.screenshot(path="./screen-shots/replit.png")
+        url_init = "https://identitytoolkit.googleapis.com/v1/accounts"
+        with page.expect_response(lambda response: url_init in response.url) as response_info:
+            page.locator(
+                "xpath=/html/body/div[1]/div/div[2]/div/main/div[2]/div/form/div[1]/input"
+            ).fill(self.__login_name)
+            page.locator(
+                "xpath=/html/body/div[1]/div/div[2]/div/main/div[2]/div/form/div[2]/div/input"
+            ).fill(self.__login_password)
+            page.locator(
+                "xpath=/html/body/div[1]/div/div[2]/div/main/div[2]/div/form/div[3]/button"
+            ).click()
+        response = response_info.value
+        if response.status != 200:
+            print(response)
+            if response.status == 400:
+                print(f"response.status = {response.status}")
+                raise ValueError("Invalid login credentials")
+            else:
+                print(f"response.status = {response.status}")
+                raise ValueError("ReplitScrapper._login_replit() something other than 401 happened")
+        page.wait_for_url("https://replit.com/~")
+        page.screenshot(path="./screen-shots/replit_after_login.png")
+
+    def _download_as_zip(self, page) -> None:
+        # Wait for page load
+        page.locator(
+            "xpath=/html/body/div[1]/div[1]/div[1]/div[2]/div/div[1]/div/div[3]/div/div[1]/button/div/span"
+        ).wait_for()
+        while page.locator(
+                "xpath=/html/body/div[1]/div[1]/div[1]/div[2]/header/div[2]/button"
+                ).text_content() != "Run":
+            print(page.locator(
+                "xpath=/html/body/div[1]/div[1]/div[1]/div[2]/header/div[2]/button"
+                ).text_content())
+            page.wait_for_timeout(2000)
+        page.screenshot(path="./screen-shots/target_page.png")
+
+        # Begin downloading
+        page.locator(
+            "xpath=/html/body/div[1]/div[1]/div[1]/div[2]/div/div[1]/div/div[2]/div[1]/div[1]/div/button[3]"
+        ).click()
+        with page.expect_download() as download_info:
+            page.locator(
+                "xpath=/html/body/div[@class='css-1o92kwk']//div[@id='item-4']//div[@class='css-1l2rn59']"
+            ).click()
+        download = download_info.value
+        self._set_downloaded_filename(download.suggested_filename)
+        download.save_as(f"./screen-shots/{download.suggested_filename}")
+
+    def run(self):
+        with sync_playwright() as p:
+            # Context setup
+            browser = p.chromium.launch(slow_mo=50)
+            # browser = p.chromium.launch(headless=False
+            #                 , slow_mo=50
+            #                 )
+            context = browser.new_context(user_agent=ReplitScrapper.ua)
+            page = context.new_page()
+            stealth_sync(page)
+
+            # Login replit
+            self._login_replit(page)
+
+            # Download repo files as zip
+            self._visit_replit_repo(page)
+            self._download_as_zip(page)
+
+            # Clean-up
+            context.close()
+            browser.close()

tests/test_replit_scrapper.py

@@ -1,49 +1,45 @@
 import unittest
 from funcs.replit_scrapper import ReplitScrapper
-from selenium.webdriver.support.wait import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.webdriver.common.by import By
+import os
+from dotenv import load_dotenv
+load_dotenv()
 
 
 class Test(unittest.TestCase):
 
-    # def test_scrapper_quit(self):
-    #     scrapper = ReplitScrapper()
-    #     scrapper.driver.get('https://www.google.com/')
-    #     scrapper.cleanup()
-    #     self.assertFalse(scrapper.driver.service.is_connectable())
-        
-    # def test_scrapper_login_replit_homepage(self):
-    #     scrapper = ReplitScrapper()
-    #     scrapper.login()
-    #     WebDriverWait(scrapper.driver, 10).until(
-    #         EC.presence_of_element_located((By.XPATH, "//div[@data-cy='home-page']"))
-    #     )
-    #     self.assertEqual(scrapper.driver.current_url, 'https://replit.com/~')
-    #     scrapper.cleanup()
-
-    # def test_scrapper_get_given_url_after_login(self):
-    #     scrapper = ReplitScrapper()
-    #     scrapper.login()
-    #     scrapper.driver.get('https://replit.com/@JustCallMeRay/Group2-Aug-23')
-    #     self.assertEqual(scrapper.driver.current_url, 'https://replit.com/@JustCallMeRay/Group2-Aug-23')
-    #     scrapper.cleanup()
-
-    # def test_scrapper_returns_list_given_empty_input(self):
-    #     scrapper = ReplitScrapper()
-    #     scrapper.login()
-    #     file_list = scrapper.get_file_list()
-    #     self.assertIsInstance(file_list, list)
-    #     scrapper.cleanup()
-    
-    def test_scrapper_returns_file_list_given_non_empty_input(self):
-        scrapper = ReplitScrapper()
-        # scrapper.login()
-        scrapper.driver.get('https://replit.com/@JustCallMeRay/Group2-Aug-23')
-        file_list = scrapper.get_file_list()
-        expected = ['main.py']
-        self.assertListEqual(file_list, expected)
-        scrapper.cleanup()
+    def test_scrapper_raise_value_error_when_replit_url_not_set(self):
+        scrapper = ReplitScrapper(login_name=None, login_password=None)
+        with self.assertRaises(ValueError) as ctx_manager:
+            scrapper.get_replit_url()
+        self.assertEqual(str(ctx_manager.exception), 'Missing replit_url')
+
+    def test_scrapper_return_replit_url(self):
+        test_url = "https://replit.com/@pythondojoarchi/SlipperyGargantuanDebuggers"
+
+        scrapper = ReplitScrapper(login_name=None, login_password=None)
+        scrapper.set_replit_url(test_url)
+        self.assertEqual(scrapper.get_replit_url(), test_url)
+
+    # Commented out to avoid acount freezes
+    # def test_scrapper_login_with_invalid_credentials(self):
+    #     scrapper = ReplitScrapper(login_name = os.environ['EMAIL'], login_password = "ThisIsNotTheCorrectPassword")
+    #     with self.assertRaises(ValueError) as ctx_manager:
+    #         scrapper.run()
+    #     self.assertEqual(str(ctx_manager.exception), 'Invalid login credentials')
+
+    def test_scrapper_download_repo_as_zip(self):
+        test_url = "https://replit.com/@pythondojoarchi/SlipperyGargantuanDebuggers"
+        target_zip_name = "SlipperyGargantuanDebuggers.zip"
+        WDIR = os.path.abspath(os.path.dirname(__name__))
+        full_target_file_path = os.path.join(WDIR, "screen-shots", target_zip_name)
+        print(full_target_file_path)
+
+        scrapper = ReplitScrapper(login_name=os.environ['EMAIL'], login_password=os.environ['PASSWORD'])
+        scrapper.set_replit_url(test_url)
+        scrapper.run()
+
+        print(scrapper.get_downloaded_filename())
+        self.assertTrue(os.path.exists(full_target_file_path))
 
 
 if __name__ == "__main__":