Staged current progress

Author: fungss

funcs/replit_scrapper.py

@@ -0,0 +1,56 @@
+from dotenv import load_dotenv
+load_dotenv()
+import os
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+import time
+import random
+from selenium.webdriver.common.action_chains import ActionChains
+
+class ReplitScrapper():
+    def __init__(self):
+        chrome_options = Options()
+        chrome_options.add_argument('incognito')
+        chrome_options.add_argument("--window-size=1920,1080")
+        # chrome_options.add_argument("--headless")
+        user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.50 Safari/537.36'
+        chrome_options.add_argument(f'user-agent={user_agent}')
+        driver = webdriver.Chrome(options=chrome_options)
+        self.driver = driver
+
+    # def login(self) -> None:
+    #     self.driver.get(os.environ['LOGINURL'])
+    #     wait = WebDriverWait(self.driver, 10)
+    #     email = wait.until(EC.visibility_of_element_located((By.ID, "1val-input")))
+    #     time.sleep(random.randint(2, 5))
+    #     email.send_keys(os.environ['EMAIL'])
+    #     password = wait.until(EC.visibility_of_element_located((By.ID, "2val-input")))
+    #     time.sleep(random.randint(2, 5))
+    #     password.send_keys(os.environ['PASSWORD'])
+    #     submit_button = wait.until(EC.visibility_of_element_located((By.XPATH, "//button[@type='submit']")))
+    #     time.sleep(random.randint(2, 5))
+    #     action = ActionChains(self.driver)
+    #     action.move_to_element(submit_button)
+    #     action.click()
+        
+    def get_file_list(self) -> list:
+        result = []
+        wait = WebDriverWait(self.driver, 10)
+        showcode_button = wait.until(EC.visibility_of_element_located((By.XPATH, "//div[@class='css-148day7']/span[@class='css-36v8q4']/button[@type='button']")))
+        time.sleep(random.randint(2, 5))
+        action = ActionChains(self.driver)
+        action.move_to_element(showcode_button)
+        action.click()
+        files = wait.until(EC.visibility_of_element_located((By.XPATH, "//div[@role='treeitem']/preceding-sibling::div")))
+        if len(files) != 0:
+            for file in files:
+                result.append(file.div.get_attribute('title'))
+        else:
+            print("File list not found.")
+        return result
+    
+    def cleanup(self) -> None:
+        self.driver.quit()

tests/test_replit_scrapper.py

@@ -0,0 +1,50 @@
+import unittest
+from funcs.replit_scrapper import ReplitScrapper
+from selenium.webdriver.support.wait import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+
+
+class Test(unittest.TestCase):
+
+    # def test_scrapper_quit(self):
+    #     scrapper = ReplitScrapper()
+    #     scrapper.driver.get('https://www.google.com/')
+    #     scrapper.cleanup()
+    #     self.assertFalse(scrapper.driver.service.is_connectable())
+        
+    # def test_scrapper_login_replit_homepage(self):
+    #     scrapper = ReplitScrapper()
+    #     scrapper.login()
+    #     WebDriverWait(scrapper.driver, 10).until(
+    #         EC.presence_of_element_located((By.XPATH, "//div[@data-cy='home-page']"))
+    #     )
+    #     self.assertEqual(scrapper.driver.current_url, 'https://replit.com/~')
+    #     scrapper.cleanup()
+
+    # def test_scrapper_get_given_url_after_login(self):
+    #     scrapper = ReplitScrapper()
+    #     scrapper.login()
+    #     scrapper.driver.get('https://replit.com/@JustCallMeRay/Group2-Aug-23')
+    #     self.assertEqual(scrapper.driver.current_url, 'https://replit.com/@JustCallMeRay/Group2-Aug-23')
+    #     scrapper.cleanup()
+
+    # def test_scrapper_returns_list_given_empty_input(self):
+    #     scrapper = ReplitScrapper()
+    #     scrapper.login()
+    #     file_list = scrapper.get_file_list()
+    #     self.assertIsInstance(file_list, list)
+    #     scrapper.cleanup()
+    
+    def test_scrapper_returns_file_list_given_non_empty_input(self):
+        scrapper = ReplitScrapper()
+        # scrapper.login()
+        scrapper.driver.get('https://replit.com/@JustCallMeRay/Group2-Aug-23')
+        file_list = scrapper.get_file_list()
+        expected = ['main.py']
+        self.assertListEqual(file_list, expected)
+        scrapper.cleanup()
+
+
+if __name__ == "__main__":
+    unittest.main()