Change web scraping library to playwright

Author: fungss

.DS_Store

@@ -1,3 +1,6 @@
+# Act bin file for local testing
+bin/ 
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -158,3 +161,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+playwright/.auth
+screen-shots

.gitignore

@@ -0,0 +1,52 @@
+from playwright.sync_api import sync_playwright
+from playwright_stealth import stealth_sync
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+ua = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/69.0.3497.100 Safari/537.36"
+    )
+
+test_url = "https://replit.com/@pythondojoarchi/SlipperyGargantuanDebuggers"
+
+with sync_playwright() as p:
+    # browser = p.chromium.launch(headless=False
+    #                             , slow_mo=50
+    #                             )
+    browser = p.chromium.launch()
+    context = browser.new_context(user_agent=ua)
+    page = context.new_page()
+    stealth_sync(page)
+    page.goto(os.environ['LOGINURL'], wait_until="domcontentloaded")
+    page.screenshot(path="./screen-shots/replit.png")
+
+    # Login
+    page.locator("xpath=/html/body/div[1]/div/div[2]/div/main/div[2]/div/form/div[1]/input").fill(os.environ['EMAIL'])
+    page.locator("xpath=/html/body/div[1]/div/div[2]/div/main/div[2]/div/form/div[2]/div/input").fill(os.environ['PASSWORD'])
+    page.locator("xpath=/html/body/div[1]/div/div[2]/div/main/div[2]/div/form/div[3]/button").click()
+    page.wait_for_url("https://replit.com/~")
+    page.screenshot(path="./screen-shots/replit_after_login.png")
+
+    # Download repo files as zip
+    page.goto(test_url, wait_until="domcontentloaded")
+    page.locator("xpath=/html/body/div[1]/div[1]/div[1]/div[2]/div/div[1]/div/div[3]/div/div[1]/button/div/span").wait_for()
+    while page.locator("xpath=/html/body/div[1]/div[1]/div[1]/div[2]/header/div[2]/button").text_content() != "Run":
+        print(page.locator("xpath=/html/body/div[1]/div[1]/div[1]/div[2]/header/div[2]/button").text_content())
+        page.wait_for_timeout(2000)
+    page.screenshot(path="./screen-shots/target_page.png")
+    
+    page.locator("xpath=/html/body/div[1]/div[1]/div[1]/div[2]/div/div[1]/div/div[2]/div[1]/div[1]/div/div[3]/button").click()
+    with page.expect_download() as download_info:
+        page.locator("xpath=/html/body/div[@class='css-1o92kwk']//div[@id='item-4']//div[@class='css-1l2rn59']").click()
+    download = download_info.value
+    download.save_as(f"./screen-shots/{download.suggested_filename}")
+
+    # Clean-up
+    context.close()
+    browser.close()
+
+
+

funcs/replit_playwright.py

@@ -1,4 +1,23 @@
+certifi==2023.11.17
+charset-normalizer==3.3.2
 flake8==6.1.0
+greenlet==3.0.1
+idna==3.6
+iniconfig==2.0.0
 mccabe==0.7.0
+packaging==23.2
+playwright==1.40.0
+playwright-stealth==1.0.6
+pluggy==1.3.0
 pycodestyle==2.11.0
+pyee==11.0.1
 pyflakes==3.1.0
+pytest==7.4.3
+pytest-base-url==2.0.0
+pytest-playwright==0.4.3
+python-dotenv==1.0.0
+python-slugify==8.0.1
+requests==2.31.0
+text-unidecode==1.3
+typing_extensions==4.9.0
+urllib3==2.1.0