chore: Raised lxml upper bound from <6.0.0 to <=6.0.2 to unblock Python 3.14 support and Bumped beautifulsoup4 minimum version from 4.7.0 to 4.13.5 to fix lxml 6.0 parsing bug with curly braces

nithisub · timm4205 · commit 356583b65e5c · 2026-03-27T20:39:06.000-07:00
diff --git a/requirements.txt b/requirements.txt
@@ -1,9 +1,9 @@
 scramp>=1.2.0,<1.5.0
 pytz>=2020.1
-beautifulsoup4>=4.7.0,<5.0.0
+beautifulsoup4>=4.13.5,<5.0.0
 boto3>=1.42.22,<2.0.0
 requests>=2.23.0,<3.0.0
-lxml>=4.6.5,<6.0.0
+lxml>=4.6.5,<=6.0.2
 botocore>=1.12.201,<2.0.0
 packaging
 setuptools
diff --git a/test/integration/datatype/_generate_test_datatype_tables.py b/test/integration/datatype/_generate_test_datatype_tables.py
@@ -11,6 +11,8 @@
     from redshift_connector import Connection
 
 SCHEMA_NAME: str = "datatype_integration"
+# toggle precision between 18 and 38 to test 8 & 16 byte. scale must be >= 8
+NUMERIC_PRECISION: str = "(18, 8)"
 CREATE_FILE_PATH: str = "{}/datatype_test_stmts.sql".format(pathlib.Path().absolute())
 TEARDOWN_FILE_PATH: str = "{}/datatype_teardown_stmts.sql".format(pathlib.Path().absolute())
 """
@@ -200,7 +202,7 @@ class Datatypes(Enum):
     int2 = auto()
     int4 = auto()
     int8 = auto()
-    numeric = "(18, 8)"  # toggle precision between 18 and 38 to test 8 & 16  byte. scale must be >= 8
+    numeric = auto()
     float4 = auto()
     float8 = auto()
     bool = auto()
@@ -460,7 +462,7 @@ def _build_table_stmts(dt: Datatypes) -> None:
 
     col_type: str = dt.name
     if dt.name == Datatypes.numeric.name:
-        col_type += dt.value
+        col_type += NUMERIC_PRECISION
 
     create_stmt: str = "create table {schema}.test_{datatype} (c1 varchar, c2 {col_type});".format(
         schema=SCHEMA_NAME, datatype=dt.name, col_type=col_type
diff --git a/test/unit/plugin/test_adfs_credentials_provider.py b/test/unit/plugin/test_adfs_credentials_provider.py
@@ -1,6 +1,7 @@
 import typing
 from unittest.mock import MagicMock, Mock, patch
 
+import bs4  # type: ignore
 import pytest  # type: ignore
 import requests
 
@@ -219,3 +220,78 @@ def test_form_based_authentication_empty_saml_response_should_fail(mocker) -> No
 
     with pytest.raises(InterfaceError, match="Failed to find ADFS access_token in authentication response payload"):
         acp.form_based_authentication()
+
+
+class TestLxmlCurlyBraceCompatibility:
+    """
+    Regression tests for lxml 6.0 + bs4 compatibility.
+    """
+
+    # This HTML triggers the actual lxml 6.0 + bs4 < 4.13.5 bug.
+    # lxml 6.0 recovers {currency: as an HTML attribute, and bs4 < 4.13.5
+    # fails to handle it in _getNsTag(), raising ValueError.
+    MALFORMED_HTML_WITH_CURLY_BRACE_ATTR = (
+        '<!DOCTYPE html><html lang="id"><head><noscript>'
+        '<img height="1" width="1" style="display:none" '
+        'src="https://www.example.com/tr?id=123&ev=PageView&noscript=1" '
+        'fbq("track", "Purchase" , {currency: "IDR" , value: 20000.00}); />'
+        '</noscript></head><body></body>'
+    )
+
+    ADFS_HTML_WITH_CURLY_BRACES = """
+    <html>
+    <head>
+        <style>
+            .illustrationClass {background-image:url(/adfs/portal/illustration/illustration.png);}
+            .f_circleG { position: absolute; height: 22px; width: 22px; }
+        </style>
+    </head>
+    <body>
+        <form method="post" id="loginForm" action="/adfs/ls/IdpInitiatedSignOn.aspx?loginToRp=urn:amazon:webservices">
+            <input id="userNameInput" name="UserName" type="email" value="" />
+            <input id="passwordInput" name="Password" type="password" />
+            <input id="optionForms" type="hidden" name="AuthMethod" value="FormsAuthentication"/>
+        </form>
+    </body>
+    </html>
+    """
+
+    ADFS_SAML_RESPONSE_WITH_CURLY_BRACES = """
+    <html>
+    <head><title>Working...</title></head>
+    <body>
+        <form method="POST" name="hiddenform" action="https://signin.aws.amazon.com:443/saml">
+            <input type="hidden" name="SAMLResponse" value="dummy_value"/>
+        </form>
+    </body>
+    </html>
+    """
+
+    def test_lxml6_curly_brace_attribute_bug(self):
+        # This must not raise. With the old bs4, it throws ValueError.
+        soup = bs4.BeautifulSoup(self.MALFORMED_HTML_WITH_CURLY_BRACE_ATTR, features="lxml")
+        assert soup is not None
+
+    def test_parse_adfs_sign_in_page_with_curly_braces(self):
+        soup = bs4.BeautifulSoup(self.ADFS_HTML_WITH_CURLY_BRACES, features="lxml")
+        username_input = soup.find("input", {"name": "UserName"})
+        assert username_input is not None
+        assert username_input.get("type") == "email"
+
+        password_input = soup.find("input", {"name": "Password"})
+        assert password_input is not None
+
+    def test_form_based_auth_with_curly_brace_html(self, mocker):
+        acp, _ = make_valid_adfs_credentials_provider()
+
+        mock_auth_form = MagicMock()
+        mock_auth_form.text = self.ADFS_HTML_WITH_CURLY_BRACES
+
+        mock_saml_response = MagicMock()
+        mock_saml_response.text = self.ADFS_SAML_RESPONSE_WITH_CURLY_BRACES
+
+        mocker.patch("requests.get", return_value=mock_auth_form)
+        mocker.patch("requests.post", return_value=mock_saml_response)
+
+        result = acp.form_based_authentication()
+        assert result == "dummy_value"
