chore: add tests for processing request payload (#688)

### Description

Correction of comments related to  PR #683
Add new tests for `http_crawler` different forms of `payload`

### Testing

The test checks that the `payload` on the client side matches the data
received by the server.
Also added verification that different types of `payload` are correctly
handled on the client side and correctly recognized by the server with
appropriate headers

### Checklist

- [x] CI passed

Author: Mantisus

src/crawlee/_request.py

@@ -138,9 +138,13 @@ class BaseRequestData(BaseModel):
     payload: Annotated[
         HttpPayload | None,
         BeforeValidator(lambda v: v.encode() if isinstance(v, str) else v),
-        PlainSerializer(lambda v: v.decode() if isinstance(v, bytes) else None),
+        PlainSerializer(lambda v: v.decode() if isinstance(v, bytes) else v),
     ] = None
-    """HTTP request payload."""
+    """HTTP request payload.
+
+    TODO: Re-check the need for `Validator` and `Serializer` once the issue is resolved.
+    https://github.com/apify/crawlee-python/issues/94
+    """
 
     user_data: Annotated[
         dict[str, JsonSerializable],  # Internally, the model contains `UserData`, this is just for convenience

tests/unit/_memory_storage_client/test_request_queue_client.py

@@ -105,7 +105,6 @@ async def test_request_state_serialization(request_queue_client: RequestQueueCli
     got_request = await request_queue_client.get_request(request.id)
 
     assert request == got_request
-    assert request.payload == got_request.payload
 
 
 async def test_add_record(request_queue_client: RequestQueueClient) -> None:

tests/unit/http_crawler/test_http_crawler.py

@@ -21,6 +21,18 @@
     from crawlee.http_crawler._http_crawling_context import HttpCrawlingContext
 
 
+# Payload, e.g. data for a form submission.
+PAYLOAD = {
+    'custname': 'John Doe',
+    'custtel': '1234567890',
+    'custemail': 'johndoe@example.com',
+    'size': 'large',
+    'topping': '["bacon", "cheese", "mushroom"]',
+    'delivery': '13:00',
+    'comments': 'Please ring the doorbell upon arrival.',
+}
+
+
 @pytest.fixture
 async def mock_request_handler() -> Callable[[HttpCrawlingContext], Awaitable[None]] | AsyncMock:
     return AsyncMock()
@@ -214,21 +226,9 @@ async def test_http_status_statistics(crawler: HttpCrawler, server: respx.MockRo
     [CurlImpersonateHttpClient, HttpxHttpClient],
     ids=['curl', 'httpx'],
 )
-async def test_sending_payload(http_client_class: type[BaseHttpClient]) -> None:
+async def test_sending_payload_as_raw_data(http_client_class: type[BaseHttpClient]) -> None:
     http_client = http_client_class()
     crawler = HttpCrawler(http_client=http_client)
-
-    # Payload, e.g. data from a form submission.
-    payload = {
-        'custname': 'John Doe',
-        'custtel': '1234567890',
-        'custemail': 'johndoe@example.com',
-        'size': 'large',
-        'topping': '["bacon", "cheese", "mushroom"]',
-        'delivery': '13:00',
-        'comments': 'Please ring the doorbell upon arrival.',
-    }
-
     responses = []
 
     @crawler.router.default_handler
@@ -237,35 +237,100 @@ async def request_handler(context: HttpCrawlingContext) -> None:
         # The httpbin.org/post endpoint returns the provided payload in the response.
         responses.append(response)
 
+    encoded_payload = urlencode(PAYLOAD).encode()
     request = Request.from_url(
         url='https://httpbin.org/post',
         method='POST',
-        payload=urlencode(payload).encode(),
+        payload=encoded_payload,
     )
 
     await crawler.run([request])
 
-    # The request handler should be called once.
-    assert len(responses) == 1, 'The request handler should be called once.'
+    assert len(responses) == 1, 'Request handler should be called exactly once.'
+    assert responses[0]['data'].encode() == encoded_payload, 'Response payload data does not match the sent payload.'
 
     # The reconstructed payload data should match the original payload. We have to flatten the values, because
     # parse_qs returns a list of values for each key.
-    response_data = {
-        k: v[0] if len(v) == 1 else v for k, v in parse_qs(responses[0]['data'].strip("b'").strip("'")).items()
-    }
+    response_data = {k: v[0] if len(v) == 1 else v for k, v in parse_qs(responses[0]['data']).items()}
+    assert response_data == PAYLOAD, 'The reconstructed payload data does not match the sent payload.'
 
-    assert response_data == payload, 'The reconstructed payload data should match the original payload.'
+    assert responses[0]['json'] is None, 'Response JSON data should be empty when only raw data is sent.'
+    assert responses[0]['form'] == {}, 'Response form data should be empty when only raw data is sent.'
 
 
 @pytest.mark.parametrize(
     'http_client_class',
     [CurlImpersonateHttpClient, HttpxHttpClient],
     ids=['curl', 'httpx'],
 )
-async def test_sending_url_query_params(http_client_class: type[BaseHttpClient]) -> None:
+async def test_sending_payload_as_form_data(http_client_class: type[BaseHttpClient]) -> None:
     http_client = http_client_class()
     crawler = HttpCrawler(http_client=http_client)
+    responses = []
 
+    @crawler.router.default_handler
+    async def request_handler(context: HttpCrawlingContext) -> None:
+        response = json.loads(context.http_response.read())
+        # The httpbin.org/post endpoint returns the provided payload in the response.
+        responses.append(response)
+
+    request = Request.from_url(
+        url='https://httpbin.org/post',
+        method='POST',
+        headers={'content-type': 'application/x-www-form-urlencoded'},
+        payload=urlencode(PAYLOAD).encode(),
+    )
+
+    await crawler.run([request])
+
+    assert len(responses) == 1, 'Request handler should be called exactly once.'
+    assert responses[0]['form'] == PAYLOAD, 'Form data in response does not match the sent payload.'
+
+    assert responses[0]['json'] is None, 'Response JSON data should be empty when only form data is sent.'
+    assert responses[0]['data'] == '', 'Response raw data should be empty when only form data is sent.'
+
+
+@pytest.mark.parametrize(
+    'http_client_class',
+    [CurlImpersonateHttpClient, HttpxHttpClient],
+    ids=['curl', 'httpx'],
+)
+async def test_sending_payload_as_json(http_client_class: type[BaseHttpClient]) -> None:
+    http_client = http_client_class()
+    crawler = HttpCrawler(http_client=http_client)
+    responses = []
+
+    @crawler.router.default_handler
+    async def request_handler(context: HttpCrawlingContext) -> None:
+        response = json.loads(context.http_response.read())
+        # The httpbin.org/post endpoint returns the provided payload in the response.
+        responses.append(response)
+
+    json_payload = json.dumps(PAYLOAD).encode()
+    request = Request.from_url(
+        url='https://httpbin.org/post',
+        method='POST',
+        payload=json_payload,
+        headers={'content-type': 'application/json'},
+    )
+
+    await crawler.run([request])
+
+    assert len(responses) == 1, 'Request handler should be called exactly once.'
+    assert responses[0]['data'].encode() == json_payload, 'Response raw JSON data does not match the sent payload.'
+    assert responses[0]['json'] == PAYLOAD, 'Response JSON data does not match the sent payload.'
+
+    assert responses[0]['form'] == {}, 'Response form data should be empty when only JSON data is sent.'
+
+
+@pytest.mark.parametrize(
+    'http_client_class',
+    [CurlImpersonateHttpClient, HttpxHttpClient],
+    ids=['curl', 'httpx'],
+)
+async def test_sending_url_query_params(http_client_class: type[BaseHttpClient]) -> None:
+    http_client = http_client_class()
+    crawler = HttpCrawler(http_client=http_client)
     responses = []
 
     @crawler.router.default_handler
@@ -280,11 +345,7 @@ async def request_handler(context: HttpCrawlingContext) -> None:
 
     await crawler.run([request])
 
-    # The request handler should be called once.
-    assert len(responses) == 1, 'The request handler should be called once.'
+    assert len(responses) == 1, 'Request handler should be called exactly once.'
 
-    # Validate the response query parameters.
     response_args = responses[0]['args']
-    assert (
-        response_args == query_params
-    ), 'The reconstructed query parameters should match the original query parameters.'
+    assert response_args == query_params, 'Reconstructed query params must match the original query params.'