docs: Add Storages & Request loaders guides (#896)

- Consolidate "Request storage" & "Result storage" into a single
"Storages" guide and further improvesit.
- Add a new "Request loaders" guide.
- This can be transferred to the Crawlee JS once the request loaders are
implemented there as well.
- Add Mermaid support to Docusaurus.
- Closes: #833

Author: vdusek

…code/request_storage/rl_basic_example.py …code/request_loaders/rl_basic_example.pydocs/guides/code/request_storage/rl_basic_example.py renamed to docs/guides/code/request_loaders/rl_basic_example.py docs/guides/code/request_storage/rl_basic_example.py renamed to docs/guides/code/request_loaders/rl_basic_example.py

@@ -8,7 +8,11 @@ async def main() -> None:
     # Leave name empty to use the default request list.
     request_list = RequestList(
         name='my-request-list',
-        requests=['https://apify.com/', 'https://crawlee.dev/', 'https://crawlee.dev/python/'],
+        requests=[
+            'https://apify.com/',
+            'https://crawlee.dev/',
+            'https://crawlee.dev/python/',
+        ],
     )
 
     # Fetch and process requests from the queue.

docs/guides/code/request_loaders/tandem_example.py

@@ -0,0 +1,26 @@
+import asyncio
+
+from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+from crawlee.request_loaders import RequestList
+
+
+async def main() -> None:
+    # Create a static request list.
+    request_list = RequestList(['https://crawlee.dev', 'https://apify.com'])
+
+    # Convert the request list to a request manager using the to_tandem method.
+    # It is a tandem with the default request queue.
+    request_manager = await request_list.to_tandem()
+
+    # Create a crawler and pass the request manager to it.
+    crawler = ParselCrawler(request_manager=request_manager)
+
+    @crawler.router.default_handler
+    async def handler(context: ParselCrawlingContext) -> None:
+        # New links will be enqueued directly to the queue.
+        await context.enqueue_links()
+
+    await crawler.run()
+
+
+asyncio.run(main())

…quest_storage/tandem_example_explicit.py …quest_loaders/tandem_example_explicit.pydocs/guides/code/request_storage/tandem_example_explicit.py renamed to docs/guides/code/request_loaders/tandem_example_explicit.py docs/guides/code/request_storage/tandem_example_explicit.py renamed to docs/guides/code/request_loaders/tandem_example_explicit.py

@@ -6,20 +6,22 @@
 
 
 async def main() -> None:
-    # Create a static request list
+    # Create a static request list.
     request_list = RequestList(['https://crawlee.dev', 'https://apify.com'])
 
-    # Open the default request queue
+    # Open the default request queue.
     request_queue = await RequestQueue.open()
 
-    crawler = ParselCrawler(
-        # Requests from the list will be processed first, but they will be enqueued in the default request queue first
-        request_manager=RequestManagerTandem(request_list, request_queue),
-    )
+    # And combine them together to a sinhle request manager.
+    request_manager = RequestManagerTandem(request_list, request_queue)
+
+    # Create a crawler and pass the request manager to it.
+    crawler = ParselCrawler(request_manager=request_manager)
 
     @crawler.router.default_handler
     async def handler(context: ParselCrawlingContext) -> None:
-        await context.enqueue_links()  # New links will be enqueued directly to the queue
+        # New links will be enqueued directly to the queue.
+        await context.enqueue_links()
 
     await crawler.run()
 

docs/guides/code/request_storage/rl_with_crawler_example.py

@@ -5,9 +5,12 @@
 
 
 async def main() -> None:
+    # Set the purge_on_start field to False to avoid purging the storage on start.
     # highlight-next-line
-    config = Configuration(purge_on_start=False)
-    crawler = HttpCrawler(configuration=config)
+    configuration = Configuration(purge_on_start=False)
+
+    # Pass the configuration to the crawler.
+    crawler = HttpCrawler(configuration=configuration)
 
     @crawler.router.default_handler
     async def request_handler(context: HttpCrawlingContext) -> None:

docs/guides/code/request_storage/tandem_example.py

@@ -1,13 +1,21 @@
 import asyncio
 
+from crawlee.crawlers import HttpCrawler
 from crawlee.storage_clients import MemoryStorageClient
 
 
 async def main() -> None:
     storage_client = MemoryStorageClient.from_config()
+
+    # Call the purge_on_start method to explicitly purge the storage.
     # highlight-next-line
     await storage_client.purge_on_start()
 
+    # Pass the storage client to the crawler.
+    crawler = HttpCrawler(storage_client=storage_client)
+
+    # ...
+
 
 if __name__ == '__main__':
     asyncio.run(main())