add text summarizer

Author: mjhea0

project/app/api/crud.py

@@ -8,7 +8,7 @@
 
 
 async def post(payload: SummaryPayloadSchema) -> int:
-    summary = TextSummary(url=payload.url, summary="dummy summary",)
+    summary = TextSummary(url=payload.url, summary="")
     await summary.save()
     return summary.id
 

project/app/api/summaries.py

@@ -3,7 +3,7 @@
 
 from typing import List
 
-from fastapi import APIRouter, HTTPException, Path
+from fastapi import APIRouter, HTTPException, Path, BackgroundTasks
 
 from app.api import crud
 from app.models.pydantic import (
@@ -12,15 +12,20 @@
     SummaryUpdatePayloadSchema,
 )
 from app.models.tortoise import SummarySchema
+from app.summarizer import generate_summary
 
 
 router = APIRouter()
 
 
 @router.post("/", response_model=SummaryResponseSchema, status_code=201)
-async def create_summary(payload: SummaryPayloadSchema) -> SummaryResponseSchema:
+async def create_summary(
+    payload: SummaryPayloadSchema, background_tasks: BackgroundTasks
+) -> SummaryResponseSchema:
     summary_id = await crud.post(payload)
 
+    background_tasks.add_task(generate_summary, summary_id, payload.url)
+
     response_object = {"id": summary_id, "url": payload.url}
     return response_object
 

project/app/summarizer.py

@@ -0,0 +1,24 @@
+# project/app/summarizer.py
+
+
+import nltk
+from newspaper import Article
+
+from app.models.tortoise import TextSummary
+
+
+async def generate_summary(summary_id: int, url: str) -> None:
+    article = Article(url)
+    article.download()
+    article.parse()
+
+    try:
+        nltk.data.find("tokenizers/punkt")
+    except LookupError:
+        nltk.download("punkt")
+    finally:
+        article.nlp()
+
+    summary = article.summary
+
+    await TextSummary.filter(id=summary_id).update(summary=summary)

project/requirements.txt

@@ -4,8 +4,10 @@ fastapi==0.55.1
 flake8===3.8.2
 gunicorn==20.0.4
 isort==4.3.21
+newspaper3k
 pytest==5.4.2
 pytest-cov==2.9.0
+pytest-xdist==1.32.0
 requests==2.23.0
 tortoise-orm==0.16.12
 uvicorn==0.11.5

project/tests/test_summaries.py

@@ -5,8 +5,15 @@
 
 import pytest
 
+from app.api import summaries
+
+
+def test_create_summary(test_app_with_db, monkeypatch):
+    def mock_generate_summary(summary_id, url):
+        return None
+
+    monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary)
 
-def test_create_summary(test_app_with_db):
     response = test_app_with_db.post(
         "/summaries/", data=json.dumps({"url": "https://foo.bar"})
     )
@@ -33,7 +40,12 @@ def test_create_summaries_invalid_json(test_app):
     assert response.json()["detail"][0]["msg"] == "URL scheme not permitted"
 
 
-def test_read_summary(test_app_with_db):
+def test_read_summary(test_app_with_db, monkeypatch):
+    def mock_generate_summary(summary_id, url):
+        return None
+
+    monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary)
+
     response = test_app_with_db.post(
         "/summaries/", data=json.dumps({"url": "https://foo.bar"})
     )
@@ -45,7 +57,6 @@ def test_read_summary(test_app_with_db):
     response_dict = response.json()
     assert response_dict["id"] == summary_id
     assert response_dict["url"] == "https://foo.bar"
-    assert response_dict["summary"]
     assert response_dict["created_at"]
 
 
@@ -68,7 +79,12 @@ def test_read_summary_incorrect_id(test_app_with_db):
     }
 
 
-def test_read_all_summaries(test_app_with_db):
+def test_read_all_summaries(test_app_with_db, monkeypatch):
+    def mock_generate_summary(summary_id, url):
+        return None
+
+    monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary)
+
     response = test_app_with_db.post(
         "/summaries/", data=json.dumps({"url": "https://foo.bar"})
     )
@@ -81,7 +97,12 @@ def test_read_all_summaries(test_app_with_db):
     assert len(list(filter(lambda d: d["id"] == summary_id, response_list))) == 1
 
 
-def test_remove_summary(test_app_with_db):
+def test_remove_summary(test_app_with_db, monkeypatch):
+    def mock_generate_summary(summary_id, url):
+        return None
+
+    monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary)
+
     response = test_app_with_db.post(
         "/summaries/", data=json.dumps({"url": "https://foo.bar"})
     )
@@ -111,7 +132,12 @@ def test_remove_summary_incorrect_id(test_app_with_db):
     }
 
 
-def test_update_summary(test_app_with_db):
+def test_update_summary(test_app_with_db, monkeypatch):
+    def mock_generate_summary(summary_id, url):
+        return None
+
+    monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary)
+
     response = test_app_with_db.post(
         "/summaries/", data=json.dumps({"url": "https://foo.bar"})
     )

project/tests/test_summaries_unit.py

@@ -0,0 +1,228 @@
+# project/tests/test_summaries_unit.py
+
+
+import json
+from datetime import datetime
+
+import pytest
+
+from app.api import crud, summaries
+
+
+def test_create_summary(test_app, monkeypatch):
+    test_request_payload = {"url": "https://foo.bar"}
+    test_response_payload = {"id": 1, "url": "https://foo.bar"}
+
+    async def mock_post(payload):
+        return 1
+
+    monkeypatch.setattr(crud, "post", mock_post)
+
+    def mock_generate_summary(summary_id, url):
+        return None
+
+    monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary)
+
+    response = test_app.post("/summaries/", data=json.dumps(test_request_payload),)
+
+    assert response.status_code == 201
+    assert response.json() == test_response_payload
+
+
+def test_create_summaries_invalid_json(test_app):
+    response = test_app.post("/summaries/", data=json.dumps({}))
+    assert response.status_code == 422
+    assert response.json() == {
+        "detail": [
+            {
+                "loc": ["body", "payload", "url"],
+                "msg": "field required",
+                "type": "value_error.missing",
+            }
+        ]
+    }
+
+    response = test_app.post("/summaries/", data=json.dumps({"url": "invalid://url"}))
+    assert response.status_code == 422
+    assert response.json()["detail"][0]["msg"] == "URL scheme not permitted"
+
+
+def test_read_summary(test_app, monkeypatch):
+    test_data = {
+        "id": 1,
+        "url": "https://foo.bar",
+        "summary": "summary",
+        "created_at": datetime.utcnow().isoformat(),
+    }
+
+    async def mock_get(id):
+        return test_data
+
+    monkeypatch.setattr(crud, "get", mock_get)
+
+    response = test_app.get("/summaries/1/")
+    assert response.status_code == 200
+    assert response.json() == test_data
+
+
+def test_read_summary_incorrect_id(test_app, monkeypatch):
+    async def mock_get(id):
+        return None
+
+    monkeypatch.setattr(crud, "get", mock_get)
+
+    response = test_app.get("/summaries/999/")
+    assert response.status_code == 404
+    assert response.json()["detail"] == "Summary not found"
+
+
+def test_read_all_summaries(test_app, monkeypatch):
+    test_data = [
+        {
+            "id": 1,
+            "url": "https://foo.bar",
+            "summary": "summary",
+            "created_at": datetime.utcnow().isoformat(),
+        },
+        {
+            "id": 2,
+            "url": "https://testdrivenn.io",
+            "summary": "summary",
+            "created_at": datetime.utcnow().isoformat(),
+        },
+    ]
+
+    async def mock_get_all():
+        return test_data
+
+    monkeypatch.setattr(crud, "get_all", mock_get_all)
+
+    response = test_app.get("/summaries/")
+    assert response.status_code == 200
+    assert response.json() == test_data
+
+
+def test_remove_summary(test_app, monkeypatch):
+    async def mock_get(id):
+        return {
+            "id": 1,
+            "url": "https://foo.bar",
+            "summary": "summary",
+            "created_at": datetime.utcnow().isoformat(),
+        }
+
+    monkeypatch.setattr(crud, "get", mock_get)
+
+    async def mock_delete(id):
+        return id
+
+    monkeypatch.setattr(crud, "delete", mock_delete)
+
+    response = test_app.delete("/summaries/1/")
+    assert response.status_code == 200
+    assert response.json() == {"id": 1, "url": "https://foo.bar"}
+
+
+def test_remove_summary_incorrect_id(test_app, monkeypatch):
+    async def mock_get(id):
+        return None
+
+    monkeypatch.setattr(crud, "get", mock_get)
+
+    response = test_app.delete("/summaries/999/")
+    assert response.status_code == 404
+    assert response.json()["detail"] == "Summary not found"
+
+
+def test_update_summary(test_app, monkeypatch):
+    test_request_payload = {"url": "https://foo.bar", "summary": "updated"}
+    test_response_payload = {
+        "id": 1,
+        "url": "https://foo.bar",
+        "summary": "summary",
+        "created_at": datetime.utcnow().isoformat(),
+    }
+
+    async def mock_put(id, payload):
+        return test_response_payload
+
+    monkeypatch.setattr(crud, "put", mock_put)
+
+    response = test_app.put("/summaries/1/", data=json.dumps(test_request_payload),)
+    assert response.status_code == 200
+    assert response.json() == test_response_payload
+
+
+@pytest.mark.parametrize(
+    "summary_id, payload, status_code, detail",
+    [
+        [
+            999,
+            {"url": "https://foo.bar", "summary": "updated!"},
+            404,
+            "Summary not found",
+        ],
+        [
+            0,
+            {"url": "https://foo.bar", "summary": "updated!"},
+            422,
+            [
+                {
+                    "loc": ["path", "id"],
+                    "msg": "ensure this value is greater than 0",
+                    "type": "value_error.number.not_gt",
+                    "ctx": {"limit_value": 0},
+                }
+            ],
+        ],
+        [
+            1,
+            {},
+            422,
+            [
+                {
+                    "loc": ["body", "payload", "url"],
+                    "msg": "field required",
+                    "type": "value_error.missing",
+                },
+                {
+                    "loc": ["body", "payload", "summary"],
+                    "msg": "field required",
+                    "type": "value_error.missing",
+                },
+            ],
+        ],
+        [
+            1,
+            {"url": "https://foo.bar"},
+            422,
+            [
+                {
+                    "loc": ["body", "payload", "summary"],
+                    "msg": "field required",
+                    "type": "value_error.missing",
+                }
+            ],
+        ],
+    ],
+)
+def test_update_summary_invalid(
+    test_app, monkeypatch, summary_id, payload, status_code, detail
+):
+    async def mock_put(id, payload):
+        return None
+
+    monkeypatch.setattr(crud, "put", mock_put)
+
+    response = test_app.put(f"/summaries/{summary_id}/", data=json.dumps(payload))
+    assert response.status_code == status_code
+    assert response.json()["detail"] == detail
+
+
+def test_update_summary_invalid_url(test_app):
+    response = test_app.put(
+        "/summaries/1/",
+        data=json.dumps({"url": "invalid://url", "summary": "updated!"}),
+    )
+    assert response.status_code == 422
+    assert response.json()["detail"][0]["msg"] == "URL scheme not permitted"