3.12 support, drop 3.7, uprev dependencies (#439)

* support python 3.12, drop 3.7

* fix linting and type checking

* add optional deps to requirements/pyproject.txt

* make test_max_bursts_multiple less flakey :fingers_crossed:

* mypy strict, ruff improvement

* fix github release job

Author: samuelcolvin

.github/workflows/ci.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - v0.23-branch
     tags:
       - '**'
   pull_request: {}
@@ -14,12 +13,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: set up python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.11'
 
     - run: pip install -r requirements/linting.txt -r requirements/pyproject.txt pre-commit
 
@@ -29,20 +28,20 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: set up python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
+        python-version: '3.11'
 
     - run: pip install -r requirements/docs.txt -r requirements/pyproject.txt
     - run: pip install .
 
     - run: make docs
 
     - name: Store docs site
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: docs
         path: docs/_build/
@@ -53,21 +52,21 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu]
-        python: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
         redis: ['5']
         include:
-          - python: '3.10'
+          - python: '3.11'
             redis: '6'
             os: 'ubuntu'
-          - python: '3.10'
+          - python: '3.11'
             redis: '7'
             os: 'ubuntu'
 
     env:
       PYTHON: ${{ matrix.python }}
       OS: ${{ matrix.os }}
 
-    runs-on: ${{ format('{0}-latest', matrix.os) }}
+    runs-on: ${{ matrix.os }}-latest
 
     services:
       redis:
@@ -77,10 +76,10 @@ jobs:
         options: --entrypoint redis-server
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: set up python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python }}
 
@@ -90,50 +89,61 @@ jobs:
 
     - run: coverage xml
 
-    - uses: codecov/codecov-action@v2
+    - uses: codecov/codecov-action@v4
       with:
         file: ./coverage.xml
         env_vars: PYTHON,OS
 
-  deploy:
-    name: Deploy
+  check:
+    if: always()
     needs: [lint, docs, test]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Decide whether the needed jobs succeeded or failed
+        uses: re-actors/alls-green@release/v1
+        id: all-green
+        with:
+          jobs: ${{ toJSON(needs) }}
+
+  release:
+    name: Release
+    needs: [check]
     if: "success() && startsWith(github.ref, 'refs/tags/')"
     runs-on: ubuntu-latest
+    environment: release
+
+    permissions:
+      id-token: write
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: get docs
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: docs
           path: docs/_build/
 
       - name: set up python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.11'
 
       - name: install
-        run: pip install -U twine build packaging
+        run: pip install -U build
 
       - name: check version
         id: check-version
-        run: python <(curl -Ls https://gist.githubusercontent.com/samuelcolvin/4e1ad439c5489e8d6478cdee3eb952ef/raw/check_version.py)
-        env:
-          VERSION_PATH: 'arq/version.py'
+        uses: samuelcolvin/check-python-version@v3.2
+        with:
+          version_file_path: 'arq/version.py'
 
       - name: build
         run: python -m build
 
-      - run: twine check dist/*
-
-      - name: upload to pypi
-        run: twine upload dist/*
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.pypi_token }}
+      - name: Upload package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
 
       - name: publish docs
         if: '!fromJSON(steps.check-version.outputs.IS_PRERELEASE)'

.pre-commit-config.yaml

@@ -11,9 +11,9 @@ repos:
 
 - repo: local
   hooks:
-  - id: lint
-    name: Lint
-    entry: make lint
+  - id: format
+    name: Format
+    entry: make format
     types: [python]
     language: system
     pass_filenames: false

Makefile

@@ -1,24 +1,36 @@
 .DEFAULT_GOAL := all
-isort = isort arq tests
-black = black arq tests
+sources = arq tests
 
 .PHONY: install
 install:
-	pip install -U pip pre-commit
+	pip install -U pip pre-commit pip-tools
 	pip install -r requirements/all.txt
 	pip install -e .[watch]
 	pre-commit install
 
+.PHONY: refresh-lockfiles
+refresh-lockfiles:
+	find requirements/ -name '*.txt' ! -name 'all.txt' -type f -delete
+	make update-lockfiles
+
+.PHONY: update-lockfiles
+update-lockfiles:
+	@echo "Updating requirements/*.txt files using pip-compile"
+	pip-compile -q --strip-extras -o requirements/linting.txt requirements/linting.in
+	pip-compile -q --strip-extras -o requirements/testing.txt requirements/testing.in
+	pip-compile -q --strip-extras -o requirements/docs.txt requirements/docs.in
+	pip-compile -q --strip-extras -o requirements/pyproject.txt pyproject.toml --all-extras
+	pip install --dry-run -r requirements/all.txt
+
 .PHONY: format
 format:
-	$(isort)
-	$(black)
+	ruff check --fix $(sources)
+	ruff format $(sources)
 
 .PHONY: lint
 lint:
-	flake8 --max-complexity 10 --max-line-length 120 --ignore E203,W503 arq/ tests/
-	$(isort) --check-only --df
-	$(black) --check
+	ruff check $(sources)
+	ruff format --check $(sources)
 
 .PHONY: test
 test:

arq/connections.py

@@ -4,7 +4,7 @@
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from operator import attrgetter
-from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, Union, cast
 from urllib.parse import parse_qs, urlparse
 from uuid import uuid4
 
@@ -163,8 +163,8 @@ async def enqueue_job(
 
             job = serialize_job(function, args, kwargs, _job_try, enqueue_time_ms, serializer=self.job_serializer)
             pipe.multi()
-            pipe.psetex(job_key, expires_ms, job)  # type: ignore[no-untyped-call]
-            pipe.zadd(_queue_name, {job_id: score})  # type: ignore[unused-coroutine]
+            pipe.psetex(job_key, expires_ms, job)
+            pipe.zadd(_queue_name, {job_id: score})
             try:
                 await pipe.execute()
             except WatchError:
@@ -210,7 +210,7 @@ async def queued_jobs(self, *, queue_name: Optional[str] = None) -> List[JobDef]
 
 
 async def create_pool(
-    settings_: RedisSettings = None,
+    settings_: Optional[RedisSettings] = None,
     *,
     retry: int = 0,
     job_serializer: Optional[Serializer] = None,
@@ -237,7 +237,8 @@ def pool_factory(*args: Any, **kwargs: Any) -> ArqRedis:
                 ssl=settings.ssl,
                 **kwargs,
             )
-            return client.master_for(settings.sentinel_master, redis_class=ArqRedis)
+            redis = client.master_for(settings.sentinel_master, redis_class=ArqRedis)
+            return cast(ArqRedis, redis)
 
     else:
         pool_factory = functools.partial(
@@ -288,10 +289,10 @@ def pool_factory(*args: Any, **kwargs: Any) -> ArqRedis:
 
 async def log_redis_info(redis: 'Redis[bytes]', log_func: Callable[[str], Any]) -> None:
     async with redis.pipeline(transaction=False) as pipe:
-        pipe.info(section='Server')  # type: ignore[unused-coroutine]
-        pipe.info(section='Memory')  # type: ignore[unused-coroutine]
-        pipe.info(section='Clients')  # type: ignore[unused-coroutine]
-        pipe.dbsize()  # type: ignore[unused-coroutine]
+        pipe.info(section='Server')
+        pipe.info(section='Memory')
+        pipe.info(section='Clients')
+        pipe.dbsize()
         info_server, info_memory, info_clients, key_count = await pipe.execute()
 
     redis_version = info_server.get('redis_version', '?')

arq/jobs.py

@@ -83,7 +83,7 @@ def __init__(
         self._deserializer = _deserializer
 
     async def result(
-        self, timeout: Optional[float] = None, *, poll_delay: float = 0.5, pole_delay: float = None
+        self, timeout: Optional[float] = None, *, poll_delay: float = 0.5, pole_delay: Optional[float] = None
     ) -> Any:
         """
         Get the result of the job or, if the job raised an exception, reraise it.
@@ -103,8 +103,8 @@ async def result(
 
         async for delay in poll(poll_delay):
             async with self._redis.pipeline(transaction=True) as tr:
-                tr.get(result_key_prefix + self.job_id)  # type: ignore[unused-coroutine]
-                tr.zscore(self._queue_name, self.job_id)  # type: ignore[unused-coroutine]
+                tr.get(result_key_prefix + self.job_id)
+                tr.zscore(self._queue_name, self.job_id)
                 v, s = await tr.execute()
 
             if v:
@@ -154,9 +154,9 @@ async def status(self) -> JobStatus:
         Status of the job.
         """
         async with self._redis.pipeline(transaction=True) as tr:
-            tr.exists(result_key_prefix + self.job_id)  # type: ignore[unused-coroutine]
-            tr.exists(in_progress_key_prefix + self.job_id)  # type: ignore[unused-coroutine]
-            tr.zscore(self._queue_name, self.job_id)  # type: ignore[unused-coroutine]
+            tr.exists(result_key_prefix + self.job_id)
+            tr.exists(in_progress_key_prefix + self.job_id)
+            tr.zscore(self._queue_name, self.job_id)
             is_complete, is_in_progress, score = await tr.execute()
 
         if is_complete:
@@ -180,8 +180,8 @@ async def abort(self, *, timeout: Optional[float] = None, poll_delay: float = 0.
         job_info = await self.info()
         if job_info and job_info.score and job_info.score > timestamp_ms():
             async with self._redis.pipeline(transaction=True) as tr:
-                tr.zrem(self._queue_name, self.job_id)  # type: ignore[unused-coroutine]
-                tr.zadd(self._queue_name, {self.job_id: 1})  # type: ignore[unused-coroutine]
+                tr.zrem(self._queue_name, self.job_id)
+                tr.zadd(self._queue_name, {self.job_id: 1})
                 await tr.execute()
 
         await self._redis.zadd(abort_jobs_ss, {self.job_id: timestamp_ms()})

arq/typing.py

@@ -19,8 +19,8 @@
 
 
 if TYPE_CHECKING:
-    from .cron import CronJob  # noqa F401
-    from .worker import Function  # noqa F401
+    from .cron import CronJob
+    from .worker import Function
 
 OptionType = Union[None, Set[int], int]
 WEEKDAYS = 'mon', 'tues', 'wed', 'thurs', 'fri', 'sat', 'sun'