add updater

Author: sargunv

updater/Dockerfile

@@ -0,0 +1,22 @@
+FROM docker:dind
+
+RUN apk update
+RUN apk add curl python3 git bash dos2unix openssh build-base python3-dev
+
+RUN ln -sf $(ls /usr/bin/easy_install*) /usr/bin/easy_install
+RUN easy_install pip
+RUN pip install docker-compose
+
+RUN mkdir /updater
+WORKDIR /updater
+COPY . /updater/
+RUN dos2unix cmd.bash
+
+ENV COMMIT_NAME 'Updater Bot'
+ENV COMMIT_EMAIL ''
+ENV COMMIT_MESSAGE '[Updater Bot] Regenerate data'
+ENV BRANCH_NAME 'updater-bot'
+ENV REPO_POKEAPI 'git@github.com:PokeAPI/pokeapi'
+ENV REPO_DATA 'git@github.com:PokeAPI/api-data'
+
+CMD bash cmd.bash

updater/README.md

@@ -0,0 +1,29 @@
+# Updater Bot
+
+## Usage
+
+First, make sure you can read/write the target repository over SSH.
+Launch the bot with a volume containing the SSH keys to `/root/.ssh` and an environment variable for email address.
+Since this container runs Docker within itself, it needs to run in privileged mode.
+
+```
+docker run --privileged -v ~/.ssh:/root/.ssh -e COMMIT_EMAIL=example@example.com sargunv/pokeapi-ditto:updater
+```
+
+**Note:** Due to lack of support for file permissions, this does not work on Docker for Windows.
+
+## Environment Variables
+
+### Required
+
+ - `COMMIT_EMAIL`
+
+### Optional
+
+See [the Dockerfile](updater/Dockerfile) for the defaults.
+
+ - `COMMIT_NAME`
+ - `COMMIT_MESSAGE`
+ - `BRANCH_NAME`
+ - `REPO_POKEAPI`
+ - `REPO_DITTO`

updater/cmd.bash

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+[ -z "${COMMIT_NAME}" ] && { echo "Need to set COMMIT_NAME"; exit 1; }
+[ -z "${COMMIT_EMAIL}" ] && { echo "Need to set COMMIT_EMAIL"; exit 1; }
+[ -z "${COMMIT_MESSAGE}" ] && { echo "Need to set COMMIT_MESSAGE"; exit 1; }
+[ -z "${REPO_POKEAPI}" ] && { echo "Need to set REPO_POKEAPI"; exit 1; }
+[ -z "${REPO_DATA}" ] && { echo "Need to set REPO_DATA"; exit 1; }
+[ -z "${BRANCH_NAME}" ] && { echo "Need to set BRANCH_NAME"; exit 1; }
+
+set -e
+set -o pipefail
+set -x
+
+export COMPOSE_INTERACTIVE_NO_CLI=1
+
+dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 &> /dev/null &
+
+git clone --depth=1 "$REPO_POKEAPI" pokeapi
+git clone --depth=1 "$REPO_DATA" api-data
+
+# set up the pokeapi side
+cd pokeapi
+git apply ../disable-rate-limit.patch
+
+docker volume create --name=redis_data
+docker volume create --name=pg_data
+docker-compose up -d
+
+docker-compose exec -T app python manage.py migrate
+docker-compose exec -T app sh -c 'echo "from data.v2.build import build_all; build_all()" | python manage.py shell'
+
+# set up the data side
+cd ../api-data
+git branch -D "$BRANCH_NAME" || true
+git branch "$BRANCH_NAME"
+git checkout "$BRANCH_NAME"
+
+pip install -r requirements.txt
+rm -r ./data
+ditto clone --src-url http://localhost/ --dest-dir ./data
+ditto analyze --api-dir ./data/api --schema-dir ./data/schema
+
+# commit and push
+git add data
+git config user.name "$COMMIT_NAME"
+git config user.email "$COMMIT_EMAIL"
+git commit -m "$COMMIT_MESSAGE"
+git push -fu origin "$BRANCH_NAME"

updater/disable-rate-limit.patch

@@ -0,0 +1,45 @@
+diff --git a/Resources/nginx/nginx.conf b/Resources/nginx/nginx.conf
+index a8841fe..8346c0a 100644
+--- a/Resources/nginx/nginx.conf
++++ b/Resources/nginx/nginx.conf
+@@ -38,7 +38,7 @@ http {
+         server app:8000 fail_timeout=0;
+     }
+ 
+-    limit_req_zone $binary_remote_addr zone=api:10m rate=2r/s;
++    # limit_req_zone $binary_remote_addr zone=api:10m rate=2r/s;
+ 
+     server {
+         listen 80 deferred;
+@@ -72,7 +72,7 @@ http {
+             add_header Cache-Control "public";
+             add_header Pragma public;
+ 
+-            limit_req zone=api burst=10;
++            # limit_req zone=api burst=10;
+ 
+             proxy_set_header X-Real-IP $remote_addr;
+             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+diff --git a/config/settings.py b/config/settings.py
+index 1b2731b..44bab3e 100755
+--- a/config/settings.py
++++ b/config/settings.py
+@@ -161,12 +161,12 @@ REST_FRAMEWORK = {
+     'PAGE_SIZE': 20,
+ 
+     'PAGINATE_BY': 20,
+-    'DEFAULT_THROTTLE_CLASSES': (
+-        'rest_framework.throttling.AnonRateThrottle',
+-    ),
+-    'DEFAULT_THROTTLE_RATES': {
+-        'anon': '1000/hour'
+-    }
++#    'DEFAULT_THROTTLE_CLASSES': (
++#        'rest_framework.throttling.AnonRateThrottle',
++#    ),
++#    'DEFAULT_THROTTLE_RATES': {
++#        'anon': '1000/hour'
++#    }
+ }
+ 
+ MARKDOWN_DEUX_STYLES = {