pagination support (#56)

Follows pagination for supported endpoints. 

* helpers.py: get_next_page parses headers for "next" url
* rest/transport: send methods take an optional pagination_handler,
  if present and a next page is found, accumulate the results via
  the handler function
* rest/zones.py: add pagination_handlers for list and retrieve, pass
  them to make_request
* try to write some actually useful tests exercising twisted transport.
  basic and ugly but don't rely on twisted test frameworks to run.

Author: rupa

README.rst

@@ -70,7 +70,7 @@ Contributions
 Pull Requests and issues are welcome. See the [NS1 Contribution Guidelines](https://github.com/ns1/community) for more information.
 
 Our CI process will lint and check for formatting issues with `flake8` and
-`black`. 
+`black`.
 It is suggested to run these checks prior to submitting a pull request and fix
 any issues:
 ```

examples/zones.py

@@ -15,6 +15,12 @@
 # to load an alternate configuration file:
 # api = NS1(configFile='/etc/ns1/api.json')
 
+# turn on "follow pagination". This will handle paginated responses for
+# zone list and the records for a zone retrieve. It's off by default to
+# avoid a breaking change
+config = api.config
+config["follow_pagination"] = True
+
 ######################
 # LOAD / CREATE ZONE #
 ######################

ns1/config.py

@@ -65,6 +65,9 @@ def _doDefaults(self):
         if "ddi" not in self._data:
             self._data["ddi"] = False
 
+        if "follow_pagination" not in self._data:
+            self._data["follow_pagination"] = False
+
     def createFromAPIKey(self, apikey, maybeWriteDefault=False):
         """
         Create a basic config from a single API key

ns1/helpers.py

@@ -1,3 +1,5 @@
+import re
+
 from threading import Lock
 
 
@@ -13,3 +15,35 @@ def __new__(cls, *args, **kwargs):
                 if cls._instance is None:
                     cls._instance = object.__new__(cls, *args, **kwargs)
         return cls._instance
+
+
+def get_next_page(headers):
+    headers = {k.lower(): v for k, v in headers.items()}
+    links = _parse_header_links(headers.get("link", ""))
+    for link in links:
+        if link.get("rel") == "next":
+            return link.get("url").replace("http://", "https://")
+
+
+# cribbed from requests, since we don't want to require it as a dependency
+def _parse_header_links(value):
+    """Return a dict of parsed link headers proxies.
+    i.e. Link: <http:/.../front.jpeg>; rel=front; type="image/jpeg",<http://.../back.jpeg>; rel=back;type="image/jpeg"
+    """
+    links = []
+    replace_chars = " '\""
+    for val in re.split(", *<", value):
+        try:
+            url, params = val.split(";", 1)
+        except ValueError:
+            url, params = val, ""
+        link = {}
+        link["url"] = url.strip("<> '\"")
+        for param in params.split(";"):
+            try:
+                key, value = param.split("=")
+            except ValueError:
+                break
+            link[key.strip(replace_chars)] = value.strip(replace_chars)
+        links.append(link)
+    return links

ns1/rest/transport/base.py

@@ -18,6 +18,7 @@ def __init__(self, config, module):
             "ignore-ssl-errors", self._config.get("ignore-ssl-errors", False)
         )
         self._rate_limit_func = self._config.getRateLimitingFunc()
+        self._follow_pagination = self._config.get("follow_pagination", False)
 
     def _logHeaders(self, headers):
         if self._config["verbosity"] > 0:

ns1/rest/transport/basic.py

@@ -5,6 +5,7 @@
 #
 from __future__ import absolute_import
 
+from ns1.helpers import get_next_page
 from ns1.rest.transport.base import TransportBase
 from ns1.rest.errors import (
     ResourceException,
@@ -54,30 +55,7 @@ def _set_opener(self):
             return build_opener(HTTPSHandler(context=context))
         return build_opener(HTTPSHandler)
 
-    def send(
-        self,
-        method,
-        url,
-        headers=None,
-        data=None,
-        files=None,
-        params=None,
-        callback=None,
-        errback=None,
-    ):
-        if headers is None:
-            headers = {}
-        if files is not None:
-            # XXX
-            raise Exception("file uploads not supported in BasicTransport yet")
-        self._logHeaders(headers)
-        self._log.debug("%s %s %s" % (method, url, data))
-
-        if sys.version_info.major >= 3 and isinstance(data, str):
-            data = data.encode("utf-8")
-        request = Request(url, headers=headers, data=data)
-        request.get_method = lambda: method
-
+    def _send(self, url, headers, data, method, errback):
         def handleProblem(code, resp, msg):
             if errback:
                 errback((resp, msg))
@@ -103,6 +81,9 @@ def handleProblem(code, resp, msg):
                     body=msg,
                 )
 
+        request = Request(url, headers=headers, data=data)
+        request.get_method = lambda: method
+
         # Handle error and responses the same so we can
         # always pass the body to the handleProblem function
         try:
@@ -127,22 +108,54 @@ def handleProblem(code, resp, msg):
             except AttributeError:
                 pass
             try:
-                jsonOut = json.loads(body)
+                return headers, json.loads(body)
             except ValueError:
                 if errback:
                     errback(resp)
-                    return
                 else:
                     raise ResourceException(
                         "invalid json in response", resp, body
                     )
         else:
-            jsonOut = None
+            return headers, None
+
+    def send(
+        self,
+        method,
+        url,
+        headers=None,
+        data=None,
+        files=None,
+        params=None,
+        callback=None,
+        errback=None,
+        pagination_handler=None,
+    ):
+        if headers is None:
+            headers = {}
+        if files is not None:
+            # XXX
+            raise Exception("file uploads not supported in BasicTransport yet")
+        self._logHeaders(headers)
+        self._log.debug("%s %s %s" % (method, url, data))
+
+        if sys.version_info.major >= 3 and isinstance(data, str):
+            data = data.encode("utf-8")
+
+        resp_headers, jsonOut = self._send(url, headers, data, method, errback)
+        if self._follow_pagination and pagination_handler is not None:
+            next_page = get_next_page(resp_headers)
+            while next_page is not None:
+                self._log.debug("following pagination to: %s" % (next_page))
+                next_headers, next_json = self._send(
+                    next_page, headers, data, method, errback
+                )
+                jsonOut = pagination_handler(jsonOut, next_json)
+                next_page = get_next_page(next_headers)
 
         if callback:
             return callback(jsonOut)
-        else:
-            return jsonOut
+        return jsonOut
 
     def _get_headers(self, response):
         # works for 2 and 3

ns1/rest/transport/requests.py

@@ -5,6 +5,7 @@
 #
 from __future__ import absolute_import
 
+from ns1.helpers import get_next_page
 from ns1.rest.transport.base import TransportBase
 from ns1.rest.errors import (
     ResourceException,
@@ -43,18 +44,7 @@ def _rateLimitHeaders(self, headers):
             "remaining": int(headers.get("X-RateLimit-Remaining", 100)),
         }
 
-    def send(
-        self,
-        method,
-        url,
-        headers=None,
-        data=None,
-        params=None,
-        files=None,
-        callback=None,
-        errback=None,
-    ):
-        self._logHeaders(headers)
+    def _send(self, method, url, headers, data, files, params, errback):
         resp = self.REQ_MAP[method](
             url,
             headers=headers,
@@ -92,7 +82,7 @@ def send(
         # TODO make sure json is valid if a body is returned
         if resp.text:
             try:
-                jsonOut = resp.json()
+                return response_headers, resp.json()
             except ValueError:
                 if errback:
                     errback(resp)
@@ -102,12 +92,38 @@ def send(
                         "invalid json in response", resp, resp.text
                     )
         else:
-            jsonOut = None
+            return response_headers, None
+
+    def send(
+        self,
+        method,
+        url,
+        headers=None,
+        data=None,
+        params=None,
+        files=None,
+        callback=None,
+        errback=None,
+        pagination_handler=None,
+    ):
+        self._logHeaders(headers)
+
+        resp_headers, jsonOut = self._send(
+            method, url, headers, data, files, params, errback
+        )
+        if self._follow_pagination and pagination_handler is not None:
+            next_page = get_next_page(resp_headers)
+            while next_page is not None:
+                self._log.debug("following pagination to: %s" % next_page)
+                next_headers, next_json = self._send(
+                    method, next_page, headers, data, files, params, errback
+                )
+                jsonOut = pagination_handler(jsonOut, next_json)
+                next_page = get_next_page(next_headers)
 
         if callback:
             return callback(jsonOut)
-        else:
-            return jsonOut
+        return jsonOut
 
 
 TransportBase.REGISTRY["requests"] = RequestsTransport