improve geofeed validation and add lru_cache to API calling functions

Author: jonathan-kosgei

setup.cfg

@@ -32,7 +32,6 @@ install_requires =
 [options.entry_points]
 console_scripts =
     ipdata = ipdata.cli:cli
-    geofeed_validate = ipdata.cli:validate
 
 [options.packages.find]
 where = src

src/ipdata/cli.py

@@ -248,10 +248,18 @@ def usage(ctx, api_key):
 @cli.command(default=True)
 @click.argument("resource", required=False, type=str, default="")
 @click.option(
-    "--fields", "-f", required=False, multiple=True, default=[],
+    "--fields",
+    "-f",
+    required=False,
+    multiple=True,
+    default=[],
 )
 @click.option(
-    "--exclude", "-e", required=False, multiple=True, default=[],
+    "--exclude",
+    "-e",
+    required=False,
+    multiple=True,
+    default=[],
 )
 @click.option("--api-key", "-k", required=False, default=None, help="ipdata API Key")
 @click.option(
@@ -295,7 +303,9 @@ def lookup(ctx, resource, fields, api_key, pretty_print, raw, copy, exclude):
 
     # enforce mutual exclusivity of fields and exclude
     if exclude and fields:
-        raise click.ClickException("'--fields / -f' and '--exclude / -e' are mutually exclusive.")
+        raise click.ClickException(
+            "'--fields / -f' and '--exclude / -e' are mutually exclusive."
+        )
 
     # if the user wants to exclude some fields, get all the fields in fields that are not in exclude
     if exclude:
@@ -342,11 +352,13 @@ def process(resources, processor, fields):
 
 @cli.command()
 @click.argument("input", required=True, type=click.File(mode="r", encoding="utf-8"))
+@click.option("--fields", "-f", required=False, multiple=True)
 @click.option(
-    "--fields", "-f", required=False, multiple=True
-)
-@click.option(
-    "--exclude", "-e", required=False, multiple=True, default=[],
+    "--exclude",
+    "-e",
+    required=False,
+    multiple=True,
+    default=[],
 )
 @click.option(
     "--output", "-o", required=True, type=click.File(mode="w", encoding="utf-8")
@@ -378,13 +390,14 @@ def batch(ctx, input, fields, output, format, exclude):
 
     # enforce mutual exclusivity of fields and exclude
     if exclude and fields:
-        raise click.ClickException("'--fields / -f' and '--exclude / -e' are mutually exclusive.")
+        raise click.ClickException(
+            "'--fields / -f' and '--exclude / -e' are mutually exclusive."
+        )
 
     # if the user wants to exclude some fields, get all the fields in fields that are not in exclude
     if exclude:
         fields = set(ipdata.valid_fields).difference(set(exclude))
 
-
     # Prepare requests
     ipdata = IPData(ctx.obj["api-key"])
     resources = [resource.strip() for resource in input.readlines()]

src/ipdata/geofeeds.py

@@ -50,7 +50,7 @@ def _download(self):
         :raises Exception: if a failure occurs when downloading the geofeed
         """
         random_name = hashlib.sha224( str(random.getrandbits(256)).encode('utf-8') ).hexdigest()[:16]
-        cache_path = f"{pwd}/{self.dir}/{random_name}.csv"
+        cache_path = f"{self.dir}/{random_name}.csv"
 
         try:
             response = requests.get(self.source, timeout=60)
@@ -82,11 +82,10 @@ def entries(self):
         if "://" in self.source and not self.cache_path:
             self._download()
             path = self.cache_path
-
-        # if path still contains a url it means the download was not successful in which case we yield nothing!
-        if "://" in path:
-            log.warning(f"No entries found {self.source}")
-            yield
+            # if path still contains a url it means the download was not successful in which case we yield nothing!
+            if not path:
+                log.warning(f"No entries found {self.source}")
+                return
 
         with open(path) as f:
             reader = csv.reader(f)
@@ -99,6 +98,7 @@ def entries(self):
                     yield GeofeedValidationError(
                         f"[{entry}] is not a valid geofeed entry. The 'IP Range' and 'Country' fields are required however you must have the requisite minimum number of commas (currently 4) present"
                     )
+                    return
 
                 # instantiate an Entry object
                 geofeed_entry = Entry(*entry)
@@ -108,12 +108,15 @@ def entries(self):
                     yield GeofeedValidationError(
                         f"Duplicate prefixes found {geofeed_entry.ip_range}"
                     )
+                    return
 
                 self.prefixes.add(geofeed_entry.ip_range)
 
                 # generate entry
                 yield geofeed_entry
 
+    def __iter__(self):
+        yield from self.entries()
 
 class Entry(object):
     """

src/ipdata/ipdata.py

@@ -22,6 +22,7 @@
 import requests
 import logging
 import urllib3
+import functools
 
 from requests.adapters import HTTPAdapter, Retry
 from rich.logging import RichHandler
@@ -135,6 +136,7 @@ def __init__(
         self._session = requests.Session()
         self._session.mount("http", adapter)
 
+    @functools.lru_cache(maxsize=100)
     def _validate_fields(self, fields):
         """
         Validates the fields passed in by the user, first ensuring it's a collection. In prior versions 'fields' was a string, however it now needs to be a collection.
@@ -152,6 +154,7 @@ def _validate_fields(self, fields):
                 f"The field(s) {diff} are not supported. Only {self.valid_fields} are supported."
             )
 
+    @functools.lru_cache(maxsize=100)
     def _validate_ip_address(self, ip):
         """
         Checks that 'ip' is a valid IP Address.
@@ -163,6 +166,7 @@ def _validate_ip_address(self, ip):
         if request_ip.is_private or request_ip.is_reserved or request_ip.is_multicast:
             raise ValueError(f"{ip} is a reserved IP Address")
 
+    @functools.lru_cache(maxsize=100)
     def lookup(self, resource="", fields=[]):
         """
         Makes a GET request to the IPData API for the specified 'resource' and the given 'fields'.
@@ -212,6 +216,7 @@ def lookup(self, resource="", fields=[]):
 
         return data
 
+    @functools.lru_cache(maxsize=100)
     def bulk(self, resources, fields=[]):
         """
         Lookup up to 100 resources in one request. Makes a POST request wth the resources as a JSON array and the specified fields.