Skip to content

Commit a7b3509

Browse files
alexbourretalexbourret
committed
modifications following review, first pass
1 parent 3745cbb commit a7b3509

2 files changed

Lines changed: 34 additions & 35 deletions

File tree

‎python-lib/pagination.py‎

Lines changed: 33 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,11 @@
77

88
class Pagination(object):
99

10-
def __init__(self, config=None, skip_key=None, limit_key=None, total_key=None, next_page_key=None):
10+
def __init__(self):
1111
self.next_page_key = None
1212
self.next_page_url_base = None
1313
self.skip_key = None
1414
self.next_page_url = None
15-
self.remaining_records = None
1615
self.records_to_skip = None
1716
self.pagination_type = ""
1817
self.counting_key = None
@@ -25,7 +24,8 @@ def __init__(self, config=None, skip_key=None, limit_key=None, total_key=None, n
2524
self.data_is_list = None
2625
self.update_next_page = self.update_next_page_default
2726

28-
def configure_paging(self, config=None, skip_key=None, limit_key=None, total_key=None, next_page_key=None, next_page_url_base=None, url=None, pagination_type="na"):
27+
def configure_paging(self, config=None, skip_key=None,
28+
next_page_key=None, next_page_url_base=None, pagination_type="na"):
2929
config = {} if config is None else config
3030
self.pagination_type = config.get("pagination_type", pagination_type)
3131
if self.pagination_type == "next_page":
@@ -49,7 +49,6 @@ def configure_paging(self, config=None, skip_key=None, limit_key=None, total_key
4949
self.update_next_page = self.update_next_page_default
5050

5151
def reset_paging(self, counting_key=None, url=None):
52-
self.remaining_records = 0
5352
self.records_to_skip = 0
5453
self.counting_key = counting_key
5554
self.counter = 0
@@ -70,7 +69,7 @@ def set_counting_key(self, counting_key):
7069
def update_next_page_offset(self, data, response_links=None):
7170
self.is_first_batch = False
7271
self.counter += 1
73-
self.next_page_number = self.next_page_number + 1
72+
#self.next_page_number = self.next_page_number + 1
7473
self.data_is_list = False
7574
if isinstance(data, list):
7675
self.data_is_list = True
@@ -91,26 +90,28 @@ def update_next_page_offset(self, data, response_links=None):
9190
self.is_last_batch_empty = True
9291
else:
9392
batch_size = 1
94-
if self.skip_key:
95-
self.skip = data.get(self.skip_key)
96-
logger.info("update_next_page_offset:skip=data[{}]={}".format(self.skip_key, self.skip))
93+
#if self.skip_key:
94+
# self.skip = data.get(self.skip_key)
95+
# logger.info("update_next_page_offset:skip=data[{}]={}".format(self.skip_key, self.skip))
9796
self.records_to_skip = self.records_to_skip + batch_size
98-
logger.info("update_next_page_offset:records_to_skip={}, batch_size={}".format(self.records_to_skip, batch_size))
99-
97+
logger.info("update_next_page_offset:data_is_list={}, records_to_skip={}, batch_size={}, is_last_batch_empty={}".format(
98+
self.data_is_list, self.records_to_skip, batch_size, self.is_last_batch_empty
99+
))
100100

101101
def update_next_page_per_page(self, data, response_links=None):
102102
self.is_first_batch = False
103103
self.counter += 1
104104
self.next_page_number = self.next_page_number + 1
105105
self.data_is_list = False
106+
106107
if isinstance(data, list):
107108
self.data_is_list = True
108109
batch_size = len(data)
109-
self.records_to_skip = self.records_to_skip + batch_size
110+
#self.records_to_skip = self.records_to_skip + batch_size
110111
if batch_size == 0:
111112
self.is_last_batch_empty = True
112-
logger.info("update_next_page_per_page:data is list:batch_size={}, records_to_skip={}, is_last_batch_empty={}".format(
113-
batch_size, self.records_to_skip, self.is_last_batch_empty
113+
logger.info("update_next_page_per_page:data is list:batch_size={}, next_page_number={}, is_last_batch_empty={}".format(
114+
batch_size, self.next_page_number, self.is_last_batch_empty
114115
))
115116
return
116117
elif self.counting_key:
@@ -122,27 +123,20 @@ def update_next_page_per_page(self, data, response_links=None):
122123
self.is_last_batch_empty = True
123124
else:
124125
batch_size = 1
125-
if self.skip_key:
126-
self.skip = data.get(self.skip_key)
127-
logger.info("update_next_page_per_page:skip=data[{}]={}".format(self.skip_key, self.skip))
128-
self.records_to_skip = self.records_to_skip + batch_size
129-
logger.info("update_next_page_per_page:records_to_skip={}, batch_size={}".format(self.records_to_skip, batch_size))
130-
126+
#if self.skip_key:
127+
# self.skip = data.get(self.skip_key)
128+
# logger.info("update_next_page_per_page:skip=data[{}]={}".format(self.skip_key, self.skip))
129+
#self.records_to_skip = self.records_to_skip + batch_size
130+
logger.info("update_next_page_per_page:data_is_list={}, next_page_number={}, batch_size={}, is_last_batch_empty={}".format(
131+
self.data_is_list, self.next_page_number, batch_size, self.is_last_batch_empty
132+
))
131133

132134
def update_next_page_link(self, data, response_links=None):
133-
self.next_page_url = None
134-
response_links = response_links or {}
135-
next_link = response_links.get('next', {})
136-
next_page_url = next_link.get("url")
137135
self.is_first_batch = False
138136
self.counter += 1
139-
if next_page_url:
140-
self.next_page_url = next_page_url
141-
self.params_must_be_blanked = True
142-
logger.info("update_next_page_link:next_link={}, next_page_url={}, params_must_be_blanked={}, next_page_number={}, counter={}".format(
143-
next_link, self.next_page_url, self.params_must_be_blanked, self.next_page_number, self.counter
144-
))
137+
self.next_page_url = None
145138
self.data_is_list = False
139+
146140
if self.next_page_key:
147141
next_page_path = extract_key_using_json_path(data, self.next_page_key)
148142
if self.next_page_url_base and next_page_path:
@@ -152,11 +146,19 @@ def update_next_page_link(self, data, response_links=None):
152146
logger.info("update_next_page_link:next_page_url_base={}, next_page_path={}, next_page_url={}".format(
153147
self.next_page_url_base, next_page_path, self.next_page_url
154148
))
149+
else:
150+
response_links = response_links or {}
151+
next_link = response_links.get('next', {})
152+
next_page_url = next_link.get("url")
153+
if next_page_url:
154+
self.next_page_url = next_page_url
155+
self.params_must_be_blanked = True
156+
logger.info("update_next_page_link:next_link={}, next_page_url={}, params_must_be_blanked={}, next_page_number={}, counter={}".format(
157+
next_link, self.next_page_url, self.params_must_be_blanked, self.next_page_number, self.counter
158+
))
155159

156160
def update_next_page_default(self, data, response_links=None):
157161
self.is_first_batch = False
158-
self.counter += 1
159-
self.next_page_number = self.next_page_number + 1
160162

161163
def has_next_page(self):
162164
if self.is_last_batch_empty:

‎python-lib/rest_api_client.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,17 +64,14 @@ def __init__(self, credential, endpoint, custom_key_values={}):
6464
is_next_page_url_relative = endpoint.get("is_next_page_url_relative", False)
6565
next_page_url_base = endpoint.get("next_page_url_base", None) if is_next_page_url_relative else None
6666
next_page_url_base = format_template(next_page_url_base, **self.presets_variables)
67-
top_key = endpoint.get("top_key")
6867
skip_key = endpoint.get("skip_key")
6968
pagination_type = endpoint.get("pagination_type", "na")
70-
if pagination_type=="next_page" and is_next_page_url_relative and not next_page_url_base:
69+
if pagination_type == "next_page" and is_next_page_url_relative and not next_page_url_base:
7170
raise RestAPIClientError("Pagination's 'Next page URL' is relative but no 'Base URL to next page' has been set")
7271
self.pagination.configure_paging(
7372
skip_key=skip_key,
74-
limit_key=top_key,
7573
next_page_key=next_page_url_key,
7674
next_page_url_base=next_page_url_base,
77-
url=self.endpoint_url,
7875
pagination_type=pagination_type
7976
)
8077
self.last_interaction = None

0 commit comments

Comments
 (0)