1-
1+ import json
22import requests
33import time
4- # import configparser
5-
4+ import multiprocessing as mp
65from random import randint , choice , sample
7- from multiprocessing import Process , Queue
86
9- # from . import SETTINGS, IMAGES
7+ from . import API
108
119
1210class ImageCache :
1311 '''Class used for caching images'''
14- image_api = 'https://api.thecatapi.com/v1/images/search'
15- profile_api = "https://www.pawclub.com.au/assets/js/namesTemp.json"
16- hobbies_api = (
17- "https://gist.githubusercontent.com/mbejda/453fdb77ef8d4d3b3a67/raw/"
18- "e8334f09109dc212892406e25fdee03efdc23f56/hobbies.txt"
19- )
20- ratelimit = 0.1
12+
13+ ratelimit = 0.05
14+ api = json .load (API )
2115
2216 def __init__ (self , size ):
23- self .queue = Queue (size )
17+ self .queue = mp . Queue (size )
2418 self .worker = None
2519
2620 def __del__ (self ):
2721 self .stop ()
2822
29- def get_image (self ):
30- res = requests .get (self .image_api , stream = True )
31- data = res .json ()
23+ def __get (self , url , ** kwargs ):
24+ try :
25+ return requests .get (url , ** kwargs )
26+ except ConnectionError as e :
27+ return e
28+
29+ def __parse_image (self , response ):
30+ data = response .json ()
3231 url = data [0 ]['url' ]
33- res = requests . get (url )
34- return res .content
32+ response = self . __get (url )
33+ return { 'image' : response .content }
3534
36- def get_hobbies (self ):
37- res = requests .get (self .hobbies_api )
38- all_hobbies = res .text .split ("\n " )
39- return sample (all_hobbies , 5 )
35+ def __parse_hobbies (self , response ):
36+ all_hobbies = response .text .split ("\n " )
37+ return {'hobbies' : sample (all_hobbies , 5 )}
4038
41- def get_profile (self ):
42- res = requests .get (self .profile_api )
43- data = res .json ()
39+ def __parse_info (self , response ):
40+ data = response .json ()
4441 letter = choice ('acdefghijklmnopqrstuvwxyz' )
4542 data = choice (data [letter ])
4643 return {
47- 'name ' : data [ 'name' ],
48- 'gender ' : data ['gender ' ],
49- 'age ' : randint ( 1 , 42 ) ,
50- 'location ' : f' { randint (1 , 9999 ) } miles away' ,
51- 'image ' : self . get_image (),
52- 'hobbies' : self . get_hobbies ()
44+ 'info ' : {
45+ 'name ' : data ['name ' ],
46+ 'gender ' : data [ 'gender' ] ,
47+ 'age ' : randint (1 , 42 ) ,
48+ 'location ' : f' { randint ( 1 , 9999 ) } miles away'
49+ }
5350 }
5451
52+ def get_profile (self ):
53+ poolsize = len (self .api )
54+ with mp .Pool (poolsize ) as pool :
55+ responses = pool .map (self .__get , self .api .values ())
56+ if ConnectionError not in responses : # TODO Record connection errors
57+ return dict (* responses )
58+
5559 def next (self ):
5660 return self .queue .get ()
5761
5862 def mainloop (self , queue ):
5963 while True :
60- queue .put (self .get_profile ())
64+ profile = self .get_profile ()
65+ if profile is not None :
66+ queue .put (profile )
6167 time .sleep (self .ratelimit )
6268
6369 def start (self ):
@@ -67,4 +73,5 @@ def start(self):
6773 self .worker .start ()
6874
6975 def stop (self ):
70- self .worker .terminate ()
76+ for worker in mp .active_children ():
77+ worker .terminate ()
0 commit comments