Skip to content

Commit 615e155

Browse files
caryosceluscaryoscelus
authored
Merge pull request ZeroNetX#219 from caryoscelus/bootstrap
Bootstrap 0net trackers on first launch
2 parents a8c2117 + 70f2eea commit 615e155

8 files changed

Lines changed: 99 additions & 191 deletions

File tree

‎bootstrap.url‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.9/data-default-2023-07-23.zip

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ maxminddb
1414
rich
1515
defusedxml>=0.7
1616
pyaes
17+
requests

‎src/Config.py‎

Lines changed: 8 additions & 178 deletions
Original file line numberDiff line numberDiff line change
@@ -9,178 +9,6 @@
99
import stat
1010
import time
1111

12-
trackers = [
13-
'zero://188.242.242.224:26474',
14-
'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:23141',
15-
'zero://200:1e7a:5100:ef7c:6fa4:d8ae:b91c:a74:15441',
16-
'zero://23.184.48.134:15441',
17-
'zero://57hzgtu62yzxqgbvgxs7g3lfck3za4zrda7qkskar3tlak5recxcebyd.onion:15445',
18-
'zero://6i54dd5th73oelv636ivix6sjnwfgk2qsltnyvswagwphub375t3xcad.onion:15441',
19-
'zero://f2hnjbggc3c2u2apvxdugirnk6bral54ibdoul3hhvu7pd4fso5fq3yd.onion:15441',
20-
'zero://gugt43coc5tkyrhrc3esf6t6aeycvcqzw7qafxrjpqbwt4ssz5czgzyd.onion:15441',
21-
'zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441',
22-
'zero://ow7in4ftwsix5klcbdfqvfqjvimqshbm2o75rhtpdnsderrcbx74wbad.onion:15441',
23-
'zero://pn4q2zzt2pw4nk7yidxvsxmydko7dfibuzxdswi6gu6ninjpofvqs2id.onion:15441',
24-
'zero://skdeywpgm5xncpxbbr4cuiip6ey4dkambpanog6nruvmef4f3e7o47qd.onion:15441',
25-
'zero://wlxav3szbrdhest4j7dib2vgbrd7uj7u7rnuzg22cxbih7yxyg2hsmid.onion:15441',
26-
'zero://zy7wttvjtsijt5uwmlar4yguvjc2gppzbdj4v6bujng6xwjmkdg7uvqd.onion:15441',
27-
'http://bt.okmp3.ru:2710/announce',
28-
'http://fxtt.ru:80/announce',
29-
'http://incine.ru:6969/announce',
30-
'http://moeweb.pw:6969/announce',
31-
'http://open.acgnxtracker.com:80/announce',
32-
'http://t.acg.rip:6699/announce',
33-
'http://t.nyaatracker.com:80/announce',
34-
'http://t.overflow.biz:6969/announce',
35-
'http://tracker.files.fm:6969/announce',
36-
'http://tracker.mywaifu.best:6969/announce',
37-
'http://tracker.vrpnet.org:6969/announce',
38-
'http://vps02.net.orel.ru:80/announce',
39-
'udp://960303.xyz:6969/announce',
40-
'udp://aarsen.me:6969/announce',
41-
'udp://astrr.ru:6969/announce',
42-
'udp://ben.kerbertools.xyz:6969/announce',
43-
'udp://bt1.archive.org:6969/announce',
44-
'udp://bt2.archive.org:6969/announce',
45-
'udp://bt.ktrackers.com:6666/announce',
46-
'udp://bubu.mapfactor.com:6969/announce',
47-
'udp://c.ns.cluefone.com:6969/announce',
48-
'udp://cutscloud.duckdns.org:6969/announce',
49-
'udp://download.nerocloud.me:6969/announce',
50-
'udp://epider.me:6969/announce',
51-
'udp://exodus.desync.com:6969/announce',
52-
'udp://htz3.noho.st:6969/announce',
53-
'udp://ipv4.tracker.harry.lu:80/announce',
54-
'udp://laze.cc:6969/announce',
55-
'udp://mail.artixlinux.org:6969/announce',
56-
'udp://mirror.aptus.co.tz:6969/announce',
57-
'udp://moonburrow.club:6969/announce',
58-
'udp://movies.zsw.ca:6969/announce',
59-
'udp://mts.tvbit.co:6969/announce',
60-
'udp://new-line.net:6969/announce',
61-
'udp://open.demonii.com:1337/announce',
62-
'udp://open.stealth.si:80/announce',
63-
'udp://opentracker.i2p.rocks:6969/announce',
64-
'udp://p4p.arenabg.com:1337/announce',
65-
'udp://psyco.fr:6969/announce',
66-
'udp://public.publictracker.xyz:6969/announce',
67-
'udp://rep-art.ynh.fr:6969/announce',
68-
'udp://run.publictracker.xyz:6969/announce',
69-
'udp://sanincode.com:6969/announce',
70-
'udp://slicie.icon256.com:8000/announce',
71-
'udp://tamas3.ynh.fr:6969/announce',
72-
'udp://thouvenin.cloud:6969/announce',
73-
'udp://torrentclub.space:6969/announce',
74-
'udp://tracker.0x.tf:6969/announce',
75-
'udp://tracker1.bt.moack.co.kr:80/announce',
76-
'udp://tracker.4.babico.name.tr:3131/announce',
77-
'udp://tracker.altrosky.nl:6969/announce',
78-
'udp://tracker.artixlinux.org:6969/announce',
79-
'udp://tracker.farted.net:6969/announce',
80-
'udp://tracker.jonaslsa.com:6969/announce',
81-
'udp://tracker.joybomb.tw:6969/announce',
82-
'udp://tracker.monitorit4.me:6969/announce',
83-
'udp://tracker.opentrackr.org:1337/announce',
84-
'udp://tracker.pomf.se:80/announce',
85-
'udp://tracker.publictracker.xyz:6969/announce',
86-
'udp://tracker.srv00.com:6969/announce',
87-
'udp://tracker.tcp.exchange:6969/announce',
88-
'udp://tracker.theoks.net:6969/announce',
89-
'udp://transkaroo.joustasie.net:6969/announce',
90-
'udp://uploads.gamecoast.net:6969/announce',
91-
'udp://v2.iperson.xyz:6969/announce',
92-
'udp://vibe.sleepyinternetfun.xyz:1738/announce',
93-
'udp://www.skynetcenter.me:6969/announce',
94-
'udp://www.torrent.eu.org:451/announce',
95-
'zero://194.5.98.39:15441',
96-
'zero://145.239.95.38:15441',
97-
'zero://178.128.34.249:26117',
98-
'zero://217.18.217.143:39288',
99-
'zero://83.246.141.203:22207',
100-
'zero://syncronite.loki:15441',
101-
'zero://2a05:dfc1:4000:1e00::a:15441',
102-
'zero://2400:6180:100:d0::8fd:8001:21697',
103-
'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:30530',
104-
'zero://73pyhfwfwsrhfw76knkjfnw6o3lk53zfo7hlxdmxbj75sjcnol5cioad.onion:15442',
105-
'zero://fzlzmxuz2bust72cuy5g4w6d62tx624xcjaupf2kp7ffuitbiniy2hqd.onion:15441',
106-
'zero://rlcjomszyitxpwv7kzopmqgzk3bdpsxeull4c3s6goszkk6h2sotfoad.onion:15441',
107-
'zero://tqmo2nffqo4qc5jgmz3me5eri3zpgf3v2zciufzmhnvznjve5c3argad.onion:15441',
108-
'http://107.189.31.134:6969/announce',
109-
'http://119.28.71.45:8080/announce',
110-
'http://129.146.193.240:6699/announce',
111-
'http://159.69.65.157:6969/announce',
112-
'http://163.172.29.130:80/announce',
113-
'http://185.130.47.2:6969/announce',
114-
'http://45.67.35.111:6969/announce',
115-
'http://61.222.178.254:6969/announce',
116-
'http://83.31.30.182:6969/announce',
117-
'http://93.158.213.92:1337/announce',
118-
'http://95.217.167.10:6969/announce',
119-
'udp://102.223.180.235:6969/announce',
120-
'udp://103.122.21.50:6969/announce',
121-
'udp://104.131.98.232:6969/announce',
122-
'udp://104.244.77.87:6969/announce',
123-
'udp://107.189.11.58:6969/announce',
124-
'udp://107.189.31.134:6969/announce',
125-
'udp://139.144.68.88:6969/announce',
126-
'udp://149.28.239.70:6969/announce',
127-
'udp://15.204.205.14:6969/announce',
128-
'udp://156.234.201.18:80/announce',
129-
'udp://158.101.161.60:3131/announce',
130-
'udp://163.172.29.130:80/announce',
131-
'udp://167.99.185.219:6969/announce',
132-
'udp://176.31.250.174:6969/announce',
133-
'udp://176.56.4.238:6969/announce',
134-
'udp://178.32.222.98:3391/announce',
135-
'udp://184.105.151.166:6969/announce',
136-
'udp://185.102.219.163:6969/announce',
137-
'udp://185.181.60.155:80/announce',
138-
'udp://185.217.199.21:6969/announce',
139-
'udp://185.44.82.25:1337/announce',
140-
'udp://185.68.21.244:6969/announce',
141-
'udp://192.3.165.191:6969/announce',
142-
'udp://192.3.165.198:6969/announce',
143-
'udp://192.95.46.115:6969/announce',
144-
'udp://193.176.158.162:6969/announce',
145-
'udp://193.37.214.12:6969/announce',
146-
'udp://193.42.111.57:9337/announce',
147-
'udp://198.100.149.66:6969/announce',
148-
'udp://20.100.205.229:6969/announce',
149-
'udp://207.241.226.111:6969/announce',
150-
'udp://207.241.231.226:6969/announce',
151-
'udp://209.141.59.16:6969/announce',
152-
'udp://212.237.53.230:6969/announce',
153-
'udp://23.153.248.2:6969/announce',
154-
'udp://23.254.228.89:6969/announce',
155-
'udp://37.187.111.136:6969/announce',
156-
'udp://37.27.4.53:6969/announce',
157-
'udp://38.7.201.142:6969/announce',
158-
'udp://45.154.253.6:6969/announce',
159-
'udp://45.63.30.114:6969/announce',
160-
'udp://45.9.60.30:6969/announce',
161-
'udp://46.38.238.105:6969/announce',
162-
'udp://49.12.76.8:8080/announce',
163-
'udp://5.102.159.190:6969/announce',
164-
'udp://5.196.89.204:6969/announce',
165-
'udp://51.15.79.209:6969/announce',
166-
'udp://51.159.54.68:6666/announce',
167-
'udp://51.68.174.87:6969/announce',
168-
'udp://51.81.222.188:6969/announce',
169-
'udp://52.58.128.163:6969/announce',
170-
'udp://61.222.178.254:6969/announce',
171-
'udp://77.73.69.230:6969/announce',
172-
'udp://83.102.180.21:80/announce',
173-
'udp://83.31.30.182:6969/announce',
174-
'udp://85.206.172.159:6969/announce',
175-
'udp://85.239.33.28:6969/announce',
176-
'udp://86.57.161.157:6969/announce',
177-
'udp://91.216.110.52:451/announce',
178-
'udp://93.158.213.92:1337/announce',
179-
'udp://94.103.87.87:6969/announce',
180-
'udp://95.216.74.39:6969/announce',
181-
'udp://95.31.11.224:6969/announce',
182-
]
183-
18412
class Config(object):
18513

18614
def __init__(self, argv):
@@ -322,6 +150,10 @@ def createArguments(self):
322150
action.add_argument('cmd', help='API command name')
323151
action.add_argument('parameters', help='Parameters of the command', nargs='?')
324152

153+
# Import bundled sites
154+
action = self.subparsers.add_parser("importBundle", help='Import sites from a .zip bundle')
155+
action.add_argument('bundle', help='Path to a data bundle')
156+
325157
# dbRebuild
326158
action = self.subparsers.add_parser("dbRebuild", help='Rebuild site database cache')
327159
action.add_argument('address', help='Site to rebuild')
@@ -428,8 +260,10 @@ def createArguments(self):
428260
self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true')
429261
self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port')
430262
self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip')
431-
self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*')
432-
self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', metavar='path', nargs='*')
263+
self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str)
264+
self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true')
265+
self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=[], metavar='protocol://address', nargs='*')
266+
self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', default=['{data_dir}/15CEFKBRHFfAP9rmL6hhLmHoXrrgmw4B5o/cache/1/Syncronite.html'], metavar='path', nargs='*')
433267
self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable")
434268
self.parser.add_argument('--use_libsecp256k1', help='Use Libsecp256k1 liblary for speedup', type='bool', choices=[True, False], default=True)
435269
self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup', type='bool', choices=[True, False], default=True)
@@ -799,10 +633,6 @@ def initLogging(self, console_logging=None, file_logging=None):
799633
except Exception as err:
800634
print("Can't change permission of %s: %s" % (self.log_dir, err))
801635

802-
# Make warning hidden from console
803-
logging.WARNING = 15 # Don't display warnings if not in debug mode
804-
logging.addLevelName(15, "WARNING")
805-
806636
logging.getLogger('').name = "-" # Remove root prefix
807637

808638
self.error_logger = ErrorLogHandler()

‎src/Crypt/CryptBitcoin.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,3 +99,16 @@ def verify(data, valid_address, sign, lib_verify=None): # Verify data using add
9999
return sign_address in valid_address
100100
else: # One possible address
101101
return sign_address == valid_address
102+
103+
def isValidAddress(addr):
104+
'''Check if provided address is valid bitcoin address'''
105+
if addr[0] != '1':
106+
# no support for new-style addrs
107+
return False
108+
from base58 import b58decode
109+
bs = b58decode(addr)
110+
main = bs[:-4]
111+
checksum = bs[-4:]
112+
h1 = hashlib.sha256(main).digest()
113+
h2 = hashlib.sha256(h1).digest()
114+
return h2[:4] == checksum

‎src/Site/Site.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,11 @@ def __repr__(self):
8787
# Load site settings from data/sites.json
8888
def loadSettings(self, settings=None):
8989
if not settings:
90-
settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address)
90+
try:
91+
settings = json.load(open(f'{config.data_dir}/sites.json')).get(self.address)
92+
except Exception as err:
93+
logging.error(f'Error loading {config.data_dir}/sites.json: {err}')
94+
settings = {}
9195
if settings:
9296
self.settings = settings
9397
if "cache" not in settings:

‎src/Site/SiteAnnouncer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ def announceTracker(self, tracker, mode="start", num_want=10):
202202
else:
203203
raise AnnounceError("Unknown protocol: %s" % address_parts["protocol"])
204204
except Exception as err:
205-
self.site.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode))
205+
self.site.log.debug("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode))
206206
error = err
207207

208208
if error:

‎src/Site/SiteManager.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,18 @@ def __init__(self):
3131
@util.Noparallel()
3232
def load(self, cleanup=True, startup=False):
3333
from .Site import Site
34-
self.log.info("Loading sites... (cleanup: %s, startup: %s)" % (cleanup, startup))
34+
self.log.info(f'Loading sites... ({cleanup=}, {startup=})')
3535
self.loaded = False
3636
address_found = []
3737
added = 0
3838
load_s = time.time()
3939
# Load new adresses
4040
try:
41-
json_path = "%s/sites.json" % config.data_dir
41+
json_path = f"{config.data_dir}/sites.json"
4242
data = json.load(open(json_path))
4343
except Exception as err:
44-
raise Exception("Unable to load %s: %s" % (json_path, err))
44+
self.log.error(f"Unable to load {json_path}: {err}")
45+
data = {}
4546

4647
sites_need = []
4748

‎src/main.py‎

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,24 +34,79 @@ def load_config():
3434

3535
load_config()
3636

37+
def importBundle(bundle):
38+
from zipfile import ZipFile
39+
from Crypt.CryptBitcoin import isValidAddress
40+
import json
41+
42+
sites_json_path = f"{config.data_dir}/sites.json"
43+
try:
44+
with open(sites_json_path) as f:
45+
sites = json.load(f)
46+
except Exception as err:
47+
sites = {}
48+
49+
with ZipFile(bundle) as zf:
50+
all_files = zf.namelist()
51+
top_files = list(set(map(lambda f: f.split('/')[0], all_files)))
52+
if len(top_files) == 1 and not isValidAddress(top_files[0]):
53+
prefix = top_files[0]+'/'
54+
else:
55+
prefix = ''
56+
top_2 = list(set(filter(lambda f: len(f)>0,
57+
map(lambda f: f.removeprefix(prefix).split('/')[0], all_files))))
58+
for d in top_2:
59+
if isValidAddress(d):
60+
logging.info(f'unpack {d} into {config.data_dir}')
61+
for fname in filter(lambda f: f.startswith(prefix+d) and not f.endswith('/'), all_files):
62+
tgt = config.data_dir + '/' + fname.removeprefix(prefix)
63+
logging.info(f'-- {fname} --> {tgt}')
64+
info = zf.getinfo(fname)
65+
info.filename = tgt
66+
zf.extract(info)
67+
logging.info(f'add site {d}')
68+
sites[d] = {}
69+
else:
70+
logging.info(f'Warning: unknown file in a bundle: {prefix+d}')
71+
with open(sites_json_path, 'w') as f:
72+
json.dump(sites, f)
73+
3774
def init_dirs():
38-
if not os.path.isdir(config.data_dir):
39-
os.mkdir(config.data_dir)
75+
data_dir = config.data_dir
76+
if not os.path.isdir(data_dir):
77+
os.mkdir(data_dir)
4078
try:
41-
os.chmod(config.data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
79+
os.chmod(data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
4280
except Exception as err:
43-
startupError("Can't change permission of %s: %s" % (config.data_dir, err))
44-
45-
sites_json = f"{config.data_dir}/sites.json"
81+
startupError(f"Can't change permission of {data_dir}: {err}")
82+
83+
# download latest bootstrap bundle
84+
if not config.disable_bootstrap and not config.offline:
85+
import requests
86+
from io import BytesIO
87+
88+
print(f'fetching {config.bootstrap_url}')
89+
response = requests.get(config.bootstrap_url)
90+
if response.status_code != 200:
91+
startupError(f"Cannot load bootstrap bundle (response status: {response.status_code})")
92+
url = response.text
93+
print(f'got {url}')
94+
response = requests.get(url)
95+
if response.status_code < 200 or response.status_code >= 300:
96+
startupError(f"Cannot load boostrap bundle (response status: {response.status_code})")
97+
importBundle(BytesIO(response.content))
98+
99+
sites_json = f"{data_dir}/sites.json"
46100
if not os.path.isfile(sites_json):
47101
with open(sites_json, "w") as f:
48102
f.write("{}")
49-
users_json = f"{config.data_dir}/users.json"
103+
users_json = f"{data_dir}/users.json"
50104
if not os.path.isfile(users_json):
51105
with open(users_json, "w") as f:
52106
f.write("{}")
53107

54108
# TODO: GET RID OF TOP-LEVEL CODE!!!
109+
config.initConsoleLogger()
55110

56111
try:
57112
init_dirs()
@@ -73,7 +128,7 @@ def init_dirs():
73128
r = proc.wait()
74129
sys.exit(r)
75130

76-
config.initLogging()
131+
config.initLogging(console_logging=False)
77132

78133
# Debug dependent configuration
79134
from Debug import DebugHook
@@ -415,6 +470,9 @@ def siteCmd(self, address, cmd, parameters):
415470
else:
416471
return res
417472

473+
def importBundle(self, bundle):
474+
importBundle(bundle)
475+
418476
def getWebsocket(self, site):
419477
import websocket
420478

0 commit comments

Comments
 (0)