Skip to content

Commit f93cb03

Browse files
CadairCadair
committed
add back Python helpers
1 parent 3c1bb51 commit f93cb03

3 files changed

Lines changed: 327 additions & 0 deletions

File tree

‎affiliated/validate_registry.py‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import os
2+
import sys
3+
import json
4+
from termcolor import cprint
5+
from datetime import datetime
6+
7+
import requests
8+
9+
REQUIRED_KEYS = {'name', 'maintainer', 'stable', 'home_url', 'repo_url', 'coordinated'}
10+
11+
OPTIONAL_KEYS = {'provisional', 'stable', 'pypi_name', 'image', 'review', 'description'}
12+
13+
ALL_KEYS = REQUIRED_KEYS | OPTIONAL_KEYS
14+
15+
REVIEW_KEYS = {'functionality', 'ecointegration', 'documentation', 'testing',
16+
'devstatus', 'python3', 'last-updated'}
17+
18+
REVIEW_FUNCTIONALITY = {'Specialized package', 'General package'}
19+
REVIEW_DEVSTATUS = {'Unmaintained', 'Functional but low activity',
20+
'Functional but unmaintained', 'Heavy development', 'Good'}
21+
REVIEW_PYTHON3 = {'No', 'Yes'}
22+
REVIEW_GENERIC = {'Needs work', 'Partial', 'Good'}
23+
24+
25+
jsonfn = os.path.join(os.path.dirname(__file__), "registry.json")
26+
try:
27+
registry = json.load(open(jsonfn))
28+
except json.decoder.JSONDecodeError as e:
29+
cprint(jsonfn + ' : ' + e.args[0], color='red')
30+
cprint("*** JSON file appears to be malformed - see above ***", color='red')
31+
sys.exit(2)
32+
33+
error = 0
34+
35+
for package in registry['packages']:
36+
37+
if 'name' in package:
38+
name = package['name']
39+
else:
40+
cprint("ERROR: Missing package name: {0}".format(package), file=sys.stderr, color='red')
41+
error += 1
42+
continue
43+
44+
cprint("Checking {0}".format(name), color='blue')
45+
46+
print(" - verifying keys")
47+
48+
difference = set(package.keys()) - (REQUIRED_KEYS | OPTIONAL_KEYS)
49+
if difference:
50+
cprint(f" ERROR: Unrecognized key(s) for {name}: {difference}. "
51+
f"Valid options are {', '.join(ALL_KEYS)}", file=sys.stderr, color='red')
52+
error += 1
53+
54+
difference = REQUIRED_KEYS - set(package.keys())
55+
if difference:
56+
cprint(f" ERROR: Missing key(s) for {name}: {difference}", file=sys.stderr, color='red')
57+
error += 1
58+
59+
# Check that URLs work
60+
61+
print(" - verifying URLs")
62+
63+
try:
64+
r = requests.get(package['home_url'])
65+
assert r.ok
66+
except Exception:
67+
cprint(f" ERROR: Home URL for {name} - {package['home_url']} - did not work", file=sys.stderr, color='red')
68+
error += 1
69+
70+
try:
71+
r = requests.get(package['repo_url'])
72+
assert r.ok
73+
except Exception:
74+
cprint(f" ERROR: Repository URL for {name} - {package['repo_url']} - did not work", file=sys.stderr, color='red')
75+
error += 1
76+
77+
if package.get('pypi_name'):
78+
79+
print(" - verifying PyPI name")
80+
81+
r = requests.get(f"https://pypi.python.org/pypi/{package['pypi_name']}/json")
82+
if not r.ok:
83+
cprint(f" ERROR: PyPI package {package['pypi_name']} doesn't appear to exist", file=sys.stderr, color='red')
84+
error += 1
85+
86+
if package.get('review'):
87+
88+
print(" - verifying review")
89+
90+
review = package['review']
91+
92+
difference = set(review.keys()) - REVIEW_KEYS
93+
if difference:
94+
cprint(f" ERROR: Unrecognized review key(s) for {name}: {difference}. "
95+
f"Valid options are {', '.join(REVIEW_KEYS)}", file=sys.stderr, color='red')
96+
error += 1
97+
98+
difference = REVIEW_KEYS - set(review.keys())
99+
if difference:
100+
cprint(f" ERROR: Missing review key(s) for {name}: {difference}", file=sys.stderr, color='red')
101+
error += 1
102+
103+
for key, value in review.items():
104+
105+
if key == 'functionality':
106+
if value not in REVIEW_FUNCTIONALITY:
107+
cprint(f" ERROR: Invalid functionality in review for {name}: '{value}'. "
108+
f"Valid options are {', '.join(REVIEW_FUNCTIONALITY)}", file=sys.stderr, color='red')
109+
error += 1
110+
elif key == 'last-updated':
111+
try:
112+
dt = datetime.strptime(value, "%Y-%m-%d")
113+
except Exception:
114+
cprint(f" ERROR: Could not parse date: '{value}'", file=sys.stderr, color='red')
115+
error += 1
116+
elif key == 'devstatus':
117+
if value not in REVIEW_DEVSTATUS:
118+
cprint(f" ERROR: Invalid devstatus in review for {name}: '{value}'. "
119+
f"Valid options are {', '.join(REVIEW_DEVSTATUS)}", file=sys.stderr, color='red')
120+
error += 1
121+
elif key == 'python3':
122+
if value not in REVIEW_PYTHON3:
123+
cprint(f" ERROR: Invalid python3 in review for {name}: '{value}'. "
124+
f"Valid options are {', '.join(REVIEW_PYTHON3)}", file=sys.stderr, color='red')
125+
error += 1
126+
else:
127+
if value not in REVIEW_GENERIC:
128+
cprint(f" ERROR: Invalid {key} in review for {name}: '{value}'. "
129+
f"Valid options are {', '.join(REVIEW_GENERIC)}", file=sys.stderr, color='red')
130+
error += 1
131+
132+
if error > 0:
133+
sornot = 's' if error > 1 else ''
134+
cprint(f"** {error} error{sornot} occurred - see above for details **", file=sys.stderr, color='red')
135+
sys.exit(1)

‎getteam.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
"""
2+
A command line script that updates "The team" in the ``about.html`` file to
3+
reflect the current ``credits.rst`` file from the astropy repository.
4+
5+
Note that this first looks for the ``ASTROPY_REPO_PATH`` environment
6+
variable to try to find a local copy of the astropy repo.
7+
"""
8+
9+
10+
def get_astropy_credits(warner=print):
11+
"""
12+
Looks for the ``credits.rst`` file in the astropy repo and returns it, or
13+
returns False if the repo can't be found.
14+
"""
15+
import os
16+
import requests
17+
18+
creditspath = os.environ.get('ASTROPY_REPO_PATH', 'https://raw.githubusercontent.com/astropy/astropy/main/docs/credits.rst')
19+
20+
if creditspath.startswith('http'):
21+
#url - download page from web
22+
u = None
23+
try:
24+
return requests.get(creditspath).content
25+
except Exception as e:
26+
warner('Could not download credits.rst from requested path: "{0}" Using placeholder for "The Team" page.'.format(e))
27+
return False
28+
finally:
29+
if u is not None:
30+
u.close()
31+
else:
32+
if not os.path.isfile(creditspath):
33+
warner('Credits.rst file at "{0}" is not a file! Using placeholder for "The Team" page.'.format(creditspath))
34+
return False
35+
36+
with open(creditspath) as f:
37+
return f.read()
38+
39+
40+
def extract_names_list(docs, sectionname, warner=print):
41+
from docutils import nodes
42+
from docutils.core import publish_doctree
43+
44+
if not isinstance(docs, nodes.document):
45+
docs = publish_doctree(docs)
46+
47+
assert isinstance(docs, nodes.document)
48+
49+
foundsections = []
50+
for c in docs.children:
51+
titleidx = c.first_child_matching_class(nodes.title)
52+
if titleidx is not None:
53+
title = str(c.children[titleidx].children[0])
54+
if title == sectionname:
55+
section = c
56+
break
57+
else:
58+
foundsections.append(title)
59+
else:
60+
warner("No section found with name {0}. Sections are:{1!s}".format(sectionname, foundsections))
61+
return None
62+
63+
listidx = section.first_child_matching_class(nodes.bullet_list)
64+
litems = section.children[listidx].children
65+
66+
names = []
67+
for litem in litems:
68+
# Use astext() to get the concatenated text content of the list item
69+
# instead of joining node objects which can produce unexpected
70+
# characters when their string representation is used.
71+
names.append(litem.astext())
72+
73+
return names
74+
75+
76+
def process_html(fn, newcontributors, indent='\t\t\t'):
77+
"""
78+
Returns a string of html mean to look like the input, but with content from
79+
the credits file.
80+
"""
81+
lines = []
82+
incoord = incontrib = False
83+
with open(fn) as fr:
84+
for l in fr:
85+
if l.endswith('\n'):
86+
l = l[:-1] # strip newline
87+
88+
if incontrib:
89+
if '</ul>' in l:
90+
lines.extend([(indent + '<li>' + c + '</li>') for c in newcontributors])
91+
lines.append(l)
92+
incontrib = False
93+
else:
94+
if '<ul class="team">' in l:
95+
lines.append(l)
96+
lines.append(f"{<!--- the list below is auto-generated from getteam.py, use that to update --->:>12}")
97+
#skip otherwise
98+
else:
99+
# if '<ul class="coordinators">' in l:
100+
# incoord = True
101+
if '<h3 id="core-package-contributors">' in l:
102+
incontrib = True
103+
lines.append(l)
104+
105+
return '\n'.join(lines)
106+
107+
108+
if __name__ == '__main__':
109+
from docutils.core import publish_doctree
110+
111+
dt = publish_doctree(get_astropy_credits())
112+
113+
contributors = extract_names_list(dt, 'Core Package Contributors')
114+
115+
newhtml = process_html('team.html', contributors)
116+
print('Replacing "team.html" with updated version. Be sure to "git diff '
117+
'team.html" before committing to ensure no funny business happened.')
118+
with open('team.html', 'wb') as f:
119+
f.write(newhtml.encode('UTF-8'))

‎validate_roles.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import os
2+
import sys
3+
import json
4+
from termcolor import cprint
5+
6+
REQUIRED_KEYS = {"role", "url", "role-head", "responsibilities"}
7+
8+
9+
def assert_is_list(i, key, value):
10+
if isinstance(value, list):
11+
return 0
12+
else:
13+
cprint(f" ERROR: Key \"{key}\" for role #{i} should be a list but instead is a {type(value)}", file=sys.stderr, color='red')
14+
return 1
15+
16+
17+
def assert_is_string(i, key, value):
18+
if isinstance(value, str):
19+
return 0
20+
else:
21+
cprint(f" ERROR: Key \"{key}\" for role #{i} should be a string but instead is a {type(value)}", file=sys.stderr, color='red')
22+
return 1
23+
24+
25+
jsonfn = "roles.json"
26+
try:
27+
roles = json.load(open(jsonfn))
28+
except json.decoder.JSONDecodeError as e:
29+
cprint(jsonfn + ' : ' + e.args[0], color='red')
30+
cprint("*** JSON file appears to be malformed - see above ***", color='red')
31+
sys.exit(2)
32+
33+
error = 0
34+
35+
for i, role in enumerate(roles):
36+
if not isinstance(role, dict):
37+
cprint(f" ERROR: Role #{i} is not a key/value set", file=sys.stderr, color='red')
38+
error += 1
39+
else:
40+
key_difference = REQUIRED_KEYS - set(role.keys())
41+
if key_difference:
42+
cprint(f" ERROR: Missing key(s) for role #{i}: {key_difference}", file=sys.stderr, color='red')
43+
error += 1
44+
45+
error += assert_is_string(i, 'role', role['role'])
46+
error += assert_is_string(i, 'url', role['url'])
47+
error += assert_is_string(i, 'role-head', role['role-head'])
48+
49+
if 'sub-roles' in role:
50+
if 'people' in role:
51+
cprint(f" ERROR: people should not be defined at top level for role #{i} since sub-roles are defined")
52+
error += 1
53+
for sub_role in role['sub-roles']:
54+
error += assert_is_string(i, 'sub-roles[role]', sub_role['role'])
55+
error += assert_is_list(i, 'sub-roles[people]', sub_role['people'])
56+
else:
57+
error += assert_is_list(i, 'people', role['people'])
58+
59+
if isinstance(role['responsibilities'], list):
60+
for resp in role['responsibilities']:
61+
error += assert_is_string(i, 'responsibilities[description]', resp['description'])
62+
for detail in resp['details']:
63+
error += assert_is_string(i, 'responsibilities[detail]', detail)
64+
else:
65+
resp = role['responsibilities']
66+
error += assert_is_string(i, 'responsibilities[description]', resp['description'])
67+
for detail in resp['details']:
68+
error += assert_is_string(i, 'responsibilities[detail]', detail)
69+
70+
if error > 0:
71+
sornot = 's' if error > 1 else ''
72+
cprint(f"** {error} error{sornot} occurred - see above for details **", file=sys.stderr, color='red')
73+
sys.exit(1)

0 commit comments

Comments
 (0)