37 lines
2.4 KiB
Python
37 lines
2.4 KiB
Python
"""Fetch every exported page of a site live, then check every referenced asset/link on the site, its CDN and ajax files."""
|
|
import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures, collections
|
|
root, prod = sys.argv[1], sys.argv[2].rstrip('/')
|
|
host = urllib.parse.urlsplit(prod).netloc
|
|
class NoRedir(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, *a): return None
|
|
op = urllib.request.build_opener(NoRedir)
|
|
def get(url, body=False):
|
|
try:
|
|
with op.open(urllib.request.Request(url, headers={'User-Agent': 'ep-static-check', 'Accept': 'image/webp,*/*'}), timeout=60) as r:
|
|
return r.status, (r.read().decode('utf-8', 'replace') if body else '')
|
|
except urllib.error.HTTPError as e: return e.code, ''
|
|
except Exception as e: return str(e)[:60], ''
|
|
pages = []
|
|
for d, _, files in os.walk(root):
|
|
if '/_ajax' in d: continue
|
|
for f in files:
|
|
if f == 'index.html': pages.append('/' + os.path.relpath(d, root).replace('.', '', 1).lstrip('/') + ('/' if d != root else ''))
|
|
pages = sorted(set(p.replace('//', '/') for p in pages))
|
|
with concurrent.futures.ThreadPoolExecutor(12) as ex:
|
|
res = dict(zip(pages, ex.map(lambda p: get(prod + p, True), pages)))
|
|
refs = set()
|
|
for p, (st, body) in res.items():
|
|
for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', body):
|
|
for c in html.unescape(v).split(','):
|
|
c = c.strip().split(' ')[0]; u = urllib.parse.urlsplit(c)
|
|
if u.netloc in ('', host, 'cdn.ep-reisen.de') and u.path.startswith('/') and not u.path.startswith('//'):
|
|
base = 'https://' + (u.netloc or host)
|
|
refs.add(base + u.path + ('?' + u.query if u.path.startswith('/_ajax/') else ''))
|
|
with concurrent.futures.ThreadPoolExecutor(16) as ex:
|
|
rr = dict(zip(sorted(refs), ex.map(lambda u: get(u)[0], sorted(refs))))
|
|
print('pages: %d → %s' % (len(res), dict(collections.Counter(st for st, _ in res.values()))))
|
|
by = collections.Counter((urllib.parse.urlsplit(u).netloc, s if s in (200, 404) else ('3xx' if isinstance(s, int) and 300 <= s < 400 else s)) for u, s in rr.items())
|
|
print('references by host/status:'); [print(' %-26s %-5s %d' % (h, s, n)) for (h, s), n in sorted(by.items(), key=str)]
|
|
bad = [(s, u) for u, s in rr.items() if not (isinstance(s, int) and (s == 200 or 300 <= s < 400))]
|
|
for s, u in sorted(bad, key=str)[:20]: print(' FAIL %s %s' % (s, u))
|