"""Fetch every exported page of a site live, then check every referenced asset/link on the site, its CDN and ajax files.""" import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures, collections root, prod = sys.argv[1], sys.argv[2].rstrip('/') host = urllib.parse.urlsplit(prod).netloc class NoRedir(urllib.request.HTTPRedirectHandler): def redirect_request(self, *a): return None op = urllib.request.build_opener(NoRedir) def get(url, body=False): try: with op.open(urllib.request.Request(url, headers={'User-Agent': 'ep-static-check', 'Accept': 'image/webp,*/*'}), timeout=60) as r: return r.status, (r.read().decode('utf-8', 'replace') if body else '') except urllib.error.HTTPError as e: return e.code, '' except Exception as e: return str(e)[:60], '' pages = [] for d, _, files in os.walk(root): if '/_ajax' in d: continue for f in files: if f == 'index.html': pages.append('/' + os.path.relpath(d, root).replace('.', '', 1).lstrip('/') + ('/' if d != root else '')) pages = sorted(set(p.replace('//', '/') for p in pages)) with concurrent.futures.ThreadPoolExecutor(12) as ex: res = dict(zip(pages, ex.map(lambda p: get(prod + p, True), pages))) refs = set() for p, (st, body) in res.items(): for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', body): for c in html.unescape(v).split(','): c = c.strip().split(' ')[0]; u = urllib.parse.urlsplit(c) if u.netloc in ('', host, 'cdn.ep-reisen.de') and u.path.startswith('/') and not u.path.startswith('//'): base = 'https://' + (u.netloc or host) refs.add(base + u.path + ('?' + u.query if u.path.startswith('/_ajax/') else '')) with concurrent.futures.ThreadPoolExecutor(16) as ex: rr = dict(zip(sorted(refs), ex.map(lambda u: get(u)[0], sorted(refs)))) print('pages: %d → %s' % (len(res), dict(collections.Counter(st for st, _ in res.values())))) by = collections.Counter((urllib.parse.urlsplit(u).netloc, s if s in (200, 404) else ('3xx' if isinstance(s, int) and 300 <= s < 400 else s)) for u, s in rr.items()) print('references by host/status:'); [print(' %-26s %-5s %d' % (h, s, n)) for (h, s), n in sorted(by.items(), key=str)] bad = [(s, u) for u, s in rr.items() if not (isinstance(s, int) and (s == 200 or 300 <= s < 400))] for s, u in sorted(bad, key=str)[:20]: print(' FAIL %s %s' % (s, u))