Files
website-typo3/.ddev/static-export/checks/check_site.py
T

31 lines
2.0 KiB
Python

"""Request every exported page and every local URL referenced in pages/fragments via a local Apache."""
import os, re, sys, html, urllib.parse, urllib.request, urllib.error, concurrent.futures
root, host, base = sys.argv[1], sys.argv[2], 'http://127.0.0.1:8099'
class NoRedir(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *a): return None
op = urllib.request.build_opener(NoRedir)
def get(path):
try:
with op.open(urllib.request.Request(base + path, headers={'Host': host}), timeout=30) as r: return r.status
except urllib.error.HTTPError as e: return e.code
except Exception as e: return str(e)
pages, refs = [], set()
for d, _, files in os.walk(root):
for f in files:
if not f.endswith('.html'): continue
rel = '/' + os.path.relpath(os.path.join(d, f), root)
if not rel.startswith('/_ajax/') and rel != '/404.html': pages.append(rel[:-10] if rel.endswith('/index.html') else rel)
t = open(os.path.join(d, f), encoding='utf-8').read()
for v in re.findall(r'(?:href|src|srcset|hx-get|data-[\w-]+)="([^"]*)"', t):
for c in html.unescape(v).split(','):
c = c.strip().split(' ')[0]; p = urllib.parse.urlsplit(c)
if p.netloc in ('', host, 'cdn.ep-reisen.de') and p.path.startswith('/') and not p.path.startswith('//'):
refs.add(p.path + ('?' + p.query if p.path.startswith('/_ajax/') else ''))
with concurrent.futures.ThreadPoolExecutor(16) as ex:
pr = dict(zip(pages, ex.map(get, pages))); rr = dict(zip(sorted(refs), ex.map(get, sorted(refs))))
bad_p = {k: v for k, v in pr.items() if v != 200}
bad_r = {k: v for k, v in rr.items() if v not in (200, 301, 302, 303, 307, 308)}
print('pages: %d, not 200: %d' % (len(pr), len(bad_p)))
print('referenced local URLs: %d (ajax: %d), failing: %d' % (len(rr), sum(1 for k in rr if k.startswith('/_ajax/')), len(bad_r)))
for k, v in list(bad_p.items())[:10] + list(bad_r.items())[:25]: print(' %s %s' % (v, k))