Developer Utility Scripts

A collection of command-line tools and scripts for asset optimisation, static web building, and environment maintenance. Each script is short enough to read in full, and the code below is exactly what was run and tested. Requirements are listed with each tool.

1-Bit Dither Pass (dither-pass.py)

Converts a directory of images to 1-bit Atkinson-dithered PNGs, flattening any transparency onto white. ImageMagick has no Atkinson option, so the algorithm is implemented directly (the same kernel is explained on the dithering page). Requires Python 3.8 or later and Pillow (pip install pillow).

#!/usr/bin/env python3
"""dither-pass.py - Convert images to 1-bit Atkinson-dithered PNGs.

Requires: Python 3.8+ and Pillow (pip install pillow).
Usage: python3 dither-pass.py <input_dir> <output_dir> [max_width]
"""
import sys
from pathlib import Path
from PIL import Image

EXTS = {".png", ".jpg", ".jpeg", ".gif", ".bmp"}
# Atkinson kernel: each of these six neighbours receives 1/8 of the error.
KERNEL = [(1, 0), (2, 0), (-1, 1), (0, 1), (1, 1), (0, 2)]


def to_grey(img):
    """Greyscale, with any transparency flattened onto white."""
    if img.mode in ("RGBA", "LA") or "transparency" in img.info:
        img = img.convert("RGBA")
        flat = Image.new("RGBA", img.size, (255, 255, 255, 255))
        flat.alpha_composite(img)
        img = flat
    return img.convert("L")


def atkinson(img):
    w, h = img.size
    buf = [float(v) for v in img.tobytes()]
    for y in range(h):
        for x in range(w):
            i = y * w + x
            old = buf[i]
            new = 255.0 if old >= 128.0 else 0.0
            buf[i] = new
            err = (old - new) / 8.0
            for dx, dy in KERNEL:
                nx, ny = x + dx, y + dy
                if 0 <= nx < w and ny < h:
                    buf[ny * w + nx] += err
    # Every value is now exactly 0 or 255, so the 1-bit conversion is lossless.
    return Image.frombytes("L", (w, h), bytes(int(v) for v in buf)).convert("1")


def main():
    if len(sys.argv) < 3:
        print("Usage: dither-pass.py <input_dir> <output_dir> [max_width]")
        return 1
    src, dst = Path(sys.argv[1]), Path(sys.argv[2])
    max_w = int(sys.argv[3]) if len(sys.argv) > 3 else None
    if not src.is_dir():
        print(f"Error: input directory '{src}' does not exist.", file=sys.stderr)
        return 1
    dst.mkdir(parents=True, exist_ok=True)
    files = sorted(p for p in src.iterdir() if p.suffix.lower() in EXTS and p.is_file())
    print(f"dither-pass: processing {len(files)} files from {src}")
    done = 0
    for path in files:
        try:
            img = to_grey(Image.open(path))
            if max_w and img.width > max_w:
                img = img.resize((max_w, round(img.height * max_w / img.width)), Image.LANCZOS)
            out = dst / f"{path.stem}-1bit.png"
            atkinson(img).save(out, optimize=True)
        except OSError as e:
            print(f"  {path.name}: skipped ({e})", file=sys.stderr)
            continue
        print(f"  {path.name}: {path.stat().st_size}B -> {out.stat().st_size}B")
        done += 1
    print(f"dither-pass: complete. {done} of {len(files)} files written to {dst}")
    return 0


if __name__ == "__main__":
    sys.exit(main())

Usage: python3 dither-pass.py ./raw-images ./output 120 (the last argument, a maximum width in pixels, is optional)

HTML Lint & Size Check (html-lite-check.py)

Checks plain-HTML pages for a doctype, charset, viewport, language, title and landmarks, for images without alt, and for scripts or external requests. It also checks the size of the HTML file against the 14.6 KB budget from our System Load Time specification. Requires Python 3.6 or later and nothing else.

#!/usr/bin/env python3
"""html-lite-check.py - lint plain-HTML pages and check a size budget.

Checks: doctype, charset, viewport, lang, title, header/nav/main/footer
landmarks, img alt/width/height, scripts and external requests, and the
size of the HTML file itself (default budget 14.6 KB).

Usage: python3 html-lite-check.py [--budget KB] file.html [file.html ...]
Exit status is 1 if any file fails.
"""
import argparse
import re
import sys
from html.parser import HTMLParser
from pathlib import Path

REQUIRED = [
    ("DOCTYPE", r"<!doctype html>"),
    ("lang attribute", r'<html[^>]*\slang="'),
    ("meta charset", r'<meta\s+charset="utf-8"'),
    ("meta viewport", r'<meta\s+name="viewport"'),
    ("title", r"<title>[^<]+</title>"),
    ("header", r"<header[\s>]"),
    ("nav", r"<nav[\s>]"),
    ("main", r"<main[\s>]"),
    ("footer", r"<footer[\s>]"),
]
OBSOLETE = [(r"<font[\s>]", "<font>"), (r"<center[\s>]", "<center>"), (r"\sbgcolor=", "bgcolor")]


class Tags(HTMLParser):
    def __init__(self):
        super().__init__()
        self.imgs, self.scripts, self.external = [], 0, []

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag == "img":
            self.imgs.append(a)
        elif tag == "script":
            self.scripts += 1
        for key in ("src", "href"):
            v = a.get(key) or ""
            if v.startswith(("http://", "https://")) and tag in ("script", "img", "link"):
                self.external.append(v)


def check(path, budget):
    p = Path(path)
    if not p.is_file():
        print(f"{path}: ERROR file not found")
        return False
    text = p.read_text(encoding="utf-8")
    size = p.stat().st_size
    errors, warns = [], []
    for name, pat in REQUIRED:
        if not re.search(pat, text, re.I):
            errors.append(f"missing {name}")
    if size > budget:
        errors.append(f"{size} bytes exceeds budget of {budget} bytes")
    t = Tags()
    t.feed(text)
    for img in t.imgs:
        if "alt" not in img:
            errors.append(f"img without alt: {img.get('src')}")
        if "width" not in img or "height" not in img:
            warns.append(f"img without width/height: {img.get('src')}")
    if t.scripts:
        warns.append(f"{t.scripts} <script> element(s)")
    for url in t.external:
        warns.append(f"external request: {url}")
    for pat, name in OBSOLETE:
        if re.search(pat, text, re.I):
            warns.append(f"obsolete presentational markup: {name}")
    print(f"{path}: {size} bytes ({size / budget * 100:.0f}% of budget)")
    for w in warns:
        print(f"  WARN: {w}")
    for e in errors:
        print(f"  FAIL: {e}")
    return not errors


def main():
    ap = argparse.ArgumentParser(description="Lint plain-HTML pages.")
    ap.add_argument("files", nargs="+")
    ap.add_argument("--budget", type=float, default=14.6, help="size budget in KB")
    args = ap.parse_args()
    budget = int(args.budget * 1000)
    results = [check(f, budget) for f in args.files]
    return 0 if all(results) else 1


if __name__ == "__main__":
    sys.exit(main())

Usage: python3 html-lite-check.py index.html doc/*.html or python3 html-lite-check.py --budget 20 index.html

Static Index Generator (mkindex.sh)

POSIX shell script that lists a directory as a plain HTML page, for static servers without directory indexes. File names are HTML-escaped and URL-encoded, and the output file is not listed in itself.

#!/bin/sh
# mkindex.sh - Generate a plain HTML directory index
# Usage: ./mkindex.sh <directory> [output_file]
# The output defaults to <directory>/index.html and is not listed in itself.
# File names containing newlines are not supported.
set -eu

TARGET_DIR="${1:-.}"
OUTPUT="${2:-$TARGET_DIR/index.html}"

if [ ! -d "$TARGET_DIR" ]; then
    echo "Error: '$TARGET_DIR' is not a directory." >&2
    exit 1
fi

# Escape text for HTML, and percent-encode names for use in an href.
esc() {
    printf '%s' "$1" | sed -e 's/&/\&amp;/g' -e 's/</\&lt;/g' -e 's/>/\&gt;/g' -e 's/"/\&quot;/g'
}
enc() {
    printf '%s' "$1" | sed -e 's/%/%25/g' -e 's/ /%20/g' -e 's/#/%23/g' -e 's/?/%3F/g' \
        -e 's/"/%22/g' -e 's/</%3C/g' -e 's/>/%3E/g'
}

DIR_NAME=$(basename "$(cd "$TARGET_DIR" && pwd)")
OUT_NAME=$(basename "$OUTPUT")
TMP=$(mktemp)
trap 'rm -f "$TMP"' EXIT

{
    cat <<ENDOFFILE
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Index of $(esc "$DIR_NAME")</title>
</head>
<body>
<h1>Index of $(esc "$DIR_NAME")</h1>
<hr>
<ul>
ENDOFFILE

    find "$TARGET_DIR" -maxdepth 1 -type f -name "*.html" ! -name "$OUT_NAME" | sort |
    while IFS= read -r FILE; do
        NAME=$(basename "$FILE")
        printf '<li><a href="%s">%s</a></li>\n' "$(enc "$NAME")" "$(esc "$NAME")"
    done

    find "$TARGET_DIR" -maxdepth 1 -type f ! -name "*.html" ! -name "$OUT_NAME" | sort |
    while IFS= read -r FILE; do
        NAME=$(basename "$FILE")
        SIZE=$(wc -c < "$FILE" | tr -d ' ')
        printf '<li>%s (%s bytes)</li>\n' "$(esc "$NAME")" "$SIZE"
    done

    cat <<ENDOFFILE
</ul>
<hr>
<footer>
<address>Generated by mkindex.sh</address>
</footer>
</body>
</html>
ENDOFFILE
} > "$TMP"

mv "$TMP" "$OUTPUT"
chmod 644 "$OUTPUT"
trap - EXIT
echo "mkindex: wrote $OUTPUT ($(wc -c < "$OUTPUT" | tr -d ' ') bytes)"

Usage: chmod +x mkindex.sh && ./mkindex.sh /var/www/html

Navigation & Repository Links: