gen1recomp/tools/compare_shots.py

#!/usr/bin/env python3
"""Golden-screenshot diff (21-testing-and-ci "golden screenshots").

Compares captured PNGs against committed goldens with a per-pixel channel
tolerance and a budget for how many pixels may differ at all, and writes a
side-by-side image for every regression so a CI artifact shows what moved.

The two thresholds are separate on purpose.  --tolerance absorbs the
harmless drift a different GPU or LOVE build puts into a blend; --max-diff
is what actually fails the run.  A shot where one sprite moved trips the
pixel budget even though every differing pixel is far outside the channel
tolerance, and a shot that got globally half a shade darker trips neither
-- which is the intent, because that is not a regression a reviewer wants
to bless a hundred goldens over.

Usage:
  tools/compare_shots.py GOLDEN_DIR SHOT_DIR [--tolerance N] [--max-diff N]
                         [--diff-dir DIR] [--bless]

Exits 0 when every shot matches, 1 on any regression, 2 on a usage or
missing-file problem.
"""

import argparse
import os
import shutil
import sys

try:
    from PIL import Image, ImageChops
except ImportError:
    sys.stderr.write(
        "compare_shots needs Pillow: python3 -m pip install pillow\n")
    sys.exit(2)


def load_rgb(path):
    """Goldens and captures must compare in one colour space; LOVE writes
    RGBA and a hand-made golden is often RGB."""
    with Image.open(path) as image:
        return image.convert("RGB")


def compare(golden_path, shot_path, tolerance, max_diff):
    """Returns (ok, message, diff_image_or_None)."""
    golden = load_rgb(golden_path)
    shot = load_rgb(shot_path)

    if golden.size != shot.size:
        return (False,
                "size %dx%d != golden %dx%d" % (shot.width, shot.height,
                                                golden.width, golden.height),
                side_by_side(golden, shot, None))

    delta = ImageChops.difference(golden, shot)
    # collapse the three channels to the worst single-channel deviation, so
    # a pixel counts as differing on its loudest channel rather than an
    # average that would hide a red-only shift
    worst = max_channel(delta)

    histogram = worst.histogram()
    differing = sum(histogram[tolerance + 1:])

    if differing > max_diff:
        return (False,
                "%d pixels differ by more than %d (budget %d)"
                % (differing, tolerance, max_diff),
                side_by_side(golden, shot, amplify(worst)))

    return (True, "%d pixels differ by more than %d" % (differing, tolerance), None)


def max_channel(delta):
    """Per-pixel max across R/G/B as an L image."""
    red, green, blue = delta.split()
    return ImageChops.lighter(ImageChops.lighter(red, green), blue)


def amplify(mask):
    """Difference masks are near-black; stretch so the diff is visible."""
    return mask.point(lambda value: 255 if value else 0)


def side_by_side(golden, shot, mask):
    """golden | shot | mask, for the CI artifact."""
    panels = [golden, shot]
    if mask is not None:
        panels.append(mask.convert("RGB"))
    width = sum(panel.width for panel in panels) + 4 * (len(panels) - 1)
    height = max(panel.height for panel in panels)
    canvas = Image.new("RGB", (width, height), (255, 0, 255))
    offset = 0
    for panel in panels:
        canvas.paste(panel, (offset, 0))
        offset += panel.width + 4
    return canvas


def shots_in(directory):
    if not os.path.isdir(directory):
        return None
    return sorted(name for name in os.listdir(directory)
                  if name.lower().endswith(".png"))


def main(argv):
    parser = argparse.ArgumentParser(description="diff captured shots against goldens")
    parser.add_argument("golden_dir")
    parser.add_argument("shot_dir")
    parser.add_argument("--tolerance", type=int, default=2,
                        help="per-channel value a pixel may drift without counting (default 2)")
    parser.add_argument("--max-diff", type=int, default=0,
                        help="how many pixels may exceed the tolerance (default 0)")
    parser.add_argument("--diff-dir", default=None,
                        help="where to write side-by-side images (default SHOT_DIR/diffs)")
    parser.add_argument("--bless", action="store_true",
                        help="overwrite the goldens with the captures instead of comparing")
    args = parser.parse_args(argv)

    goldens = shots_in(args.golden_dir)
    shots = shots_in(args.shot_dir)

    if shots is None:
        sys.stderr.write("no shot directory: %s\n" % args.shot_dir)
        return 2

    if args.bless:
        os.makedirs(args.golden_dir, exist_ok=True)
        for name in shots:
            shutil.copyfile(os.path.join(args.shot_dir, name),
                            os.path.join(args.golden_dir, name))
            print("blessed %s" % name)
        print("\n%d goldens written to %s" % (len(shots), args.golden_dir))
        return 0

    if goldens is None:
        sys.stderr.write(
            "no golden directory: %s (capture some and re-run with --bless)\n"
            % args.golden_dir)
        return 2

    if not goldens:
        sys.stderr.write(
            "no goldens in %s -- refusing to pass vacuously\n" % args.golden_dir)
        return 2

    diff_dir = args.diff_dir or os.path.join(args.shot_dir, "diffs")
    failures = 0

    for name in goldens:
        golden_path = os.path.join(args.golden_dir, name)
        shot_path = os.path.join(args.shot_dir, name)

        if not os.path.exists(shot_path):
            print("FAIL %s: no capture for this golden" % name)
            failures += 1
            continue

        ok, message, diff = compare(golden_path, shot_path,
                                    args.tolerance, args.max_diff)
        if ok:
            print("ok   %s (%s)" % (name, message))
        else:
            failures += 1
            print("FAIL %s: %s" % (name, message))
            if diff is not None:
                os.makedirs(diff_dir, exist_ok=True)
                out = os.path.join(diff_dir, name)
                diff.save(out)
                print("     diff written to %s" % out)

    # a capture with no golden is a new screen nobody blessed; report it,
    # but it is not a regression in an existing one
    for name in shots:
        if name not in goldens:
            print("note %s: captured but no golden (bless it to pin it)" % name)

    print("\n%d/%d shots matched" % (len(goldens) - failures, len(goldens)))
    print("ALL SHOTS MATCHED" if failures == 0 else "%d FAILURES" % failures)
    return 0 if failures == 0 else 1


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))