Find the screen buffers in the firmware instead of hardcoding one build's

The page was told --frame-addr 0x200012BE --status-addr 0x2000163E and used them as a fallback. The firmware the user actually flashed keeps its buffers at 0x2000129E/0x2000161E, 32 bytes earlier, so every line landed 32 bytes off: that is the "other firmware looks shifted" report. The images here are minimal ELFs with no symbol table, so there is nothing to read -- but the firmware's own buffers hold the same bytes the controller holds, and tools/uvk5_buffers.py finds them by matching (1024/1024 bytes for that file).

The two address flags are optional now, work/run-webui.ps1 passes no machine-specific values at all, and the page reports what it found in /api/status and /api/panel. tools/uvk5_testenv.qemu() also looks in the sibling qemu-7.2/build the rest of the repo assumes. Fixed /api/panel's emulator-off branch, which called jsonify with both a dict and kwargs and 500'd.

Tests: test_uvk5_buffers (the search must count matches, not pairs -- its first version scored every offset full marks and always answered the first one).

tools/test_uvk5_buffers.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""The buffer search must count *matches*, not just bytes compared.
+
+Its first version summed one per zipped pair and forgot `if a == b`, so every
+offset scored the full total and the search always answered "the first offset" --
+which looked like a working discovery and pointed the fallback at whatever happened
+to sit at the start of SRAM. These tests use synthetic memory, so they need no
+emulator and fail on that code.
+"""
+import struct
+import unittest
+
+import uvk5_buffers
+from uvk5_lcd import FRAME_BYTES, STATUS_BYTES
+
+
+def panel_memory(seed=1):
+    """A controller memory with a recognisable, non-repeating-ish pattern."""
+    frame = bytes(((i * 13) + seed) & 0xFF for i in range(FRAME_BYTES))
+    status = bytes(((i * 29) + seed) & 0xFF for i in range(STATUS_BYTES))
+    return status + frame          # the panel's order: the status page comes first
+
+
+class TestLocate(unittest.TestCase):
+    def test_it_finds_the_buffers_where_they_are(self):
+        gram = panel_memory()
+        pad = 0x1234
+        # In SRAM the frame comes first and the status line follows it.
+        sram = bytes(pad) + gram[STATUS_BYTES:] + gram[:STATUS_BYTES] + bytes(0x100)
+
+        frame, status, best, total = uvk5_buffers.locate(sram, gram)
+
+        self.assertEqual(frame, uvk5_buffers.SRAM_BASE + pad)
+        self.assertEqual(status, uvk5_buffers.SRAM_BASE + pad + FRAME_BYTES)
+        self.assertEqual(best, total, "a perfect match must score the whole window")
+        self.assertEqual(total, FRAME_BYTES + STATUS_BYTES)
+
+    def test_a_single_match_does_not_win_by_being_first(self):
+        """The bug this guards: scoring pairs instead of matches picked offset 0."""
+        gram = panel_memory()
+        pad = 0x0800
+        sram = bytes(pad) + gram[STATUS_BYTES:] + gram[:STATUS_BYTES] + bytes(0x40)
+        _, _, best, total = uvk5_buffers.locate(sram, gram)
+        self.assertGreater(best, total * 0.9)
+        self.assertNotEqual(uvk5_buffers.score(sram, 0, gram), total,
+                            "offset 0 is blank here and must not score full marks")
+
+    def test_a_partly_stale_buffer_still_wins(self):
+        """The panel can be a frame ahead of the buffer it was copied from."""
+        gram = panel_memory()
+        pad = 0x0400
+        frame = bytearray(gram[STATUS_BYTES:])
+        for i in range(0, 60):
+            frame[i] ^= 0x5A
+        sram = bytes(pad) + bytes(frame) + gram[:STATUS_BYTES] + bytes(0x80)
+        found, _, best, total = uvk5_buffers.locate(sram, gram)
+        self.assertEqual(found, uvk5_buffers.SRAM_BASE + pad)
+        # What matters is that the right offset wins, not that it is perfect: the
+        # panel can be a frame ahead of the buffer it was copied from.
+        runner_up = max(uvk5_buffers.score(sram, off, gram)
+                        for off in range(0, len(sram) - total + 1)
+                        if off != pad)
+        self.assertGreater(best, runner_up)
+
+
+class TestSymbols(unittest.TestCase):
+    def test_a_minimal_elf_has_no_symbols_to_offer(self):
+        """tools/bin2elf.py writes program headers only, which is the usual case."""
+        elf = bytearray(0x34 + 32 + 0x40)
+        elf[0:4] = b"\x7fELF"
+        elf[4] = 1                     # 32-bit
+        elf[5] = 1                     # little endian
+        struct.pack_into("<I", elf, 0x1C, 0x34)     # one program header
+        struct.pack_into("<H", elf, 0x2A, 32)
+        struct.pack_into("<H", elf, 0x2C, 1)
+        struct.pack_into("<I", elf, 0x20, 0)        # no section headers
+        struct.pack_into("<H", elf, 0x30, 0)
+        import os
+        import tempfile
+        path = os.path.join(tempfile.mkdtemp(), "minimal.elf")
+        with open(path, "wb") as fh:
+            fh.write(bytes(elf))
+        self.assertEqual(uvk5_buffers.from_symbols(path), (None, None))
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/uvk5_buffers.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Where a firmware keeps its screen: found by looking, not assumed.
+
+The display controller's memory is the screen: every firmware pushes its pixels
+through the same controller, so the panel needs no per-build knowledge and is the
+path the page uses. The *guest* buffers are the fallback, and they move between
+builds -- so passing one build's addresses for another renders a picture that is
+plausible and wrong, which is exactly how "the other firmware looks shifted" arrived.
+
+The firmware files here are minimal ELFs: one program header, no section headers and
+no symbol table (tools/bin2elf.py writes them), so there are no `gFrameBuffer`
+symbols to read. What there *is*, is behaviour: the firmware's own buffers hold the
+same bytes the controller holds, because that is where the driver copied them from.
+So the addresses are found by sliding the controller's memory through SRAM and
+keeping the offset that agrees.
+
+    tools/uvk5_buffers.py --qmp 127.0.0.1:4444      # say where this firmware keeps it
+
+Agreement is not assumed to be perfect: the panel can be a frame ahead of the buffer
+it was copied from, so the score is reported and a caller decides what to trust.
+"""
+import argparse
+import os
+import struct
+import sys
+import tempfile
+
+import uvk5_lcd
+
+# 16 KB of SRAM. The buffers are inside it; nothing else is near.
+SRAM_BASE = 0x20000000
+SRAM_SIZE = 0x4000
+
+# The frame is seven pages and the status line is one, and in SRAM the *frame* comes
+# first (gFrameBuffer at 0x200012BE, gStatusLine at 0x2000163E for the 5.9.0.CN build).
+# The panel's own memory has them the other way round, because page 0 is the top line.
+FRAME_BYTES = uvk5_lcd.FRAME_BYTES
+STATUS_BYTES = uvk5_lcd.STATUS_BYTES
+
+# Below this fraction of bytes agreeing, the match is a coincidence rather than a
+# buffer. Measured: a real match scores above 0.99 unless the screen is mid-update.
+CONFIDENT = 0.90
+
+
+def score(sram: bytes, offset: int, gram: bytes) -> int:
+    """How many bytes at @offset match the controller's memory, frame then status."""
+    if offset < 0 or offset + FRAME_BYTES + STATUS_BYTES > len(sram):
+        return -1
+    frame = sram[offset:offset + FRAME_BYTES]
+    status = sram[offset + FRAME_BYTES:offset + FRAME_BYTES + STATUS_BYTES]
+    return (sum(1 for a, b in zip(frame, gram[STATUS_BYTES:]) if a == b)
+            + sum(1 for a, b in zip(status, gram[:STATUS_BYTES]) if a == b))
+
+
+def locate(sram: bytes, gram: bytes):
+    """(frame address, status address, matching bytes, total) for the best offset."""
+    total = FRAME_BYTES + STATUS_BYTES
+    best, best_at = -1, None
+    for offset in range(0, len(sram) - total + 1):
+        value = score(sram, offset, gram)
+        if value > best:
+            best, best_at = value, offset
+    if best_at is None:
+        return None, None, 0, total
+    return SRAM_BASE + best_at, SRAM_BASE + best_at + FRAME_BYTES, best, total
+
+
+def from_symbols(path: str):
+    """(frame, status) from a real ELF symbol table, or (None, None).
+
+    Kept because it is authoritative when it works -- a fully linked ELF (the CW
+    timing build, for instance) does name these buffers. The images this project
+    usually runs do not.
+    """
+    try:
+        data = open(path, "rb").read()
+    except OSError:
+        return None, None
+    if len(data) < 52 or data[0] != 0x7F or data[1:4] != b"ELF" or data[4] != 1:
+        return None, None
+    e_shoff, = struct.unpack_from("<I", data, 0x20)
+    e_shentsize, e_shnum, _ = struct.unpack_from("<HHH", data, 0x2E)
+    if not e_shoff or not e_shnum:
+        return None, None
+    sections = []
+    for i in range(e_shnum):
+        fields = struct.unpack_from("<IIIIIIIIII", data, e_shoff + i * e_shentsize)
+        sections.append(dict(type=fields[1], offset=fields[4], size=fields[5],
+                             link=fields[6], entsize=fields[9]))
+    found = {}
+    for section in sections:
+        if section["type"] not in (2, 11):
+            continue
+        strings = sections[section["link"]]
+        blob = data[strings["offset"]:strings["offset"] + strings["size"]]
+        step = section["entsize"] or 16
+        for k in range(section["size"] // step):
+            name, value = struct.unpack_from("<II", data, section["offset"] + k * step)
+            if not name or not value:
+                continue
+            end = blob.find(b"\x00", name)
+            found[blob[name:end].decode("ascii", "replace")] = value
+    frame = found.get("gFrameBuffer")
+    status = found.get("gStatusLine")
+    if frame and status:
+        return frame, status
+    return None, None
+
+
+def sram_from(client) -> bytes:
+    """Read the whole of SRAM through QMP memsave."""
+    path = os.path.join(tempfile.mkdtemp(prefix="uvk5-buffers-"), "sram.bin")
+    client.command("memsave", val=SRAM_BASE, size=SRAM_SIZE, filename=path)
+    with open(path, "rb") as fh:
+        return fh.read()
+
+
+def discover(client, gram: bytes = None, image_path: str = None) -> dict:
+    """{frame, status, how, score, total} -- symbols first, then the search."""
+    if image_path:
+        frame, status = from_symbols(image_path)
+        if frame:
+            return dict(frame=frame, status=status, how="elf symbols",
+                        score=None, total=None)
+    if client is None:
+        return dict(frame=None, status=None, how="no emulator", score=0, total=0)
+    gram = gram if gram is not None else uvk5_lcd.FrameGrabber(client, 0, 0).panel_gram()
+    frame, status, best, total = locate(sram_from(client), gram)
+    how = "sram search" if best >= CONFIDENT * total else "sram search (weak match)"
+    return dict(frame=frame, status=status, how=how, score=best, total=total)
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--qmp", default="127.0.0.1:4444")
+    ap.add_argument("--image", help="firmware file, to try its symbols first")
+    args = ap.parse_args(argv)
+
+    from uvk5_qmp import QmpClient
+    try:
+        client = QmpClient(args.qmp, timeout=20)
+    except Exception as exc:
+        print("cannot reach QMP at %s: %s" % (args.qmp, str(exc).splitlines()[0]), file=sys.stderr)
+        print("if the web UI is running it holds the one QMP client", file=sys.stderr)
+        return 2
+    found = discover(client, image_path=args.image)
+    print("frame  0x%08X" % found["frame"] if found["frame"] else "frame  (not found)")
+    print("status 0x%08X" % found["status"] if found["status"] else "status (not found)")
+    print("how    %s" % found["how"])
+    if found["score"] is not None:
+        print("match  %d/%d bytes" % (found["score"], found["total"]))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/uvk5_stream.py

@@ -82,6 +82,21 @@ class FramePump:
                 self._raw = None
             self._generation += 1
 
+    def set_buffers(self, frame_addr: int, status_addr: int):
+        """Point the guest-RAM fallback at this firmware's buffers.
+
+        The addresses move between builds, so they are discovered from the firmware
+        itself (tools/uvk5_buffers.py) rather than hardcoded. Only the fallback uses
+        them: the panel path needs none.
+        """
+        with self._lock:
+            self._frame_addr = frame_addr
+            self._status_addr = status_addr
+            grabber = self._grabber
+            if grabber is not None:
+                self._grabber = FrameGrabber(grabber._client, frame_addr, status_addr,
+                                             self._spool_dir)
+
     def _run(self):
         while not self._stop.is_set():
             started = time.monotonic()

tools/uvk5_testenv.py

@@ -36,7 +36,16 @@ def qemu():
         if value and os.path.exists(value):
             return pathlib.Path(value)
     found = shutil.which("qemu-system-arm")
-    return pathlib.Path(found) if found else None
+    if found:
+        return pathlib.Path(found)
+    # The sibling tree tools/setup_qemu.sh builds into, which is also what run_tests.sh
+    # assumes. Not a machine-specific path: it is this checkout's own convention, so a
+    # fresh clone that followed the README finds its QEMU without being told.
+    sibling = pathlib.Path(__file__).resolve().parent.parent.parent / "qemu-7.2" / "build"
+    for name in ("qemu-system-arm", "qemu-system-arm.exe"):
+        if (sibling / name).exists():
+            return sibling / name
+    return None
 
 
 def gdb():

tools/webui.py

@@ -144,7 +144,7 @@ def image_has_multiboot(path):
     return any(marker in blob for marker in MULTIBOOT_MARKERS)
 
 
-def create_app(client, frame_addr: int, status_addr: int, scale: int = 4,
+def create_app(client, frame_addr: int = None, status_addr: int = None, scale: int = 4,
                supervisor=None, log=None, image=None, boot_key=None, flash=None):
     app = Flask(__name__)
 
@@ -154,6 +154,46 @@ def create_app(client, frame_addr: int, status_addr: int, scale: int = 4,
 
     # One background grabber for every client. client may be None: the emulator
     # can be powered off, and the page still has to load.
+    told_addresses = frame_addr is not None and status_addr is not None
+    frame_addr = frame_addr or 0
+    status_addr = status_addr or 0
+    buffers = {"info": None}
+
+    def ensure_buffers():
+        """Ask the firmware where it keeps its screen, instead of being told.
+
+        The addresses move between builds, and passing one build's for another is how
+        the page ended up drawing a picture that was plausible and offset. The panel
+        path needs no addresses at all, so this concerns only the guest-RAM fallback:
+        it stays unset, and says so, rather than guessing. A --frame-addr on the
+        command line skips the search and is trusted.
+        """
+        if told_addresses:
+            return {"frame": frame_addr, "status": status_addr, "how": "command line"}
+        if buffers["info"] is not None:
+            return buffers["info"]
+        target = active_client()
+        if target is None:
+            return None
+        try:
+            import uvk5_buffers
+            path = image.current.path if image is not None and image.current else None
+            info = uvk5_buffers.discover(target, image_path=path)
+        except Exception as exc:
+            info = {"frame": None, "status": None,
+                    "how": "discovery failed: %s" % exc, "score": 0, "total": 0}
+        buffers["info"] = info
+        if info.get("frame"):
+            pump.set_buffers(info["frame"], info["status"])
+            log.add("qemu", "screen buffers read from the firmware: frame 0x%08X, "
+                            "status 0x%08X (%s; %s/%s bytes agree)"
+                    % (info["frame"], info["status"], info["how"],
+                       info["score"], info["total"]))
+        else:
+            log.add("qemu", "no screen buffers found for this firmware (%s); the panel "
+                            "path does not need them" % info["how"])
+        return info
+
     pump = FramePump(client, frame_addr, status_addr, fps=TARGET_FPS, scale=scale,
                      on_fallback=lambda note: log.add(
                          "qemu", "panel unavailable, drawing from guest RAM at "
@@ -274,7 +314,7 @@ def create_app(client, frame_addr: int, status_addr: int, scale: int = 4,
             return jsonify(powered=False, status="unreachable", error=str(exc))
         return jsonify(powered=True, speaker=speaker_on(),
                        panel=panel_state(), firmware=firmware_info(),
-                       frame_source=pump.source()[0], **info)
+                       frame_source=pump.source()[0], buffers=ensure_buffers(), **info)
 
     @app.get("/api/panel")
     def api_panel():
@@ -288,11 +328,11 @@ def create_app(client, frame_addr: int, status_addr: int, scale: int = 4,
         can be compared without guessing.
         """
         source, note = pump.source()
-        body = {"source": source, "note": note,
+        body = {"source": source, "note": note, "buffers": ensure_buffers(),
                 "frame_addr": frame_addr, "status_addr": status_addr}
         target = active_client()
         if target is None:
-            return jsonify(body, powered=False)
+            return jsonify(dict(body, powered=False))
         try:
             body["gram"] = target.command("qom-get", path=PANEL_PATH, property="gram")
             body["invert"] = bool(target.command("qom-get", path=PANEL_PATH,
@@ -1220,10 +1260,14 @@ def _default_firmware():
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     ap.add_argument("--qmp", default="/tmp/uvk5-qmp.sock")
-    ap.add_argument("--frame-addr", type=lambda v: int(v, 0), required=True,
-                    help="address of gFrameBuffer (moves between builds)")
-    ap.add_argument("--status-addr", type=lambda v: int(v, 0), required=True,
-                    help="address of gStatusLine")
+    # Optional, and normally omitted: the addresses move between builds, so the page
+    # finds them by asking the firmware (tools/uvk5_buffers.py). They are only needed
+    # by the guest-RAM fallback -- the panel path, which is what the page uses, needs
+    # no addresses at all. Passing one skips discovery and trusts the value.
+    ap.add_argument("--frame-addr", type=lambda v: int(v, 0), default=None,
+                    help="address of gFrameBuffer; omit to find it in the firmware")
+    ap.add_argument("--status-addr", type=lambda v: int(v, 0), default=None,
+                    help="address of gStatusLine; omit to find it in the firmware")
     ap.add_argument("--host", default="127.0.0.1")
     ap.add_argument("--port", type=int, default=8080)
     ap.add_argument("--scale", type=int, default=4)