Apps on the page: list, install and erase overlay apps in the flash image

The Labs edition's apps live in the external flash, in the region its own App/apps/app_overlay.h defines (16 slots of 8 KiB from 0x102000, a 64-byte FAP1 header at the slot base, code one 4 KiB sector later), and upstream installs them from UVStudio over WebSerial. The page owns the image, so this adds GET /api/apps, POST /api/apps/<n> and POST /api/apps/<n>/erase, which put the same bytes at the same offsets with no serial protocol and no browser permission.

Measured on a real image while wiring it up: every one of the 16 slots already held data that is neither empty nor an app -- the localised build's factory resource block overlaps 0x102000 -- so install now refuses to overwrite anything that is not an app unless asked (--force, ?force=1), naming what is there. And _edit_flash edits a copy, which is why the source image shows no changed bytes; the first test read that as 'the install did nothing' and now checks FlashSlot.path. Tests: test_uvk5_apps grew to 20, test_webui.TestAppEndpoints adds 7 over the endpoints.

tools/test_uvk5_apps.py

@@ -115,11 +115,41 @@ class TestInstall(unittest.TestCase):
         A.erase(img, 2)
         self.assertIsNone(A.read_slot(bytes(img), 2))
 
+    def test_it_refuses_to_overwrite_something_that_is_not_an_app(self):
+        """Measured on a real image: the factory resource block can overlap the region."""
+        img = image()
+        base = A.REGION_BASE
+        img[base:base + 8] = b"RESDATA1"
+        with self.assertRaises(A.AppError) as caught:
+            A.install(img, 0, A.build(b"\\x01" * 64, "Beam"))
+        self.assertIn("already holds", str(caught.exception))
+        A.install(img, 0, A.build(b"\\x01" * 64, "Beam"), force=True)
+        self.assertEqual(A.read_slot(bytes(img), 0)["name"], "Beam")
+
+    def test_installing_over_an_app_needs_no_force(self):
+        img = image()
+        A.install(img, 0, A.build(b"\\x11" * 64, "Old"))
+        A.install(img, 0, A.build(b"\\x22" * 64, "New"))
+        self.assertEqual(A.read_slot(bytes(img), 0)["name"], "New")
+
     def test_a_slot_outside_the_region_is_refused(self):
         with self.assertRaises(A.AppError):
             A.install(image(), 16, A.build(b"x", "A"))
 
 
+class TestResolve(unittest.TestCase):
+    def test_a_bare_name_is_found_under_work_apps(self):
+        """The first attempt at this used 'Beam.app' from the repository root."""
+        path = A.resolve("Beam.app")
+        self.assertTrue(os.path.exists(path), path)
+
+    def test_a_missing_name_says_where_it_looked(self):
+        with self.assertRaises(A.AppError) as caught:
+            A.resolve("NoSuchGame.app")
+        self.assertIn("looked in", str(caught.exception))
+        self.assertIn("work", str(caught.exception))
+
+
 class TestRealFile(unittest.TestCase):
     """Runs against a downloaded Beam.app when one is present; skipped otherwise."""
 

tools/test_webui.py

@@ -1052,3 +1052,81 @@ class TestPageScriptParses(unittest.TestCase):
                              "the page's script does not parse:\n" + done.stderr)
         finally:
             os.unlink(path)
+
+
+class TestAppEndpoints(unittest.TestCase):
+    """The Labs edition's overlay apps, written into the external flash image.
+
+    No emulator: create_app is handed the flash path directly and the endpoints edit
+    that file. The region and the header are the firmware's own, from its
+    App/apps/app_overlay.h: 16 slots of 8 KiB from 0x102000, a 64-byte FAP1 header at
+    the slot base and the code one 4 KiB sector later.
+    """
+
+    def setUp(self):
+        import uvk5_apps
+        self.dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.dir, True)
+        self.flash = os.path.join(self.dir, "flash.img")
+        with open(self.flash, "wb") as fh:
+            fh.write(b"\xff" * 0x200000)
+        self.flashslot = FlashSlot(self.flash)
+        self.app = webui.create_app(None, frame_addr=0x1000, status_addr=0x2000,
+                                    flash=self.flashslot)
+        self.client = self.app.test_client()
+        self.blob = uvk5_apps.build(b"\x01" * 64, "Beam", "1.0", shortcut="beam")
+
+    def test_the_listing_names_the_region_and_every_slot(self):
+        body = self.client.get("/api/apps").get_json()
+        self.assertEqual(body["region"], 0x102000)
+        self.assertEqual(body["slot_count"], 16)
+        self.assertEqual(len(body["slots"]), 16)
+        self.assertEqual(body["slots"][0]["state"], "empty")
+
+    def test_installing_an_app_puts_it_where_the_firmware_looks(self):
+        r = self.client.post("/api/apps/3", data=self.blob)
+        self.assertEqual(r.status_code, 200)
+        self.assertEqual(r.get_json()["app"]["name"], "Beam")
+        row = r.get_json()["apps"]["slots"][3]
+        self.assertEqual(row["state"], "app")
+        self.assertEqual(row["base"], 0x102000 + 3 * 0x2000)
+        self.assertEqual(row["code_size"], 64)
+
+    def test_the_bytes_reach_the_image_the_emulator_will_boot(self):
+        # _edit_flash copies the image and repoints the slot at the copy, so the
+        # file it did not touch is expected to be untouched: check the one it uses.
+        self.client.post("/api/apps/0", data=self.blob)
+        with open(self.flashslot.path, "rb") as fh:
+            image = fh.read()
+        self.assertEqual(image[0x102000:0x102004], b"FAP1")
+        self.assertEqual(image[0x103000:0x103004], b"\x01" * 4)
+
+    def test_a_blob_that_is_not_an_app_is_refused_with_the_reason(self):
+        r = self.client.post("/api/apps/0", data=b"this is not an app at all")
+        self.assertEqual(r.status_code, 400)
+        reason = r.get_json()["error"]
+        self.assertTrue("FAP1" in reason or "too short" in reason, reason)
+
+    def test_installing_over_other_data_is_refused_unless_forced(self):
+        """Measured on a real image: the factory resource block can overlap the region."""
+        image = bytearray(b"\xff" * 0x200000)
+        image[0x106000:0x106008] = b"RESDATA1"
+        with open(self.flash, "wb") as fh:
+            fh.write(image)
+        r = self.client.post("/api/apps/2", data=self.blob)
+        self.assertEqual(r.status_code, 400)
+        self.assertIn("already holds", r.get_json()["error"])
+        r = self.client.post("/api/apps/2?force=1", data=self.blob)
+        self.assertEqual(r.status_code, 200)
+
+    def test_erase_clears_the_slot(self):
+        self.assertEqual(self.client.post("/api/apps/2", data=self.blob).status_code, 200)
+        r = self.client.post("/api/apps/2/erase")
+        body = r.get_json()
+        self.assertEqual(r.status_code, 200, body)
+        self.assertEqual(body["apps"]["slots"][2]["state"], "empty")
+
+    def test_a_slot_outside_the_region_is_refused(self):
+        r = self.client.post("/api/apps/16", data=self.blob)
+        self.assertEqual(r.status_code, 400)
+        self.assertIn("out of range", r.get_json()["error"])

tools/uvk5_apps.py

@@ -30,6 +30,7 @@ in slot N" in this page's slot table touch the same external flash.
     tools/uvk5_apps.py erase   work/user-flash.img 1
 """
 import argparse
+import os
 import struct
 import sys
 import zlib
@@ -45,6 +46,12 @@ REGION_BASE = 0x00102000
 SLOT_STRIDE = 0x0002000
 CODE_OFFSET = 0x00001000
 SLOT_COUNT = 16
+# The firmware's own names for these, so a reader can hold both side by side.
+APP_REGION_BASE = REGION_BASE
+APP_SLOT_STRIDE = SLOT_STRIDE
+APP_CODE_OFFSET = CODE_OFFSET
+APP_SLOT_COUNT = SLOT_COUNT
+APP_OVERLAY_MAX_BYTES = APP_OVERLAY_MAX
 
 FLAG_COMMITTED = 0x0001
 FLAG_SCREEN_SAVER = 0x0002
@@ -166,16 +173,28 @@ def list_apps(image: bytes):
     return [info for info in (read_slot(image, i) for i in range(SLOT_COUNT)) if info]
 
 
-def install(image: bytearray, slot: int, blob: bytes) -> dict:
+def install(image: bytearray, slot: int, blob: bytes, force: bool = False) -> dict:
     """Write @blob into @slot: header at the base, code at +0x1000, rest erased.
 
     The header sector is erased first, the way the flash would be: an install must not
     leave a byte of the previous app behind for the loader to trip over.
+
+    Refuses to overwrite a slot that holds something which is neither empty nor an app.
+    Measured on a real image: 0x102000..0x122000 can already carry data (the factory
+    resource block of a localised build overlaps it), and an install there silently
+    destroys it. `force` is for when that is what you meant.
     """
     info = parse(blob)
     base = slot_base(slot)
     if base + SLOT_STRIDE > len(image):
         raise AppError("the image is too small for slot %d" % slot)
+    if not force:
+        occupied = read_slot(bytes(image), slot)
+        if occupied is not None and occupied.get("kind") not in ("app",):
+            raise AppError(
+                "slot %d at 0x%06X already holds %s (%r), not an app; erase it first "
+                "or pass force to overwrite" % (slot, base, occupied.get("kind", "data"),
+                                                (occupied.get("name") or "")[:16]))
     for i in range(base, base + SLOT_STRIDE):
         image[i] = 0xFF
     image[base:base + HDR_SIZE] = blob[:HDR_SIZE]
@@ -193,6 +212,73 @@ def erase(image: bytearray, slot: int) -> int:
     return base
 
 
+def resolve(name: str) -> str:
+    """A path to the .app named @name, looking in the places it usually is.
+
+    The first person to try this typed `tools/uvk5_apps.py install image 1 Beam.app` from the
+    repository root, where no such file exists: the tool said FileNotFoundError and nothing
+    else. The apps live in work/apps/ once downloaded, and the bare name is what everybody
+    types, so both are accepted and the refusal lists where it looked.
+    """
+    if os.path.isabs(name) or os.path.exists(name):
+        return name
+    roots = [os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "work", "apps"),
+             os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "assets", "apps"),
+             "."]
+    tried = []
+    for root in roots:
+        candidate = os.path.join(root, name)
+        tried.append(candidate)
+        if os.path.exists(candidate):
+            return candidate
+    raise AppError("no %s; looked in %s" % (name, ", ".join(os.path.normpath(t) for t in tried)))
+
+
+def apps_json(path: str) -> dict:
+    """Every app slot as a row, for the page: one entry per slot, occupied or not.
+
+    The page shows all of them rather than only the occupied ones, because choosing the
+    slot is part of installing, and the radio itself prints Empty for a free one.
+    """
+    with open(path, "rb") as fh:
+        image = fh.read()
+    occupied = {info["slot"]: info for info in list_apps(image)}
+    rows = []
+    for slot in range(SLOT_COUNT):
+        row = dict(slot=slot, base=slot_base(slot))
+        info = occupied.get(slot)
+        if info is None:
+            row["state"] = "empty"
+        elif info.get("kind") == "app":
+            row.update(state="app", name=info["name"], version=info["version"],
+                       code_size=info["code_size"], shortcut=info["shortcut"],
+                       committed=info["committed"], crc32=info["crc32"])
+        else:
+            row.update(state=info.get("kind", "unknown"), name=info.get("magic", "?"))
+        rows.append(row)
+    return dict(region=REGION_BASE, slot_count=SLOT_COUNT, stride=SLOT_STRIDE,
+                code_offset=CODE_OFFSET, overlay_max=APP_OVERLAY_MAX, slots=rows)
+
+
+def install_file(path: str, slot: int, blob: bytes, force: bool = False) -> dict:
+    """Install @blob into @slot of the flash image at @path, in place."""
+    image = _read(path)
+    info = install(image, slot, blob, force=force)
+    with open(path, "wb") as fh:
+        fh.write(image)
+    return info
+
+
+def erase_file(path: str, slot: int) -> int:
+    """Clear @slot of the flash image at @path."""
+    image = _read(path)
+    base = erase(image, slot)
+    with open(path, "wb") as fh:
+        fh.write(image)
+    return base
+
+
+
 def _read(path: str) -> bytearray:
     with open(path, "rb") as fh:
         return bytearray(fh.read())
@@ -208,6 +294,8 @@ def main(argv=None) -> int:
     p.add_argument("image")
     p.add_argument("slot", type=int)
     p.add_argument("app")
+    p.add_argument("--force", action="store_true",
+                   help="overwrite a slot holding something other than an app")
     p = sub.add_parser("erase", help="clear a slot")
     p.add_argument("image")
     p.add_argument("slot", type=int)
@@ -217,7 +305,7 @@ def main(argv=None) -> int:
 
     try:
         if args.cmd == "info":
-            with open(args.app, "rb") as fh:
+            with open(resolve(args.app), "rb") as fh:
                 info = parse(fh.read())
             print("%s %s  %d bytes of code (blob %d)  ABI %d  api>=%d  shortcut %s  CRC 0x%08X"
                   % (info["name"], info["version"], info["code_size"], info["total"], info["abi"],
@@ -236,10 +324,10 @@ def main(argv=None) -> int:
                 else:
                     print("slot %2d @0x%06X  %s" % (info["slot"], info["base"], info["kind"]))
             return 0
-        with open(args.app, "rb") as fh:
+        with open(resolve(args.app), "rb") as fh:
             blob = fh.read()
         if args.cmd == "install":
-            info = install(image, args.slot, blob)
+            info = install(image, args.slot, blob, force=args.force)
             with open(args.image, "wb") as fh:
                 fh.write(image)
             print("installed %s %s into slot %d at 0x%06X (%d bytes)"

tools/webui.py

@@ -29,6 +29,7 @@ from uvk5_image import ImageError, detect as detect_image
 from uvk5_slots import (SLOT_COUNT, erase_slot_file, slots_json,
                         write_slot_file)
 from uvk5_keys import KEYS, is_valid, normalise
+import uvk5_apps
 from uvk5_lcd import PANEL_PATH
 from uvk5_logs import LogBuffer
 from uvk5_stream import FramePump
@@ -515,6 +516,66 @@ def create_app(client, frame_addr: int = None, status_addr: int = None, scale: i
         log.add("slots", "slot %d erased" % slot, ip=client_ip())
         return jsonify(slot=row)
 
+    # ------------------------------------------------------------- overlay apps
+    #
+    # The Labs edition's small apps (Tetris, Breakout, Beam, Plasma, ...) live in the
+    # *external* flash, in the region its own App/apps/app_overlay.h describes: 16 slots
+    # of 8 KiB from 0x102000, a 64-byte FAP1 header at the slot base and the code one
+    # 4 KiB sector later. Upstream installs them from UVStudio over WebSerial; this page
+    # owns the image, so the same bytes go to the same offsets with no serial protocol
+    # and no browser permission. The 64-byte header is shared with the multiboot firmware
+    # slots -- FMB1 is a firmware, FAP1 is an app -- which is why both live in this one
+    # image and why the power-on menu can list them together.
+    @app.get("/api/apps")
+    def api_apps():
+        """Every overlay-app slot in the flash image the emulator is using."""
+        if flash is None:
+            return jsonify(error="this server was started without flash control"), 409
+        try:
+            return jsonify(uvk5_apps.apps_json(flash.path))
+        except Exception as exc:
+            return jsonify(error=str(exc)), 500
+
+    @app.post("/api/apps/<int:slot>")
+    def api_app_install(slot):
+        """Install an uploaded .app into a slot, then power the radio on again."""
+        if not 0 <= slot < uvk5_apps.APP_SLOT_COUNT:
+            return jsonify(error="app slot %d is out of range (0..%d)"
+                           % (slot, uvk5_apps.APP_SLOT_COUNT - 1)), 400
+        data = request.get_data(cache=False, as_text=False)
+        if not data:
+            return jsonify(error="no .app in the request body"), 400
+        if len(data) > MAX_UPLOAD_BYTES:
+            return jsonify(error="%d bytes is too large" % len(data)), 413
+        try:
+            force = request.args.get("force", "").lower() in ("1", "true", "yes")
+            info = _edit_flash(lambda p: uvk5_apps.install_file(p, slot, data, force))
+        except Exception as exc:
+            # The tool refuses what the firmware would show as APP ERROR, so the reason
+            # reaches the page instead of becoming a silent no-op on the radio.
+            log.add("apps", "slot %d refused: %s" % (slot, exc), ip=client_ip())
+            return jsonify(error=str(exc)), 400
+        log.add("apps", "slot %d <- %s %s (%d bytes of code)"
+                % (slot, info["name"], info["version"], info["code_size"]), ip=client_ip())
+        return jsonify(app=dict(slot=slot, name=info["name"], version=info["version"],
+                                code_size=info["code_size"], crc32=info["crc32"],
+                                shortcut=info["shortcut"]),
+                       apps=uvk5_apps.apps_json(flash.path))
+
+    @app.post("/api/apps/<int:slot>/erase")
+    def api_app_erase(slot):
+        """Clear one app slot."""
+        if not 0 <= slot < uvk5_apps.APP_SLOT_COUNT:
+            return jsonify(error="app slot %d is out of range (0..%d)"
+                           % (slot, uvk5_apps.APP_SLOT_COUNT - 1)), 400
+        try:
+            _edit_flash(lambda p: uvk5_apps.erase_file(p, slot))
+        except Exception as exc:
+            log.add("apps", "slot %d erase failed: %s" % (slot, exc), ip=client_ip())
+            return jsonify(error=str(exc)), 400
+        log.add("apps", "slot %d erased" % slot, ip=client_ip())
+        return jsonify(apps=uvk5_apps.apps_json(flash.path))
+
     @app.post("/api/flash")
     def api_flash_upload():
         """Use an uploaded image as the external flash, slots and all."""