From 911e11a3721179583a82bf8f48562d331cd08f25 Mon Sep 17 00:00:00 2001 From: HighDrexler Date: Thu, 20 Aug 2026 17:14:32 -0400 Subject: [PATCH] test(mods): document and cover streamed import access --- ...-streamed-mod-imports-and-install-cache.md | 141 ++++++++++++++ src/mods/ImportAccess.lua | 174 ++++++++++++++++++ src/mods/StreamMD5.lua | 101 ++++++++++ .../engine/mod_import_access_parity_test.lua | 42 +++++ tests/engine/mod_import_access_test.lua | 105 +++++++++++ .../engine/required_import_streaming_test.lua | 97 ++++++++++ tests/engine/stream_md5_test.lua | 33 ++++ 7 files changed, 693 insertions(+) create mode 100644 docs/rfcs/0008-streamed-mod-imports-and-install-cache.md create mode 100644 src/mods/ImportAccess.lua create mode 100644 src/mods/StreamMD5.lua create mode 100644 tests/engine/mod_import_access_parity_test.lua create mode 100644 tests/engine/mod_import_access_test.lua create mode 100644 tests/engine/required_import_streaming_test.lua create mode 100644 tests/engine/stream_md5_test.lua diff --git a/docs/rfcs/0008-streamed-mod-imports-and-install-cache.md b/docs/rfcs/0008-streamed-mod-imports-and-install-cache.md new file mode 100644 index 00000000..22240416 --- /dev/null +++ b/docs/rfcs/0008-streamed-mod-imports-and-install-cache.md @@ -0,0 +1,141 @@ +# RFC 0008 — Streamed mod imports and installation-scoped generated cache + +## Motivation + +`required_imports`/`optional_imports` can now describe files up to 2 GiB, but +the existing launcher and public mod API still assume imported bytes are small: + +* the Windows desktop picker stages a selected required import through a fixed + `%TEMP%/pokeport_required_import.bin` path before validation; +* the fallback import path materializes the selected file as one Lua string; +* after validation a mod can only use `mod:read("baseroms/...")`, which also + materializes the whole file; +* `mod.storage` is intentionally scoped to one Pokémon playthrough, so it is + not an appropriate home for a one-time generated asset cache shared by every + save using the same installed mod. + +This makes optical-disc-sized user sources impractical even though the manifest +schema already accepts them. A failed temporary staging copy can also turn a +valid large source into a smaller temporary file and produce a misleading +"wrong file size" rejection. + +A mod should be able to consume its own already-validated source incrementally +and compile derived runtime data once, without receiving a host path or general +filesystem access. + +## Decision being extended + +This extends the same legal/sandbox direction as **D11 asset transforms** +(`src/mods/AssetTransform.lua`): mods distribute recipes and derive bytes from +user-owned sources rather than shipping ROM-derived data. It also follows the +**D14 parity-gate** contract referenced by `tests/harness.lua` and +`tests/engine/gate_meta_coverage.lua` (the `21-testing-and-ci` plan): additive +extension points ship public-API coverage, no-mod parity coverage, and docs in +the same change. + +The historical D11 plan document is referenced by source comments but is not +present in the current repository tree; this RFC is the checked-in design +record for the new surface. + +## Exact API delta + +No manifest field changes. Existing `required_imports` and `optional_imports` +remain the declaration/validation authority. + +Two additive facades are added to the `mod` object. + +### `mod.imports` + +```lua +local info, err = mod.imports:info("source_id") +local bytes, err = mod.imports:read("source_id", offset, length) +``` + +* `source_id` must name an import declared by the calling mod. +* the import is rechecked through `RequiredImports.validateStored` before it is + exposed, so missing, replaced, or invalid optional imports are not readable; +* `offset` and `length` are zero-based byte coordinates; +* one read is capped at 8 MiB; +* no host path or file handle is returned; +* production reads seek into the engine-owned stored copy instead of reading + the whole source. + +`info()` returns declaration metadata plus stored size. It does not expose a +host path. + +### `mod.cache` + +```lua +mod.cache:write("extract/v1/model.bin", bytes) +local bytes = mod.cache:read("extract/v1/model.bin") +local info = mod.cache:info("extract/v1/model.bin") +mod.cache:delete("extract/v1/model.bin") +``` + +The cache is rooted at `mod_cache//`, follows the engine persistence +backend, and is independent of game version, launcher slot, and playthrough. +Paths are checked with `SafePath`; `..`, absolute paths, drive paths, and other +escapes remain unavailable. A single cache write is capped at 64 MiB so large +generated datasets are naturally split into independently replaceable files. + +The engine does not interpret cache bytes. Mods own generated-format versioning, +fingerprints, transactional completion markers, and rebuild policy. + +## Launcher/import transport delta + +For large raw required imports: + +1. desktop pickers return the original selected path instead of staging it + through a fixed temporary file; +2. the engine opens that source itself; +3. bytes are copied directly to the existing engine-owned + `mods//baseroms/` destination in 4 MiB chunks; +4. MD5 is updated incrementally during the copy; +5. the normal size/MD5 validation receipt is written only after the complete + destination passes validation; +6. partial destinations are removed on short reads, write failure, size + mismatch, or digest mismatch. + +N64 imports stay on the existing canonicalization path because byte-order and +copier-header normalization require transformation rather than a raw copy. + +If a validation receipt for an already-stored large raw import is missing, the +engine rebuilds it with streaming MD5 rather than a whole-file read. + +## Backward compatibility / migration + +**Existing mods do nothing.** This is additive. + +* manifest v1/v2 fields are unchanged; +* `mod:read`, `mod.storage`, registries, events, hooks, and legacy compatibility + retain their existing behavior; +* small required imports retain the existing in-memory validation path; +* N64 imports retain canonicalization and existing accepted byte orders; +* a mod that never touches `mod.imports` or `mod.cache` creates no new cache + files and observes no new behavior. + +The mod API integer is not bumped because no existing member changes meaning or +shape. + +## Security and legal posture + +The launcher remains the authority that validates user-supplied bytes. The new +facade narrows access rather than widening it: a mod can read only ids declared +in its own manifest, only after validation, and only in bounded ranges. It does +not receive host paths, `io`, or a raw filesystem handle. + +`mod.cache` is writable only beneath the calling mod's generated-cache root. +Nothing in this RFC permits packaged ROM-derived bytes; `modkit lint/pack` +continue to enforce the existing legal posture. + +## Parity guarantee + +The change ships with: + +* a no-mod/API-v1 parity test proving an empty load and an existing v1-style + `mod:read` load do not create cache data or change the old surface; +* a public mod-API test that reaches `mod.imports` and `mod.cache` through a + real `Loader` load, including bounded reads, undeclared/missing imports, + cache isolation, and traversal rejection; +* incremental MD5 vectors and a large-import streaming regression test; +* the existing engine suite, required-import suite, and mod lint gates. diff --git a/src/mods/ImportAccess.lua b/src/mods/ImportAccess.lua new file mode 100644 index 00000000..11f32380 --- /dev/null +++ b/src/mods/ImportAccess.lua @@ -0,0 +1,174 @@ +-- Scoped access to a mod's launcher-validated required/optional imports and +-- to installation-wide generated cache data. +-- +-- This intentionally does not expose host paths or raw filesystem handles. +-- Import reads are bounded and can only address ids declared by the calling +-- mod's manifest. Cache paths are confined to mod_cache// and are not +-- tied to a Pokémon playthrough. + +local RequiredImports = require("src.mods.RequiredImports") +local SafePath = require("src.mods.SafePath") +local SaveData = require("src.core.SaveData") + +local ImportAccess = {} + +ImportAccess.MAX_READ_BYTES = 8 * 1024 * 1024 +ImportAccess.MAX_CACHE_WRITE_BYTES = 64 * 1024 * 1024 + +local function specMap(manifest) + local out = {} + for _, spec in ipairs(RequiredImports.specs(manifest)) do out[spec.id] = spec end + return out +end + +local function parentOf(path) + return path:match("^(.*)/[^/]+$") +end + +local function copyInfo(info) + if not info then return nil end + return { type = info.type, size = info.size, modtime = info.modtime } +end + +local function fsReadRange(fs, path, offset, length) + if fs and type(fs.readRange) == "function" then + return fs.readRange(path, offset, length) + end + local newFile = fs and fs.newFile + if newFile then + local file, makeErr = newFile(path) + if not file then return nil, makeErr or "could not open import" end + local ok, openErr = file:open("r") + if not ok then return nil, openErr or "could not open import" end + local seekOk, seekErr = file:seek(offset) + if seekOk == nil or seekOk == false then + file:close() + return nil, seekErr or "could not seek import" + end + local data, readErr = file:read(length) + file:close() + return data, readErr + end + -- Injectable headless filesystems may expose only read(). Production + -- love.filesystem has newFile(), so large imports are never materialized + -- into one Lua string by this fallback. + if fs and fs.read then + local data = fs.read(path) + if type(data) ~= "string" then return nil, "could not read import" end + return data:sub(offset + 1, offset + length) + end + return nil, "random-access import reads are unavailable" +end + +local function validatedInfo(manifest, spec, fs) + local ok, detail = RequiredImports.validateStored(manifest, spec, fs) + if not ok then return nil, detail or "import is not validated" end + local path = RequiredImports.path(manifest, spec) + local info = fs.getInfo and fs.getInfo(path, "file") or nil + if not info then return nil, "import is missing" end + return info, detail +end + +local function makeCache(modId, fs) + local root = "mod_cache/" .. modId + local function pathFor(rel, what) + rel = SafePath.require(rel, what or "mod.cache path") + return root .. "/" .. rel + end + local cache = {} + + function cache:write(rel, bytes) + if type(bytes) ~= "string" then + return nil, "mod.cache:write expects a byte string" + end + if #bytes > ImportAccess.MAX_CACHE_WRITE_BYTES then + return nil, "mod.cache:write payload exceeds 64 MiB; split generated data into smaller files" + end + local path = pathFor(rel, "mod.cache:write") + local parent = parentOf(path) + if parent and fs.createDirectory then + local ok = fs.createDirectory(parent) + if ok == false then return nil, "could not create cache directory" end + end + if not fs.write then return nil, "cache writes are unavailable" end + return fs.write(path, bytes) + end + + function cache:read(rel) + local path = pathFor(rel, "mod.cache:read") + if not fs.read then return nil, "cache reads are unavailable" end + return fs.read(path) + end + + function cache:info(rel) + local path = pathFor(rel, "mod.cache:info") + if not fs.getInfo then return nil end + return copyInfo(fs.getInfo(path)) + end + + function cache:exists(rel) + local info = self:info(rel) + return info ~= nil and info.type == "file" + end + + + function cache:delete(rel) + local path = pathFor(rel, "mod.cache:delete") + if not fs.remove then return nil, "cache deletion is unavailable" end + return fs.remove(path) + end + + return cache +end + +function ImportAccess.new(manifest, fs) + local specs = specMap(manifest) + local cacheFs = SaveData.persistenceFs(fs) or fs + local imports = {} + + function imports:info(id) + local spec = specs[id] + if not spec then return nil, "undeclared import: " .. tostring(id) end + local info, digestOrErr = validatedInfo(manifest, spec, fs) + if not info then return nil, digestOrErr end + return { + id = spec.id, + name = spec.name, + file = spec.file, + size = info.size, + md5 = digestOrErr, + required = spec.required ~= false, + } + end + + function imports:read(id, offset, length) + local spec = specs[id] + if not spec then return nil, "undeclared import: " .. tostring(id) end + offset, length = tonumber(offset), tonumber(length) + if not offset or offset < 0 or offset % 1 ~= 0 then + return nil, "offset must be a non-negative integer" + end + if not length or length < 0 or length % 1 ~= 0 then + return nil, "length must be a non-negative integer" + end + if length > ImportAccess.MAX_READ_BYTES then + return nil, "single import read exceeds 8 MiB" + end + + local info, err = validatedInfo(manifest, spec, fs) + if not info then return nil, err end + local size = tonumber(info.size) or tonumber(spec.size) + if size and offset + length > size then return nil, "import read is out of bounds" end + if length == 0 then return "" end + + local path = RequiredImports.path(manifest, spec) + local data, readErr = fsReadRange(fs, path, offset, length) + if not data then return nil, readErr end + if #data ~= length then return nil, "short import read" end + return data + end + + return imports, makeCache(manifest.id, cacheFs) +end + +return ImportAccess diff --git a/src/mods/StreamMD5.lua b/src/mods/StreamMD5.lua new file mode 100644 index 00000000..1990e8ec --- /dev/null +++ b/src/mods/StreamMD5.lua @@ -0,0 +1,101 @@ +local bitlib = rawget(_G, "bit") or rawget(_G, "bit32") +if not bitlib then error("StreamMD5 requires bit or bit32") end + +local band, bor, bxor, bnot = bitlib.band, bitlib.bor, bitlib.bxor, bitlib.bnot +local lshift, rshift = bitlib.lshift, bitlib.rshift +local rol = bitlib.rol or bitlib.lrotate + +local K = { + 0xd76aa478,0xe8c7b756,0x242070db,0xc1bdceee,0xf57c0faf,0x4787c62a,0xa8304613,0xfd469501, + 0x698098d8,0x8b44f7af,0xffff5bb1,0x895cd7be,0x6b901122,0xfd987193,0xa679438e,0x49b40821, + 0xf61e2562,0xc040b340,0x265e5a51,0xe9b6c7aa,0xd62f105d,0x02441453,0xd8a1e681,0xe7d3fbc8, + 0x21e1cde6,0xc33707d6,0xf4d50d87,0x455a14ed,0xa9e3e905,0xfcefa3f8,0x676f02d9,0x8d2a4c8a, + 0xfffa3942,0x8771f681,0x6d9d6122,0xfde5380c,0xa4beea44,0x4bdecfa9,0xf6bb4b60,0xbebfbc70, + 0x289b7ec6,0xeaa127fa,0xd4ef3085,0x04881d05,0xd9d4d039,0xe6db99e5,0x1fa27cf8,0xc4ac5665, + 0xf4292244,0x432aff97,0xab9423a7,0xfc93a039,0x655b59c3,0x8f0ccc92,0xffeff47d,0x85845dd1, + 0x6fa87e4f,0xfe2ce6e0,0xa3014314,0x4e0811a1,0xf7537e82,0xbd3af235,0x2ad7d2bb,0xeb86d391, +} +local S = { + 7,12,17,22, 7,12,17,22, 7,12,17,22, 7,12,17,22, + 5,9,14,20, 5,9,14,20, 5,9,14,20, 5,9,14,20, + 4,11,16,23, 4,11,16,23, 4,11,16,23, 4,11,16,23, + 6,10,15,21, 6,10,15,21, 6,10,15,21, 6,10,15,21, +} + +local function add32(a,b,c,d) + local n = (a or 0) + (b or 0) + (c or 0) + (d or 0) + return band(n, 0xffffffff) +end + +local function le32_from(s, i) + local b1,b2,b3,b4 = s:byte(i, i+3) + return bor(b1, lshift(b2,8), lshift(b3,16), lshift(b4,24)) +end + +local function le32_bytes(x) + return string.char( + band(x,0xff), band(rshift(x,8),0xff), + band(rshift(x,16),0xff), band(rshift(x,24),0xff)) +end + +local M = {} +local StreamMD5 = {} +StreamMD5.__index = StreamMD5 + +function StreamMD5.new() + return setmetatable({ + a=0x67452301, b=0xefcdab89, c=0x98badcfe, d=0x10325476, + bytes=0, buffer="", done=false, + }, StreamMD5) +end + +function StreamMD5:_block(block) + for j=1,16 do M[j] = le32_from(block, (j-1)*4+1) end + local a,b,c,d = self.a,self.b,self.c,self.d + for i=0,63 do + local f,g + if i < 16 then + f = bor(band(b,c), band(bnot(b),d)); g=i + elseif i < 32 then + f = bor(band(d,b), band(bnot(d),c)); g=(5*i+1)%16 + elseif i < 48 then + f = bxor(b,c,d); g=(3*i+5)%16 + else + f = bxor(c, bor(b,bnot(d))); g=(7*i)%16 + end + local tmp=d + d=c + c=b + b=add32(b, rol(add32(a,f,K[i+1],M[g+1]), S[i+1])) + a=tmp + end + self.a=add32(self.a,a); self.b=add32(self.b,b) + self.c=add32(self.c,c); self.d=add32(self.d,d) +end + +function StreamMD5:update(data) + assert(not self.done, "StreamMD5 context already finalized") + assert(type(data)=="string", "StreamMD5:update expects a string") + self.bytes = self.bytes + #data + local s = self.buffer .. data + local full = #s - (#s % 64) + for i=1,full,64 do self:_block(s:sub(i,i+63)) end + self.buffer = s:sub(full+1) + return self +end + +function StreamMD5:final() + assert(not self.done, "StreamMD5 context already finalized") + local originalBytes = self.bytes + local padLen = (56 - ((originalBytes + 1) % 64)) % 64 + local bits = originalBytes * 8 + local lo = bits % 4294967296 + local hi = math.floor(bits / 4294967296) % 4294967296 + self:update("\128" .. string.rep("\0", padLen) .. le32_bytes(lo) .. le32_bytes(hi)) + assert(#self.buffer == 0, "MD5 finalization left a partial block") + self.done=true + local raw = le32_bytes(self.a)..le32_bytes(self.b)..le32_bytes(self.c)..le32_bytes(self.d) + return (raw:gsub(".", function(ch) return string.format("%02x", ch:byte()) end)) +end + +return StreamMD5 diff --git a/tests/engine/mod_import_access_parity_test.lua b/tests/engine/mod_import_access_parity_test.lua new file mode 100644 index 00000000..fc2f7c73 --- /dev/null +++ b/tests/engine/mod_import_access_parity_test.lua @@ -0,0 +1,42 @@ +-- No-mod and API-v1 parity coverage for additive mod.imports/mod.cache. +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") + +-- No mod installed: no generated cache namespace is touched. +local emptyFiles = {} +local none = T.sdk.loadNone({ fs = T.sdk.memfs(emptyFiles) }) +T.eq(#none.errors, 0, "zero-mod load remains clean") +for path in pairs(emptyFiles) do + T.check(path:sub(1, 10) ~= "mod_cache/", + "zero-mod load creates no installation cache data") +end +none.release() + +-- Existing API-v1 style file access still behaves exactly as before. The new +-- facades are additive members; mod:read remains the same scoped read path. +local files = { + ["mods/v1_read_probe/manifest.json"] = [[{ + "id": "v1_read_probe", + "name": "V1 Read Probe", + "version": "1.0.0", + "entry": "main.lua", + "api": 1 + }]], + ["mods/v1_read_probe/main.lua"] = [[ + local mod = ... + mod.exports.payload = mod:read("payload.txt") + mod.exports.hasImports = type(mod.imports) == "table" + mod.exports.hasCache = type(mod.cache) == "table" + ]], + ["mods/v1_read_probe/payload.txt"] = "unchanged-v1-read", +} +local run = T.sdk.loadMods({ "mods/v1_read_probe" }, { fs = T.sdk.memfs(files) }) +T.eq(#run.errors, 0, "API-v1 probe still loads") +local out = run.loader.exports.v1_read_probe +T.eq(out.payload, "unchanged-v1-read", "mod:read keeps its v1 behavior") +T.eq(out.hasImports, true, "new import facade is additive") +T.eq(out.hasCache, true, "new cache facade is additive") +run.release() + +T.finish("mod_import_access_parity") diff --git a/tests/engine/mod_import_access_test.lua b/tests/engine/mod_import_access_test.lua new file mode 100644 index 00000000..f861e73f --- /dev/null +++ b/tests/engine/mod_import_access_test.lua @@ -0,0 +1,105 @@ +-- Public mod-API coverage for bounded validated imports + installation cache. +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") + +local digest = "00000000000000000000000000000000" +local files = { + ["mods/import_api_probe/manifest.json"] = [[{ + "id": "import_api_probe", + "name": "Import API Probe", + "version": "1.0.0", + "entry": "main.lua", + "api": 2, + "required_imports": [{ + "id": "disc", + "name": "Disc", + "file": "source.iso", + "md5": ["00000000000000000000000000000000"], + "size": 16 + }], + "optional_imports": [{ + "id": "optional", + "name": "Optional", + "file": "optional.bin", + "md5": ["11111111111111111111111111111111"], + "size": 4, + "required": false + }] + }]], + ["mods/import_api_probe/main.lua"] = [[ + local mod = ... + local info, infoErr = mod.imports:info("disc") + local slice, sliceErr = mod.imports:read("disc", 4, 6) + local oob, oobErr = mod.imports:read("disc", 15, 2) + local missing, missingErr = mod.imports:info("optional") + local undeclared, undeclaredErr = mod.imports:read("not_declared", 0, 1) + + local wrote, writeErr = mod.cache:write("extract/v1/probe.bin", "abc") + local cached, readErr = mod.cache:read("extract/v1/probe.bin") + local cacheInfo = mod.cache:info("extract/v1/probe.bin") + local escaped = pcall(function() mod.cache:write("../escape.bin", "x") end) + + mod.exports.result = { + info = info, infoErr = infoErr, + slice = slice, sliceErr = sliceErr, + oob = oob, oobErr = oobErr, + missing = missing, missingErr = missingErr, + undeclared = undeclared, undeclaredErr = undeclaredErr, + wrote = wrote, writeErr = writeErr, + cached = cached, readErr = readErr, + cacheInfo = cacheInfo, + escaped = escaped, + } + ]], + ["mods/import_api_probe/baseroms/source.iso"] = "0123456789abcdef", + ["mods/import_api_probe/baseroms/.required-import-disc.validated"] = + "v1\n" .. digest .. "\n16\n1\n", +} + +local baseFs = T.sdk.memfs(files) +local oldInfo = baseFs.getInfo +function baseFs.getInfo(path) + local info = oldInfo(path) + if info and info.type == "file" then + return { type = "file", size = #(files[path] or ""), modtime = 1 } + end + return info +end +function baseFs.readRange(path, offset, length) + local body = files[path] + if not body then return nil end + return body:sub(offset + 1, offset + length) +end +function baseFs.createDirectory() return true end +function baseFs.remove(path) files[path] = nil return true end + +local run = T.sdk.loadMods({ "mods/import_api_probe" }, { fs = baseFs }) +T.eq(#run.errors, 0, + "the import/cache probe loads through the production Loader") + +local out = run.loader.exports.import_api_probe.result +T.check(type(out.info) == "table", "declared validated import has info") +T.eq(out.info.size, 16, "import info reports stored size") +T.eq(out.slice, "456789", "bounded read seeks into the validated import") +T.eq(out.sliceErr, nil, "bounded read has no error") +T.eq(out.oob, nil, "out-of-bounds read is refused") +T.check(type(out.oobErr) == "string" and out.oobErr:find("out of bounds", 1, true), + "out-of-bounds read explains the refusal") +T.eq(out.missing, nil, "missing optional import is not exposed") +T.check(type(out.missingErr) == "string", "missing optional import returns an error") +T.eq(out.undeclared, nil, "undeclared import id is refused") +T.check(type(out.undeclaredErr) == "string" + and out.undeclaredErr:find("undeclared", 1, true), + "undeclared import refusal is explicit") + +T.check(out.wrote == true, "installation cache write succeeds") +T.eq(out.cached, "abc", "installation cache read returns exact bytes") +T.eq(out.cacheInfo and out.cacheInfo.size, 3, "installation cache info is scoped") +T.eq(out.escaped, false, "cache traversal is rejected by the public facade") +T.eq(files["mod_cache/import_api_probe/extract/v1/probe.bin"], "abc", + "cache bytes live under the calling mod id") +T.eq(files["escape.bin"], nil, "cache traversal created nothing outside its root") + +run.release() +T.finish("mod_import_access") diff --git a/tests/engine/required_import_streaming_test.lua b/tests/engine/required_import_streaming_test.lua new file mode 100644 index 00000000..ec038105 --- /dev/null +++ b/tests/engine/required_import_streaming_test.lua @@ -0,0 +1,97 @@ +-- Regression for large raw required imports: direct source -> engine-owned +-- baseroms destination, incremental MD5, no whole-file staging requirement. +package.path = "./?.lua;./?/init.lua;" .. package.path +if not _G.love then _G.love = require("tests.love_stub") end + +local T = require("tests.modkit") +local RomImporter = require("src.import.RomImporter") +local RequiredImports = require("src.mods.RequiredImports") + +-- The stock headless filesystem intentionally omits love.filesystem.newFile. +-- Add the smallest streaming-file facade this transport path needs so the +-- regression drives the same seek/read/write API production LÖVE provides. +local oldNewFile = love.filesystem.newFile +local oldGetInfo = love.filesystem.getInfo +love.filesystem.newFile = function(path) + local body, pos, mode = love.filesystem.read(path) or "", 1, nil + local file = {} + function file:open(m) + mode = m + if m == "w" then body, pos = "", 1 else pos = 1 end + return true + end + function file:read(n) + local out = body:sub(pos, pos + n - 1) + pos = pos + #out + if out == "" then return nil end + return out + end + function file:write(bytes) + if mode ~= "w" then return nil, "not open for write" end + body = body:sub(1, pos - 1) .. bytes .. body:sub(pos + #bytes) + pos = pos + #bytes + love.filesystem.write(path, body) + return true + end + function file:seek(offset) pos = offset + 1 return offset end + function file:getSize() return #body end + function file:close() + if mode == "w" then love.filesystem.write(path, body) end + mode = nil + return true + end + return file +end +love.filesystem.getInfo = function(path, filter) + local info = oldGetInfo(path, filter) + if info and info.type == "file" then + local bytes = love.filesystem.read(path) or "" + return { type = "file", size = #bytes, modtime = 1 } + end + return info +end + +local source = "required_import_streaming_source.tmp" +local f = assert(io.open(source, "wb")) +f:write("abc") +f:close() + +local manifest = { + id = "stream_probe", + name = "Stream Probe", + path = "mods/stream_probe", + required_imports = { + { id = "source", name = "Source", file = "source.bin", format = "raw", + size = 3, md5 = { "900150983cd24fb0d6963f7d28e17f72" } }, + }, + optional_imports = {}, +} + +local importer = setmetatable({ + mods = { { id = "stream_probe", manifest = manifest } }, + requiredImportNotice = nil, + modNotice = nil, + _refreshMods = function(self) self._refreshed = true end, +}, RomImporter) + +-- Exercise the exact large-file branch with tiny fixture bytes by lowering the +-- threshold for this test. Production keeps the 128 MiB confirmation/streaming +-- threshold; the copy/hash algorithm is identical. +local oldWarn = RequiredImports.LARGE_WARN_BYTES +RequiredImports.LARGE_WARN_BYTES = 2 +local ok = importer:_importRequiredSource("stream_probe", "source", source, true) +RequiredImports.LARGE_WARN_BYTES = oldWarn + +T.eq(ok, true, "large raw required import streams successfully") +T.eq(love.filesystem.read("mods/stream_probe/baseroms/source.bin"), "abc", + "streamed destination preserves exact source bytes") +T.eq(importer.requiredImportNotice, nil, "successful stream leaves no import error") +T.eq(importer._refreshed, true, "successful stream refreshes the mod list") + +love.filesystem.remove("mods/stream_probe/baseroms/source.bin") +love.filesystem.remove("mods/stream_probe/baseroms/source.iso") +os.remove(source) +love.filesystem.newFile = oldNewFile +love.filesystem.getInfo = oldGetInfo + +T.finish("required_import_streaming") diff --git a/tests/engine/stream_md5_test.lua b/tests/engine/stream_md5_test.lua new file mode 100644 index 00000000..c75879b3 --- /dev/null +++ b/tests/engine/stream_md5_test.lua @@ -0,0 +1,33 @@ +-- Incremental MD5 vectors used by large required-import streaming. +package.path = "./?.lua;./?/init.lua;" .. package.path + +-- Plain Lua runners may expose bit32 while production LuaJIT exposes bit. +if not rawget(_G, "bit") and not rawget(_G, "bit32") then + local ok, bit32 = pcall(require, "bit32") + if ok then _G.bit32 = bit32 end +end + +local T = require("tests.modkit") +local MD5 = require("src.mods.StreamMD5") + +local vectors = { + { "", "d41d8cd98f00b204e9800998ecf8427e" }, + { "a", "0cc175b9c0f1b6a831c399e269772661" }, + { "abc", "900150983cd24fb0d6963f7d28e17f72" }, + { "message digest", "f96b697d7cb7938d525a2f31aaf161d0" }, + { "abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b" }, +} + +for _, row in ipairs(vectors) do + local ctx = MD5.new() + for i = 1, #row[1], 3 do ctx:update(row[1]:sub(i, i + 2)) end + T.eq(ctx:final(), row[2], "incremental MD5 vector: " .. row[1]) +end + +-- Cross a large number of block boundaries without building one giant string. +local million = MD5.new() +for _ = 1, 1000 do million:update(string.rep("a", 1000)) end +T.eq(million:final(), "7707d6ae4e027c70eea2a935c2296f21", + "RFC 1321 million-a vector") + +T.finish("stream_md5")