Merge pull request #1616 from HighDrexler/fix/large-required-imports-v2

Fix/large required imports v2

tests/engine/mod_import_access_parity_test.lua

@@ -0,0 +1,42 @@
+-- No-mod and API-v1 parity coverage for additive mod.imports/mod.cache.
+package.path = "./?.lua;./?/init.lua;" .. package.path
+
+local T = require("tests.modkit")
+
+-- No mod installed: no generated cache namespace is touched.
+local emptyFiles = {}
+local none = T.sdk.loadNone({ fs = T.sdk.memfs(emptyFiles) })
+T.eq(#none.errors, 0, "zero-mod load remains clean")
+for path in pairs(emptyFiles) do
+  T.check(path:sub(1, 10) ~= "mod_cache/",
+    "zero-mod load creates no installation cache data")
+end
+none.release()
+
+-- Existing API-v1 style file access still behaves exactly as before. The new
+-- facades are additive members; mod:read remains the same scoped read path.
+local files = {
+  ["mods/v1_read_probe/manifest.json"] = [[{
+    "id": "v1_read_probe",
+    "name": "V1 Read Probe",
+    "version": "1.0.0",
+    "entry": "main.lua",
+    "api": 1
+  }]],
+  ["mods/v1_read_probe/main.lua"] = [[
+    local mod = ...
+    mod.exports.payload = mod:read("payload.txt")
+    mod.exports.hasImports = type(mod.imports) == "table"
+    mod.exports.hasCache = type(mod.cache) == "table"
+  ]],
+  ["mods/v1_read_probe/payload.txt"] = "unchanged-v1-read",
+}
+local run = T.sdk.loadMods({ "mods/v1_read_probe" }, { fs = T.sdk.memfs(files) })
+T.eq(#run.errors, 0, "API-v1 probe still loads")
+local out = run.loader.exports.v1_read_probe
+T.eq(out.payload, "unchanged-v1-read", "mod:read keeps its v1 behavior")
+T.eq(out.hasImports, true, "new import facade is additive")
+T.eq(out.hasCache, true, "new cache facade is additive")
+run.release()
+
+T.finish("mod_import_access_parity")

tests/engine/mod_import_access_test.lua

@@ -0,0 +1,105 @@
+-- Public mod-API coverage for bounded validated imports + installation cache.
+package.path = "./?.lua;./?/init.lua;" .. package.path
+
+local T = require("tests.modkit")
+
+local digest = "00000000000000000000000000000000"
+local files = {
+  ["mods/import_api_probe/manifest.json"] = [[{
+    "id": "import_api_probe",
+    "name": "Import API Probe",
+    "version": "1.0.0",
+    "entry": "main.lua",
+    "api": 2,
+    "required_imports": [{
+      "id": "disc",
+      "name": "Disc",
+      "file": "source.iso",
+      "md5": ["00000000000000000000000000000000"],
+      "size": 16
+    }],
+    "optional_imports": [{
+      "id": "optional",
+      "name": "Optional",
+      "file": "optional.bin",
+      "md5": ["11111111111111111111111111111111"],
+      "size": 4,
+      "required": false
+    }]
+  }]],
+  ["mods/import_api_probe/main.lua"] = [[
+    local mod = ...
+    local info, infoErr = mod.imports:info("disc")
+    local slice, sliceErr = mod.imports:read("disc", 4, 6)
+    local oob, oobErr = mod.imports:read("disc", 15, 2)
+    local missing, missingErr = mod.imports:info("optional")
+    local undeclared, undeclaredErr = mod.imports:read("not_declared", 0, 1)
+
+    local wrote, writeErr = mod.cache:write("extract/v1/probe.bin", "abc")
+    local cached, readErr = mod.cache:read("extract/v1/probe.bin")
+    local cacheInfo = mod.cache:info("extract/v1/probe.bin")
+    local escaped = pcall(function() mod.cache:write("../escape.bin", "x") end)
+
+    mod.exports.result = {
+      info = info, infoErr = infoErr,
+      slice = slice, sliceErr = sliceErr,
+      oob = oob, oobErr = oobErr,
+      missing = missing, missingErr = missingErr,
+      undeclared = undeclared, undeclaredErr = undeclaredErr,
+      wrote = wrote, writeErr = writeErr,
+      cached = cached, readErr = readErr,
+      cacheInfo = cacheInfo,
+      escaped = escaped,
+    }
+  ]],
+  ["mods/import_api_probe/baseroms/source.iso"] = "0123456789abcdef",
+  ["mods/import_api_probe/baseroms/.required-import-disc.validated"] =
+    "v1\n" .. digest .. "\n16\n1\n",
+}
+
+local baseFs = T.sdk.memfs(files)
+local oldInfo = baseFs.getInfo
+function baseFs.getInfo(path)
+  local info = oldInfo(path)
+  if info and info.type == "file" then
+    return { type = "file", size = #(files[path] or ""), modtime = 1 }
+  end
+  return info
+end
+function baseFs.readRange(path, offset, length)
+  local body = files[path]
+  if not body then return nil end
+  return body:sub(offset + 1, offset + length)
+end
+function baseFs.createDirectory() return true end
+function baseFs.remove(path) files[path] = nil return true end
+
+local run = T.sdk.loadMods({ "mods/import_api_probe" }, { fs = baseFs })
+T.eq(#run.errors, 0,
+  "the import/cache probe loads through the production Loader")
+
+local out = run.loader.exports.import_api_probe.result
+T.check(type(out.info) == "table", "declared validated import has info")
+T.eq(out.info.size, 16, "import info reports stored size")
+T.eq(out.slice, "456789", "bounded read seeks into the validated import")
+T.eq(out.sliceErr, nil, "bounded read has no error")
+T.eq(out.oob, nil, "out-of-bounds read is refused")
+T.check(type(out.oobErr) == "string" and out.oobErr:find("out of bounds", 1, true),
+  "out-of-bounds read explains the refusal")
+T.eq(out.missing, nil, "missing optional import is not exposed")
+T.check(type(out.missingErr) == "string", "missing optional import returns an error")
+T.eq(out.undeclared, nil, "undeclared import id is refused")
+T.check(type(out.undeclaredErr) == "string"
+    and out.undeclaredErr:find("undeclared", 1, true),
+  "undeclared import refusal is explicit")
+
+T.check(out.wrote == true, "installation cache write succeeds")
+T.eq(out.cached, "abc", "installation cache read returns exact bytes")
+T.eq(out.cacheInfo and out.cacheInfo.size, 3, "installation cache info is scoped")
+T.eq(out.escaped, false, "cache traversal is rejected by the public facade")
+T.eq(files["mod_cache/import_api_probe/extract/v1/probe.bin"], "abc",
+  "cache bytes live under the calling mod id")
+T.eq(files["escape.bin"], nil, "cache traversal created nothing outside its root")
+
+run.release()
+T.finish("mod_import_access")

tests/engine/required_import_streaming_test.lua

@@ -0,0 +1,183 @@
+-- Regression for large raw required imports: direct source -> engine-owned
+-- baseroms destination, incremental MD5, no whole-file staging requirement.
+package.path = "./?.lua;./?/init.lua;" .. package.path
+if not _G.love then _G.love = require("tests.love_stub") end
+
+local T = require("tests.modkit")
+local RomImporter = require("src.import.RomImporter")
+local RequiredImports = require("src.mods.RequiredImports")
+
+-- The stock headless filesystem intentionally omits love.filesystem.newFile.
+-- Add the smallest streaming-file facade this transport path needs so the
+-- regression drives the same seek/read/write API production LÖVE provides.
+local oldNewFile = love.filesystem.newFile
+local oldGetInfo = love.filesystem.getInfo
+love.filesystem.newFile = function(path)
+  local body, pos, mode = love.filesystem.read(path) or "", 1, nil
+  local file = {}
+  function file:open(m)
+    mode = m
+    if m == "w" then body, pos = "", 1 else pos = 1 end
+    return true
+  end
+  function file:read(n)
+    local out = body:sub(pos, pos + n - 1)
+    pos = pos + #out
+    if out == "" then return nil end
+    return out
+  end
+  function file:write(bytes)
+    if mode ~= "w" then return nil, "not open for write" end
+    body = body:sub(1, pos - 1) .. bytes .. body:sub(pos + #bytes)
+    pos = pos + #bytes
+    love.filesystem.write(path, body)
+    return true
+  end
+  function file:seek(offset) pos = offset + 1 return offset end
+  function file:getSize() return #body end
+  function file:close()
+    if mode == "w" then love.filesystem.write(path, body) end
+    mode = nil
+    return true
+  end
+  return file
+end
+love.filesystem.getInfo = function(path, filter)
+  local info = oldGetInfo(path, filter)
+  if info and info.type == "file" then
+    local bytes = love.filesystem.read(path) or ""
+    return { type = "file", size = #bytes, modtime = 1 }
+  end
+  return info
+end
+
+local source = "required_import_streaming_source.tmp"
+local f = assert(io.open(source, "wb"))
+f:write("abc")
+f:close()
+
+local manifest = {
+  id = "stream_probe",
+  name = "Stream Probe",
+  path = "mods/stream_probe",
+  required_imports = {
+    { id = "source", name = "Source", file = "source.bin", format = "raw",
+      size = 3, md5 = { "900150983cd24fb0d6963f7d28e17f72" } },
+  },
+  optional_imports = {},
+}
+
+local importer = setmetatable({
+  mods = { { id = "stream_probe", manifest = manifest } },
+  requiredImportNotice = nil,
+  modNotice = nil,
+  _refreshMods = function(self) self._refreshed = true end,
+}, RomImporter)
+
+-- Reproduce the Windows failure seen with a 1.46 GiB optical-disc image:
+-- after the user accepts the "Large import" confirmation, the old path calls
+-- love.filesystem.read(source) and attempts to materialize the entire source as
+-- one Lua string. A large raw source must go straight to the streaming branch.
+local ordinaryLoveRead = love.filesystem.read
+local forbiddenWholeSourceReads = 0
+love.filesystem.read = function(path, ...)
+  if path == source then
+    forbiddenWholeSourceReads = forbiddenWholeSourceReads + 1
+    error("large external required import used whole-file love.filesystem.read")
+  end
+  return ordinaryLoveRead(path, ...)
+end
+
+-- Exercise the exact large-file branch with tiny fixture bytes by lowering the
+-- threshold for this test. Production keeps the 128 MiB confirmation/streaming
+-- threshold; the copy/hash algorithm is identical.
+local oldWarn = RequiredImports.LARGE_WARN_BYTES
+RequiredImports.LARGE_WARN_BYTES = 2
+local ok = importer:_importRequiredSource("stream_probe", "source", source, true)
+RequiredImports.LARGE_WARN_BYTES = oldWarn
+love.filesystem.read = ordinaryLoveRead
+
+T.eq(ok, true, "large raw required import streams successfully")
+T.eq(forbiddenWholeSourceReads, 0,
+  "post-confirm large import never materializes the external source as one Lua string")
+T.eq(love.filesystem.read("mods/stream_probe/baseroms/source.bin"), "abc",
+  "streamed destination preserves exact source bytes")
+T.eq(importer.requiredImportNotice, nil, "successful stream leaves no import error")
+T.eq(importer._refreshed, true, "successful stream refreshes the mod list")
+
+local CacheFs = require("src.import.CacheFs")
+
+-- A thrown writer error must not leak the temporary empty CacheFs prefix.
+local workingNewFile = love.filesystem.newFile
+local workingPrefix = CacheFs.prefix
+CacheFs.prefix = "sentinel/"
+love.filesystem.newFile = function(path)
+  local file = workingNewFile(path)
+  if path == "mods/stream_probe/baseroms/source.bin" then
+    function file:write()
+      error("forced writer failure")
+    end
+  end
+  return file
+end
+importer.requiredImportNotice = nil
+RequiredImports.LARGE_WARN_BYTES = 2
+local failedWrite = importer:_importRequiredSource(
+  "stream_probe", "source", source, true)
+RequiredImports.LARGE_WARN_BYTES = oldWarn
+T.eq(failedWrite, nil, "thrown streaming writer error is contained")
+T.eq(CacheFs.prefix, "sentinel/",
+  "streaming writer error restores CacheFs.prefix")
+love.filesystem.newFile = workingNewFile
+CacheFs.prefix = workingPrefix
+
+-- acceptStoredDigest has its own prefix switch for marker/receipt I/O.
+-- Even an unexpected CacheFs failure must restore the caller's prefix.
+love.filesystem.write("mods/stream_probe/baseroms/source.bin", "abc")
+local workingRemove = CacheFs.remove
+CacheFs.prefix = "sentinel/"
+CacheFs.remove = function()
+  error("forced marker removal failure")
+end
+local accepted = RequiredImports.acceptStoredDigest(
+  manifest, "source", "900150983cd24fb0d6963f7d28e17f72", love.filesystem)
+T.eq(accepted, nil, "acceptStoredDigest contains CacheFs failure")
+T.eq(CacheFs.prefix, "sentinel/",
+  "acceptStoredDigest failure restores CacheFs.prefix")
+CacheFs.remove = workingRemove
+CacheFs.prefix = workingPrefix
+
+-- Native sources are rejected cleanly if seek-to-start fails after the
+-- size probe; never return a handle left sitting at EOF.
+local realIoOpen = io.open
+local fakeCloses = 0
+io.open = function(path, mode)
+  if path ~= source then return realIoOpen(path, mode) end
+  local calls = 0
+  return {
+    seek = function(_, whence)
+      calls = calls + 1
+      if whence == "end" then return 3 end
+      if whence == "set" then return nil, "forced rewind failure" end
+      return nil, "unexpected seek"
+    end,
+    close = function() fakeCloses = fakeCloses + 1 end,
+  }
+end
+importer.requiredImportNotice = nil
+RequiredImports.LARGE_WARN_BYTES = 2
+local failedSeek = importer:_importRequiredSource(
+  "stream_probe", "source", source, true)
+RequiredImports.LARGE_WARN_BYTES = oldWarn
+io.open = realIoOpen
+T.eq(failedSeek, nil, "failed native rewind rejects import source")
+T.eq(fakeCloses >= 2, true,
+  "failed native rewind closes probed source handles")
+
+love.filesystem.remove("mods/stream_probe/baseroms/source.bin")
+love.filesystem.remove("mods/stream_probe/baseroms/source.iso")
+os.remove(source)
+love.filesystem.newFile = oldNewFile
+love.filesystem.getInfo = oldGetInfo
+
+T.finish("required_import_streaming")

tests/engine/stream_md5_test.lua

@@ -0,0 +1,33 @@
+-- Incremental MD5 vectors used by large required-import streaming.
+package.path = "./?.lua;./?/init.lua;" .. package.path
+
+-- Plain Lua runners may expose bit32 while production LuaJIT exposes bit.
+if not rawget(_G, "bit") and not rawget(_G, "bit32") then
+  local ok, bit32 = pcall(require, "bit32")
+  if ok then _G.bit32 = bit32 end
+end
+
+local T = require("tests.modkit")
+local MD5 = require("src.mods.StreamMD5")
+
+local vectors = {
+  { "", "d41d8cd98f00b204e9800998ecf8427e" },
+  { "a", "0cc175b9c0f1b6a831c399e269772661" },
+  { "abc", "900150983cd24fb0d6963f7d28e17f72" },
+  { "message digest", "f96b697d7cb7938d525a2f31aaf161d0" },
+  { "abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b" },
+}
+
+for _, row in ipairs(vectors) do
+  local ctx = MD5.new()
+  for i = 1, #row[1], 3 do ctx:update(row[1]:sub(i, i + 2)) end
+  T.eq(ctx:final(), row[2], "incremental MD5 vector: " .. row[1])
+end
+
+-- Cross a large number of block boundaries without building one giant string.
+local million = MD5.new()
+for _ = 1, 1000 do million:update(string.rep("a", 1000)) end
+T.eq(million:final(), "7707d6ae4e027c70eea2a935c2296f21",
+  "RFC 1321 million-a vector")
+
+T.finish("stream_md5")