Merge pull request #1616 from HighDrexler/fix/large-required-imports-v2

Fix/large required imports v2

src/mods/ImportAccess.lua

@@ -0,0 +1,174 @@
+-- Scoped access to a mod's launcher-validated required/optional imports and
+-- to installation-wide generated cache data.
+--
+-- This intentionally does not expose host paths or raw filesystem handles.
+-- Import reads are bounded and can only address ids declared by the calling
+-- mod's manifest. Cache paths are confined to mod_cache/<mod-id>/ and are not
+-- tied to a Pokémon playthrough.
+
+local RequiredImports = require("src.mods.RequiredImports")
+local SafePath = require("src.mods.SafePath")
+local SaveData = require("src.core.SaveData")
+
+local ImportAccess = {}
+
+ImportAccess.MAX_READ_BYTES = 8 * 1024 * 1024
+ImportAccess.MAX_CACHE_WRITE_BYTES = 64 * 1024 * 1024
+
+local function specMap(manifest)
+  local out = {}
+  for _, spec in ipairs(RequiredImports.specs(manifest)) do out[spec.id] = spec end
+  return out
+end
+
+local function parentOf(path)
+  return path:match("^(.*)/[^/]+$")
+end
+
+local function copyInfo(info)
+  if not info then return nil end
+  return { type = info.type, size = info.size, modtime = info.modtime }
+end
+
+local function fsReadRange(fs, path, offset, length)
+  if fs and type(fs.readRange) == "function" then
+    return fs.readRange(path, offset, length)
+  end
+  local newFile = fs and fs.newFile
+  if newFile then
+    local file, makeErr = newFile(path)
+    if not file then return nil, makeErr or "could not open import" end
+    local ok, openErr = file:open("r")
+    if not ok then return nil, openErr or "could not open import" end
+    local seekOk, seekErr = file:seek(offset)
+    if seekOk == nil or seekOk == false then
+      file:close()
+      return nil, seekErr or "could not seek import"
+    end
+    local data, readErr = file:read(length)
+    file:close()
+    return data, readErr
+  end
+  -- Injectable headless filesystems may expose only read(). Production
+  -- love.filesystem has newFile(), so large imports are never materialized
+  -- into one Lua string by this fallback.
+  if fs and fs.read then
+    local data = fs.read(path)
+    if type(data) ~= "string" then return nil, "could not read import" end
+    return data:sub(offset + 1, offset + length)
+  end
+  return nil, "random-access import reads are unavailable"
+end
+
+local function validatedInfo(manifest, spec, fs)
+  local ok, detail = RequiredImports.validateStored(manifest, spec, fs)
+  if not ok then return nil, detail or "import is not validated" end
+  local path = RequiredImports.path(manifest, spec)
+  local info = fs.getInfo and fs.getInfo(path, "file") or nil
+  if not info then return nil, "import is missing" end
+  return info, detail
+end
+
+local function makeCache(modId, fs)
+  local root = "mod_cache/" .. modId
+  local function pathFor(rel, what)
+    rel = SafePath.require(rel, what or "mod.cache path")
+    return root .. "/" .. rel
+  end
+  local cache = {}
+
+  function cache:write(rel, bytes)
+    if type(bytes) ~= "string" then
+      return nil, "mod.cache:write expects a byte string"
+    end
+    if #bytes > ImportAccess.MAX_CACHE_WRITE_BYTES then
+      return nil, "mod.cache:write payload exceeds 64 MiB; split generated data into smaller files"
+    end
+    local path = pathFor(rel, "mod.cache:write")
+    local parent = parentOf(path)
+    if parent and fs.createDirectory then
+      local ok = fs.createDirectory(parent)
+      if ok == false then return nil, "could not create cache directory" end
+    end
+    if not fs.write then return nil, "cache writes are unavailable" end
+    return fs.write(path, bytes)
+  end
+
+  function cache:read(rel)
+    local path = pathFor(rel, "mod.cache:read")
+    if not fs.read then return nil, "cache reads are unavailable" end
+    return fs.read(path)
+  end
+
+  function cache:info(rel)
+    local path = pathFor(rel, "mod.cache:info")
+    if not fs.getInfo then return nil end
+    return copyInfo(fs.getInfo(path))
+  end
+
+  function cache:exists(rel)
+    local info = self:info(rel)
+    return info ~= nil and info.type == "file"
+  end
+
+
+  function cache:delete(rel)
+    local path = pathFor(rel, "mod.cache:delete")
+    if not fs.remove then return nil, "cache deletion is unavailable" end
+    return fs.remove(path)
+  end
+
+  return cache
+end
+
+function ImportAccess.new(manifest, fs)
+  local specs = specMap(manifest)
+  local cacheFs = SaveData.persistenceFs(fs) or fs
+  local imports = {}
+
+  function imports:info(id)
+    local spec = specs[id]
+    if not spec then return nil, "undeclared import: " .. tostring(id) end
+    local info, digestOrErr = validatedInfo(manifest, spec, fs)
+    if not info then return nil, digestOrErr end
+    return {
+      id = spec.id,
+      name = spec.name,
+      file = spec.file,
+      size = info.size,
+      md5 = digestOrErr,
+      required = spec.required ~= false,
+    }
+  end
+
+  function imports:read(id, offset, length)
+    local spec = specs[id]
+    if not spec then return nil, "undeclared import: " .. tostring(id) end
+    offset, length = tonumber(offset), tonumber(length)
+    if not offset or offset < 0 or offset % 1 ~= 0 then
+      return nil, "offset must be a non-negative integer"
+    end
+    if not length or length < 0 or length % 1 ~= 0 then
+      return nil, "length must be a non-negative integer"
+    end
+    if length > ImportAccess.MAX_READ_BYTES then
+      return nil, "single import read exceeds 8 MiB"
+    end
+
+    local info, err = validatedInfo(manifest, spec, fs)
+    if not info then return nil, err end
+    local size = tonumber(info.size) or tonumber(spec.size)
+    if size and offset + length > size then return nil, "import read is out of bounds" end
+    if length == 0 then return "" end
+
+    local path = RequiredImports.path(manifest, spec)
+    local data, readErr = fsReadRange(fs, path, offset, length)
+    if not data then return nil, readErr end
+    if #data ~= length then return nil, "short import read" end
+    return data
+  end
+
+  return imports, makeCache(manifest.id, cacheFs)
+end
+
+return ImportAccess

src/mods/Loader.lua

@@ -962,6 +962,8 @@ function Loader:_api(mod)
   local Storage = engineRequire("src.mods.Storage")
   local storage = Storage and Storage.new(modId, loader.fs)
   local Checkpoint = engineRequire("src.core.Checkpoint")
+  local ImportAccess = engineRequire("src.mods.ImportAccess")
+  local importApi, installCache = ImportAccess.new(mod.manifest, loader.fs)
   local api = {
     id = modId,
     version = mod.manifest.version,
@@ -1161,6 +1163,12 @@ function Loader:_api(mod)
     -- checkpoint. The
     -- engine binds version/playthrough/mod scope and portable persistence;
     -- callers never receive paths or a raw filesystem handle.
+    -- Read-only bounded access to this mod's manifest-declared, launcher-validated
+    -- imports. No host path is exposed; large sources are read in bounded ranges.
+    imports = importApi,
+    -- Installation-scoped generated data, independent from Pokémon save slots.
+    -- This is where ROM-derived caches belong; mod.storage remains playthrough-scoped.
+    cache = installCache,
     storage = {
       context = function(_, game) return storage:context(game) end,
       selected = function(_, game) return storage:selected(game) end,

src/mods/RequiredImports.lua

@@ -120,6 +120,36 @@ local function accepts(spec, digest)
   return false
 end
 
+local function specById(manifest, importId)
+  for _, candidate in ipairs(allSpecs(manifest)) do
+    if candidate.id == importId then return candidate end
+  end
+  return nil
+end
+
+RequiredImports.spec = specById
+
+local function streamDigest(fs, path, chunkBytes)
+  if not (fs and fs.newFile) then return nil, "streaming file access is unavailable" end
+  local file, makeErr = fs.newFile(path)
+  if not file then return nil, makeErr or "could not open stored import" end
+  local ok, openErr = file:open("r")
+  if not ok then return nil, openErr or "could not open stored import" end
+  local MD5 = require("src.mods.StreamMD5")
+  local ctx = MD5.new()
+  chunkBytes = chunkBytes or (4 * 1024 * 1024)
+  while true do
+    local data, readErr = file:read(chunkBytes)
+    if data and #data > 0 then ctx:update(data) end
+    if not data or #data < chunkBytes then
+      if readErr then file:close(); return nil, readErr end
+      break
+    end
+  end
+  file:close()
+  return ctx:final()
+end
+
 function RequiredImports.path(manifest, spec)
   return manifest.path .. "/baseroms/" .. spec.file
 end
@@ -180,6 +210,47 @@ local function removeReceipt(manifest, spec, fs)
   end
 end
 
+-- Finalize a caller-streamed import after the destination bytes have already
+-- been copied into the engine-owned baseroms path. This keeps large imports
+-- out of a single Lua string while preserving the same size/MD5 receipt rules.
+function RequiredImports.acceptStoredDigest(manifest, importId, digest, fs)
+  fs = fs or (love and love.filesystem)
+  local spec = specById(manifest, importId)
+  if not spec then return nil, "unknown required import: " .. tostring(importId) end
+  digest = tostring(digest or ""):lower()
+  if not accepts(spec, digest) then
+    return nil, ("MD5 mismatch (got %s)"):format(digest ~= "" and digest or "unavailable")
+  end
+  local path = RequiredImports.path(manifest, spec)
+  local info = fs and fs.getInfo and fs.getInfo(path, "file") or nil
+  if not info then return nil, "copied import is missing" end
+  local sizeErr = RequiredImports.sizeError(spec, info.size, true)
+  if sizeErr then return nil, sizeErr end
+  if love and fs == love.filesystem then
+    local savedPrefix = CacheFs.prefix
+    local ok, prefixErr = xpcall(function()
+      CacheFs.prefix = ""
+      CacheFs.remove(removedMarker(manifest, spec))
+      CacheFs.prefix = savedPrefix
+      -- writeReceipt has its own temporary CacheFs prefix switch. Keep it
+      -- inside this guard too so a write error cannot leak global state.
+      writeReceipt(manifest, spec, digest, info, fs)
+    end, function(err)
+      return tostring(err)
+    end)
+    CacheFs.prefix = savedPrefix
+    if not ok then
+      return nil, "could not finalize import receipt: " .. tostring(prefixErr)
+    end
+    return true, digest
+  elseif fs and fs.remove then
+    fs.remove(removedMarker(manifest, spec))
+  end
+  writeReceipt(manifest, spec, digest, info, fs)
+  return true, digest
+end
+
+
 -- Validate bytes against a declaration.  The returned data is canonicalized
 -- (notably for N64 byte order/header variants) and is what must be stored.
 function RequiredImports.validateData(spec, data, hashFn)
@@ -217,6 +288,22 @@ function RequiredImports.validateStored(manifest, spec, fs, hashFn)
   local cached = cachedDigest(manifest, spec, fs, info)
   if cached then return true, cached, true end
   removeReceipt(manifest, spec, fs)
+  -- Large raw imports (GameCube discs, future optical images, etc.) must never
+  -- be materialized into one Lua string merely because their validation
+  -- receipt was lost. Stream the MD5 directly from the installed file. N64
+  -- sources stay on the canonicalization path because byte-order/header
+  -- normalization is part of their validation contract.
+  if info.size and info.size > RequiredImports.LARGE_WARN_BYTES
+      and spec.format ~= "n64" and fs.newFile then
+    local digest, hashErr = streamDigest(fs, path)
+    if not digest then return nil, hashErr end
+    if not accepts(spec, digest) then
+      return nil, ("MD5 mismatch (got %s)"):format(digest)
+    end
+    info = fs.getInfo(path, "file") or info
+    writeReceipt(manifest, spec, digest, info, fs)
+    return true, digest, false
+  end
   if not fs.read then return nil, "file could not be read" end
   local data = fs.read(path)
   local normalized, detail = RequiredImports.validateStoredData(spec, data, hashFn)

src/mods/StreamMD5.lua

@@ -0,0 +1,101 @@
+local bitlib = rawget(_G, "bit") or rawget(_G, "bit32")
+if not bitlib then error("StreamMD5 requires bit or bit32") end
+
+local band, bor, bxor, bnot = bitlib.band, bitlib.bor, bitlib.bxor, bitlib.bnot
+local lshift, rshift = bitlib.lshift, bitlib.rshift
+local rol = bitlib.rol or bitlib.lrotate
+
+local K = {
+  0xd76aa478,0xe8c7b756,0x242070db,0xc1bdceee,0xf57c0faf,0x4787c62a,0xa8304613,0xfd469501,
+  0x698098d8,0x8b44f7af,0xffff5bb1,0x895cd7be,0x6b901122,0xfd987193,0xa679438e,0x49b40821,
+  0xf61e2562,0xc040b340,0x265e5a51,0xe9b6c7aa,0xd62f105d,0x02441453,0xd8a1e681,0xe7d3fbc8,
+  0x21e1cde6,0xc33707d6,0xf4d50d87,0x455a14ed,0xa9e3e905,0xfcefa3f8,0x676f02d9,0x8d2a4c8a,
+  0xfffa3942,0x8771f681,0x6d9d6122,0xfde5380c,0xa4beea44,0x4bdecfa9,0xf6bb4b60,0xbebfbc70,
+  0x289b7ec6,0xeaa127fa,0xd4ef3085,0x04881d05,0xd9d4d039,0xe6db99e5,0x1fa27cf8,0xc4ac5665,
+  0xf4292244,0x432aff97,0xab9423a7,0xfc93a039,0x655b59c3,0x8f0ccc92,0xffeff47d,0x85845dd1,
+  0x6fa87e4f,0xfe2ce6e0,0xa3014314,0x4e0811a1,0xf7537e82,0xbd3af235,0x2ad7d2bb,0xeb86d391,
+}
+local S = {
+  7,12,17,22, 7,12,17,22, 7,12,17,22, 7,12,17,22,
+  5,9,14,20, 5,9,14,20, 5,9,14,20, 5,9,14,20,
+  4,11,16,23, 4,11,16,23, 4,11,16,23, 4,11,16,23,
+  6,10,15,21, 6,10,15,21, 6,10,15,21, 6,10,15,21,
+}
+
+local function add32(a,b,c,d)
+  local n = (a or 0) + (b or 0) + (c or 0) + (d or 0)
+  return band(n, 0xffffffff)
+end
+
+local function le32_from(s, i)
+  local b1,b2,b3,b4 = s:byte(i, i+3)
+  return bor(b1, lshift(b2,8), lshift(b3,16), lshift(b4,24))
+end
+
+local function le32_bytes(x)
+  return string.char(
+    band(x,0xff), band(rshift(x,8),0xff),
+    band(rshift(x,16),0xff), band(rshift(x,24),0xff))
+end
+
+local M = {}
+local StreamMD5 = {}
+StreamMD5.__index = StreamMD5
+
+function StreamMD5.new()
+  return setmetatable({
+    a=0x67452301, b=0xefcdab89, c=0x98badcfe, d=0x10325476,
+    bytes=0, buffer="", done=false,
+  }, StreamMD5)
+end
+
+function StreamMD5:_block(block)
+  for j=1,16 do M[j] = le32_from(block, (j-1)*4+1) end
+  local a,b,c,d = self.a,self.b,self.c,self.d
+  for i=0,63 do
+    local f,g
+    if i < 16 then
+      f = bor(band(b,c), band(bnot(b),d)); g=i
+    elseif i < 32 then
+      f = bor(band(d,b), band(bnot(d),c)); g=(5*i+1)%16
+    elseif i < 48 then
+      f = bxor(b,c,d); g=(3*i+5)%16
+    else
+      f = bxor(c, bor(b,bnot(d))); g=(7*i)%16
+    end
+    local tmp=d
+    d=c
+    c=b
+    b=add32(b, rol(add32(a,f,K[i+1],M[g+1]), S[i+1]))
+    a=tmp
+  end
+  self.a=add32(self.a,a); self.b=add32(self.b,b)
+  self.c=add32(self.c,c); self.d=add32(self.d,d)
+end
+
+function StreamMD5:update(data)
+  assert(not self.done, "StreamMD5 context already finalized")
+  assert(type(data)=="string", "StreamMD5:update expects a string")
+  self.bytes = self.bytes + #data
+  local s = self.buffer .. data
+  local full = #s - (#s % 64)
+  for i=1,full,64 do self:_block(s:sub(i,i+63)) end
+  self.buffer = s:sub(full+1)
+  return self
+end
+
+function StreamMD5:final()
+  assert(not self.done, "StreamMD5 context already finalized")
+  local originalBytes = self.bytes
+  local padLen = (56 - ((originalBytes + 1) % 64)) % 64
+  local bits = originalBytes * 8
+  local lo = bits % 4294967296
+  local hi = math.floor(bits / 4294967296) % 4294967296
+  self:update("\128" .. string.rep("\0", padLen) .. le32_bytes(lo) .. le32_bytes(hi))
+  assert(#self.buffer == 0, "MD5 finalization left a partial block")
+  self.done=true
+  local raw = le32_bytes(self.a)..le32_bytes(self.b)..le32_bytes(self.c)..le32_bytes(self.d)
+  return (raw:gsub(".", function(ch) return string.format("%02x", ch:byte()) end))
+end
+
+return StreamMD5