fix(mods): support large required imports end-to-end

.github/workflows/apply-large-import-pr.yml

@@ -1,37 +0,0 @@
-name: Apply prepared large-import patch
-
-on:
-  push:
-    branches:
-      - fix/large-required-imports-v2
-
-permissions:
-  contents: write
-
-jobs:
-  apply:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: Reconstruct and apply patch
-        shell: bash
-        run: |
-          set -euo pipefail
-          base64 -d .github/pr-build/large-import.patch.b64.1 > /tmp/large-import.patch
-          base64 -d .github/pr-build/large-import.patch.b64.2 >> /tmp/large-import.patch
-          git apply --check /tmp/large-import.patch
-          git apply /tmp/large-import.patch
-          rm -rf .github/pr-build
-          rm -f .github/workflows/apply-large-import-pr.yml
-      - name: Commit prepared change
-        shell: bash
-        run: |
-          set -euo pipefail
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add -A
-          git diff --cached --check
-          git commit -m "fix(mods): support large required imports end-to-end"
-          git push origin HEAD:fix/large-required-imports-v2

docs/modding.md

@@ -122,21 +122,40 @@ Each object requires a stable `id`, a display `name`, a destination `file`
 digests. `format` is either `"raw"` (the default) or `"n64"`. An optional
 `description` gives players dump or region guidance in the import panel.
 `size` declares the exact canonical byte length; `max_size` declares a smaller
-per-import ceiling when an exact size is not appropriate. Every import also
+per-import ceiling when an exact size is not appropriate. The engine hard limit
-has an engine-enforced 128 MiB ceiling and is rejected before hashing when its
+is 2 GiB. Imports above 128 MiB receive an explicit free-space confirmation and
-filesystem reports an invalid size.
+use the launcher's streaming large-file path rather than being materialized as
+one Lua string.
 
 For `"n64"`, the launcher recognizes `.z64`, `.v64`, and `.n64` byte orders,
 strips a recognized 512-byte copier header, converts the bytes to canonical
 big-endian `.z64` order, and then checks MD5. The canonical bytes are written
 to `mods/<mod-id>/baseroms/<file>`. Each selection is a private grant to that
 mod: the launcher never scans or copies another mod's imported files merely
-because its manifest names the same digest. Mods read the result with their existing scoped `mod:read` API, for
+because its manifest names the same digest. Small sources can still be read
-example `mod:read("baseroms/stadium2.z64")`; no host path or new filesystem
+with the existing scoped `mod:read` API, for example
+`mod:read("baseroms/stadium2.z64")`. For large sources, prefer the bounded
+`mod.imports` facade described below; no host path or new general filesystem
 permission is exposed. Missing `required_imports` block the mod before its
 entry chunk runs; missing `optional_imports` remain visible in the same
 launcher panel but do not block loading.
 
+#### Bounded access to validated imports
+
+A loaded mod can address only ids declared by its own `required_imports` or
+`optional_imports` arrays:
+
+```lua
+local info, err = mod.imports:info("stadium2")
+local header, err = mod.imports:read("stadium2", 0, 4096)
+```
+
+`read` uses zero-based offsets and is capped at 8 MiB per call. The engine
+rechecks the stored import before exposing it, seeks into the engine-owned
+copy, and never gives the mod a host path or file handle. This is intended for
+large source formats whose table/index can be parsed with small reads before
+selectively reading the payloads a transform actually needs.
+
 MD5 here identifies a known dump because ROM databases commonly publish it;
 it is not a security or authenticity guarantee. Do not paste the SHA-1 used by
 Gen1Recomp's own game-ROM importer into an import's `md5` field. Mod archives
@@ -441,6 +460,28 @@ default** (1x front, 2x back).
   ball-to-pic grow multiplies your scale through each stage, so a rescaled
   mon still grows into place from the ball, grounded the whole way.
 
+## Installation-scoped generated cache
+
+Generated data derived from a validated user source often belongs to the mod
+installation rather than to one Pokémon save. `mod.cache` is that namespace:
+
+```lua
+local ok, err = mod.cache:write("extract/v1/arena.bin", encodedArena)
+local bytes, err = mod.cache:read("extract/v1/arena.bin")
+local info = mod.cache:info("extract/v1/arena.bin")
+mod.cache:delete("extract/v1/arena.bin")
+```
+
+The physical root is engine-owned (`mod_cache/<mod-id>/`) and never exposed to
+the mod. Keys are safe relative paths and a single write is capped at 64 MiB.
+The cache does not rewind with checkpoints and is not scoped to game version,
+slot, or playthrough. The mod owns its generated format, fingerprints, rebuild
+policy, and completion marker; the engine treats the bytes as opaque data.
+
+Use `mod.storage` instead when the data belongs to one playthrough. Use
+`mod.cache` when it is a reproducible installation artifact that can be rebuilt
+from a declared user source.
+
 ## Durable tool storage and runtime checkpoints
 
 `mod.save` remains the right place for state that should travel with the next

src/import/CacheFs.lua

@@ -287,6 +287,43 @@ function CacheFs.write(rel, data)
   return love.filesystem.write(rel, data)
 end
 
+-- Open a cache-relative file for streaming replacement. The returned handle
+-- has write(bytes) and close() methods and follows the same portable/save-dir
+-- routing as CacheFs.write without forcing the caller to hold the whole file
+-- in one Lua string.
+function CacheFs.openWrite(rel)
+  rel = withPrefix(rel)
+  local root = CacheFs.root()
+  if root then
+    ensureParents(root, rel)
+    local f, err = io.open(realPath(root, rel), "wb")
+    if not f then return nil, err end
+    return {
+      write = function(_, data)
+        local ok, writeErr = f:write(data)
+        if not ok then return nil, writeErr end
+        return true
+      end,
+      close = function() f:close() end,
+    }
+  end
+  if not (love and love.filesystem and love.filesystem.newFile) then
+    return nil, "streaming cache writes are unavailable"
+  end
+  local parent = rel:match("^(.*)/[^/]+$")
+  if parent and not love.filesystem.createDirectory(parent) then
+    local info = love.filesystem.getInfo(parent)
+    local reason = info and ("a " .. info.type .. " already exists there")
+      or "unknown reason"
+    return nil, "could not create " .. parent .. ": " .. reason
+  end
+  local file, makeErr = love.filesystem.newFile(rel)
+  if not file then return nil, makeErr or "could not create cache file" end
+  local ok, openErr = file:open("w")
+  if not ok then return nil, openErr or "could not open cache file" end
+  return file
+end
+
 -- read cache-relative `rel`; returns the bytes or nil
 function CacheFs.read(rel)
   rel = withPrefix(rel)

src/import/RomImporter.lua

@@ -382,6 +382,102 @@ local function externalFileSize(path)
   return size
 end
 
+local function openImportSource(path)
+  -- Desktop picker paths live outside LÖVE's virtual filesystem. Prefer the
+  -- native file handle so a 1.46 GiB disc is never copied to a temp file or
+  -- read into one Lua string before validation.
+  local native = io.open(path, "rb")
+  if native then
+    local size = native:seek("end")
+    if size then native:seek("set", 0) end
+    return {
+      size = size,
+      read = function(_, n) return native:read(n) end,
+      close = function() native:close() end,
+    }
+  end
+  if love and love.filesystem and love.filesystem.newFile then
+    local file, makeErr = love.filesystem.newFile(path)
+    if not file then return nil, makeErr or "could not open source file" end
+    local ok, openErr = file:open("r")
+    if not ok then return nil, openErr or "could not open source file" end
+    local size = file.getSize and file:getSize() or nil
+    return {
+      size = size,
+      read = function(_, n) return file:read(n) end,
+      close = function() file:close() end,
+    }
+  end
+  return nil, "streaming source access is unavailable"
+end
+
+local function streamRequiredImport(manifest, importId, source)
+  local RequiredImports = require("src.mods.RequiredImports")
+  local spec = RequiredImports.spec(manifest, importId)
+  if not spec then return nil, "Import declaration was not found." end
+  if spec.format == "n64" then
+    return nil, "streaming canonicalization is unavailable for N64 imports"
+  end
+  local input, openErr = openImportSource(source)
+  if not input then return nil, openErr end
+  local sizeErr = RequiredImports.sizeError(spec, input.size, false)
+  if sizeErr then input:close(); return nil, sizeErr end
+
+  local CacheFs = require("src.import.CacheFs")
+  local destination = RequiredImports.path(manifest, spec)
+  local savedPrefix = CacheFs.prefix
+  CacheFs.prefix = ""
+  CacheFs.remove(RequiredImports.receiptPath(manifest, spec))
+  CacheFs.remove(destination)
+  local output, makeErr = CacheFs.openWrite(destination)
+  if not output then
+    CacheFs.prefix = savedPrefix
+    input:close()
+    return nil, makeErr or "could not create imported file"
+  end
+
+  local function cleanupDestination()
+    CacheFs.remove(destination)
+    CacheFs.prefix = savedPrefix
+  end
+
+  local MD5 = require("src.mods.StreamMD5")
+  local md5 = MD5.new()
+  local total, chunkBytes = 0, 4 * 1024 * 1024
+  while true do
+    local chunk = input:read(chunkBytes)
+    if not chunk or #chunk == 0 then break end
+    md5:update(chunk)
+    local wrote, writeErr = output:write(chunk)
+    if wrote == false or wrote == nil then
+      input:close(); output:close(); cleanupDestination()
+      return nil, "could not copy import: " .. tostring(writeErr or "write failed")
+    end
+    total = total + #chunk
+    if #chunk < chunkBytes then break end
+  end
+  input:close()
+  output:close()
+
+  if input.size and total ~= input.size then
+    cleanupDestination()
+    return nil, ("source read ended early (expected %d bytes, copied %d)"):format(input.size, total)
+  end
+  local storedSizeErr = RequiredImports.sizeError(spec, total, true)
+  if storedSizeErr then cleanupDestination(); return nil, storedSizeErr end
+  local digest = md5:final()
+  CacheFs.prefix = savedPrefix
+  local accepted, detail = RequiredImports.acceptStoredDigest(
+    manifest, importId, digest, love.filesystem)
+  if not accepted then
+    CacheFs.prefix = ""
+    CacheFs.remove(destination)
+    CacheFs.prefix = savedPrefix
+    return nil, detail
+  end
+  return true, detail
+end
+
 local function readDroppedFile(file)
   local ok, openError = file:open("r")
   if not ok then return nil, openError end
@@ -1228,11 +1324,13 @@ local function chooseRequiredFile()
       "$d=New-Object System.Windows.Forms.OpenFileDialog;",
       "$d.Title='" .. prompt .. "';",
       "$d.Filter='All files (*.*)|*.*';",
+      -- Required imports can be multi-gigabyte optical-disc images. Do NOT
+      -- stage them through %TEMP%: that doubles free-space requirements and a
+      -- failed Copy-Item can leave a plausible-looking truncated temp file.
+      -- Stream the selected source directly into the mod-owned destination.
       "if($d.ShowDialog() -eq 'OK'){",
-      "$t=Join-Path $env:TEMP 'pokeport_required_import.bin';",
-      "Copy-Item -LiteralPath $d.FileName -Destination $t -Force;",
       "[Console]::OutputEncoding=[Text.Encoding]::UTF8;",
-      "[Console]::Write($t)}",
+      "[Console]::Write($d.FileName)}",
     })
     return commandOutput(
       'powershell -NoProfile -STA -Command "' .. script .. '"')
@@ -2072,8 +2170,13 @@ function RomImporter:_importRequiredSource(modId, importId, source, confirmed)
     return nil
   end
   local RequiredImports = require("src.mods.RequiredImports")
-  local info = love.filesystem.getInfo(source, "file")
+  -- A desktop picker returns a host path. Ask the host file handle first;
-  local size = info and info.size or externalFileSize(source)
+  -- love.filesystem.getInfo is only authoritative for virtual/save paths.
+  local size = externalFileSize(source)
+  if not size then
+    local info = love.filesystem.getInfo(source, "file")
+    size = info and info.size or nil
+  end
   local sizeErr = RequiredImports.sizeError(spec, size, false)
   if sizeErr then
     requiredImportNotice(self, modId, importId, sizeErr)
@@ -2096,6 +2199,20 @@ function RomImporter:_importRequiredSource(modId, importId, source, confirmed)
     }
     return nil
   end
+  if type(size) == "number" and size > RequiredImports.LARGE_WARN_BYTES
+      and spec.format ~= "n64" then
+    local ok, result = streamRequiredImport(manifest, importId, source)
+    if ok then
+      self.requiredImportNotice = nil
+      self.modNotice = { ok = true, text = "Imported " .. tostring(importId)
+        .. " for " .. tostring(manifest.name or manifest.id) .. "." }
+      self:_refreshMods()
+      return true
+    end
+    requiredImportNotice(self, modId, importId, result)
+    self.modNotice = nil
+    return nil
+  end
   local data = love.filesystem.read(source)
   if not data then data = readExternalPath(source) end
   if not data then

src/mods/Loader.lua

@@ -962,6 +962,8 @@ function Loader:_api(mod)
   local Storage = engineRequire("src.mods.Storage")
   local storage = Storage and Storage.new(modId, loader.fs)
   local Checkpoint = engineRequire("src.core.Checkpoint")
+  local ImportAccess = engineRequire("src.mods.ImportAccess")
+  local importApi, installCache = ImportAccess.new(mod.manifest, loader.fs)
   local api = {
     id = modId,
     version = mod.manifest.version,
@@ -1161,6 +1163,12 @@ function Loader:_api(mod)
     -- checkpoint. The
     -- engine binds version/playthrough/mod scope and portable persistence;
     -- callers never receive paths or a raw filesystem handle.
+    -- Read-only bounded access to this mod's manifest-declared, launcher-validated
+    -- imports. No host path is exposed; large sources are read in bounded ranges.
+    imports = importApi,
+    -- Installation-scoped generated data, independent from Pokémon save slots.
+    -- This is where ROM-derived caches belong; mod.storage remains playthrough-scoped.
+    cache = installCache,
     storage = {
       context = function(_, game) return storage:context(game) end,
       selected = function(_, game) return storage:selected(game) end,

src/mods/RequiredImports.lua

@@ -120,6 +120,36 @@ local function accepts(spec, digest)
   return false
 end
 
+local function specById(manifest, importId)
+  for _, candidate in ipairs(allSpecs(manifest)) do
+    if candidate.id == importId then return candidate end
+  end
+  return nil
+end
+
+RequiredImports.spec = specById
+
+local function streamDigest(fs, path, chunkBytes)
+  if not (fs and fs.newFile) then return nil, "streaming file access is unavailable" end
+  local file, makeErr = fs.newFile(path)
+  if not file then return nil, makeErr or "could not open stored import" end
+  local ok, openErr = file:open("r")
+  if not ok then return nil, openErr or "could not open stored import" end
+  local MD5 = require("src.mods.StreamMD5")
+  local ctx = MD5.new()
+  chunkBytes = chunkBytes or (4 * 1024 * 1024)
+  while true do
+    local data, readErr = file:read(chunkBytes)
+    if data and #data > 0 then ctx:update(data) end
+    if not data or #data < chunkBytes then
+      if readErr then file:close(); return nil, readErr end
+      break
+    end
+  end
+  file:close()
+  return ctx:final()
+end
+
 function RequiredImports.path(manifest, spec)
   return manifest.path .. "/baseroms/" .. spec.file
 end
@@ -180,6 +210,35 @@ local function removeReceipt(manifest, spec, fs)
   end
 end
 
+-- Finalize a caller-streamed import after the destination bytes have already
+-- been copied into the engine-owned baseroms path. This keeps large imports
+-- out of a single Lua string while preserving the same size/MD5 receipt rules.
+function RequiredImports.acceptStoredDigest(manifest, importId, digest, fs)
+  fs = fs or (love and love.filesystem)
+  local spec = specById(manifest, importId)
+  if not spec then return nil, "unknown required import: " .. tostring(importId) end
+  digest = tostring(digest or ""):lower()
+  if not accepts(spec, digest) then
+    return nil, ("MD5 mismatch (got %s)"):format(digest ~= "" and digest or "unavailable")
+  end
+  local path = RequiredImports.path(manifest, spec)
+  local info = fs and fs.getInfo and fs.getInfo(path, "file") or nil
+  if not info then return nil, "copied import is missing" end
+  local sizeErr = RequiredImports.sizeError(spec, info.size, true)
+  if sizeErr then return nil, sizeErr end
+  if love and fs == love.filesystem then
+    local savedPrefix = CacheFs.prefix
+    CacheFs.prefix = ""
+    CacheFs.remove(removedMarker(manifest, spec))
+    CacheFs.prefix = savedPrefix
+  elseif fs and fs.remove then
+    fs.remove(removedMarker(manifest, spec))
+  end
+  writeReceipt(manifest, spec, digest, info, fs)
+  return true, digest
+end
+
+
 -- Validate bytes against a declaration.  The returned data is canonicalized
 -- (notably for N64 byte order/header variants) and is what must be stored.
 function RequiredImports.validateData(spec, data, hashFn)
@@ -217,6 +276,22 @@ function RequiredImports.validateStored(manifest, spec, fs, hashFn)
   local cached = cachedDigest(manifest, spec, fs, info)
   if cached then return true, cached, true end
   removeReceipt(manifest, spec, fs)
+  -- Large raw imports (GameCube discs, future optical images, etc.) must never
+  -- be materialized into one Lua string merely because their validation
+  -- receipt was lost. Stream the MD5 directly from the installed file. N64
+  -- sources stay on the canonicalization path because byte-order/header
+  -- normalization is part of their validation contract.
+  if info.size and info.size > RequiredImports.LARGE_WARN_BYTES
+      and spec.format ~= "n64" and fs.newFile then
+    local digest, hashErr = streamDigest(fs, path)
+    if not digest then return nil, hashErr end
+    if not accepts(spec, digest) then
+      return nil, ("MD5 mismatch (got %s)"):format(digest)
+    end
+    info = fs.getInfo(path, "file") or info
+    writeReceipt(manifest, spec, digest, info, fs)
+    return true, digest, false
+  end
   if not fs.read then return nil, "file could not be read" end
   local data = fs.read(path)
   local normalized, detail = RequiredImports.validateStoredData(spec, data, hashFn)