Add manifest-driven required mod imports

src/mods/LauncherMods.lua

@@ -38,6 +38,7 @@ local Version = require("src.core.Version")
 local SaveData = require("src.core.SaveData")
 local GameVersion = require("src.core.GameVersion")
 local CacheFs = require("src.import.CacheFs")
+local RequiredImports = require("src.mods.RequiredImports")
 
 local LauncherMods = {}
 
@@ -459,13 +460,33 @@ function LauncherMods.list(version)
   local ok, result = pcall(function()
     local options = SaveData.loadOptions()
     local manifests = discover()
+    -- A validated copy owned by another installed mod can satisfy the same
+    -- declared MD5 without asking the player to select the ROM twice.
+    local _, importState = RequiredImports.reconcile(manifests)
     -- The first build containing game-specific switches turns the old shared
     -- state into one explicit answer per installed mod and game.  Saving here
     -- means users who only visit the launcher still receive the migration.
     if SaveData.migrateModEnablement(options, manifests) then
       SaveData.saveOptions(options)
     end
-    return LauncherMods.deriveList(manifests, options, version)
+    local rows = LauncherMods.deriveList(manifests, options, version)
+    for _, row in ipairs(rows) do
+      local state = importState[row.id]
+        or { rows = {}, missing = 0, missingOptional = 0 }
+      local imports, missing = state.rows, state.missing
+      row.requiredImports, row.missingRequiredImports = imports, missing
+      row.imports = imports
+      row.missingOptionalImports = state.missingOptional or 0
+      if missing > 0 and row.status == "ok" then
+        row.status = "needs_import"
+        local first
+        for _, import in ipairs(imports) do
+          if not import.present then first = import break end
+        end
+        row.statusDetail = "Needs import: " .. (first and first.name or "required file")
+      end
+    end
+    return rows
   end)
   if not ok then
     -- a single bad options/mod file must not blank the launcher
@@ -677,6 +698,26 @@ local function copyTree(src, dst)
   return true
 end
 
+-- User-supplied baseroms are install state, not package content.  Snapshot
+-- them before replacing a mod tree so an update cannot make the player select
+-- the same cartridge again (or destroy their only reusable copy if the new
+-- archive later fails to copy).
+local function snapshotTree(path, into, relative)
+  local fs = love.filesystem
+  into, relative = into or {}, relative or ""
+  local info = fs.getInfo(path)
+  if not info then return into end
+  if info.type == "directory" then
+    for _, name in ipairs(fs.getDirectoryItems(path) or {}) do
+      local rel = relative == "" and name or (relative .. "/" .. name)
+      snapshotTree(path .. "/" .. name, into, rel)
+    end
+  elseif relative ~= "" and into[relative] == nil then
+    into[relative] = fs.read(path)
+  end
+  return into
+end
+
 -- Delete an installed mod subtree.  Enumeration stays on love.filesystem (the
 -- portable game folder is on its read path), but the deletes go through
 -- CacheFs so a portable install's real files actually go away instead of
@@ -919,6 +960,12 @@ function LauncherMods._installZipInner(source, opts)
     return nil, ("zip is for '%s', expected '%s'")
       :format(manifest.id, opts.expectId)
   end
+  local packagedBaseroms = root .. "/baseroms"
+  if fs.getInfo(packagedBaseroms, "directory")
+      and #(fs.getDirectoryItems(packagedBaseroms) or {}) > 0 then
+    cleanup()
+    return nil, "mod archives must not include user-supplied baseroms/ files"
+  end
 
   local dest = "mods/" .. manifest.id
   local existing, installedSomewhere = sameIdTrees(fs, manifest.id)
@@ -926,7 +973,11 @@ function LauncherMods._installZipInner(source, opts)
     cleanup()
     return nil, "a mod named '" .. manifest.id .. "' is already installed"
   end
+  local preservedBaseroms = {}
   if #existing > 0 then
+    for _, path in ipairs(existing) do
+      snapshotTree(path .. "/baseroms", preservedBaseroms)
+    end
     -- drop every old tree before copy -- mods/<id> and any same-id folder
     -- under another name, or the survivor keeps winning discover()'s
     -- first-id-wins race after the "successful" update (#801).  A tree with
@@ -950,6 +1001,25 @@ function LauncherMods._installZipInner(source, opts)
   CacheFs.prefix = ""
   local copied, copyErr = copyTree(root, dest)
   if not copied then removeTree(dest) end
+  local preserveErr
+  for rel, bytes in pairs(preservedBaseroms) do
+    if bytes ~= nil then
+      local restored, restoreErr = CacheFs.write(dest .. "/baseroms/" .. rel, bytes)
+      if not restored and not preserveErr then
+        preserveErr = "could not preserve baseroms/" .. rel .. ": "
+          .. tostring(restoreErr)
+      end
+    end
+  end
+  if preserveErr then
+    -- Do not report a successful update that discarded user-owned input. Keep
+    -- a best-effort baseroms-only tree for the next retry instead.
+    removeTree(dest)
+    for rel, bytes in pairs(preservedBaseroms) do
+      if bytes ~= nil then CacheFs.write(dest .. "/baseroms/" .. rel, bytes) end
+    end
+    copied, copyErr = nil, preserveErr
+  end
   CacheFs.prefix = savedPrefix
   if not copied then
     cleanup()

src/mods/Loader.lua

@@ -4,6 +4,7 @@ local SaveData = require("src.core.SaveData")
 local Data = require("src.core.Data")
 local GameVersion = require("src.core.GameVersion")
 local Version = require("src.core.Version")
+local RequiredImports = require("src.mods.RequiredImports")
 local Assets = require("src.render.Assets")
 local ModUI = require("src.ui.ModUI")
 local DateTime = require("src.core.DateTime")
@@ -565,7 +566,24 @@ function Loader:_validate()
     elseif manifest.assets_transforms
         and not self:_exists(mod.path .. "/" .. manifest.assets_transforms) then
       reason = "assets_transforms file missing: " .. manifest.assets_transforms
-    elseif manifest.game_version and not devEngine() then
+    end
+    if not reason and #(manifest.required_imports or {}) > 0 then
+      for _, import in ipairs(manifest.required_imports) do
+        local path = mod.path .. "/baseroms/" .. import.file
+        if not self:_exists(path) then
+          reason = "required import missing: " .. import.name
+          break
+        end
+        local data = self.fs.read and self.fs.read(path)
+        local valid, importErr = RequiredImports.validateStoredData(import, data)
+        if not valid then
+          reason = "required import invalid: " .. import.name
+            .. " (" .. tostring(importErr) .. ")"
+          break
+        end
+      end
+    end
+    if not reason and manifest.game_version and not devEngine() then
       local ok, err = Semver.satisfies(Version.engine, manifest.game_version)
       if not ok then
         reason = ("needs game version %s, engine is %s")

src/mods/Manifest.lua

@@ -132,13 +132,74 @@ local function mergeConflictLists(conflicts, incompatible)
   return out
 end
 
+local scrubUtf8 -- shared by required-import labels and top-level strings
+
+-- User-supplied files a mod needs beside its own source.  The launcher owns
+-- the picker and the copy; the mod only reads the resulting
+-- <mod>/baseroms/<file> through its existing scoped mod:read surface.  MD5 is
+-- deliberately the manifest vocabulary here: ROM preservation databases and
+-- the mods consuming these files commonly identify dumps by MD5, and the
+-- digest is an identity check rather than a security boundary.
+local function parseImports(value, field, required)
+  field = field or "required_imports"
+  local out, ids, files = {}, {}, {}
+  for _, entry in ipairs(array(value)) do
+    assert(type(entry) == "table", field .. " entries must be objects")
+    local id = entry.id
+    assert(type(id) == "string" and id:match("^[%w_%-]+$"),
+      field .. " id must contain only letters, numbers, _ or -")
+    assert(not ids[id], "duplicate " .. field .. " id: " .. id)
+    ids[id] = true
+
+    local file = entry.file
+    assert(type(file) == "string" and file ~= "",
+      field .. " file is required")
+    file = SafePath.require(file, field .. " file")
+    assert(not file:find("/", 1, true),
+      field .. " file must be a filename inside baseroms")
+    assert(not files[file], "duplicate " .. field .. " file: " .. file)
+    files[file] = true
+
+    local hashes = entry.md5
+    if type(hashes) == "string" then hashes = { hashes } end
+    assert(type(hashes) == "table" and #hashes > 0,
+      field .. " md5 must be a hash or non-empty array")
+    local accepted, seen = {}, {}
+    for _, digest in ipairs(hashes) do
+      assert(type(digest) == "string" and digest:match("^[%x]+$")
+          and #digest == 32, field .. " md5 values must be 32 hex characters")
+      digest = digest:lower()
+      if not seen[digest] then
+        seen[digest] = true
+        accepted[#accepted + 1] = digest
+      end
+    end
+
+    local format = entry.format or "raw"
+    assert(format == "raw" or format == "n64",
+      field .. " format must be raw or n64")
+    local name = entry.name or id
+    assert(type(name) == "string" and name ~= "",
+      field .. " name must be a non-empty string")
+    out[#out + 1] = {
+      id = id,
+      name = scrubUtf8(name),
+      file = file,
+      md5 = accepted,
+      format = format,
+      required = required ~= false,
+    }
+  end
+  return out
+end
+
 -- Drop bytes that are not valid UTF-8 (malformed sequences, overlongs,
 -- surrogates, > U+10FFFF) and a leading BOM.  LÖVE's text renderer raises
 -- "Invalid UTF-8" from love.graphics.print/printf, so any manifest string a
 -- panel may draw must be scrubbed here -- the one place every mod manifest
 -- passes through -- or a single mangled description crashes the whole MODS
 -- panel instead of misrendering one card.
-local function scrubUtf8(s)
+scrubUtf8 = function(s)
   if type(s) ~= "string" then return s end
   s = s:gsub("^\239\187\191", "")
   local out, i, n = {}, 1, #s
@@ -290,6 +351,19 @@ function Manifest.validate(raw, path)
 
   local conflicts = mergeConflictLists(raw.conflicts, raw.incompatible)
 
+  local requiredImports = parseImports(raw.required_imports,
+    "required_imports", true)
+  local optionalImports = parseImports(raw.optional_imports,
+    "optional_imports", false)
+  local importIds, importFiles = {}, {}
+  for _, list in ipairs({ requiredImports, optionalImports }) do
+    for _, import in ipairs(list) do
+      assert(not importIds[import.id], "duplicate import id: " .. import.id)
+      assert(not importFiles[import.file], "duplicate import file: " .. import.file)
+      importIds[import.id], importFiles[import.file] = true, true
+    end
+  end
+
   return {
     id = raw.id,
     name = raw.name,
@@ -318,6 +392,8 @@ function Manifest.validate(raw, path)
     permissionSet = permissionSet,
     options_schema = optionalFile(raw.options_schema, "options_schema"),
     assets_transforms = optionalFile(raw.assets_transforms, "assets_transforms"),
+    required_imports = requiredImports,
+    optional_imports = optionalImports,
     -- an env var name, not a path, so it keeps the plain string check
     force_enable_env = optionalString(raw.force_enable_env, "force_enable_env"),
     path = path,

src/mods/RequiredImports.lua

@@ -0,0 +1,266 @@
+-- Engine-owned import handling for files declared by a mod's required or
+-- optional import arrays. Mods never receive host paths or broader
+-- filesystem access: accepted bytes are copied into their own
+-- mods/<id>/baseroms/ tree, where the existing mod:read sandbox can see them.
+
+local CacheFs = require("src.import.CacheFs")
+
+local RequiredImports = {}
+
+local function allSpecs(manifest)
+  local out = {}
+  for _, spec in ipairs((manifest and manifest.required_imports) or {}) do
+    out[#out + 1] = spec
+  end
+  for _, spec in ipairs((manifest and manifest.optional_imports) or {}) do
+    out[#out + 1] = spec
+  end
+  return out
+end
+
+local function isRequired(spec)
+  return spec.required ~= false
+end
+
+RequiredImports.specs = allSpecs
+
+local N64_MAGIC = {
+  ["\128\55\18\64"] = "z64", -- big endian / canonical
+  ["\55\128\64\18"] = "v64", -- byte-swapped
+  ["\64\18\55\128"] = "n64", -- little endian words
+}
+
+local function n64KindAt(data, offset)
+  return N64_MAGIC[data:sub(offset, offset + 3)]
+end
+
+-- Return canonical big-endian N64 bytes.  A 512-byte copier header is
+-- recognized only when valid N64 magic follows it, so arbitrary data is never
+-- shortened just because its size happens to line up.
+function RequiredImports.normalizeN64(data)
+  if type(data) ~= "string" then return nil, "selected file could not be read" end
+  local offset, kind = 1, n64KindAt(data, 1)
+  if not kind then
+    kind = n64KindAt(data, 513)
+    if kind then offset = 513 end
+  end
+  if not kind then return nil, "not a recognized Nintendo 64 ROM" end
+  data = data:sub(offset)
+  if kind == "z64" then return data end
+
+  if kind == "v64" then
+    if #data % 2 ~= 0 then return nil, "byte-swapped N64 ROM has an odd size" end
+    return (data:gsub("(.)(.)", "%2%1"))
+  else
+    if #data % 4 ~= 0 then return nil, "little-endian N64 ROM size is not word aligned" end
+    return (data:gsub("(.)(.)(.)(.)", "%4%3%2%1"))
+  end
+end
+
+function RequiredImports.normalize(spec, data)
+  if spec and spec.format == "n64" then
+    return RequiredImports.normalizeN64(data)
+  end
+  if type(data) ~= "string" then return nil, "selected file could not be read" end
+  return data
+end
+
+local function hexDigest(data, hashFn)
+  if hashFn then return hashFn(data):lower() end
+  if not (love and love.data and love.data.hash and love.data.encode) then
+    return nil, "MD5 support is unavailable in this build"
+  end
+  local digest = love.data.hash("md5", data)
+  if type(digest) == "userdata" and digest.getString then
+    digest = digest:getString()
+  end
+  return love.data.encode("string", "hex", digest):lower()
+end
+
+local function accepts(spec, digest)
+  for _, wanted in ipairs((spec and spec.md5) or {}) do
+    if wanted == digest then return true end
+  end
+  return false
+end
+
+function RequiredImports.path(manifest, spec)
+  return manifest.path .. "/baseroms/" .. spec.file
+end
+
+local function removedMarker(manifest, spec)
+  return manifest.path .. "/baseroms/." .. spec.id .. ".removed"
+end
+
+-- Validate bytes against a declaration.  The returned data is canonicalized
+-- (notably for N64 byte order/header variants) and is what must be stored.
+function RequiredImports.validateData(spec, data, hashFn)
+  local normalized, normalizeErr = RequiredImports.normalize(spec, data)
+  if not normalized then return nil, normalizeErr end
+  local digest, hashErr = hexDigest(normalized, hashFn)
+  if not digest then return nil, hashErr end
+  if not accepts(spec, digest) then
+    return nil, ("MD5 mismatch (got %s)"):format(digest)
+  end
+  return normalized, digest
+end
+
+function RequiredImports.validateStoredData(spec, data, hashFn)
+  local normalized, detail = RequiredImports.validateData(spec, data, hashFn)
+  if not normalized then return nil, detail end
+  if normalized ~= data then
+    return nil, "stored N64 ROM is not canonical; choose the source file again"
+  end
+  return normalized, detail
+end
+
+function RequiredImports.inspect(manifest, fs, hashFn)
+  fs = fs or (love and love.filesystem)
+  local rows, missing, missingOptional = {}, 0, 0
+  for _, spec in ipairs(allSpecs(manifest)) do
+    local path = RequiredImports.path(manifest, spec)
+    local suppressed = fs and fs.getInfo
+      and fs.getInfo(removedMarker(manifest, spec), "file") ~= nil
+    local data = fs and fs.read and fs.read(path) or nil
+    local normalized, detail
+    if data then
+      normalized, detail = RequiredImports.validateStoredData(spec, data, hashFn)
+    end
+    local row = { id = spec.id, name = spec.name, file = spec.file,
+      format = spec.format, path = path, present = normalized ~= nil,
+      digest = normalized and detail or nil,
+      error = data and not normalized and detail or nil,
+      suppressed = suppressed, required = isRequired(spec), spec = spec }
+    if not row.present then
+      if row.required then missing = missing + 1
+      else missingOptional = missingOptional + 1 end
+    end
+    rows[#rows + 1] = row
+  end
+  return rows, missing, missingOptional
+end
+
+function RequiredImports.importData(manifest, importId, data, opts)
+  opts = opts or {}
+  local spec
+  for _, candidate in ipairs(allSpecs(manifest)) do
+    if candidate.id == importId then spec = candidate break end
+  end
+  if not spec then return nil, "unknown required import: " .. tostring(importId) end
+  local normalized, digest = RequiredImports.validateData(spec, data, opts.hash)
+  if not normalized then return nil, digest end
+  local savedPrefix = CacheFs.prefix
+  CacheFs.prefix = ""
+  local ok, err = CacheFs.write(RequiredImports.path(manifest, spec), normalized)
+  if ok then CacheFs.remove(removedMarker(manifest, spec)) end
+  CacheFs.prefix = savedPrefix
+  if not ok then return nil, "could not copy import: " .. tostring(err) end
+  return true, digest
+end
+
+function RequiredImports.remove(manifest, importId)
+  for _, spec in ipairs(allSpecs(manifest)) do
+    if spec.id == importId then
+      local savedPrefix = CacheFs.prefix
+      CacheFs.prefix = ""
+      CacheFs.remove(RequiredImports.path(manifest, spec))
+      local marked, markErr = CacheFs.write(removedMarker(manifest, spec), "removed\n")
+      CacheFs.prefix = savedPrefix
+      if not marked then return nil, "could not remember removal: " .. tostring(markErr) end
+      return true
+    end
+  end
+  return nil, "unknown required import: " .. tostring(importId)
+end
+
+-- Fill missing imports from another installed mod when its accepted canonical
+-- MD5 overlaps.  The source remains inside the engine-owned mods tree, and a
+-- fresh validation is performed before every copy.
+function RequiredImports.reconcile(manifests, fs, hashFn)
+  fs = fs or (love and love.filesystem)
+  if not (fs and fs.read) then return {}, {} end
+  local available, state, declaredPaths = {}, {}, {}
+  for _, manifest in ipairs(manifests or {}) do
+    local rows, missing, missingOptional = {}, 0, 0
+    for _, spec in ipairs(allSpecs(manifest)) do
+      local path = RequiredImports.path(manifest, spec)
+      declaredPaths[path] = true
+      local data = fs.read(path)
+      local suppressed = fs.getInfo
+        and fs.getInfo(removedMarker(manifest, spec), "file") ~= nil
+      local normalized, detail
+      if data then
+        normalized, detail = RequiredImports.validateStoredData(spec, data, hashFn)
+        if normalized then available[detail] = normalized end
+      end
+      local row = { id = spec.id, name = spec.name, file = spec.file,
+        format = spec.format, path = path, present = normalized ~= nil,
+        digest = normalized and detail or nil,
+        error = data and not normalized and detail or nil,
+        suppressed = suppressed, required = isRequired(spec), spec = spec }
+      if not row.present then
+        if row.required then missing = missing + 1
+        else missingOptional = missingOptional + 1 end
+      end
+      rows[#rows + 1] = row
+    end
+    state[manifest.id] = { rows = rows, missing = missing,
+      missingOptional = missingOptional }
+  end
+
+
+  -- Compatibility with mods that already maintained their own baseroms
+  -- folder before this manifest field existed: index every other file in an
+  -- installed mod's folder by both raw and (when recognizable) canonical N64
+  -- MD5. Nothing outside the engine-owned mods tree is searched.
+  if fs.getDirectoryItems and fs.getInfo then
+    for _, manifest in ipairs(manifests or {}) do
+      local dir = manifest.path .. "/baseroms"
+      if fs.getInfo(dir, "directory") then
+        for _, name in ipairs(fs.getDirectoryItems(dir) or {}) do
+          local path = dir .. "/" .. name
+          if name:sub(1, 1) ~= "." and not declaredPaths[path]
+              and fs.getInfo(path, "file") then
+            local data = fs.read(path)
+            if data then
+              local rawDigest = hexDigest(data, hashFn)
+              if rawDigest then available[rawDigest] = data end
+              local canonical = RequiredImports.normalizeN64(data)
+              if canonical then
+                local canonicalDigest = hexDigest(canonical, hashFn)
+                if canonicalDigest then available[canonicalDigest] = canonical end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+
+  local copied = {}
+  for _, manifest in ipairs(manifests or {}) do
+    local entry = state[manifest.id]
+    for _, row in ipairs(entry.rows) do
+      if not row.present and not row.suppressed then
+        for _, digest in ipairs(row.spec.md5) do
+          local data = available[digest]
+          if data then
+            local ok = RequiredImports.importData(manifest, row.id, data,
+              { hash = hashFn })
+            if ok then
+              copied[#copied + 1] = { mod = manifest.id, import = row.id,
+                digest = digest }
+              row.present, row.digest, row.error = true, digest, nil
+              if row.required then entry.missing = entry.missing - 1
+              else entry.missingOptional = entry.missingOptional - 1 end
+            end
+            break
+          end
+        end
+      end
+    end
+  end
+  return copied, state
+end
+
+return RequiredImports