bazinga

src/mods/Job.lua

@@ -0,0 +1,209 @@
+-- Background compute for sandboxed mods, behind the "background" permission.
+--
+-- mod.fetch covers work that is waiting on a server.  This covers work that is
+-- waiting on the CPU: a mod hands over a script from its own folder plus a
+-- table of plain data, and gets the return value back through the same
+-- handle/poll/release shape mod.fetch uses.
+--
+-- The worker (src/mods/job_worker.lua) builds the SAME Sandbox.envFor
+-- environment the main thread does before it loads the mod's chunk, so this
+-- is not the love.thread hole reopened: the mod's code still cannot see
+-- io, os, debug, ffi, package or love.filesystem, and require is refused
+-- outright inside a job.
+--
+-- One thread per job rather than a pool.  A pooled state would carry one
+-- mod's globals into the next mod's job, and resetting it properly is the
+-- same work as making a new one.
+
+local SafePath = require("src.mods.SafePath")
+
+local Job = {}
+
+Job.MAX_INFLIGHT = 2       -- per mod
+Job.MAX_GLOBAL = 4         -- across all mods, so jobs cannot eat every core
+Job.DEFAULT_SECONDS = 5
+Job.MAX_SECONDS = 30
+-- Depth cap on the data crossing the channel.  A cycle is caught by the seen
+-- set; this catches the merely absurd.
+Job.MAX_DEPTH = 16
+
+local nextId = 0
+local liveGlobal = 0
+
+-- Only plain data crosses a thread boundary: a function or userdata cannot be
+-- serialised, and letting one through would fail deep inside LÖVE instead of
+-- at the call the mod made.
+local function plain(value, depth, seen)
+  local t = type(value)
+  if t == "nil" or t == "boolean" or t == "number" or t == "string" then
+    return value
+  end
+  if t ~= "table" then
+    return nil, ("a job cannot carry a %s, only plain data"):format(t)
+  end
+  depth = (depth or 0) + 1
+  if depth > Job.MAX_DEPTH then
+    return nil, "a job's data is nested too deeply"
+  end
+  seen = seen or {}
+  if seen[value] then return nil, "a job cannot carry a cycle" end
+  seen[value] = true
+  local out = {}
+  for k, v in pairs(value) do
+    local kt = type(k)
+    if kt ~= "string" and kt ~= "number" then
+      return nil, ("a job cannot carry a %s key"):format(kt)
+    end
+    local copied, err = plain(v, depth, seen)
+    if err then return nil, err end
+    out[k] = copied
+  end
+  seen[value] = nil
+  return out
+end
+Job.plain = plain
+
+function Job.available()
+  return (love and love.thread and love.thread.newThread) ~= nil
+end
+
+local function bucket(loader, modId)
+  loader.jobs = loader.jobs or {}
+  local b = loader.jobs[modId]
+  if not b then b = {}; loader.jobs[modId] = b end
+  return b
+end
+
+local function inflight(b)
+  local n = 0
+  for _, job in pairs(b) do
+    if job.status == "pending" then n = n + 1 end
+  end
+  return n
+end
+
+-- `script` is relative to the mod's own folder, and goes through the same
+-- SafePath rules mod:read does -- a job is not a way to name a path.
+-- Argument checks come BEFORE the host check: a bad path or an unserialisable
+-- argument is the mod author's bug and should read the same on every machine,
+-- not be masked into "unavailable" on a build without threads.
+function Job.run(loader, modId, modPath, script, arg, opts)
+  if type(script) ~= "string" or script == "" then
+    return nil, "a job needs a script path inside your mod"
+  end
+  -- SafePath.require raises rather than returning, so the mod's bad path
+  -- comes back as a value here instead of unwinding its caller.
+  local okPath, safe = pcall(SafePath.join, modPath, script, "a job script")
+  if not okPath then return nil, tostring(safe) end
+  local payload, dataErr = plain(arg)
+  if dataErr then return nil, dataErr end
+  if not Job.available() then return nil, "background jobs are unavailable" end
+
+  local b = bucket(loader, modId)
+  if inflight(b) >= Job.MAX_INFLIGHT then
+    return nil, ("too many jobs in flight (limit %d); poll and release the "
+      .. "ones you have"):format(Job.MAX_INFLIGHT)
+  end
+  if liveGlobal >= Job.MAX_GLOBAL then
+    return nil, "the machine is already running as many jobs as it will"
+  end
+
+  opts = type(opts) == "table" and opts or {}
+  local seconds = tonumber(opts.maxSeconds) or Job.DEFAULT_SECONDS
+  if seconds > Job.MAX_SECONDS then seconds = Job.MAX_SECONDS end
+  if seconds < 1 then seconds = 1 end
+
+  nextId = nextId + 1
+  local argName = "modjob_arg_" .. nextId
+  local resultName = "modjob_result_" .. nextId
+  local argCh = love.thread.getChannel(argName)
+  local resCh = love.thread.getChannel(resultName)
+  argCh:clear()
+  resCh:clear()
+  argCh:push(payload == nil and false or payload)
+
+  local okNew, thread = pcall(love.thread.newThread, "src/mods/job_worker.lua")
+  if not okNew or not thread then return nil, "could not start a job thread" end
+  local Json = require("src.link.Json")
+  local permissions = select(2, pcall(Json.encode,
+    loader.mods and loader.mods[modId]
+      and loader.mods[modId].manifest.permissionSet or {})) or "{}"
+  local started = pcall(thread.start, thread, modId, safe, argName, resultName,
+    permissions)
+  if not started then return nil, "could not start a job thread" end
+
+  liveGlobal = liveGlobal + 1
+  local handle = {}
+  b[handle] = { thread = thread, resultCh = resCh, status = "pending",
+                deadline = love.timer.getTime() + seconds, seconds = seconds }
+  return handle
+end
+
+local function settle(job, status, value, err)
+  if job.status == "pending" then liveGlobal = math.max(0, liveGlobal - 1) end
+  job.status, job.value, job.err = status, value, err
+end
+
+function Job.poll(loader, modId, handle)
+  local job = bucket(loader, modId)[handle]
+  if not job then return { status = "error", err = "unknown job" } end
+  if job.status == "pending" then
+    local msg = job.resultCh:pop()
+    if msg then
+      if msg.ok then settle(job, "ok", msg.result)
+      else settle(job, "error", nil, msg.err) end
+    else
+      -- A worker that died before pushing anything (an error outside its own
+      -- pcall) would otherwise leave the mod polling forever.
+      local threadErr = job.thread.getError and job.thread:getError()
+      if threadErr then
+        settle(job, "error", nil, tostring(threadErr))
+      elseif love.timer.getTime() > job.deadline then
+        -- The budget bounds how long the MOD waits, not how long the work
+        -- runs: there is no way to stop a LÖVE thread, and every in-worker
+        -- attempt made things worse (see job_worker.lua).  A job that
+        -- overruns is reported here and its result dropped if it ever lands.
+        settle(job, "error", nil, ("job exceeded its %gs budget")
+          :format(job.seconds))
+      end
+    end
+  end
+  if job.status == "ok" then
+    -- A copy, so a mod cannot edit what a later poll returns.
+    return { status = "ok", result = (plain(job.value)) }
+  end
+  return { status = job.status, err = job.err }
+end
+
+function Job.release(loader, modId, handle)
+  local b = bucket(loader, modId)
+  local job = b[handle]
+  if not job then return false end
+  if job.status == "pending" then liveGlobal = math.max(0, liveGlobal - 1) end
+  b[handle] = nil
+  return true
+end
+
+-- There is no way to kill a LÖVE thread, so cancelling drops the result
+-- rather than stopping the work; the worker's own time budget is what bounds
+-- how long an abandoned job can run.
+function Job.cancel(loader, modId, handle)
+  local job = bucket(loader, modId)[handle]
+  if not job then return false end
+  if job.status == "pending" then
+    settle(job, "cancelled")
+  end
+  return true
+end
+
+function Job.releaseAll(loader, modId)
+  local b = loader.jobs and loader.jobs[modId]
+  if not b then return end
+  for handle, job in pairs(b) do
+    if job.status == "pending" then liveGlobal = math.max(0, liveGlobal - 1) end
+    b[handle] = nil
+  end
+  loader.jobs[modId] = nil
+end
+
+return Job

src/mods/Loader.lua

@@ -23,6 +23,8 @@ local Hooks = require("src.mods.Hooks")
 local LegacyCompat = require("src.mods.LegacyCompat")
 local Runtime = require("src.mods.Runtime")
 local Steps = require("src.mods.Steps")
+local Net = require("src.mods.Net")
+local Job = require("src.mods.Job")
 
 local Loader = {}
 Loader.__index = Loader
@@ -1070,6 +1072,51 @@ function Loader:_api(mod)
       return { available = function() return false end,
                sync = refuse, poll = refuse }
     end)(),
+    -- Background HTTP, behind the "network" permission the player already
+    -- sees.  This is what love.thread is NOT: the worker runs engine code in
+    -- an engine-owned pool, so a mod gets asynchrony without getting a Lua
+    -- state the sandbox cannot reach.  get() hands back an opaque handle;
+    -- poll() is non-blocking, so nothing here can hang a frame.
+    fetch = (function()
+      if mod.manifest.permissionSet.network then
+        return {
+          available = function() return Net.available() end,
+          get = function(_, url, opts) return Net.get(loader, modId, url, opts) end,
+          poll = function(_, handle) return Net.poll(loader, modId, handle) end,
+          release = function(_, handle) return Net.release(loader, modId, handle) end,
+          cancel = function(_, handle) return Net.cancel(loader, modId, handle) end,
+        }
+      end
+      local function refuse()
+        error(('[%s] mod.fetch needs the "network" permission in '
+          .. "manifest.json"):format(modId), 2)
+      end
+      return { available = function() return false end,
+               get = refuse, poll = refuse, release = refuse, cancel = refuse }
+    end)(),
+    -- Background compute, behind the "background" permission.  The worker
+    -- rebuilds this mod's sandbox before loading the script, so a job is the
+    -- one thing love.thread is not: off the main thread without a Lua state
+    -- that escapes the sandbox.  Plain data in, plain data out.
+    job = (function()
+      if mod.manifest.permissionSet.background then
+        return {
+          available = function() return Job.available() end,
+          run = function(_, script, arg, opts)
+            return Job.run(loader, modId, mod.path, script, arg, opts)
+          end,
+          poll = function(_, handle) return Job.poll(loader, modId, handle) end,
+          release = function(_, handle) return Job.release(loader, modId, handle) end,
+          cancel = function(_, handle) return Job.cancel(loader, modId, handle) end,
+        }
+      end
+      local function refuse()
+        error(('[%s] mod.job needs the "background" permission in '
+          .. "manifest.json"):format(modId), 2)
+      end
+      return { available = function() return false end,
+               run = refuse, poll = refuse, release = refuse, cancel = refuse }
+    end)(),
     -- namespaced per mod; M11 backs these with save.modData /
     -- options.modOptions, the shape mods compile against is already final
     save = {
@@ -1342,6 +1389,8 @@ function Loader:_rollback(modId)
   self.migrations[modId] = nil
   self.modSave[modId] = nil
   self.stepsQueues[modId] = nil
+  Net.releaseAll(self, modId)
+  Job.releaseAll(self, modId)
 end
 
 -- a mod that explicitly swears it stays link-compatible while writing into a

src/mods/Manifest.lua

@@ -11,7 +11,8 @@ local Manifest = {}
 
 Manifest.PROFILES = { content = true, overhaul = true, total_conversion = true }
 Manifest.PERMISSIONS = { network = true, filesystem = true,
-                         engine_internals = true, steps = true }
+                         engine_internals = true, steps = true,
+                         background = true }
 
 -- link-relevant registries; a mod that writes into one of these while
 -- declaring affects_link = false gets an attributed warning from the loader

src/mods/Net.lua

@@ -0,0 +1,143 @@
+-- Background HTTP for sandboxed mods, behind the "network" permission.
+--
+-- The sandbox blocks love.thread because newThread boots a Lua state with a
+-- full standard library that none of the sandbox's rules reach -- one call and
+-- a mod has io back.  That is correct, but it left mods with no way to do
+-- anything off the main thread at all: the only reachable transports
+-- (socket, http) block, so a mod that wanted to fetch something had to hang
+-- the game to do it.
+--
+-- This is the narrow replacement.  src/net/Fetch.lua already runs a pool of
+-- engine-owned worker threads, and those workers run OUR code, not the mod's,
+-- so handing a mod a job in that pool grants no new reach.  A mod submits a
+-- URL and polls for the body; it never gets a thread, a path, or a raw handle
+-- into the shared job table.
+--
+-- WHAT THIS FILE HAS TO GET RIGHT, because Fetch itself is shared with the
+-- launcher:
+--   * Handles are opaque tables owned per mod.  Fetch keys jobs by integer,
+--     and the launcher's own ROM download and index fetches live in the same
+--     table; an integer handed to a mod would let it poll (or cancel) work
+--     that is not its own.  A forged table simply misses the lookup.
+--   * Only http and https.  The transport is curl, which also speaks file://,
+--     scp:// and ftp://; without this check mod.fetch would be a filesystem
+--     read and the sandbox would be back to square one.
+--   * A per-mod ceiling on jobs in flight, so one mod cannot fill the shared
+--     three-worker pool and starve the launcher's own fetches.
+
+local Net = {}
+
+-- Per mod, not global: the pool is shared with the launcher and a mod should
+-- never be able to monopolise it.
+Net.MAX_INFLIGHT = 4
+-- Clamp on the caller's timeout, so a mod cannot pin a worker indefinitely.
+Net.MAX_SECONDS = 30
+
+local function fetch()
+  return require("src.net.Fetch")
+end
+
+-- http/https only, and a host must actually be present -- "http://" alone
+-- reaches curl as a malformed URL rather than being refused here.
+function Net.urlDenial(url)
+  if type(url) ~= "string" or url == "" then return "url must be a string" end
+  local scheme, rest = url:match("^(%a[%w+.-]*)://(.*)$")
+  if not scheme then return "url must start with http:// or https://" end
+  scheme = scheme:lower()
+  if scheme ~= "http" and scheme ~= "https" then
+    return ("%s:// is not allowed; mod.fetch speaks http and https only")
+      :format(scheme)
+  end
+  if rest == "" or rest:match("^/") then return "url has no host" end
+  return nil
+end
+
+local function bucket(loader, modId)
+  loader.netJobs = loader.netJobs or {}
+  local b = loader.netJobs[modId]
+  if not b then b = {}; loader.netJobs[modId] = b end
+  return b
+end
+
+local function inflight(b)
+  local n = 0
+  for _, id in pairs(b) do
+    if fetch().isPending(id) then n = n + 1 end
+  end
+  return n
+end
+
+function Net.available()
+  local ok, F = pcall(fetch)
+  if not ok then return false end
+  local okAvail, avail = pcall(F.available)
+  return okAvail and avail and true or false
+end
+
+-- Returns an opaque handle, or nil plus a reason.
+function Net.get(loader, modId, url, opts)
+  local denial = Net.urlDenial(url)
+  if denial then return nil, denial end
+  opts = type(opts) == "table" and opts or {}
+  local b = bucket(loader, modId)
+  if inflight(b) >= Net.MAX_INFLIGHT then
+    return nil, ("too many requests in flight (limit %d); poll and release "
+      .. "the ones you have"):format(Net.MAX_INFLIGHT)
+  end
+  local maxSeconds = tonumber(opts.maxSeconds) or Net.MAX_SECONDS
+  if maxSeconds > Net.MAX_SECONDS then maxSeconds = Net.MAX_SECONDS end
+  if maxSeconds < 1 then maxSeconds = 1 end
+  -- The mod is named in the agent string so a server operator can see which
+  -- mod is calling them, and a mod cannot pretend to be the launcher.
+  local id = fetch().get(url, {
+    userAgent = "gen1recomp-mod/" .. tostring(modId),
+    accept = type(opts.accept) == "string" and opts.accept or nil,
+    maxSeconds = maxSeconds,
+  })
+  local handle = {}
+  b[handle] = id
+  return handle
+end
+
+-- A copy of the job's state, never the engine's own table.  An unknown or
+-- forged handle reads as an error rather than nil, so a mod that lost track of
+-- one cannot spin waiting on it forever.
+function Net.poll(loader, modId, handle)
+  local id = bucket(loader, modId)[handle]
+  if not id then return { status = "error", err = "unknown request" } end
+  local st = fetch().poll(id)
+  return { status = st.status, body = st.body, err = st.err,
+           progress = st.progress }
+end
+
+function Net.release(loader, modId, handle)
+  local b = bucket(loader, modId)
+  local id = b[handle]
+  if not id then return false end
+  fetch().release(id)
+  b[handle] = nil
+  return true
+end
+
+function Net.cancel(loader, modId, handle)
+  local id = bucket(loader, modId)[handle]
+  if not id then return false end
+  fetch().cancel(id)
+  return true
+end
+
+-- Drop everything this mod still holds.  Called when a mod unloads, so a
+-- disabled mod cannot leave jobs accumulating in the shared table.
+function Net.releaseAll(loader, modId)
+  local b = loader.netJobs and loader.netJobs[modId]
+  if not b then return end
+  local F = fetch()
+  for handle, id in pairs(b) do
+    pcall(F.cancel, id)
+    pcall(F.release, id)
+    b[handle] = nil
+  end
+  loader.netJobs[modId] = nil
+end
+
+return Net

src/mods/Sandbox.lua

@@ -68,7 +68,11 @@ end
 -- without an edit here.
 -- value is the replacement to name in the error, or true when there is none
 local BLOCKED_LOVE = {
-  filesystem = "mod.storage, mod:read and mod:list", thread = true,
+  filesystem = "mod.storage, mod:read and mod:list",
+  -- newThread's state has a full standard library and none of this file's
+  -- rules, so it stays blocked -- but the reason mods reached for it was
+  -- background work, and mod.fetch is that without the escape.
+  thread = 'mod.fetch for background HTTP (needs the "network" permission)',
   system = "mod.device:powerInfo() for battery information, mod.steps for "
     .. "the step bridge", event = true,
 }

src/mods/job_worker.lua

@@ -0,0 +1,73 @@
+-- Worker state behind src/mods/Job.lua.  One per job, not a pool: a reused
+-- state would carry one mod's globals into another mod's job.
+--
+-- This is the file that makes running mod Lua off the main thread safe.  The
+-- mod's chunk is loaded into the SAME sandbox environment the main thread
+-- builds (Sandbox.envFor), so love.filesystem, io, os, debug, ffi and package
+-- are as absent here as they are there -- even though this state required
+-- love.filesystem to bootstrap itself.
+--
+-- A job is pure compute: plain data in, plain data out, no engine API, no
+-- game state, no storage.  require is refused outright rather than reaching
+-- src.* -- an engine module loaded in a second state would be a second
+-- instance writing the same files as the main thread's.
+
+require("love.thread")
+require("love.filesystem")
+require("love.timer")
+
+local modId, scriptPath, argChannel, resultChannel, permissionsJson = ...
+
+-- Fresh love threads have no "src.*" searcher (see src/net/fetch_worker.lua),
+-- so install one before Sandbox's own requires run.
+table.insert(package.loaders or package.searchers, function(name)
+  local path = name:gsub("%.", "/") .. ".lua"
+  if not love.filesystem.getInfo(path) then return nil end
+  return love.filesystem.load(path)
+end)
+
+local resCh = love.thread.getChannel(resultChannel)
+
+local function fail(err)
+  resCh:push({ ok = false, err = tostring(err) })
+end
+
+local ok, err = pcall(function()
+  local Sandbox = require("src.mods.Sandbox")
+  local Json = require("src.link.Json")
+
+  local permissions = {}
+  if type(permissionsJson) == "string" and permissionsJson ~= "" then
+    local decoded = select(2, pcall(Json.decode, permissionsJson))
+    if type(decoded) == "table" then permissions = decoded end
+  end
+
+  local env = Sandbox.envFor({ modId = modId, permissions = permissions })
+  -- A job cannot reach the engine.  Anything it needs comes in through its
+  -- argument and goes back through its return value.
+  env.require = function(name)
+    error(("[%s] require(%q) is not available inside a background job; a job "
+      .. "takes plain data and returns plain data"):format(modId,
+      tostring(name)), 2)
+  end
+
+  local chunk, loadErr = Sandbox.loadFile(love.filesystem, scriptPath, env)
+  if not chunk then error(loadErr or ("could not load " .. scriptPath), 0) end
+
+  local arg = love.thread.getChannel(argChannel):pop()
+
+  -- NO in-worker time budget, deliberately.  A debug count hook was the
+  -- obvious way to stop a runaway, and it does not work: LuaJIT swallows an
+  -- error raised from a hook (measured: ~5000 raises a second, the loop
+  -- running straight through them), and the raising itself wedged the whole
+  -- process -- the main thread stopped being scheduled at all.  Without the
+  -- hook a runaway job simply spins on its own core, the game stays
+  -- responsive, and it quits normally.  Job.poll enforces maxSeconds on the
+  -- main thread so the MOD is never left waiting; the work itself runs to its
+  -- own end.
+  local ranOk, result = pcall(chunk, arg)
+  if not ranOk then error(result, 0) end
+  resCh:push({ ok = true, result = result })
+end)
+
+if not ok then fail(err) end