From b286e6584f6867235e6c3aa2bd3ac53fa82b87f2 Mon Sep 17 00:00:00 2001 From: MaxTomahawk Date: Fri, 21 Aug 2026 15:00:45 +0200 Subject: [PATCH 1/6] feat(mod-api): add imported dataset views --- docs/modding.md | 36 ++++ docs/rfcs/0012-imported-dataset-views.md | 113 ++++++++++ src/import/CacheFormat.lua | 22 ++ src/import/RomImporter.lua | 4 +- src/mods/DatasetViews.lua | 228 ++++++++++++++++++++ src/mods/Loader.lua | 9 + tests/modkit/cases/dataset_views.lua | 251 +++++++++++++++++++++++ 7 files changed, 661 insertions(+), 2 deletions(-) create mode 100644 docs/rfcs/0012-imported-dataset-views.md create mode 100644 src/import/CacheFormat.lua create mode 100644 src/mods/DatasetViews.lua create mode 100644 tests/modkit/cases/dataset_views.lua diff --git a/docs/modding.md b/docs/modding.md index 7c724404..f68ce215 100644 --- a/docs/modding.md +++ b/docs/modding.md @@ -200,6 +200,42 @@ the adapter's own coverage table: python3 tools/modkit.py gen2check mods/my_mod ``` +## Imported version datasets + +A mod can inspect semantic content from another game the player has already +imported without switching the active game or reaching into engine cache +internals: + +```lua +local gold, reason = mod.datasets:open("gold") +if not gold then + -- reason is "unknown_version" or "not_imported" + return +end + +local chikorita = gold.content.pokemon:get("CHIKORITA") +local normalVsGhost = gold.content.type_chart:get("NORMAL>GHOST") +local spritePath = gold.assets:path(chikorita.spriteFront) + +for id, record in gold.content.pokemon:each() do + -- ids are returned in deterministic lexical order +end +``` + +`view.version` and `view.generation` identify the selected dataset. +`view.content` exposes the same registry names and generation-specific record +shapes as `mod.content`, but only `get`, `has`, and `each`; returned +records are detached copies and cannot mutate either dataset. Each open call +receives an independent facade, so one mod cannot replace another mod view method. Engine-provided +semantic records, such as `NORMAL>GHOST` type matchups, are included. + +Only a cache carrying the current engine completion marker is opened. Missing +and stale imports both return `nil, "not_imported"`; no raw ROM bytes or +generated Lua modules are exposed. `view.assets:path(relative)` and +`view.assets:info(relative)` accept only `assets/generated/...` paths and +keep them under the selected version cache prefix. The API never changes +`mod.game`, the active `Data` table, `GameVersion`, or cache mount state. + ## Editing maps in Tiled Maps are data, not assets, so they can be authored in a real map editor and diff --git a/docs/rfcs/0012-imported-dataset-views.md b/docs/rfcs/0012-imported-dataset-views.md new file mode 100644 index 00000000..dc854e31 --- /dev/null +++ b/docs/rfcs/0012-imported-dataset-views.md @@ -0,0 +1,113 @@ +# RFC 0012: Read-only imported dataset views + +## Status + +Proposed. + +## Motivation + +A cross-version content mod can read only the active merged dataset through +`mod.content`. Even when the player has already imported another supported +game, a mod cannot inspect that version's semantic species, moves, items, type +chart, or generated asset namespace. The available alternatives are private: +mutating `CacheFs.prefix`, mounting another cache over the active one, loading +generated Lua directly, or asking for a second raw-ROM import. They compose +poorly with the active game, expose unstable import layout, and make safe +read-only use impossible from the sandbox. + +The immediate example is an optional content pack derived from an already +verified Gold import while Red, Blue, or Yellow remains active. The capability +is generic and useful to randomizers, compatibility inspectors, dex tools, and +other cross-version content mods. + +## Decision and plan extended + +This implements **D-AT-004: optional Kanto+ content consumes an +active-independent semantic dataset view**. The consuming design is tracked in +the Adaptive Trainers implementation plan, +[`docs/superpowers/plans/2026-08-14-adaptive-trainers.md`](https://github.com/MaxTomahawk/gen1recomp-adaptive-trainers/blob/main/docs/superpowers/plans/2026-08-14-adaptive-trainers.md), +Task 8. The delta is a generic, additive public API and contains no trainer +pools, scaling, boss identities, version-mixing rules, or Adaptive Trainers +policy. + +## Exact API delta + +Every sandboxed mod receives: + +```lua +local view, reason = mod.datasets:open("gold") +``` + +A known version with the current completed import marker returns a read-only view: + +```lua +view = { + version = "gold", + generation = 2, + content = { + pokemon = { + get = function(self, id) end, + has = function(self, id) end, + each = function(self) end, + }, + -- every public registry name and alias + }, + assets = { + path = function(self, generatedPath) end, + info = function(self, generatedPath) end, + }, +} +``` + +`get` returns a detached copy or nil. `has` reports semantic presence. +`each` returns ids in lexical order and detached values. The registries use +the selected version's generation routing and the engine's existing +`Schemas`, `Registry`, and `Builtins` normalization, so structured sources +such as type matchups retain the same public ids used by the active +`mod.content` facade. No register, patch, override, or remove verb is exposed. Each call returns an +independent facade over the cached internal dataset, so facade mutation cannot cross +mod boundaries. + +`assets:path` returns the selected cache-prefixed virtual path. +`assets:info` returns sanitized `type` and optional `size` metadata. Both +accept only relative paths below `assets/generated/`, reject control +characters, absolute paths, backslashes, and traversal, and expose no byte +reader. + +An unknown version returns `nil, "unknown_version"`. A missing or stale +completion marker returns `nil, "not_imported"`. Generated tables are parsed +under an empty environment and cached per version; the API never exposes raw +ROM bytes, generated source, host paths, or a cache mount. + +## Migration and compatibility + +Existing mods change nothing. `mod.datasets` is additive and requires no +permission. The service is allocated lazily on the first explicit +`mod.datasets:open` call. A boot with no mods, or with mods that do not call +it, performs no cross-version cache reads. + +Opening a view does not change `GameVersion`, `CacheFs.prefix`, the active +`Data` table, PhysFS mounts, save state, or the selected game's behavior. +Red, Blue, Yellow, Gold, and Silver keep their existing active data paths. + +The completion-marker format moves to the pure `CacheFormat` helper shared by +the importer and dataset service. The literal marker and import readiness +behavior are unchanged. + +## Verification + +- `tests/modkit/cases/dataset_views.lua` loads a sandboxed fixture mod through + the public API and covers Red, Blue, Yellow, and Gold independently. +- The test proves semantic registry normalization, deterministic iteration, + detached records, read-only facades, cross-mod facade isolation, + version-prefixed generated assets, + traversal rejection, stable failure reasons, and stale-marker rejection. +- The same test proves that cross-version reads do not change the active data, + active game, or cache prefix, and that a no-mod boot performs zero + cross-version cache reads. +- The existing importer and engine/modkit suites prove unchanged marker + readiness and no-mod behavior. + +## Deprecation etiquette + +Nothing is removed, renamed, superseded, or deprecated. diff --git a/src/import/CacheFormat.lua b/src/import/CacheFormat.lua new file mode 100644 index 00000000..54d3b926 --- /dev/null +++ b/src/import/CacheFormat.lua @@ -0,0 +1,22 @@ +-- Current completion-marker contract for generated ROM caches. +-- Pure and version-aware so read-only consumers can verify an import without +-- loading the launcher or changing CacheFs.prefix. + +local GameVersion = require("src.core.GameVersion") + +local CacheFormat = {} + +CacheFormat.PREFIX = "rom-cache-v10:" + +function CacheFormat.markerFor(version) + local info = GameVersion.VERSIONS[version] + if not info then return nil end + return CacheFormat.PREFIX .. info.sha1 +end + +function CacheFormat.matches(version, marker) + local expected = CacheFormat.markerFor(version) + return expected ~= nil and marker == expected +end + +return CacheFormat diff --git a/src/import/RomImporter.lua b/src/import/RomImporter.lua index aa6ea4cf..9d5d2f39 100644 --- a/src/import/RomImporter.lua +++ b/src/import/RomImporter.lua @@ -1,4 +1,5 @@ local GameVersion = require("src.core.GameVersion") +local CacheFormat = require("src.import.CacheFormat") local GamepadMap = require("src.core.GamepadMap") local Logger = require("src.core.Logger") local Strings = require("src.core.Strings") @@ -41,7 +42,6 @@ end -- the stage, which is what the Yellow markers below already do for #439/#557. -- Reach for a bump when the change spans versions or has no single file to -- point at. -local CACHE_FORMAT = "rom-cache-v10:" -- The completion marker is written under each version's cache prefix -- (red/rom-cache.complete, blue/rom-cache.complete, ...). local MARKER_PATH = "rom-cache.complete" @@ -49,7 +49,7 @@ local MARKER_PATH = "rom-cache.complete" -- The marker a finished import writes for a version: the generation tag plus -- that version's ROM hash, so both a format bump and a swapped ROM invalidate. local function markerFor(version) - return CACHE_FORMAT .. GameVersion.info(version).sha1 + return CacheFormat.markerFor(version) end local COMMUNITY_URL = "https://bois.icu" local TRUST_WARNING = "if you did not get this from bryanthaboi's github " .. diff --git a/src/mods/DatasetViews.lua b/src/mods/DatasetViews.lua new file mode 100644 index 00000000..8576df96 --- /dev/null +++ b/src/mods/DatasetViews.lua @@ -0,0 +1,228 @@ +-- Read-only views over verified, version-scoped generated datasets. +-- +-- A view reads semantic registry records from another imported version without +-- changing GameVersion, CacheFs.prefix, the active Data table, or any PhysFS +-- mount. Records are detached copies and generated assets stay under the +-- selected version's virtual cache prefix. + +local CacheFormat = require("src.import.CacheFormat") +local GameVersion = require("src.core.GameVersion") +local Builtins = require("src.mods.Builtins") +local Merge = require("src.mods.Merge") +local Registry = require("src.mods.Registry") +local Schemas = require("src.mods.Schemas") + +local DatasetViews = {} +DatasetViews.__index = DatasetViews + +local GEN1_MODULES = { + constants = "constants", maps = "maps", tilesets = "tilesets", + text = "text", text_pointers = "text_pointers", + trainer_headers = "trainer_headers", font = "font", sprites = "sprites", + pokemon = "pokemon", moves = "moves", items = "items", + type_chart = "type_chart", trainers = "trainers", + encounters = "encounters", field = "field", + battle_anims = "battle_anims", audio = "audio", + palettes = "palettes", icons = "icons", +} + +local GEN2_MODULES = { + pokemon = "pokemon", moves = "moves", items = "items", + type_chart = "type_chart", audio = "audio", font = "font", + gen2Maps = "maps", gen2Tilesets = "tilesets", gen2Text = "text", + gen2Trainers = "trainers", gen2Encounters = "encounters", + gen2Sprites = "sprites", gen2Palettes = "palettes", + gen2Icons = "icons", gen2BattleAnims = "battle_anims", + gen2Constants = "constants", gen2Landmarks = "landmarks", +} + +local function compileTable(source, chunkname) + if type(source) ~= "string" or source:byte(1) == 27 then return nil end + local env = {} + local chunk, err + if setfenv then + chunk, err = loadstring(source, chunkname) + if chunk then setfenv(chunk, env) end + else + chunk, err = load(source, chunkname, "t", env) + end + if not chunk then return nil, err end + local ok, value = pcall(chunk) + if not ok or type(value) ~= "table" then + return nil, ok and "generated module did not return a table" or value + end + return value +end + +local function resolvePath(root, suffix) + local node = root + for key in suffix:gmatch("[^.]+") do + if type(node) ~= "table" then return nil end + node = node[key] + end + return node +end + +local function assetRelative(path) + if type(path) ~= "string" or path == "" then + error("dataset asset path is required", 3) + end + if path:find("\\", 1, true) or path:sub(1, 1) == "/" + or path:find("[%z\1-\31]") then + error("dataset asset path must be a generated relative path", 3) + end + if path:sub(1, 17) ~= "assets/generated/" then + error("dataset assets are limited to assets/generated/", 3) + end + for segment in path:gmatch("[^/]+") do + if segment == "." or segment == ".." then + error("dataset asset path may not traverse directories", 3) + end + end + return path +end + +function DatasetViews.new(fs) + assert(fs and fs.read, "DatasetViews.new requires a readable filesystem") + return setmetatable({ fs = fs, datasets = {} }, DatasetViews) +end + +function DatasetViews:_module(view, root) + local moduleName = view.modules[root] + if not moduleName then return nil end + local cached = view.moduleCache[moduleName] + if cached ~= nil then return cached or nil end + local path = view.prefix .. "data/generated/" .. moduleName .. ".lua" + local source = self.fs.read(path) + local value = compileTable(source, "@" .. path) + view.moduleCache[moduleName] = value or false + return value +end + +function DatasetViews:_data(view) + if view.data then return view.data end + local data = {} + for root in pairs(view.modules) do + local value = self:_module(view, root) + if value ~= nil then data[root] = value end + end + view.data = data + return data +end + +local function registryBase(data, target) + return function() + return resolvePath(data, target) + end +end + +function DatasetViews:_registries(view) + if view.registries then return view.registries end + local data = self:_data(view) + local registries = {} + for name, catalogSpec in pairs(Schemas.REGISTRIES) do + local spec = Schemas.shapeFor(name, catalogSpec, view.generation) + local registry = Registry.new(name, spec) + if spec.target then registry.base = registryBase(data, spec.target) end + registries[name] = registry + end + Builtins.install(registries, data, view.generation) + for _, registry in pairs(registries) do registry:freeze() end + view.registries = registries + return registries +end + +function DatasetViews:_registry(view, name) + local registry = self:_registries(view)[name] + + local function valueAt(id) + local value = registry and registry:get(id) + if value == nil then return nil end + return Merge.deepCopy(value) + end + + return { + get = function(_, id) + if type(id) ~= "string" or id == "" then return nil end + return valueAt(id) + end, + has = function(_, id) + return valueAt(id) ~= nil + end, + each = function() + local ids = {} + if registry then + for id in registry:each() do + if type(id) == "string" then ids[#ids + 1] = id end + end + end + table.sort(ids) + local index = 0 + return function() + index = index + 1 + local id = ids[index] + if id == nil then return nil end + return id, valueAt(id) + end + end, + } +end + +function DatasetViews:_assets(view) + local service = self + local assets = {} + + function assets:path(path) + return view.prefix .. assetRelative(path) + end + + function assets:info(path) + local full = self:path(path) + local info = service.fs.getInfo and service.fs.getInfo(full, "file") + if not info then return nil end + local out = { type = info.type } + if info.size ~= nil then out.size = info.size end + return out + end + + return assets +end + +function DatasetViews:open(version) + if type(version) ~= "string" or not GameVersion.VERSIONS[version] then + return nil, "unknown_version" + end + local internal = self.datasets[version] + if not internal then + local prefix = GameVersion.cachePrefix(version) + local marker = self.fs.read(prefix .. "rom-cache.complete") + if not CacheFormat.matches(version, marker) then + return nil, "not_imported" + end + + local generation = GameVersion.generation(version) + internal = { + version = version, generation = generation, prefix = prefix, + modules = generation == 2 and GEN2_MODULES or GEN1_MODULES, + moduleCache = {}, + } + internal.registries = self:_registries(internal) + self.datasets[version] = internal + end + local view = { + version = version, + generation = internal.generation, + content = {}, + } + for name in pairs(Schemas.REGISTRIES) do + view.content[name] = self:_registry(internal, name) + end + for alias, canonical in pairs(Schemas.ALIASES) do + view.content[alias] = view.content[canonical] + end + view.assets = self:_assets(internal) + + return view +end + +return DatasetViews diff --git a/src/mods/Loader.lua b/src/mods/Loader.lua index 84b087cd..3dbba98e 100644 --- a/src/mods/Loader.lua +++ b/src/mods/Loader.lua @@ -983,6 +983,15 @@ function Loader:_api(mod) -- a deep copy: what a mod does to its own view never reaches the loader manifest = Merge.deepCopy(mod.manifest), content = {}, + datasets = { + open = function(_, version) + if not loader.datasetViews then + local DatasetViews = engineRequire("src.mods.DatasetViews") + loader.datasetViews = DatasetViews.new(loader.fs) + end + return loader.datasetViews:open(version) + end, + }, exports = {}, DELETE = Registry.DELETE, events = { diff --git a/tests/modkit/cases/dataset_views.lua b/tests/modkit/cases/dataset_views.lua new file mode 100644 index 00000000..59edb614 --- /dev/null +++ b/tests/modkit/cases/dataset_views.lua @@ -0,0 +1,251 @@ +-- Cross-version imported datasets through the public sandboxed mod API. +-- The view is semantic and read-only: registry-shaped records in, detached +-- copies out, with generated assets kept inside the selected version namespace. + +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") +local GameVersion = require("src.core.GameVersion") +local CacheFs = require("src.import.CacheFs") +local CacheFormat = require("src.import.CacheFormat") + +local function serialized(value) + local function encode(v) + if type(v) == "string" then return string.format("%q", v) end + if type(v) == "number" or type(v) == "boolean" then return tostring(v) end + local keys = {} + for key in pairs(v) do keys[#keys + 1] = key end + table.sort(keys, function(a, b) return tostring(a) < tostring(b) end) + local out = { "{" } + for _, key in ipairs(keys) do + out[#out + 1] = "[" .. encode(key) .. "]=" .. encode(v[key]) .. "," + end + out[#out + 1] = "}" + return table.concat(out) + end + return "return " .. encode(value) +end + +local files = { + ["mods/dataset_probe/manifest.json"] = [[{ + "id": "dataset_probe", + "name": "Dataset Probe", + "version": "1.0.0", + "entry": "main.lua", + "api": 2, + "games": ["all"] + }]], + ["mods/dataset_probe/main.lua"] = [[ +local mod = ... +local out = {} +for _, version in ipairs({ "red", "blue", "yellow", "gold" }) do + local view, reason = mod.datasets:open(version) + if not view then + out[version] = { reason = reason } + else + local registry = view.content.pokemon + local first = registry:get("FIXMON") + first.name = "MUTATED" + local ids = {} + for id in registry:each() do ids[#ids + 1] = id end + local immune = view.content.type_chart:get("NORMAL>GHOST") + local steel = view.content.type_chart:get("STEEL") + out[version] = { + version = view.version, + generation = view.generation, + name = registry:get("FIXMON").name, + has = registry:has("FIXMON"), + ids = ids, + writable = registry.register ~= nil or registry.patch ~= nil + or registry.override ~= nil or registry.remove ~= nil, + sprite = view.assets:path(registry:get("FIXMON").spriteFront), + spriteInfo = view.assets:info(registry:get("FIXMON").spriteFront), + normalGhost = immune and immune.multiplier, + steelCategory = steel and steel.category, + } + end +end +local missing, missingReason = mod.datasets:open("silver") +local unknown, unknownReason = mod.datasets:open("crystal") +out.silver = { present = missing ~= nil, reason = missingReason } +out.crystal = { present = unknown ~= nil, reason = unknownReason } +local gold = mod.datasets:open("gold") +out.assetEscape = pcall(function() gold.assets:path("../save.lua") end) +out.rawAssetRead = gold.assets.read ~= nil +mod.exports.result = out + ]], +} + +local isolationMods = { + ["mods/dataset_mutator/manifest.json"] = [[{ + "id": "dataset_mutator", + "name": "Dataset Mutator", + "version": "1.0.0", + "entry": "main.lua", + "api": 2, + "games": ["all"] + }]], + ["mods/dataset_mutator/main.lua"] = [[ +local mod = ... +local view = assert(mod.datasets:open("red")) +view.content.pokemon.get = function() return { name = "POISONED" } end + ]], + ["mods/dataset_observer/manifest.json"] = [[{ + "id": "dataset_observer", + "name": "Dataset Observer", + "version": "1.0.0", + "entry": "main.lua", + "api": 2, + "games": ["all"], + "dependencies": ["dataset_mutator"] + }]], + ["mods/dataset_observer/main.lua"] = [[ +local mod = ... +local view = assert(mod.datasets:open("red")) +mod.exports.name = view.content.pokemon:get("FIXMON").name + ]], +} + +local labels = { + red = "RED SOURCE", blue = "BLUE SOURCE", + yellow = "YELLOW SOURCE", gold = "GOLD SOURCE", +} + +for _, version in ipairs({ "red", "blue", "yellow", "gold" }) do + local prefix = GameVersion.cachePrefix(version) + files[prefix .. "rom-cache.complete"] = + "rom-cache-v10:" .. GameVersion.info(version).sha1 + files[prefix .. "data/generated/pokemon.lua"] = serialized({ + FIXMON = { + id = "FIXMON", name = labels[version], dex = version == "gold" and 252 or 152, + spriteFront = "assets/generated/battle/front/fixmon.png", + }, + ALPHA = { id = "ALPHA", name = "ALPHA", dex = 1, + spriteFront = "assets/generated/battle/front/alpha.png" }, + }) + files[prefix .. "data/generated/type_chart.lua"] = serialized({ + matchups = { { attacker = "NORMAL", defender = "GHOST", multiplier = 0 } }, + types = version == "gold" and { + STEEL = { name = "STEEL", category = "physical", index = 9 }, + } or {}, + }) + files[prefix .. "assets/generated/battle/front/fixmon.png"] = "png-" .. version +end + +-- An otherwise plausible Silver tree has no completion marker. A view must +-- not execute or expose it. +files[GameVersion.cachePrefix("silver") .. "data/generated/pokemon.lua"] = + serialized({ FIXMON = { id = "FIXMON", name = "UNVERIFIED" } }) + +T.eq(CacheFormat.markerFor("red"), + "rom-cache-v10:" .. GameVersion.info("red").sha1, + "cache marker follows the importer format and version hash") +T.eq(CacheFormat.matches("silver", "rom-cache-v9:stale"), false, + "stale cache formats are rejected") +T.eq(CacheFormat.markerFor("crystal"), nil, + "unknown versions have no valid cache marker") + +local originalVersion = GameVersion.get() +local originalPrefix = CacheFs.prefix +GameVersion.set("red") + +local function isVersionCachePath(path) + for _, version in ipairs(GameVersion.ORDER) do + local prefix = GameVersion.cachePrefix(version) + if path:sub(1, #prefix) == prefix then return true end + end + return false +end + +-- No mod means the new service is completely cold. Count only cross-version +-- cache reads so the loader's ordinary discovery/state reads do not matter. +local vanillaFiles = {} +for path, body in pairs(files) do + if not path:match("^mods/") then + vanillaFiles[path] = body + end +end +local vanillaFs = T.sdk.memfs(vanillaFiles) +local vanillaReads = 0 +local vanillaRead = vanillaFs.read +vanillaFs.read = function(path) + if isVersionCachePath(path) then + vanillaReads = vanillaReads + 1 + end + return vanillaRead(path) +end +local vanillaData = { pokemon = { ACTIVE = { id = "ACTIVE", name = "ACTIVE" } } } +local vanilla = T.sdk.loadNone({ fs = vanillaFs, data = vanillaData }) +T.eq(#vanilla.errors, 0, "no-mod loader remains clean") +T.eq(vanillaReads, 0, "no mod performs no imported-dataset reads") +T.eq(vanillaData.pokemon.ACTIVE.name, "ACTIVE", + "no mod leaves the active dataset unchanged") +vanilla.release() + +local activeData = { pokemon = { ACTIVE = { id = "ACTIVE", name = "ACTIVE" } } } +local run = T.sdk.loadMods({ "mods/dataset_probe" }, { + fs = T.sdk.memfs(files), data = activeData, generation = 1, +}) +T.eq(#run.errors, 0, + "sandboxed dataset probe loads clean: " .. tostring(run.errors[1])) +local out = run.loader.exports.dataset_probe + and run.loader.exports.dataset_probe.result or {} + +for _, version in ipairs({ "red", "blue", "yellow", "gold" }) do + local got = out[version] or {} + T.eq(got.version, version, version .. " view reports its selected version") + T.eq(got.generation, version == "gold" and 2 or 1, + version .. " view reports the selected generation") + T.eq(got.name, labels[version], + version .. " returns its own semantic species record") + T.eq(got.has, true, version .. " registry has() reads the imported view") + T.same(got.ids, { "ALPHA", "FIXMON" }, + version .. " registry each() is deterministic") + T.eq(got.writable, false, version .. " registry exposes no write verbs") + T.eq(got.sprite, + GameVersion.cachePrefix(version) .. "assets/generated/battle/front/fixmon.png", + version .. " asset path stays in its cache namespace") + T.same(got.spriteInfo, { type = "file" }, + version .. " asset info is sanitized metadata") + T.eq(got.normalGhost, 0, + version .. " exposes structured matchup ids semantically") +end +T.eq(out.gold and out.gold.steelCategory, "physical", + "Gold exposes imported Steel as a semantic type record") + +T.same(out.silver, { present = false, reason = "not_imported" }, + "an unverified cache is unavailable") +T.same(out.crystal, { present = false, reason = "unknown_version" }, + "an unknown version fails with a stable reason") +T.eq(out.assetEscape, false, "dataset asset paths reject traversal") +T.eq(out.rawAssetRead, false, "dataset assets expose no raw byte reader") +T.eq(activeData.pokemon.ACTIVE.name, "ACTIVE", + "cross-version reads do not mutate the active game dataset") +T.eq(activeData.pokemon.FIXMON, nil, + "cross-version records are not merged into active game data") +T.eq(GameVersion.get(), "red", "dataset reads do not switch the active game") +T.eq(CacheFs.prefix, originalPrefix, "dataset reads do not change cache prefix state") + +run.release() + +local isolationFiles = {} +for path, body in pairs(files) do + if not path:match("^mods/") then + isolationFiles[path] = body + end +end +for path, body in pairs(isolationMods) do isolationFiles[path] = body end +local isolationRun = T.sdk.loadMods({ + "mods/dataset_mutator", "mods/dataset_observer", +}, { fs = T.sdk.memfs(isolationFiles), + data = { pokemon = {} }, generation = 1 }) +T.eq(#isolationRun.errors, 0, "cross-mod isolation fixture loads cleanly") +T.eq(isolationRun.loader.exports.dataset_observer + and isolationRun.loader.exports.dataset_observer.name, + labels.red, "one mod cannot mutate another mod dataset facade") +isolationRun.release() + +GameVersion.set(originalVersion) +CacheFs.prefix = originalPrefix + +T.finish("dataset_views") From ca700c44d7047dc089d8ff77cec0b803b0b00489 Mon Sep 17 00:00:00 2001 From: MaxTomahawk Date: Mon, 24 Aug 2026 09:33:51 +0200 Subject: [PATCH 2/6] fix(mod-api): harden imported dataset views --- docs/modding.md | 24 +- docs/modding/reference/registries.md | 3 +- ...iews.md => 0015-imported-dataset-views.md} | 57 +- src/core/Data.lua | 16 +- src/core/DatasetHydration.lua | 29 + src/core/Game2.lua | 6 +- src/core/SaveSerializer.lua | 85 ++- src/import/CacheContract.lua | 160 ++++++ src/import/CacheFs.lua | 19 +- src/import/RomImporter.lua | 183 +------ src/mods/Builtins.lua | 10 +- src/mods/DatasetViews.lua | 270 +++++++--- src/mods/Loader.lua | 2 +- tests/engine/dataset_views_no_mod_parity.lua | 22 + tests/engine/generated_data_decoder_test.lua | 37 ++ .../engine/rom_importer_source_tree_test.lua | 50 +- tests/engine/trade_art_import.lua | 18 +- tests/modkit/cases/dataset_views.lua | 497 ++++++++++-------- .../cases/dataset_views_nontermination.lua | 21 + tests/modkit/dataset_view_fixture.lua | 121 +++++ 20 files changed, 1051 insertions(+), 579 deletions(-) rename docs/rfcs/{0012-imported-dataset-views.md => 0015-imported-dataset-views.md} (56%) create mode 100644 src/core/DatasetHydration.lua create mode 100644 src/import/CacheContract.lua create mode 100644 tests/engine/dataset_views_no_mod_parity.lua create mode 100644 tests/engine/generated_data_decoder_test.lua create mode 100644 tests/modkit/cases/dataset_views_nontermination.lua create mode 100644 tests/modkit/dataset_view_fixture.lua diff --git a/docs/modding.md b/docs/modding.md index f68ce215..16e28741 100644 --- a/docs/modding.md +++ b/docs/modding.md @@ -209,7 +209,7 @@ internals: ```lua local gold, reason = mod.datasets:open("gold") if not gold then - -- reason is "unknown_version" or "not_imported" + -- reason is "unknown_version", "not_imported", or "invalid_cache" return end @@ -223,15 +223,21 @@ end ``` `view.version` and `view.generation` identify the selected dataset. -`view.content` exposes the same registry names and generation-specific record -shapes as `mod.content`, but only `get`, `has`, and `each`; returned -records are detached copies and cannot mutate either dataset. Each open call -receives an independent facade, so one mod cannot replace another mod view method. Engine-provided -semantic records, such as `NORMAL>GHOST` type matchups, are included. +`view.content` exposes the same registry names, aliases, generation routing, +and data-only record shapes as `mod.content`, but only `get`, `has`, and +`each`. Returned records are detached copies and cannot mutate either dataset. +Records containing functions, userdata, threads, metatables, or cycles are not +exposed. Each open call receives an independent facade, so one mod cannot +replace another mod view method. Canonical boot shaping is included, such as +Gen 1 defaults and Yellow corrections, and Gold's Foresight matchup rows and +derived `held_items`. -Only a cache carrying the current engine completion marker is opened. Missing -and stale imports both return `nil, "not_imported"`; no raw ROM bytes or -generated Lua modules are exposed. `view.assets:path(relative)` and +The marker and every required generated module for the selected version are +rechecked on each open. Missing, partial, and stale imports return +`nil, "not_imported"`; malformed or resource-limit-breaking generated data +returns `nil, "invalid_cache"`. Generated modules are decoded with a bounded +literal-only grammar and are never executed. No raw ROM bytes or generated +source are exposed. `view.assets:path(relative)` and `view.assets:info(relative)` accept only `assets/generated/...` paths and keep them under the selected version cache prefix. The API never changes `mod.game`, the active `Data` table, `GameVersion`, or cache mount state. diff --git a/docs/modding/reference/registries.md b/docs/modding/reference/registries.md index 929299f7..be3ed622 100644 --- a/docs/modding/reference/registries.md +++ b/docs/modding/reference/registries.md @@ -342,6 +342,7 @@ accepted and merged as-is. | `source` | string | | `swarmGrass` | map of string -> {map?, rates, slots} | | `swarmWater` | map of string -> {map?, rate, slots} | +| `timeFishGroups` | map of string | integer 0..255 -> {day, nite} | | `treeSets` | map of string -> {common, rare} | | `trees` | map of string -> string | | `water` | map of string -> {map?, rate, slots} | @@ -598,7 +599,7 @@ mod.content.map_songs:override("PALLET_TOWN", "Music_Routes1") | `id` | string | yes | | `index` | integer >= 0 | no | | `label` | string | no | -| `objects` | list of any value | no | +| `objects` | list of {pokemon?, ...} | no | | `palette` | string | no | | `signs` | list of any value | no | | `tileset` | tilesets id | yes | diff --git a/docs/rfcs/0012-imported-dataset-views.md b/docs/rfcs/0015-imported-dataset-views.md similarity index 56% rename from docs/rfcs/0012-imported-dataset-views.md rename to docs/rfcs/0015-imported-dataset-views.md index dc854e31..b1aa7c88 100644 --- a/docs/rfcs/0012-imported-dataset-views.md +++ b/docs/rfcs/0015-imported-dataset-views.md @@ -1,4 +1,4 @@ -# RFC 0012: Read-only imported dataset views +# RFC 0015: Read-only imported dataset views ## Status @@ -15,10 +15,15 @@ generated Lua directly, or asking for a second raw-ROM import. They compose poorly with the active game, expose unstable import layout, and make safe read-only use impossible from the sandbox. -The immediate example is an optional content pack derived from an already -verified Gold import while Red, Blue, or Yellow remains active. The capability -is generic and useful to randomizers, compatibility inspectors, dex tools, and -other cross-version content mods. +The concrete consumer is **Adaptive Trainers**, whose approved Phase G Kanto+ +sidecar must derive nine Kanto-line continuations, Steel/type and move +definitions, and generated sprites from the player's existing verified Gold +cache while Red, Blue, or Yellow remains active. `mod.content` exposes only the +active R/B/Y dataset; `mod.imports` can read only separately declared raw mod +imports; and `mod.cache` is mod-private generated output. None can inspect the +launcher-owned Gold semantic dataset without a second ROM import and mod-side +ROM interpretation. The engine API remains generic and contains no Adaptive +Trainers policy. ## Decision and plan extended @@ -59,7 +64,8 @@ view = { } ``` -`get` returns a detached copy or nil. `has` reports semantic presence. +`get` returns a bounded detached data-only copy or nil. `has` reports data-only +semantic presence. `each` returns ids in lexical order and detached values. The registries use the selected version's generation routing and the engine's existing `Schemas`, `Registry`, and `Builtins` normalization, so structured sources @@ -74,10 +80,16 @@ accept only relative paths below `assets/generated/`, reject control characters, absolute paths, backslashes, and traversal, and expose no byte reader. -An unknown version returns `nil, "unknown_version"`. A missing or stale -completion marker returns `nil, "not_imported"`. Generated tables are parsed -under an empty environment and cached per version; the API never exposes raw -ROM bytes, generated source, host paths, or a cache mount. +An unknown version returns `nil, "unknown_version"`. A missing, partial, or +stale cache returns `nil, "not_imported"`. A required module that is malformed +or exceeds the limits (8 MiB per module, 48 MiB aggregate, depth 64, 500,000 +values, 2 MiB per string, or 250,000 entries per table) returns +`nil, "invalid_cache"`; actionable detail is engine-logged but not exposed to +the mod. Generated Lua is decoded with the existing restricted +literal grammar and never executed. Functions, userdata, threads, metatables, +cycles, non-table roots, binary chunks, and trailing syntax cannot cross the +facade. Successful roots are decoded lazily and cached per selected version. +The API never exposes raw ROM bytes, generated source, host paths, or a mount. ## Migration and compatibility @@ -90,23 +102,28 @@ Opening a view does not change `GameVersion`, `CacheFs.prefix`, the active `Data` table, PhysFS mounts, save state, or the selected game's behavior. Red, Blue, Yellow, Gold, and Silver keep their existing active data paths. -The completion-marker format moves to the pure `CacheFormat` helper shared by -the importer and dataset service. The literal marker and import readiness -behavior are unchanged. +The completion marker and per-version required-file rules live in the pure, +injected `CacheContract` shared by the importer and dataset service. It also +defines source-tree behavior. Neither consumer mutates `CacheFs.prefix` while +checking readiness. Every `open` revalidates the contract and required module +shapes; a stale/remove/reimport transition evicts the previous semantic view. ## Verification -- `tests/modkit/cases/dataset_views.lua` loads a sandboxed fixture mod through - the public API and covers Red, Blue, Yellow, and Gold independently. +- `tests/modkit/cases/dataset_views.lua` loads sandboxed fixture mods through + the public API and covers Red, Blue, Yellow, Gold, and Silver independently. - The test proves semantic registry normalization, deterministic iteration, detached records, read-only facades, cross-mod facade isolation, version-prefixed generated assets, traversal rejection, stable failure reasons, and stale-marker rejection. -- The same test proves that cross-version reads do not change the active data, - active game, or cache prefix, and that a no-mod boot performs zero - cross-version cache reads. -- The existing importer and engine/modkit suites prove unchanged marker - readiness and no-mod behavior. +- It also proves missing/empty/partial/stale/remove/reimport behavior, hostile + generated-source rejection, canonical Gen 1/Yellow/Gold hydration, and the + approved Kanto+ Gold records and assets. +- `tests/modkit/cases/dataset_views_nontermination.lua` proves generated code + is rejected rather than executed; `tests/engine/generated_data_decoder_test.lua` + proves every decoder resource bound. +- `tests/engine/dataset_views_no_mod_parity.lua` is the separate guarded no-mod + parity suite and proves the service stays unallocated with zero cache reads. ## Deprecation etiquette diff --git a/src/core/Data.lua b/src/core/Data.lua index 31d7932a..935739a9 100644 --- a/src/core/Data.lua +++ b/src/core/Data.lua @@ -82,8 +82,9 @@ local function copy(value) return out end -function Data:applyVersionedFieldData() - if require("src.core.GameVersion").isYellow() then +function Data:applyVersionedFieldData(version) + version = version or require("src.core.GameVersion").get() + if version == "yellow" then self.field.trades = copy(YELLOW_TRADES) -- The old man's catch demo is a RATTATA in Yellow -- (scripts/ViridianCity.asm ViridianCityOldManStartCatchTrainingScript @@ -106,7 +107,8 @@ end -- Fills only what the cache is missing, so an importer that learns to -- stamp one of these keys silently takes over from the engine. -function Data:seedDefaults() +function Data:seedDefaults(version) + version = version or require("src.core.GameVersion").get() local constants = self.constants or {} self.constants = constants self.field = self.field or {} @@ -131,7 +133,7 @@ function Data:seedDefaults() if constants.dexDigits == nil then constants.dexDigits = math.max(3, #tostring(constants.dexSize)) end - self:applyVersionedFieldData() + Data.applyVersionedFieldData(self, version) local boot = self.field.boot if boot == nil then boot = {} @@ -144,7 +146,7 @@ function Data:seedDefaults() -- only the un-overridden default flips, so a total conversion that set -- field.boot.screens.splash keeps its choice on any version. if boot.screens.splash == BOOT_DEFAULTS.screens.splash - and require("src.core.GameVersion").isYellow() then + and version == "yellow" then boot.screens.splash = "YellowIntro" end -- the naming screen presets the importer already extracts but nothing @@ -163,11 +165,11 @@ function Data:seedDefaults() -- extractor never writes headers for them. Seed the EVENT_BEAT_* / -- after-battle rows so Blaine's SetEventRange deactivation and talk -- after-text work like the other gyms (scripts/CinnabarGym.asm). - self:seedCinnabarGymTrainerHeaders() + Data.seedCinnabarGymTrainerHeaders(self) -- #197: the Fighting Dojo Karate Master is text_asm, so the extractor -- writes no header for him -- seed one so he engages on sight and has -- his defeat / re-talk lines (same idea as the Cinnabar seed above). - self:seedFightingDojoKarateMaster() + Data.seedFightingDojoKarateMaster(self) -- #189: 1F cabin door order vs rooms map (survey zoom) require("src.world.SsAnneLayout").apply(self.maps) end diff --git a/src/core/DatasetHydration.lua b/src/core/DatasetHydration.lua new file mode 100644 index 00000000..003cc3bf --- /dev/null +++ b/src/core/DatasetHydration.lua @@ -0,0 +1,29 @@ +-- Canonical pure shaping shared by the active boot and inactive dataset views. +-- The caller supplies both the data table and selected version; no global +-- GameVersion, cache prefix, mount, or active Data table is changed. + +local GameVersion = require("src.core.GameVersion") + +local DatasetHydration = {} + +function DatasetHydration.applyGen2(data, moduleLoader) + local chart = data.type_chart or {} + chart.matchups = chart.matchups or {} + for _, row in ipairs(chart.foresightMatchups or {}) do + chart.matchups[#chart.matchups + 1] = row + end + data.type_chart = chart + data.gen2HeldItems = + (moduleLoader or require)("src.core.gen2.ItemEffects").heldItemsFrom(data.items) + return data +end + +function DatasetHydration.apply(data, version, moduleLoader) + if GameVersion.generation(version) == 2 then + return DatasetHydration.applyGen2(data, moduleLoader) + end + require("src.core.Data").seedDefaults(data, version) + return data +end + +return DatasetHydration diff --git a/src/core/Game2.lua b/src/core/Game2.lua index 4ab8a479..c0821818 100644 --- a/src/core/Game2.lua +++ b/src/core/Game2.lua @@ -894,10 +894,7 @@ function Game2:load() self.data.type_chart = loadGenerated("data/generated/type_chart.lua") or {} -- data/types/type_matchups.asm:112-116: the rows after the `db -2` marker -- apply by default; Foresight is what cuts the table short at it. - local chart = self.data.type_chart - for _, row in ipairs(chart.foresightMatchups or {}) do - chart.matchups[#chart.matchups + 1] = row - end + require("src.core.DatasetHydration").applyGen2(self.data) -- The `held_items` registry's merge target: ItemAttributes' last two columns -- as their own table, so a mod can give an item a held behaviour without -- owning the whole item record. Built BEFORE mods:load so the registry @@ -906,7 +903,6 @@ function Game2:load() -- changed from what a mod reached through the shared `items` registry -- instead. Both halves live in src/core/gen2/ItemEffects.lua. local ItemEffects = require("src.core.gen2.ItemEffects") - self.data.gen2HeldItems = ItemEffects.heldItemsFrom(self.data.items) local heldBefore = ItemEffects.heldSnapshot(self.data.gen2HeldItems) -- Gen 2-only tables the menus read. Namespaced so nothing collides with the -- Gen 1 keys of the same idea (data.palettes, data.icons). diff --git a/src/core/SaveSerializer.lua b/src/core/SaveSerializer.lua index 9f821f67..7adbae86 100644 --- a/src/core/SaveSerializer.lua +++ b/src/core/SaveSerializer.lua @@ -70,9 +70,27 @@ local function fail(state, why) error(("parse error at byte %d: %s"):format(state.pos, why), 0) end +local function limit(state, name) + return state.limits and state.limits[name] +end + +local function bumpNode(state) + state.nodes = state.nodes + 1 + local maximum = limit(state, "maxNodes") + if maximum and state.nodes > maximum then fail(state, "too many values") end +end + local function skip(state) - local _, last = state.src:find("^[ \t\r\n]*", state.pos) - state.pos = last + 1 + while true do + local _, last = state.src:find("^[ \t\r\n]*", state.pos) + state.pos = last + 1 + if not (state.limits and state.limits.allowComments + and state.src:sub(state.pos, state.pos + 1) == "--") then + return + end + local newline = state.src:find("\n", state.pos + 2, true) + state.pos = newline and newline + 1 or (#state.src + 1) + end end local function peek(state) @@ -90,7 +108,10 @@ local function readString(state) fail(state, "unterminated string") elseif c == '"' then state.pos = i + 1 - return table.concat(out) + local value = table.concat(out) + local maximum = limit(state, "maxStringBytes") + if maximum and #value > maximum then fail(state, "string too long") end + return value elseif c == "\\" then local nxt = src:sub(i + 1, i + 1) if nxt:match("%d") then @@ -136,10 +157,14 @@ end local readValue local function readTable(state) + bumpNode(state) state.depth = state.depth + 1 - if state.depth > MAX_DEPTH then fail(state, "table nesting too deep") end + if state.depth > (limit(state, "maxDepth") or MAX_DEPTH) then + fail(state, "table nesting too deep") + end state.pos = state.pos + 1 local out = {} + local entries, nextArray = 0, 1 skip(state) if peek(state) == "}" then state.pos = state.pos + 1 @@ -148,7 +173,7 @@ local function readTable(state) end while true do skip(state) - local key + local key, value local c = peek(state) if c == "[" then state.pos = state.pos + 1 @@ -156,15 +181,38 @@ local function readTable(state) skip(state) if peek(state) ~= "]" then fail(state, "expected ]") end state.pos = state.pos + 1 + skip(state) + if peek(state) ~= "=" then fail(state, "expected =") end + state.pos = state.pos + 1 + value = readValue(state) elseif c:match("[%a_]") then - key = readIdent(state) + local start = state.pos + local ident = readIdent(state) + skip(state) + if peek(state) == "=" then + key = ident + state.pos = state.pos + 1 + value = readValue(state) + elseif state.limits and state.limits.allowArray then + state.pos = start + key = nextArray + value = readValue(state) + else + fail(state, "expected =") + end + elseif state.limits and state.limits.allowArray then + key = nextArray + value = readValue(state) else fail(state, "expected key") end - skip(state) - if peek(state) ~= "=" then fail(state, "expected =") end - state.pos = state.pos + 1 - out[key] = readValue(state) + if key == nil then fail(state, "nil table key") end + if out[key] ~= nil then fail(state, "duplicate table key") end + entries = entries + 1 + local maximum = limit(state, "maxTableEntries") + if maximum and entries > maximum then fail(state, "too many table entries") end + out[key] = value + if type(key) == "number" and key == nextArray then nextArray = nextArray + 1 end skip(state) local sep = peek(state) if sep == "," then @@ -189,25 +237,30 @@ readValue = function(state) skip(state) local c = peek(state) if c == '"' then + bumpNode(state) return readString(state) elseif c == "{" then return readTable(state) elseif c:match("[%a_]") then -- the only bare words in the grammar are the boolean literals local word = readIdent(state) - if word == "true" then return true end - if word == "false" then return false end + if word == "true" then bumpNode(state); return true end + if word == "false" then bumpNode(state); return false end state.pos = state.pos - #word fail(state, "unexpected name '" .. word .. "'") elseif c:match("[%-%d%.]") then + bumpNode(state) return readNumber(state) end fail(state, c == "" and "unexpected end of input" or "unexpected character") end -function SaveSerializer.decode(str) +function SaveSerializer.decode(str, limits) if type(str) ~= "string" then return nil, "save must be a string" end - local state = { src = str, pos = 1, depth = 0 } + if limits and limits.maxBytes and #str > limits.maxBytes then + return nil, ("input is %d bytes (max %d)"):format(#str, limits.maxBytes) + end + local state = { src = str, pos = 1, depth = 0, nodes = 0, limits = limits } local ok, result = pcall(function() skip(state) local word = state.src:match("^[%a_][%w_]*", state.pos) @@ -219,7 +272,9 @@ function SaveSerializer.decode(str) return value end) if not ok then return nil, result end - if type(result) ~= "table" then return nil, "save root must be a table" end + if type(result) ~= "table" then + return nil, (limits and limits.rootName or "save") .. " root must be a table" + end return result end diff --git a/src/import/CacheContract.lua b/src/import/CacheContract.lua new file mode 100644 index 00000000..9b8d6413 --- /dev/null +++ b/src/import/CacheContract.lua @@ -0,0 +1,160 @@ +-- Pure version-scoped readiness rules shared by the importer and read-only +-- dataset consumers. Callers inject a filesystem; no global cache prefix or +-- active game state is changed while inspecting another version. + +local CacheFormat = require("src.import.CacheFormat") +local GameVersion = require("src.core.GameVersion") + +local CacheContract = {} + +CacheContract.MARKER_PATH = "rom-cache.complete" + +local REQUIRED_FILES = { + "data/generated/constants.lua", "data/generated/maps.lua", + "data/generated/text.lua", "data/generated/field.lua", + "data/generated/battle_anims.lua", + "assets/generated/title/pokemon_logo.png", + "assets/generated/fonts/font.png", + "assets/generated/battle/front/pikachu.png", + "assets/generated/battle/anims/move_anim_0.png", + "assets/generated/battle/anims/move_anim_1.png", + "assets/generated/audio/programs.bin", + "assets/generated/trade/game_boy.png", +} + +local VERSION_REQUIRED_FILES = { + yellow = { + "assets/generated/battle/trainers/jessie_james.png", + "assets/generated/battle/profoakb.png", + "assets/generated/pikachu/pikapic_1.png", + }, +} + +local GOLD_REQUIRED_FILES = { + "data/generated/constants.lua", "data/generated/maps.lua", + "data/generated/roofs.lua", "data/generated/sprites.lua", + "data/generated/scripts.lua", "data/generated/text.lua", + "data/generated/rom_text.lua", + "data/generated/pokemon.lua", "data/generated/tilesets.lua", + "data/generated/audio.lua", "data/generated/marts.lua", + "assets/generated/fonts/font.png", "assets/generated/fonts/frames.png", + "assets/generated/title/pokemon_logo.png", + "assets/generated/title/title_screen.png", "assets/generated/title/hooh.png", + "assets/generated/title/hooh_5.png", "assets/generated/title/clouds.png", + "assets/generated/title/copyright_splash.png", + "data/generated/oak_speech.lua", "assets/generated/intro/oak.png", + "assets/generated/intro/cal.png", "assets/generated/tilesets/johto.png", + "assets/generated/tilesets/roofs/new_bark.png", + "assets/generated/sprites/chris.png", + "assets/generated/battle/front/chikorita.png", + "assets/generated/battle/front/pikachu.png", + "assets/generated/battle/front/marill.png", + "assets/generated/battle/trainers/falkner.png", + "assets/generated/battle/hud/balls.png", + "assets/generated/audio/programs.bin", + "assets/generated/slots/gold_slots_1.png", + "assets/generated/card_flip/card_flip_1.png", + "assets/generated/pc/mail_item.png", +} + +local SEMANTIC_MODULES = { + [1] = { + "constants", "maps", "tilesets", "text", "text_pointers", + "trainer_headers", "font", "sprites", "pokemon", "moves", "items", + "type_chart", "trainers", "encounters", "field", "battle_anims", + }, + [2] = { + "pokemon", "moves", "items", "type_chart", "audio", "font", "maps", + "tilesets", "text", "trainers", "encounters", "sprites", "palettes", + "icons", "battle_anims", "constants", "landmarks", + }, +} + +local OPTIONAL_SEMANTIC_MODULES = { [1] = { "audio", "palettes", "icons" }, [2] = {} } + +local function copy(values) + local out = {} + for index, value in ipairs(values or {}) do out[index] = value end + return out +end + +function CacheContract.requiredFiles(version, semantic) + local files + if version == "gold" or version == "silver" then + files = copy(GOLD_REQUIRED_FILES) + else + files = copy(REQUIRED_FILES) + for _, path in ipairs(VERSION_REQUIRED_FILES[version] or {}) do + files[#files + 1] = path + end + end + if semantic then + local generation = GameVersion.generation(version) + for _, name in ipairs(SEMANTIC_MODULES[generation] or {}) do + files[#files + 1] = "data/generated/" .. name .. ".lua" + end + end + local unique, out = {}, {} + for _, path in ipairs(files) do + if not unique[path] then unique[path], out[#out + 1] = true, path end + end + return out +end + +function CacheContract.semanticModules(version) + return copy(SEMANTIC_MODULES[GameVersion.generation(version)]) +end + +function CacheContract.optionalSemanticModules(version) + return copy(OPTIONAL_SEMANTIC_MODULES[GameVersion.generation(version)]) +end + +local function isFile(fs, path) + local info = fs.getInfo and fs.getInfo(path, "file") + if info then return info.type == nil or info.type == "file" end + return false +end + +local function hasFiles(version, fs, prefix, semantic) + for _, path in ipairs(CacheContract.requiredFiles(version, semantic)) do + if not isFile(fs, prefix .. path) then return false, path end + end + return true +end + +local function sourcePrefix(version) + return version == "red" and "" or GameVersion.cachePrefix(version) +end + +local function sourceReady(version, fs, semantic) + if not (fs.getRealDirectory and fs.getSource) then return nil end + local prefix = sourcePrefix(version) + local ok = hasFiles(version, fs, prefix, semantic) + if not ok then return nil end + local first = prefix .. CacheContract.requiredFiles(version, semantic)[1] + if fs.getRealDirectory(first) ~= fs.getSource() then return nil end + return { kind = "source", prefix = prefix } +end + +function CacheContract.inspect(version, fs, opts) + opts = opts or {} + if not (GameVersion.VERSIONS[version] and fs and fs.read and fs.getInfo) then + return nil, "not_imported", "unsupported cache inspection" + end + if opts.allowSource then + local source = sourceReady(version, fs, opts.semantic) + if source then return source end + end + local prefix = GameVersion.cachePrefix(version) + local marker = fs.read(prefix .. CacheContract.MARKER_PATH) + if not CacheFormat.matches(version, marker) then + return nil, "not_imported", "completion marker is missing or stale" + end + local complete, missing = hasFiles(version, fs, prefix, opts.semantic) + if not complete then + return nil, "not_imported", "required cache file is missing: " .. missing + end + return { kind = "cache", prefix = prefix, marker = marker } +end + +return CacheContract diff --git a/src/import/CacheFs.lua b/src/import/CacheFs.lua index 4b86d5cb..deedb517 100644 --- a/src/import/CacheFs.lua +++ b/src/import/CacheFs.lua @@ -324,9 +324,9 @@ function CacheFs.openWrite(rel) return file end --- read cache-relative `rel`; returns the bytes or nil -function CacheFs.read(rel) - rel = withPrefix(rel) +-- Read an exact version-qualified path without consulting CacheFs.prefix. +-- Readiness checks use this to inspect another version without global state. +function CacheFs.readAt(rel) local root = CacheFs.root() if root then local f = io.open(realPath(root, rel), "rb") @@ -341,6 +341,11 @@ function CacheFs.read(rel) return love.filesystem.read(rel) end +-- read cache-relative `rel`; returns the bytes or nil +function CacheFs.read(rel) + return CacheFs.readAt(withPrefix(rel)) +end + -- Read cache-relative `rel` for the active GameVersion when PhysFS may hide -- prefixed Blue/Yellow/Gold trees (fused NX mount hole). Same order Data:load -- already used: active version prefix with CacheFs.prefix cleared, then @@ -386,8 +391,7 @@ function CacheFs.loadActive(rel) end -- does cache-relative `rel` exist as a file? -function CacheFs.exists(rel) - rel = withPrefix(rel) +function CacheFs.existsAt(rel) local root = CacheFs.root() if root then local f = io.open(realPath(root, rel), "rb") @@ -398,6 +402,11 @@ function CacheFs.exists(rel) return love.filesystem.getInfo(rel, "file") ~= nil end + +function CacheFs.exists(rel) + return CacheFs.existsAt(withPrefix(rel)) +end + -- remove a single cache-relative file function CacheFs.remove(rel) rel = withPrefix(rel) diff --git a/src/import/RomImporter.lua b/src/import/RomImporter.lua index 9d5d2f39..0e1dfc76 100644 --- a/src/import/RomImporter.lua +++ b/src/import/RomImporter.lua @@ -1,5 +1,6 @@ local GameVersion = require("src.core.GameVersion") local CacheFormat = require("src.import.CacheFormat") +local CacheContract = require("src.import.CacheContract") local GamepadMap = require("src.core.GamepadMap") local Logger = require("src.core.Logger") local Strings = require("src.core.Strings") @@ -38,13 +39,13 @@ end -- the same unbootable save as before. -- Deliberately NOT bumped for the Gold trainer-pic gap: this tag invalidates -- every version at once, and that gap is Gold-only. A per-version marker in --- VERSION_REQUIRED_FILES_OVERRIDE.gold re-imports exactly the caches that lack --- the stage, which is what the Yellow markers below already do for #439/#557. +-- CacheContract's Gold required-file list re-imports exactly the caches that +-- lack the stage, the same per-version mechanism used for Yellow #439/#557. -- Reach for a bump when the change spans versions or has no single file to -- point at. -- The completion marker is written under each version's cache prefix -- (red/rom-cache.complete, blue/rom-cache.complete, ...). -local MARKER_PATH = "rom-cache.complete" +local MARKER_PATH = CacheContract.MARKER_PATH -- The marker a finished import writes for a version: the generation tag plus -- that version's ROM hash, so both a format bump and a swapped ROM invalidate. @@ -56,109 +57,6 @@ local TRUST_WARNING = "if you did not get this from bryanthaboi's github " .. "or a link from the discord that bryanthaboi himself posted, just know " .. "it might have been tampered with. go to the discord to verify " .. COMMUNITY_URL .. " (or click the logo above)" -local REQUIRED_FILES = { - "data/generated/constants.lua", - "data/generated/maps.lua", - "data/generated/text.lua", - "data/generated/field.lua", - "data/generated/battle_anims.lua", - "assets/generated/title/pokemon_logo.png", - "assets/generated/fonts/font.png", - "assets/generated/battle/front/pikachu.png", - "assets/generated/battle/anims/move_anim_0.png", - "assets/generated/battle/anims/move_anim_1.png", - "assets/generated/audio/programs.bin", - -- The trade cinematic's Game Boy / cable art. Caches built before #750 - -- carry none of it and fall back to plain rectangles, so listing one of - -- the files re-imports them without a CACHE_FORMAT bump. - "assets/generated/trade/game_boy.png", -} - --- Files only one version's cache carries. A version that predates one of --- them re-imports on its own, without dragging the other versions through a --- CACHE_FORMAT bump. -local VERSION_REQUIRED_FILES = { - yellow = { - "assets/generated/battle/trainers/jessie_james.png", -- #439 - -- Oak's own back pic and the pikapic base frames only exist in caches - -- built after their manifest symbols landed, so an older Yellow cache - -- has to re-import to stop falling back to the old man's back pic and - -- to the battle front pic (#557, #561). Both are gated on manifest - -- symbols in RomExtractor, so these markers must only ever list files - -- tools/rom_manifest_yellow.json can actually produce -- otherwise the - -- cache reads as incomplete and re-importing cannot clear it. - "assets/generated/battle/profoakb.png", - "assets/generated/pikachu/pikapic_1.png", - }, -} - --- Gold Phase 1 writes a thinner cache than Gen 1 (no battle anim sheets, --- trade art, or field.lua payload yet -- see docs/gold-phase1.md). This --- list replaces REQUIRED_FILES entirely for that version so a successful --- Gen 2 extract is not stuck as "incomplete" waiting on Gen 1 markers. -local VERSION_REQUIRED_FILES_OVERRIDE = { - gold = { - "data/generated/constants.lua", - "data/generated/maps.lua", - "data/generated/roofs.lua", -- Phase 2: forces re-import of Phase 1 caches - "data/generated/sprites.lua", -- OW sheets (Chris + NPCs) - "data/generated/scripts.lua", -- disassembled map scripts - "data/generated/text.lua", -- decoded Gen 2 dialogue strings - -- The engine's own strings, keyed by label rather than by address. A - -- cache built before RomExtractorGen2:extractText has none, and every - -- line that reads through src/core/RomText.lua would silently keep - -- printing its Lua fallback, so this re-imports those caches rather than - -- bumping CACHE_FORMAT and dragging Red, Blue and Yellow through it too. - "data/generated/rom_text.lua", - "data/generated/pokemon.lua", - "data/generated/tilesets.lua", - "data/generated/audio.lua", - -- Mart shelves + the heal machine art ride the same import, so listing - -- marts.lua alone re-imports the caches from before either existed - -- (empty shop shelves, no Pokecenter light show). - "data/generated/marts.lua", - "assets/generated/fonts/font.png", - "assets/generated/fonts/frames.png", -- the seven other OPTION textbox frames - "assets/generated/title/pokemon_logo.png", - "assets/generated/title/title_screen.png", -- TitleScreenTilemap composition - "assets/generated/title/hooh.png", - "assets/generated/title/hooh_5.png", -- wing-flap frames force re-import - "assets/generated/title/clouds.png", - "assets/generated/title/copyright_splash.png", - "data/generated/oak_speech.lua", -- Oak texts + trainer pics - "assets/generated/intro/oak.png", - "assets/generated/intro/cal.png", - "assets/generated/tilesets/johto.png", - "assets/generated/tilesets/roofs/new_bark.png", - "assets/generated/sprites/chris.png", - "assets/generated/battle/front/chikorita.png", - "assets/generated/battle/front/pikachu.png", - "assets/generated/battle/front/marill.png", -- Oak speech demo mon - -- The trainer class pics (TrainerPicPointers). FALKNER is row 0 of that - -- table, so a cache that produced any class pic at all produced this one. - -- Listed for the reason the Yellow markers above are: a cache built before - -- the stage existed reads as INCOMPLETE and re-imports itself, so this - -- particular gap cannot survive a tag bump being forgotten again. It - -- costs nothing on a current cache and is the difference between every - -- trainer battle opening with a picture and opening with none. - "assets/generated/battle/trainers/falkner.png", - -- BattleStart_TrainerHuds cannot draw its party rows from a cache made - -- before the four ball tiles were extracted (#1502). - "assets/generated/battle/hud/balls.png", - "assets/generated/audio/programs.bin", - -- Goldenrod Game Corner reel + board art (#1581). menu_gfx.lua used to - -- advertise these paths even when Slots*LZ / CardFlip* were absent from - -- the manifest, so a cache that never wrote the PNGs still looked - -- complete and SlotMachine crashed on its labelled-cell fallback. - "assets/generated/slots/gold_slots_1.png", - "assets/generated/card_flip/card_flip_1.png", - -- PCMailGFX (engine/pokemon/bills_pc.asm:2170-2173) - "assets/generated/pc/mail_item.png", - }, -} --- Same Gen 2 extract, so a Silver cache is complete when the same files exist. -VERSION_REQUIRED_FILES_OVERRIDE.silver = VERSION_REQUIRED_FILES_OVERRIDE.gold - -- "Split-screen ROM selector" first-run palette (matches the FirstRun mockup): -- a dark neon arcade panel, one column per game. -- Red, Blue, and Yellow share the same importer flow once listed in @@ -211,58 +109,6 @@ local PAL = { chipInkGold = { 58, 44, 0 }, -- #3a2c00 dark "Y" on the gold chip } --- Per-version required cache files. Gold replaces the Gen 1 list entirely --- (VERSION_REQUIRED_FILES_OVERRIDE); Yellow adds a few extra markers. -local function requiredFilesFor(version) - local override = VERSION_REQUIRED_FILES_OVERRIDE[version] - if override then return override, true end - return REQUIRED_FILES, false -end - --- CacheFs.exists checks the game folder directly for a portable install, --- otherwise the save directory through love.filesystem. It honors --- CacheFs.prefix, so we point it at the version's cache subtree (red/, --- blue/, yellow/, gold/). -local function allRequiredFilesExist(version) - local CacheFs = require("src.import.CacheFs") - local saved = CacheFs.prefix - CacheFs.prefix = GameVersion.cachePrefix(version) - local ok = true - local required, isOverride = requiredFilesFor(version) - for _, path in ipairs(required) do - if not CacheFs.exists(path) then ok = false; break end - end - if ok and not isOverride then - for _, path in ipairs(VERSION_REQUIRED_FILES[version] or {}) do - if not CacheFs.exists(path) then ok = false; break end - end - end - CacheFs.prefix = saved - return ok -end - --- A developer checkout / Python build leaves generated data in the physfs --- source: Red at the historical root, Blue/Yellow/Gold in their versioned --- trees. Imported Red caches still live under red/. Check source paths --- directly so that cache prefix cannot hide Red's source tree, and keep --- save-dir caches from counting as current source data. -local function sourceTreeHasData(version) - if not love.filesystem.getRealDirectory then return false end - local prefix = version == "red" and "" or GameVersion.cachePrefix(version) - local required, isOverride = requiredFilesFor(version) - for _, path in ipairs(required) do - if love.filesystem.getInfo(prefix .. path, "file") == nil then return false end - end - if not isOverride then - for _, path in ipairs(VERSION_REQUIRED_FILES[version] or {}) do - if love.filesystem.getInfo(prefix .. path, "file") == nil then return false end - end - end - local path = prefix .. required[1] - local real = love.filesystem.getRealDirectory(path) - return real == love.filesystem.getSource() -end - -- ------- ROM cache location -- -- The extracted cache (data/generated, assets/generated) plus the @@ -318,7 +164,8 @@ local function purgeSaveDirCache() -- / yellow/ / gold/ prefix) so it cannot shadow the portable game-folder cache. for _, version in ipairs(GameVersion.ORDER) do local prefix = GameVersion.cachePrefix(version) - if saveDirHas(prefix .. MARKER_PATH) or saveDirHas(prefix .. REQUIRED_FILES[1]) then + local probe = CacheContract.requiredFiles(version)[1] + if saveDirHas(prefix .. MARKER_PATH) or saveDirHas(prefix .. probe) then removeTree(prefix .. "data/generated") removeTree(prefix .. "assets/generated") love.filesystem.remove(prefix .. MARKER_PATH) @@ -336,13 +183,17 @@ function RomImporter.isReady(version) -- save-directory copy that would otherwise shadow it at runtime. purgeSaveDirCache() end - -- Generated data in a developer checkout / Python build is always current. - if sourceTreeHasData(version) then return true end - local saved = CacheFs.prefix - CacheFs.prefix = GameVersion.cachePrefix(version) - local marker = CacheFs.read(MARKER_PATH) - CacheFs.prefix = saved - return marker == markerFor(version) and allRequiredFilesExist(version) + local backing = { + read = CacheFs.readAt, + getInfo = function(path) + return CacheFs.existsAt(path) and { type = "file" } or nil + end, + } + if love and love.filesystem then + backing.getRealDirectory = love.filesystem.getRealDirectory + backing.getSource = love.filesystem.getSource + end + return CacheContract.inspect(version, backing, { allowSource = true }) ~= nil end function RomImporter.syncAndroidShortcuts(activeVersion) diff --git a/src/mods/Builtins.lua b/src/mods/Builtins.lua index 66695a17..b5cd4143 100644 --- a/src/mods/Builtins.lua +++ b/src/mods/Builtins.lua @@ -163,8 +163,8 @@ end -- a module the build dropped disables its registry rather than the game: -- the consumer still reads its own table, so vanilla keeps working -local function load(path) - local ok, module = pcall(require, path) +local function load(path, moduleLoader) + local ok, module = pcall(moduleLoader or require, path) if ok then return module end Logger.warn("builtin registrations skipped for %s (%s)", path, tostring(module)) return nil @@ -187,14 +187,14 @@ local function isolate(registry) }, { __index = registry }) end -function Builtins.install(content, data, generation) +function Builtins.install(content, data, generation, moduleLoader) for _, entry in ipairs(registrantsFor(generation)) do local registry = content[entry.name] and isolate(content[entry.name]) if registry then if entry.install then local modules, complete = {}, true for i, path in ipairs(entry.modules) do - modules[i] = load(path) + modules[i] = load(path, moduleLoader) if modules[i] == nil then complete = false end end -- data is the fourth argument, not the second, so the existing @@ -204,7 +204,7 @@ function Builtins.install(content, data, generation) entry.install(registry, modules, Builtins.OWNER, data) end else - local module = load(entry.from) + local module = load(entry.from, moduleLoader) -- entry.fn names the entry point for a module that owns more than one -- registry (Gold's Battle owns statuses and move_effects); the default -- is the registerInto every single-registry module exposes diff --git a/src/mods/DatasetViews.lua b/src/mods/DatasetViews.lua index 8576df96..eb0364b9 100644 --- a/src/mods/DatasetViews.lua +++ b/src/mods/DatasetViews.lua @@ -1,21 +1,32 @@ --- Read-only views over verified, version-scoped generated datasets. --- --- A view reads semantic registry records from another imported version without --- changing GameVersion, CacheFs.prefix, the active Data table, or any PhysFS --- mount. Records are detached copies and generated assets stay under the --- selected version's virtual cache prefix. +-- Read-only, bounded views over version-scoped generated datasets. -local CacheFormat = require("src.import.CacheFormat") +local CacheContract = require("src.import.CacheContract") +local DatasetHydration = require("src.core.DatasetHydration") local GameVersion = require("src.core.GameVersion") +local Logger = require("src.core.Logger") +local SaveSerializer = require("src.core.SaveSerializer") local Builtins = require("src.mods.Builtins") -local Merge = require("src.mods.Merge") local Registry = require("src.mods.Registry") local Schemas = require("src.mods.Schemas") local DatasetViews = {} DatasetViews.__index = DatasetViews -local GEN1_MODULES = { +-- A generated module is ROM-sized data, never an arbitrary program. These +-- caps bound both one malicious file and the aggregate work one open performs. +local DECODE_LIMITS = { + allowArray = true, + allowComments = true, + maxBytes = 8 * 1024 * 1024, + maxDepth = 64, + maxNodes = 500000, + maxStringBytes = 2 * 1024 * 1024, + maxTableEntries = 250000, + rootName = "generated data", +} +local MAX_AGGREGATE_BYTES = 48 * 1024 * 1024 + +local GEN1_ROOTS = { constants = "constants", maps = "maps", tilesets = "tilesets", text = "text", text_pointers = "text_pointers", trainer_headers = "trainer_headers", font = "font", sprites = "sprites", @@ -26,7 +37,7 @@ local GEN1_MODULES = { palettes = "palettes", icons = "icons", } -local GEN2_MODULES = { +local GEN2_ROOTS = { pokemon = "pokemon", moves = "moves", items = "items", type_chart = "type_chart", audio = "audio", font = "font", gen2Maps = "maps", gen2Tilesets = "tilesets", gen2Text = "text", @@ -36,22 +47,8 @@ local GEN2_MODULES = { gen2Constants = "constants", gen2Landmarks = "landmarks", } -local function compileTable(source, chunkname) - if type(source) ~= "string" or source:byte(1) == 27 then return nil end - local env = {} - local chunk, err - if setfenv then - chunk, err = loadstring(source, chunkname) - if chunk then setfenv(chunk, env) end - else - chunk, err = load(source, chunkname, "t", env) - end - if not chunk then return nil, err end - local ok, value = pcall(chunk) - if not ok or type(value) ~= "table" then - return nil, ok and "generated module did not return a table" or value - end - return value +local function decode(source) + return SaveSerializer.decode(source, DECODE_LIMITS) end local function resolvePath(root, suffix) @@ -82,38 +79,124 @@ local function assetRelative(path) return path end -function DatasetViews.new(fs) - assert(fs and fs.read, "DatasetViews.new requires a readable filesystem") - return setmetatable({ fs = fs, datasets = {} }, DatasetViews) +local function copyData(value, state, depth) + state = state or { seen = {}, nodes = 0 } + state.nodes = state.nodes + 1 + if state.nodes > DECODE_LIMITS.maxNodes then return nil, false end + local kind = type(value) + if kind == "nil" or kind == "boolean" or kind == "number" then return value end + if kind == "string" then + if #value > DECODE_LIMITS.maxStringBytes then return nil, false end + return value + end + if kind ~= "table" or getmetatable(value) ~= nil then return nil, false end + depth = (depth or 0) + 1 + if depth > DECODE_LIMITS.maxDepth or state.seen[value] then return nil, false end + state.seen[value] = true + local out, entries = {}, 0 + for key, child in pairs(value) do + entries = entries + 1 + if entries > DECODE_LIMITS.maxTableEntries then return nil, false end + local keyCopy, keyOk = copyData(key, state, depth) + local childCopy, childOk = copyData(child, state, depth) + if keyOk == false or childOk == false or keyCopy == nil then return nil, false end + out[keyCopy] = childCopy + end + state.seen[value] = nil + return out, true +end + +local function isDataOnly(value, state, depth) + state = state or { seen = {}, nodes = 0 } + state.nodes = state.nodes + 1 + if state.nodes > DECODE_LIMITS.maxNodes then return false end + local kind = type(value) + if kind == "nil" or kind == "boolean" or kind == "number" then return true end + if kind == "string" then return #value <= DECODE_LIMITS.maxStringBytes end + if kind ~= "table" or getmetatable(value) ~= nil then return false end + depth = (depth or 0) + 1 + if depth > DECODE_LIMITS.maxDepth or state.seen[value] then return false end + state.seen[value] = true + local entries = 0 + for key, child in pairs(value) do + entries = entries + 1 + if entries > DECODE_LIMITS.maxTableEntries + or not isDataOnly(key, state, depth) + or not isDataOnly(child, state, depth) then + return false + end + end + state.seen[value] = nil + return true +end + +function DatasetViews.new(fs, engineRequire) + assert(fs and fs.read and fs.getInfo, + "DatasetViews.new requires a readable filesystem") + return setmetatable({ fs = fs, engineRequire = engineRequire or require, + datasets = {} }, DatasetViews) +end + +function DatasetViews:_validate(version, inspected) + local sources, aggregate = {}, 0 + local modules = CacheContract.semanticModules(version) + for _, name in ipairs(CacheContract.optionalSemanticModules(version)) do + local path = inspected.prefix .. "data/generated/" .. name .. ".lua" + if self.fs.getInfo(path, "file") then modules[#modules + 1] = name end + end + for _, name in ipairs(modules) do + local path = inspected.prefix .. "data/generated/" .. name .. ".lua" + local info = self.fs.getInfo(path, "file") + local size = info and info.size + if size and size > DECODE_LIMITS.maxBytes then return nil, name .. ": size limit" end + aggregate = aggregate + (size or 0) + if aggregate > MAX_AGGREGATE_BYTES then return nil, "aggregate size limit" end + end + aggregate = 0 + for _, name in ipairs(modules) do + local path = inspected.prefix .. "data/generated/" .. name .. ".lua" + local source = self.fs.read(path) + if type(source) ~= "string" then return nil, "missing " .. name end + aggregate = aggregate + #source + if aggregate > MAX_AGGREGATE_BYTES then return nil, "aggregate size limit" end + local value, err = decode(source) + if type(value) ~= "table" then + return nil, name .. ": " .. tostring(err or "non-table root") + end + sources[name] = source + end + return sources end function DatasetViews:_module(view, root) local moduleName = view.modules[root] if not moduleName then return nil end + local source = view.sources[moduleName] + if source == nil then return nil end local cached = view.moduleCache[moduleName] - if cached ~= nil then return cached or nil end - local path = view.prefix .. "data/generated/" .. moduleName .. ".lua" - local source = self.fs.read(path) - local value = compileTable(source, "@" .. path) - view.moduleCache[moduleName] = value or false + if cached and cached.source == source then return cached.value end + local value = assert(decode(source)) + view.moduleCache[moduleName] = { source = source, value = value } return value end function DatasetViews:_data(view) if view.data then return view.data end local data = {} - for root in pairs(view.modules) do - local value = self:_module(view, root) - if value ~= nil then data[root] = value end - end + setmetatable(data, { + __index = function(target, root) + local value = self:_module(view, root) + if value ~= nil then rawset(target, root, value) end + return value + end, + }) + DatasetHydration.apply(data, view.version, self.engineRequire) view.data = data return data end local function registryBase(data, target) - return function() - return resolvePath(data, target) - end + return function() return resolvePath(data, target) end end function DatasetViews:_registries(view) @@ -123,10 +206,11 @@ function DatasetViews:_registries(view) for name, catalogSpec in pairs(Schemas.REGISTRIES) do local spec = Schemas.shapeFor(name, catalogSpec, view.generation) local registry = Registry.new(name, spec) - if spec.target then registry.base = registryBase(data, spec.target) end + local target = Schemas.targetFor(name, catalogSpec, view.generation) + if target then registry.base = registryBase(data, target) end registries[name] = registry end - Builtins.install(registries, data, view.generation) + Builtins.install(registries, data, view.generation, self.engineRequire) for _, registry in pairs(registries) do registry:freeze() end view.registries = registries return registries @@ -134,26 +218,30 @@ end function DatasetViews:_registry(view, name) local registry = self:_registries(view)[name] - - local function valueAt(id) + local function rawAt(id) local value = registry and registry:get(id) - if value == nil then return nil end - return Merge.deepCopy(value) + if value == nil or not isDataOnly(value) then return nil end + return value + end + local function valueAt(id) + local value = rawAt(id) + if value == nil then return nil end + return (copyData(value)) end - return { get = function(_, id) if type(id) ~= "string" or id == "" then return nil end return valueAt(id) end, has = function(_, id) - return valueAt(id) ~= nil + if type(id) ~= "string" or id == "" then return false end + return rawAt(id) ~= nil end, each = function() local ids = {} if registry then - for id in registry:each() do - if type(id) == "string" then ids[#ids + 1] = id end + for id, value in registry:each() do + if type(id) == "string" and isDataOnly(value) then ids[#ids + 1] = id end end end table.sort(ids) @@ -171,20 +259,15 @@ end function DatasetViews:_assets(view) local service = self local assets = {} - - function assets:path(path) - return view.prefix .. assetRelative(path) - end - + function assets:path(path) return view.prefix .. assetRelative(path) end function assets:info(path) local full = self:path(path) local info = service.fs.getInfo and service.fs.getInfo(full, "file") - if not info then return nil end - local out = { type = info.type } + if not info or (info.type and info.type ~= "file") then return nil end + local out = { type = "file" } if info.size ~= nil then out.size = info.size end return out end - return assets end @@ -192,28 +275,54 @@ function DatasetViews:open(version) if type(version) ~= "string" or not GameVersion.VERSIONS[version] then return nil, "unknown_version" end - local internal = self.datasets[version] - if not internal then - local prefix = GameVersion.cachePrefix(version) - local marker = self.fs.read(prefix .. "rom-cache.complete") - if not CacheFormat.matches(version, marker) then - return nil, "not_imported" - end - - local generation = GameVersion.generation(version) - internal = { - version = version, generation = generation, prefix = prefix, - modules = generation == 2 and GEN2_MODULES or GEN1_MODULES, - moduleCache = {}, - } - internal.registries = self:_registries(internal) - self.datasets[version] = internal + local inspected, reason, detail = CacheContract.inspect(version, self.fs, { + allowSource = true, semantic = true, + }) + if not inspected then + self.datasets[version] = nil + Logger.warn("dataset %s unavailable: %s", version, detail) + return nil, reason end - local view = { - version = version, - generation = internal.generation, - content = {}, - } + local sources, invalid = self:_validate(version, inspected) + if not sources then + self.datasets[version] = nil + Logger.warn("dataset %s cache rejected: %s", version, invalid) + return nil, "invalid_cache" + end + + local internal = self.datasets[version] + local changed = not internal or internal.prefix ~= inspected.prefix + if internal and not changed then + for name, source in pairs(internal.sources) do + if sources[name] ~= source then changed = true; break end + end + end + if internal and not changed then + for name, source in pairs(sources) do + if internal.sources[name] ~= source then changed = true; break end + end + end + if changed then + internal = { + version = version, + generation = GameVersion.generation(version), + prefix = inspected.prefix, + modules = GameVersion.generation(version) == 2 and GEN2_ROOTS or GEN1_ROOTS, + moduleCache = {}, sources = sources, + } + local ok, buildError = pcall(function() internal.registries = self:_registries(internal) end) + if not ok then + Logger.warn("dataset %s semantic hydration failed: %s", version, + tostring(buildError)) + self.datasets[version] = nil + return nil, "invalid_cache" + end + self.datasets[version] = internal + else + internal.sources = sources + end + + local view = { version = version, generation = internal.generation, content = {} } for name in pairs(Schemas.REGISTRIES) do view.content[name] = self:_registry(internal, name) end @@ -221,7 +330,6 @@ function DatasetViews:open(version) view.content[alias] = view.content[canonical] end view.assets = self:_assets(internal) - return view end diff --git a/src/mods/Loader.lua b/src/mods/Loader.lua index 3dbba98e..caba3b90 100644 --- a/src/mods/Loader.lua +++ b/src/mods/Loader.lua @@ -987,7 +987,7 @@ function Loader:_api(mod) open = function(_, version) if not loader.datasetViews then local DatasetViews = engineRequire("src.mods.DatasetViews") - loader.datasetViews = DatasetViews.new(loader.fs) + loader.datasetViews = DatasetViews.new(loader.fs, engineRequire) end return loader.datasetViews:open(version) end, diff --git a/tests/engine/dataset_views_no_mod_parity.lua b/tests/engine/dataset_views_no_mod_parity.lua new file mode 100644 index 00000000..1b7a4cbe --- /dev/null +++ b/tests/engine/dataset_views_no_mod_parity.lua @@ -0,0 +1,22 @@ +-- Dedicated Route B no-mod parity: the additive facade remains cold. +package.path = "./?.lua;./?/init.lua;" .. package.path +local T = require("tests.modkit") + +local reads = 0 +local fs = T.sdk.memfs({}) +local read = fs.read +fs.read = function(path) + if path:match("^[a-z]+/data/generated/") + or path:match("^[a-z]+/rom%-cache%.complete$") then + reads = reads + 1 + end + return read(path) +end +local data = { pokemon = { ACTIVE = { id = "ACTIVE", nested = { n = 1 } } } } +local run = T.sdk.loadNone({ fs = fs, data = data, generation = 1 }) +T.eq(#run.errors, 0, "no-mod load remains clean") +T.eq(run.loader.datasetViews, nil, "no-mod load does not allocate dataset service") +T.eq(reads, 0, "no-mod load performs no imported dataset reads") +T.eq(data.pokemon.ACTIVE.nested.n, 1, "no-mod data remains unchanged") +run.release() +T.finish("dataset_views_no_mod_parity") diff --git a/tests/engine/generated_data_decoder_test.lua b/tests/engine/generated_data_decoder_test.lua new file mode 100644 index 00000000..dd0672f3 --- /dev/null +++ b/tests/engine/generated_data_decoder_test.lua @@ -0,0 +1,37 @@ +-- Restricted generated-data grammar and resource limits. +package.path = "./?.lua;./?/init.lua;" .. package.path +local T = require("tests.modkit") +local SaveSerializer = require("src.core.SaveSerializer") + +local limits = { + allowArray = true, allowComments = true, + maxBytes = 256, maxDepth = 4, maxNodes = 7, + maxStringBytes = 8, maxTableEntries = 5, rootName = "generated data", +} + +local value, err = SaveSerializer.decode( + '-- Generated by tools/build_data.py. DO NOT EDIT.\n' + .. 'return { rows = { "A", "B" }, [3] = true }\n', limits) +T.same(value, { rows = { "A", "B" }, [3] = true }, + "LuaWriter array and keyed-table grammar decodes as data") +T.eq(err, nil, "valid generated data has no error") + +local invalid = { + { "return function() end", "function literal" }, + { "owned = true; return {}", "global assignment" }, + { "return {}; print('x')", "trailing executable syntax" }, + { string.char(27) .. "Lua", "binary chunk" }, + { "return " .. string.rep(" ", 260) .. "{}", "source-size limit" }, + { 'return { a = { b = { c = { d = { e = {} } } } } }', "depth limit" }, + { 'return { a = "123456789" }', "string limit" }, + { 'return { 1, 2, 3, 4, 5, 6 }', "table-entry limit" }, + { 'return { a = 1, b = 2, c = 3, d = 4, e = { f = 6, g = 7 } }', + "node limit" }, + { "return 7", "non-table root" }, +} +for _, row in ipairs(invalid) do + local got = SaveSerializer.decode(row[1], limits) + T.eq(got, nil, row[2] .. " fails closed") +end + +T.finish("generated_data_decoder") diff --git a/tests/engine/rom_importer_source_tree_test.lua b/tests/engine/rom_importer_source_tree_test.lua index 508ba043..f5490421 100644 --- a/tests/engine/rom_importer_source_tree_test.lua +++ b/tests/engine/rom_importer_source_tree_test.lua @@ -1,32 +1,36 @@ --- sourceTreeHasData must use each version's required-file list. Gold's --- cache has no Gen 1 trade art / pikachu.png; validating it against --- REQUIRED_FILES made a Gold source tree look incomplete forever. +-- Shared cache readiness uses the selected version's source-tree contract. package.path = "./?.lua;./?/init.lua;" .. package.path local T = require("tests.harness") -local check = T.check +local CacheContract = require("src.import.CacheContract") +local GameVersion = require("src.core.GameVersion") -local f = assert(io.open("src/import/RomImporter.lua", "r")) -local src = f:read("*a") -f:close() +local present = {} +for _, path in ipairs(CacheContract.requiredFiles("gold")) do + present[GameVersion.cachePrefix("gold") .. path] = true +end +local fs = { + read = function() return nil end, + getInfo = function(path) return present[path] and { type = "file" } or nil end, + getRealDirectory = function(path) return present[path] and "/source" or nil end, + getSource = function() return "/source" end, +} -local start = src:find("local function sourceTreeHasData", 1, true) -check(start ~= nil, "sourceTreeHasData is defined") -local finish = src:find("\nfunction RomImporter.isReady", start, true) -check(finish ~= nil, "sourceTreeHasData ends before isReady") -local body = src:sub(start, finish) +local inspected = CacheContract.inspect("gold", fs, { allowSource = true }) +T.eq(inspected and inspected.kind, "source", + "Gold source tree uses Gold's required-file contract") -check(body:find("requiredFilesFor", 1, true) ~= nil, - "sourceTreeHasData uses requiredFilesFor (Gold override, not Gen 1 only)") -check(body:find("ipairs(REQUIRED_FILES)", 1, true) == nil, - "sourceTreeHasData does not iterate the Gen 1 REQUIRED_FILES list raw") +present["gold/assets/generated/battle/hud/balls.png"] = nil +T.eq(CacheContract.inspect("gold", fs, { allowSource = true }), nil, + "missing Gold-only trainer HUD asset makes the source tree incomplete") -local helperStart = src:find("local function requiredFilesFor", 1, true) -check(helperStart ~= nil, "requiredFilesFor helper exists") -local helper = src:sub(helperStart, start) -check(helper:find("VERSION_REQUIRED_FILES_OVERRIDE", 1, true) ~= nil, - "requiredFilesFor consults VERSION_REQUIRED_FILES_OVERRIDE") -check(src:find('"assets/generated/battle/hud/balls.png"', 1, true) ~= nil, - "Gold caches require the trainer HUD ball sheet") +local required = {} +for _, path in ipairs(CacheContract.requiredFiles("gold")) do required[path] = true end +T.eq(required["data/generated/rom_text.lua"], true, + "Gold requires generated ROM text") +T.eq(required["assets/generated/pc/mail_item.png"], true, + "Gold requires PC mail art") +T.eq(required["assets/generated/trade/game_boy.png"], nil, + "Gold does not inherit the Gen 1 trade-art contract") T.finish() diff --git a/tests/engine/trade_art_import.lua b/tests/engine/trade_art_import.lua index 71736adf..8e2318cd 100644 --- a/tests/engine/trade_art_import.lua +++ b/tests/engine/trade_art_import.lua @@ -83,16 +83,12 @@ if extractor then "field.lua publishes tradeArt") end --- a cache imported before #750 has none of the art; listing one of the --- files in REQUIRED_FILES is what makes it re-import -local importer = readFile("src/import/RomImporter.lua") -T.check(importer ~= nil, "src/import/RomImporter.lua is readable") -if importer then - local required = importer:match("local REQUIRED_FILES = {(.-)\n}") - T.check(required ~= nil, "REQUIRED_FILES parses") - T.check(required ~= nil and required:find( - '"assets/generated/trade/game_boy.png"', 1, true) ~= nil, - "REQUIRED_FILES makes pre-#750 caches re-import the trade art") -end +-- a cache imported before #750 has none of the art; the shared readiness +-- contract is what makes it re-import. +local CacheContract = require("src.import.CacheContract") +local required = {} +for _, path in ipairs(CacheContract.requiredFiles("red")) do required[path] = true end +T.eq(required["assets/generated/trade/game_boy.png"], true, + "required-file contract makes pre-#750 caches re-import the trade art") T.finish("trade art import") diff --git a/tests/modkit/cases/dataset_views.lua b/tests/modkit/cases/dataset_views.lua index 59edb614..d655fdae 100644 --- a/tests/modkit/cases/dataset_views.lua +++ b/tests/modkit/cases/dataset_views.lua @@ -1,251 +1,288 @@ --- Cross-version imported datasets through the public sandboxed mod API. --- The view is semantic and read-only: registry-shaped records in, detached --- copies out, with generated assets kept inside the selected version namespace. - +-- Sandboxed public-API coverage for imported semantic dataset views. package.path = "./?.lua;./?/init.lua;" .. package.path local T = require("tests.modkit") +local Fixture = require("tests.modkit.dataset_view_fixture") local GameVersion = require("src.core.GameVersion") local CacheFs = require("src.import.CacheFs") -local CacheFormat = require("src.import.CacheFormat") - -local function serialized(value) - local function encode(v) - if type(v) == "string" then return string.format("%q", v) end - if type(v) == "number" or type(v) == "boolean" then return tostring(v) end - local keys = {} - for key in pairs(v) do keys[#keys + 1] = key end - table.sort(keys, function(a, b) return tostring(a) < tostring(b) end) - local out = { "{" } - for _, key in ipairs(keys) do - out[#out + 1] = "[" .. encode(key) .. "]=" .. encode(v[key]) .. "," - end - out[#out + 1] = "}" - return table.concat(out) - end - return "return " .. encode(value) -end - -local files = { - ["mods/dataset_probe/manifest.json"] = [[{ - "id": "dataset_probe", - "name": "Dataset Probe", - "version": "1.0.0", - "entry": "main.lua", - "api": 2, - "games": ["all"] - }]], - ["mods/dataset_probe/main.lua"] = [[ -local mod = ... -local out = {} -for _, version in ipairs({ "red", "blue", "yellow", "gold" }) do - local view, reason = mod.datasets:open(version) - if not view then - out[version] = { reason = reason } - else - local registry = view.content.pokemon - local first = registry:get("FIXMON") - first.name = "MUTATED" - local ids = {} - for id in registry:each() do ids[#ids + 1] = id end - local immune = view.content.type_chart:get("NORMAL>GHOST") - local steel = view.content.type_chart:get("STEEL") - out[version] = { - version = view.version, - generation = view.generation, - name = registry:get("FIXMON").name, - has = registry:has("FIXMON"), - ids = ids, - writable = registry.register ~= nil or registry.patch ~= nil - or registry.override ~= nil or registry.remove ~= nil, - sprite = view.assets:path(registry:get("FIXMON").spriteFront), - spriteInfo = view.assets:info(registry:get("FIXMON").spriteFront), - normalGhost = immune and immune.multiplier, - steelCategory = steel and steel.category, - } - end -end -local missing, missingReason = mod.datasets:open("silver") -local unknown, unknownReason = mod.datasets:open("crystal") -out.silver = { present = missing ~= nil, reason = missingReason } -out.crystal = { present = unknown ~= nil, reason = unknownReason } -local gold = mod.datasets:open("gold") -out.assetEscape = pcall(function() gold.assets:path("../save.lua") end) -out.rawAssetRead = gold.assets.read ~= nil -mod.exports.result = out - ]], -} - -local isolationMods = { - ["mods/dataset_mutator/manifest.json"] = [[{ - "id": "dataset_mutator", - "name": "Dataset Mutator", - "version": "1.0.0", - "entry": "main.lua", - "api": 2, - "games": ["all"] - }]], - ["mods/dataset_mutator/main.lua"] = [[ -local mod = ... -local view = assert(mod.datasets:open("red")) -view.content.pokemon.get = function() return { name = "POISONED" } end - ]], - ["mods/dataset_observer/manifest.json"] = [[{ - "id": "dataset_observer", - "name": "Dataset Observer", - "version": "1.0.0", - "entry": "main.lua", - "api": 2, - "games": ["all"], - "dependencies": ["dataset_mutator"] - }]], - ["mods/dataset_observer/main.lua"] = [[ -local mod = ... -local view = assert(mod.datasets:open("red")) -mod.exports.name = view.content.pokemon:get("FIXMON").name - ]], -} - -local labels = { - red = "RED SOURCE", blue = "BLUE SOURCE", - yellow = "YELLOW SOURCE", gold = "GOLD SOURCE", -} - -for _, version in ipairs({ "red", "blue", "yellow", "gold" }) do - local prefix = GameVersion.cachePrefix(version) - files[prefix .. "rom-cache.complete"] = - "rom-cache-v10:" .. GameVersion.info(version).sha1 - files[prefix .. "data/generated/pokemon.lua"] = serialized({ - FIXMON = { - id = "FIXMON", name = labels[version], dex = version == "gold" and 252 or 152, - spriteFront = "assets/generated/battle/front/fixmon.png", - }, - ALPHA = { id = "ALPHA", name = "ALPHA", dex = 1, - spriteFront = "assets/generated/battle/front/alpha.png" }, - }) - files[prefix .. "data/generated/type_chart.lua"] = serialized({ - matchups = { { attacker = "NORMAL", defender = "GHOST", multiplier = 0 } }, - types = version == "gold" and { - STEEL = { name = "STEEL", category = "physical", index = 9 }, - } or {}, - }) - files[prefix .. "assets/generated/battle/front/fixmon.png"] = "png-" .. version -end - --- An otherwise plausible Silver tree has no completion marker. A view must --- not execute or expose it. -files[GameVersion.cachePrefix("silver") .. "data/generated/pokemon.lua"] = - serialized({ FIXMON = { id = "FIXMON", name = "UNVERIFIED" } }) - -T.eq(CacheFormat.markerFor("red"), - "rom-cache-v10:" .. GameVersion.info("red").sha1, - "cache marker follows the importer format and version hash") -T.eq(CacheFormat.matches("silver", "rom-cache-v9:stale"), false, - "stale cache formats are rejected") -T.eq(CacheFormat.markerFor("crystal"), nil, - "unknown versions have no valid cache marker") +local Schemas = require("src.mods.Schemas") local originalVersion = GameVersion.get() local originalPrefix = CacheFs.prefix GameVersion.set("red") -local function isVersionCachePath(path) - for _, version in ipairs(GameVersion.ORDER) do - local prefix = GameVersion.cachePrefix(version) - if path:sub(1, #prefix) == prefix then return true end - end - return false +local files = {} +for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver" }) do + Fixture.cache(files, version) end +local probe = Fixture.addMod(files, "dataset_probe", [[ +local mod = ... +local out = {} +for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver" }) do + local view, reason = mod.datasets:open(version) + local mon = view and view.content.pokemon:get("FIXMON") + if mon then mon.name = "MUTATED" end + local ids = {} + if view then + for id, value in view.content.pokemon:each() do + ids[#ids + 1] = id + if value then value.name = "EACH_MUTATED" end + end + end + out[version] = { + reason = reason, generation = view and view.generation, + name = view and view.content.pokemon:get("FIXMON").name, + has = view and view.content.pokemon:has("FIXMON"), ids = ids, + aliasScripts = view and view.content.scripts == view.content.map_scripts, + aliasUi = view and view.content.ui == view.content.screens, + readOnly = view and view.content.pokemon.register == nil + and view.content.pokemon.patch == nil and view.content.pokemon.remove == nil, + } +end +local yellow = assert(mod.datasets:open("yellow")) +out.yellowOldMan = yellow.content.field:get("oldManBattle") +out.yellowBoot = yellow.content.field:get("boot") +local red = assert(mod.datasets:open("red")) +out.redBoot = red.content.field:get("boot") +out.redConstants = red.content.constants:get("dexSize") +local gold = assert(mod.datasets:open("gold")) +out.registryCount = 0 +for _, registry in pairs(gold.content) do + if type(registry) == "table" then out.registryCount = out.registryCount + 1 end +end +out.foresight = gold.content.type_chart:get("NORMAL>GHOST") +out.held = gold.content.held_items:get("LEFTOVERS") +out.continuations, out.moves, out.assets = {}, {}, {} +for _, id in ipairs({ "CROBAT", "BELLOSSOM", "POLITOED", "SLOWKING", + "STEELIX", "SCIZOR", "KINGDRA", "PORYGON2", "BLISSEY" }) do + local row = gold.content.pokemon:get(id) + out.continuations[id] = row and row.dex + out.assets[id] = row and { + gold.assets:info(row.spriteFront), gold.assets:info(row.spriteBack) } +end +for _, id in ipairs({ "IRON_TAIL", "METAL_CLAW", "STEEL_WING", "RAIN_DANCE", + "SUNNY_DAY", "SANDSTORM", "SLUDGE_BOMB", "SHADOW_BALL" }) do + out.moves[id] = gold.content.moves:has(id) +end +out.steel = gold.content.type_chart:has("STEEL") +out.executableBuiltinHidden = gold.content.statuses:get("sleep") == nil +out.assetDirectory = gold.assets:info("assets/generated/battle/front") +out.assetRejects = {} +for _, path in ipairs({ "/assets/generated/x", "assets\\generated\\x", + "assets/generated/../x", "assets/generated/./x", "other/x", + "assets/generated/x\1" }) do + out.assetRejects[#out.assetRejects + 1] = pcall(gold.assets.path, gold.assets, path) +end +local unknown, unknownReason = mod.datasets:open("crystal") +out.unknown = { unknown ~= nil, unknownReason } +mod.exports.result = out +]]) --- No mod means the new service is completely cold. Count only cross-version --- cache reads so the loader's ordinary discovery/state reads do not matter. -local vanillaFiles = {} -for path, body in pairs(files) do - if not path:match("^mods/") then - vanillaFiles[path] = body - end -end -local vanillaFs = T.sdk.memfs(vanillaFiles) -local vanillaReads = 0 -local vanillaRead = vanillaFs.read -vanillaFs.read = function(path) - if isVersionCachePath(path) then - vanillaReads = vanillaReads + 1 - end - return vanillaRead(path) -end -local vanillaData = { pokemon = { ACTIVE = { id = "ACTIVE", name = "ACTIVE" } } } -local vanilla = T.sdk.loadNone({ fs = vanillaFs, data = vanillaData }) -T.eq(#vanilla.errors, 0, "no-mod loader remains clean") -T.eq(vanillaReads, 0, "no mod performs no imported-dataset reads") -T.eq(vanillaData.pokemon.ACTIVE.name, "ACTIVE", - "no mod leaves the active dataset unchanged") -vanilla.release() - -local activeData = { pokemon = { ACTIVE = { id = "ACTIVE", name = "ACTIVE" } } } -local run = T.sdk.loadMods({ "mods/dataset_probe" }, { - fs = T.sdk.memfs(files), data = activeData, generation = 1, +local active = { pokemon = { ACTIVE = { id = "ACTIVE", nested = { n = 1 } } } } +local run = T.sdk.loadMods({ probe }, { + fs = T.sdk.memfs(files), data = active, generation = 1, }) -T.eq(#run.errors, 0, - "sandboxed dataset probe loads clean: " .. tostring(run.errors[1])) -local out = run.loader.exports.dataset_probe - and run.loader.exports.dataset_probe.result or {} - -for _, version in ipairs({ "red", "blue", "yellow", "gold" }) do - local got = out[version] or {} - T.eq(got.version, version, version .. " view reports its selected version") - T.eq(got.generation, version == "gold" and 2 or 1, - version .. " view reports the selected generation") - T.eq(got.name, labels[version], - version .. " returns its own semantic species record") - T.eq(got.has, true, version .. " registry has() reads the imported view") - T.same(got.ids, { "ALPHA", "FIXMON" }, - version .. " registry each() is deterministic") - T.eq(got.writable, false, version .. " registry exposes no write verbs") - T.eq(got.sprite, - GameVersion.cachePrefix(version) .. "assets/generated/battle/front/fixmon.png", - version .. " asset path stays in its cache namespace") - T.same(got.spriteInfo, { type = "file" }, - version .. " asset info is sanitized metadata") - T.eq(got.normalGhost, 0, - version .. " exposes structured matchup ids semantically") +T.eq(#run.errors, 0, "dataset public probe loads clean") +local out = run.loader.exports.dataset_probe.result +for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver" }) do + T.eq(out[version].reason, nil, version .. " opens") + T.eq(out[version].generation, + (version == "gold" or version == "silver") and 2 or 1, + version .. " reports selected generation") + T.eq(out[version].name, "FIXMON", version .. " get/each values are detached") + T.eq(out[version].has, true, version .. " has reads without exposing a copy") + T.eq(out[version].aliasScripts, true, version .. " scripts alias is canonical") + T.eq(out[version].aliasUi, true, version .. " ui alias is canonical") + T.eq(out[version].readOnly, true, version .. " registry has no write verbs") end -T.eq(out.gold and out.gold.steelCategory, "physical", - "Gold exposes imported Steel as a semantic type record") - -T.same(out.silver, { present = false, reason = "not_imported" }, - "an unverified cache is unavailable") -T.same(out.crystal, { present = false, reason = "unknown_version" }, - "an unknown version fails with a stable reason") -T.eq(out.assetEscape, false, "dataset asset paths reject traversal") -T.eq(out.rawAssetRead, false, "dataset assets expose no raw byte reader") -T.eq(activeData.pokemon.ACTIVE.name, "ACTIVE", - "cross-version reads do not mutate the active game dataset") -T.eq(activeData.pokemon.FIXMON, nil, - "cross-version records are not merged into active game data") -T.eq(GameVersion.get(), "red", "dataset reads do not switch the active game") -T.eq(CacheFs.prefix, originalPrefix, "dataset reads do not change cache prefix state") - +T.eq(out.redConstants, 1, "Red derives dexSize through canonical defaults") +T.eq(out.redBoot and out.redBoot.screens and out.redBoot.screens.splash, + "IntroMovie", "Red uses canonical boot defaults") +T.eq(out.yellowOldMan and out.yellowOldMan.species, "RATTATA", + "Yellow applies canonical correction") +T.eq(out.yellowBoot and out.yellowBoot.screens and out.yellowBoot.screens.splash, + "YellowIntro", + "Yellow uses its canonical splash") +T.eq(out.foresight and out.foresight.multiplier, 0, + "Gold appends Foresight matchup rows") +T.eq(out.held and out.held.heldEffect, "HELD_LEFTOVERS", "Gold derives held_items") +for _, id in ipairs(Fixture.CONTINUATIONS) do + T.check(out.continuations[id] ~= nil, "Gold exposes " .. id) + T.same(out.assets[id], { { type = "file" }, { type = "file" } }, + "Gold exposes namespaced front/back assets for " .. id) +end +for _, id in ipairs(Fixture.MOVES) do T.eq(out.moves[id], true, "Gold exposes " .. id) end +T.eq(out.steel, true, "Gold exposes Steel") +local registryCount = 0 +for _ in pairs(Schemas.REGISTRIES) do registryCount = registryCount + 1 end +for _ in pairs(Schemas.ALIASES) do registryCount = registryCount + 1 end +T.eq(out.registryCount, registryCount, + "view exposes every canonical registry and alias") +T.eq(out.executableBuiltinHidden, true, + "engine records containing closures do not cross the data-only facade") +T.eq(out.assetDirectory, nil, "asset info does not expose directories") +for index, accepted in ipairs(out.assetRejects) do + T.eq(accepted, false, "asset rejection " .. index) +end +T.same(out.unknown, { false, "unknown_version" }, "unknown version fails stably") +T.eq(active.pokemon.ACTIVE.nested.n, 1, "active Data stays unchanged") +T.eq(active.pokemon.FIXMON, nil, "inactive records do not merge into active Data") +T.eq(GameVersion.get(), "red", "active GameVersion stays unchanged") +T.eq(CacheFs.prefix, originalPrefix, "CacheFs prefix stays unchanged") run.release() -local isolationFiles = {} -for path, body in pairs(files) do - if not path:match("^mods/") then - isolationFiles[path] = body - end -end -for path, body in pairs(isolationMods) do isolationFiles[path] = body end -local isolationRun = T.sdk.loadMods({ - "mods/dataset_mutator", "mods/dataset_observer", -}, { fs = T.sdk.memfs(isolationFiles), - data = { pokemon = {} }, generation = 1 }) -T.eq(#isolationRun.errors, 0, "cross-mod isolation fixture loads cleanly") -T.eq(isolationRun.loader.exports.dataset_observer - and isolationRun.loader.exports.dataset_observer.name, - labels.red, "one mod cannot mutate another mod dataset facade") +-- Facade replacement and nested record mutation do not cross mod boundaries. +local isolation = {} +Fixture.cache(isolation, "red") +local mutator = Fixture.addMod(isolation, "dataset_mutator", [[ +local mod = ... +local view = assert(mod.datasets:open("red")) +view.content.pokemon.get = function() return { name = "POISONED" } end +local row = view.content.pokemon:get("FIXMON") +if row then row.name = "CHANGED" end +]]) +local observer = Fixture.addMod(isolation, "dataset_observer", [[ +local mod = ... +local view = assert(mod.datasets:open("red")) +mod.exports.name = view.content.pokemon:get("FIXMON").name +]]) +isolation["mods/dataset_observer/manifest.json"] = [[{ + "id": "dataset_observer", "name": "dataset_observer", "version": "1.0.0", + "entry": "main.lua", "api": 2, "games": ["all"], + "dependencies": ["dataset_mutator"] +}]] +local isolationRun = T.sdk.loadMods({ mutator, observer }, { + fs = T.sdk.memfs(isolation), data = { pokemon = {} }, generation = 1, +}) +T.eq(isolationRun.loader.exports.dataset_observer.name, "FIXMON", + "facade and record mutation cannot cross mods") isolationRun.release() +-- Marker-only, empty, partial, and stale-after-first-open caches fail closed. +local readiness = {} +local emptyPrefix = GameVersion.cachePrefix("red") +readiness[emptyPrefix .. "rom-cache.complete"] = + "rom-cache-v10:" .. GameVersion.info("red").sha1 +Fixture.cache(readiness, "blue") +readiness[GameVersion.cachePrefix("blue") .. "data/generated/moves.lua"] = nil +Fixture.cache(readiness, "gold") +local staleReads = 0 +local readinessFs = T.sdk.memfs(readiness) +local rawRead = readinessFs.read +readinessFs.read = function(path) + if path == GameVersion.cachePrefix("gold") .. "rom-cache.complete" then + staleReads = staleReads + 1 + if staleReads > 1 then return "rom-cache-v9:stale" end + end + return rawRead(path) +end +local readinessMod = Fixture.addMod(readiness, "readiness_probe", [[ +local mod = ... +local out = {} +for _, version in ipairs({ "red", "blue" }) do + local view, reason = mod.datasets:open(version) + out[version] = { view ~= nil, reason } +end +local first, firstReason = mod.datasets:open("gold") +local second, secondReason = mod.datasets:open("gold") +out.gold = { first ~= nil, firstReason, second ~= nil, secondReason } +mod.exports.result = out +]]) +local readinessRun = T.sdk.loadMods({ readinessMod }, { + fs = readinessFs, data = { pokemon = {} }, generation = 1, +}) +local readyOut = readinessRun.loader.exports.readiness_probe.result +T.same(readyOut.red, { false, "not_imported" }, "marker-only empty cache fails") +T.same(readyOut.blue, { false, "not_imported" }, "partial cache fails") +T.same(readyOut.gold, { true, nil, false, "not_imported" }, + "cached view is invalidated when marker becomes stale") +readinessRun.release() + +-- Removal followed by a fresh import cannot resurrect the pre-removal view. +local transition = {} +Fixture.cache(transition, "red") +local transitionFs = T.sdk.memfs(transition) +local transitionRead = transitionFs.read +local transitionMarkers = 0 +local reimportedPokemon = require("src.import.LuaWriter").encode({ + FIXMON = { id = "FIXMON", name = "REIMPORTED", dex = 1, + spriteFront = "assets/generated/battle/front/fixmon.png", + spriteBack = "assets/generated/battle/back/fixmon.png", frontSize = 5 }, +}) +transitionFs.read = function(path) + if path == GameVersion.cachePrefix("red") .. "rom-cache.complete" then + transitionMarkers = transitionMarkers + 1 + if transitionMarkers == 2 then return nil end + end + local body = transitionRead(path) + if transitionMarkers >= 3 + and path == GameVersion.cachePrefix("red") .. "data/generated/pokemon.lua" then + return reimportedPokemon + end + return body +end +local transitionMod = Fixture.addMod(transition, "transition_probe", [[ +local mod = ... +local out = {} +for index = 1, 3 do + local view, reason = mod.datasets:open("red") + out[index] = { view and view.content.pokemon:get("FIXMON").name, reason } +end +mod.exports.result = out +]]) +local transitionRun = T.sdk.loadMods({ transitionMod }, { + fs = transitionFs, data = { pokemon = {} }, generation = 1, +}) +T.same(transitionRun.loader.exports.transition_probe.result, { + { "FIXMON" }, { nil, "not_imported" }, { "REIMPORTED" }, +}, "remove and reimport transition rebuilds the semantic view") +transitionRun.release() + +-- Every hostile source is rejected as data, with one stable public reason. +local hostile = { + { "red", "return { BAD = function() return 1 end }" }, + { "blue", "owned = true; return {}" }, + { "yellow", "return 7" }, + { "gold", string.char(27) .. "Lua" }, + { "silver", "return {}; while true do end" }, +} +local hostileFiles, hostilePaths = {}, {} +for _, row in ipairs(hostile) do + Fixture.cache(hostileFiles, row[1], { pokemon = row[2] }) +end +local hostileMod = Fixture.addMod(hostileFiles, "hostile_probe", [[ +local mod = ... +local out = {} +for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver" }) do + local view, reason = mod.datasets:open(version) + out[version] = { view ~= nil, reason } +end +mod.exports.result = out +]]) +hostilePaths[1] = hostileMod +local previousHook, previousMask, previousCount +local beforeHook +if debug.gethook and debug.sethook then + previousHook, previousMask, previousCount = debug.gethook() + beforeHook = function() end + debug.sethook(beforeHook, "", 1000) +end +local hostileRun = T.sdk.loadMods(hostilePaths, { + fs = T.sdk.memfs(hostileFiles), data = { pokemon = {} }, generation = 1, +}) +local hostileOut = hostileRun.loader.exports.hostile_probe.result +for _, row in ipairs(hostile) do + T.same(hostileOut[row[1]], { false, "invalid_cache" }, + row[1] .. " hostile generated source fails closed") +end +T.eq(debug.gethook and debug.gethook(), beforeHook, + "dataset decoding preserves the caller debug hook") +if debug.sethook then + if previousHook then debug.sethook(previousHook, previousMask, previousCount) + else debug.sethook() end +end +hostileRun.release() + GameVersion.set(originalVersion) CacheFs.prefix = originalPrefix - T.finish("dataset_views") diff --git a/tests/modkit/cases/dataset_views_nontermination.lua b/tests/modkit/cases/dataset_views_nontermination.lua new file mode 100644 index 00000000..e895f06a --- /dev/null +++ b/tests/modkit/cases/dataset_views_nontermination.lua @@ -0,0 +1,21 @@ +-- A generated chunk that would loop if executed must be rejected as syntax. +package.path = "./?.lua;./?/init.lua;" .. package.path +local T = require("tests.modkit") +local Fixture = require("tests.modkit.dataset_view_fixture") + +local files = {} +Fixture.cache(files, "red", { + pokemon = "while true do end; return {}", +}) +local modPath = Fixture.addMod(files, "nontermination_probe", [[ +local mod = ... +local view, reason = mod.datasets:open("red") +mod.exports.result = { view ~= nil, reason } +]]) +local run = T.sdk.loadMods({ modPath }, { + fs = T.sdk.memfs(files), data = { pokemon = {} }, generation = 1, +}) +T.same(run.loader.exports.nontermination_probe.result, + { false, "invalid_cache" }, "generated code is never executed") +run.release() +T.finish("dataset_views_nontermination") diff --git a/tests/modkit/dataset_view_fixture.lua b/tests/modkit/dataset_view_fixture.lua new file mode 100644 index 00000000..5d2e80bf --- /dev/null +++ b/tests/modkit/dataset_view_fixture.lua @@ -0,0 +1,121 @@ +local GameVersion = require("src.core.GameVersion") +local LuaWriter = require("src.import.LuaWriter") +local CacheContract = require("src.import.CacheContract") + +local Fixture = {} + +local GEN1_MODULES = { + "constants", "maps", "tilesets", "text", "text_pointers", + "trainer_headers", "font", "sprites", "pokemon", "moves", "items", + "type_chart", "trainers", "encounters", "field", "battle_anims", +} +local GEN1_OPTIONAL_MODULES = { "audio", "palettes", "icons" } + +local GEN2_MODULES = { + "pokemon", "moves", "items", "type_chart", "audio", "font", "maps", + "tilesets", "text", "trainers", "encounters", "sprites", "palettes", + "icons", "battle_anims", "constants", "landmarks", +} + +local CONTINUATIONS = { + "CROBAT", "BELLOSSOM", "POLITOED", "SLOWKING", "STEELIX", + "SCIZOR", "KINGDRA", "PORYGON2", "BLISSEY", +} + +local MOVES = { + "IRON_TAIL", "METAL_CLAW", "STEEL_WING", "RAIN_DANCE", "SUNNY_DAY", + "SANDSTORM", "SLUDGE_BOMB", "SHADOW_BALL", +} + +local function species(id, dex, generation) + local row = { + id = id, name = id, dex = dex, + spriteFront = "assets/generated/battle/front/" .. id:lower() .. ".png", + spriteBack = "assets/generated/battle/back/" .. id:lower() .. ".png", + } + if generation == 2 then row.picSize = 5 else row.frontSize = 5 end + return row +end + +local function defaults(version) + local generation = GameVersion.generation(version) + local pokemon = { FIXMON = species("FIXMON", 1, generation) } + local moves = {} + local items = {} + local typeChart = { + matchups = { { attacker = "NORMAL", defender = "ROCK", multiplier = 5 } }, + types = {}, + } + if generation == 2 then + for index, id in ipairs(CONTINUATIONS) do + pokemon[id] = species(id, 168 + index, generation) + end + for index, id in ipairs(MOVES) do + moves[id] = { id = id, name = id, index = index, type = "STEEL", + power = 50, accuracy = 100, pp = 15, effect = "NO_ADDITIONAL_EFFECT" } + end + typeChart.types.STEEL = { name = "STEEL", category = "physical", index = 9 } + typeChart.foresightMatchups = { + { attacker = "NORMAL", defender = "GHOST", multiplier = 0 }, + } + items.LEFTOVERS = { id = "LEFTOVERS", name = "LEFTOVERS", price = 0, + heldEffect = "HELD_LEFTOVERS", heldParameter = 0 } + end + return { + constants = {}, maps = {}, tilesets = {}, text = {}, text_pointers = {}, + trainer_headers = {}, font = {}, sprites = {}, pokemon = pokemon, + moves = moves, items = items, type_chart = typeChart, trainers = {}, + encounters = {}, field = { oakSpeech = {} }, battle_anims = {}, + audio = {}, palettes = {}, icons = {}, landmarks = {}, + } +end + +function Fixture.cache(files, version, overrides) + local prefix = GameVersion.cachePrefix(version) + local generation = GameVersion.generation(version) + local values = defaults(version) + for name, value in pairs(overrides or {}) do values[name] = value end + files[prefix .. "rom-cache.complete"] = + "rom-cache-v10:" .. GameVersion.info(version).sha1 + for _, name in ipairs(generation == 2 and GEN2_MODULES or GEN1_MODULES) do + local value = values[name] + files[prefix .. "data/generated/" .. name .. ".lua"] = + type(value) == "string" and value or LuaWriter.encode(value or {}) + end + if generation == 1 then + for _, name in ipairs(GEN1_OPTIONAL_MODULES) do + files[prefix .. "data/generated/" .. name .. ".lua"] = + LuaWriter.encode(values[name] or {}) + end + end + for _, path in ipairs(CacheContract.requiredFiles(version)) do + if files[prefix .. path] == nil then + files[prefix .. path] = path:match("%.lua$") and LuaWriter.encode({}) or "fixture" + end + end + if type(values.pokemon) == "table" then + for _, row in pairs(values.pokemon) do + if type(row) == "table" and row.spriteFront then + files[prefix .. row.spriteFront] = "front" + files[prefix .. row.spriteBack] = "back" + end + end + end + return files +end + +function Fixture.addMod(files, id, body) + files["mods/" .. id .. "/manifest.json"] = ([[{ + "id": %q, "name": %q, "version": "1.0.0", "entry": "main.lua", + "api": 2, "games": ["all"] + }]]):format(id, id) + files["mods/" .. id .. "/main.lua"] = body + return "mods/" .. id +end + +Fixture.GEN1_MODULES = GEN1_MODULES +Fixture.GEN2_MODULES = GEN2_MODULES +Fixture.CONTINUATIONS = CONTINUATIONS +Fixture.MOVES = MOVES + +return Fixture From 7e069df74062ae4ef24f0289b0c03e1b3b121fc6 Mon Sep 17 00:00:00 2001 From: MaxTomahawk Date: Mon, 24 Aug 2026 09:57:57 +0200 Subject: [PATCH 3/6] fix(mod-api): validate dataset roots lazily --- docs/modding.md | 22 ++- docs/modding/reference/registries.md | 48 +++++ docs/rfcs/0015-imported-dataset-views.md | 55 ++++-- src/mods/DatasetViews.lua | 177 +++++++++++++----- src/mods/Schemas.lua | 29 +++ .../engine/dataset_views_lazy_validation.lua | 46 +++++ tests/modkit/cases/dataset_views.lua | 116 +++++++++++- .../cases/dataset_views_nontermination.lua | 10 +- tests/modkit/dataset_view_fixture.lua | 19 +- 9 files changed, 442 insertions(+), 80 deletions(-) create mode 100644 tests/engine/dataset_views_lazy_validation.lua diff --git a/docs/modding.md b/docs/modding.md index 16e28741..2845f9f2 100644 --- a/docs/modding.md +++ b/docs/modding.md @@ -226,18 +226,28 @@ end `view.content` exposes the same registry names, aliases, generation routing, and data-only record shapes as `mod.content`, but only `get`, `has`, and `each`. Returned records are detached copies and cannot mutate either dataset. +Every generated base record passes the selected generation's existing public +schema before it is returned; extractor metadata beside record maps stays out +of the registry id space. A malformed record makes `get` return nil, `has` +return false, and `each` return no rows, and invalidates that dataset view. Records containing functions, userdata, threads, metatables, or cycles are not exposed. Each open call receives an independent facade, so one mod cannot replace another mod view method. Canonical boot shaping is included, such as Gen 1 defaults and Yellow corrections, and Gold's Foresight matchup rows and derived `held_items`. -The marker and every required generated module for the selected version are -rechecked on each open. Missing, partial, and stale imports return -`nil, "not_imported"`; malformed or resource-limit-breaking generated data -returns `nil, "invalid_cache"`. Generated modules are decoded with a bounded -literal-only grammar and are never executed. No raw ROM bytes or generated -source are exposed. `view.assets:path(relative)` and +`open` checks the completion marker, exact version-specific file inventory, +source/cache boundary, and file-size bounds without reading or decoding the +semantic modules. A root is read, bounded-decoded, normalized, and cached only +when a content operation first needs it. Each later view operation rechecks +readiness and the source bytes behind already cached roots; unchanged roots are +not decoded again. Missing, partial, and stale imports return +`nil, "not_imported"`. Malformed syntax, a resource-limit violation, or a +record that fails the public schema is discovered on first root access and +fails that operation closed; a later `open` against the same source returns +`nil, "invalid_cache"`. Generated modules use a bounded literal-only grammar +and are never executed. No raw ROM bytes or generated source are exposed. +`view.assets:path(relative)` and `view.assets:info(relative)` accept only `assets/generated/...` paths and keep them under the selected version cache prefix. The API never changes `mod.game`, the active `Data` table, `GameVersion`, or cache mount state. diff --git a/docs/modding/reference/registries.md b/docs/modding/reference/registries.md index be3ed622..96693e63 100644 --- a/docs/modding/reference/registries.md +++ b/docs/modding/reference/registries.md @@ -510,6 +510,26 @@ mod.content.item_effects:register("MOON_FLUTE", { use = fn, field = true }) mod.content.items:patch("POTION", { price = 100 }) ``` +### On Gold (Gen 2) + +- semantics: `record` +- target: `Data.items` + +The record differs; the registry name, the verbs and the id space +do not. + +| field | type | required | +|---|---|---| +| `ball` | balls id | no | +| `effect` | item_effects id | no | +| `id` | string | yes | +| `index` | integer 0..255 | no | +| `machine` | {kind, move, number} | no | +| `name` | string | yes | +| `needsTarget` | boolean | no | +| `price` | integer >= 0 | yes | +| `tossable` | boolean | no | + ## landmarks - semantics: `record` @@ -674,6 +694,34 @@ mod.content.move_effects:register("DRAIN_PP_EFFECT", { kind = "primary", run = f mod.content.moves:patch("BLIZZARD", { accuracy = 70 }) ``` +### On Gold (Gen 2) + +- semantics: `record` +- target: `Data.moves` + +The record differs; the registry name, the verbs and the id space +do not. + +| field | type | required | +|---|---|---| +| `accuracy` | integer 0..100 | yes | +| `anim` | any value | no | +| `category` | one of "physical" | "special" | "status" | no | +| `chargeText` | string | no | +| `counterable` | boolean | no | +| `effect` | move_effects id | yes | +| `fixedDamage` | integer >= 1 | function | no | +| `highCrit` | boolean | no | +| `id` | string | yes | +| `index` | integer 0..255 | no | +| `multiHit` | integer >= 1 | list of integer >= 1 | no | +| `name` | string | yes | +| `power` | integer 0..255 | yes | +| `pp` | integer 0..64 | yes | +| `priority` | integer -7..7 | no | +| `semiInvulnerable` | boolean | no | +| `type` | type_chart id | yes | + ## music - semantics: `record` diff --git a/docs/rfcs/0015-imported-dataset-views.md b/docs/rfcs/0015-imported-dataset-views.md index b1aa7c88..0bf7d9d9 100644 --- a/docs/rfcs/0015-imported-dataset-views.md +++ b/docs/rfcs/0015-imported-dataset-views.md @@ -70,9 +70,13 @@ semantic presence. the selected version's generation routing and the engine's existing `Schemas`, `Registry`, and `Builtins` normalization, so structured sources such as type matchups retain the same public ids used by the active -`mod.content` facade. No register, patch, override, or remove verb is exposed. Each call returns an -independent facade over the cached internal dataset, so facade mutation cannot cross -mod boundaries. +`mod.content` facade and extractor metadata beside record maps is not exposed +as a record id. Every generated base record is checked with that selected +generation's existing public schema before it can cross `get`, `has`, or +`each`; there is no dataset-specific duplicate schema. No register, patch, +override, or remove verb is exposed. Each call returns an independent facade +over the cached internal dataset, so facade mutation cannot cross mod +boundaries. `assets:path` returns the selected cache-prefixed virtual path. `assets:info` returns sanitized `type` and optional `size` metadata. Both @@ -80,16 +84,29 @@ accept only relative paths below `assets/generated/`, reject control characters, absolute paths, backslashes, and traversal, and expose no byte reader. -An unknown version returns `nil, "unknown_version"`. A missing, partial, or -stale cache returns `nil, "not_imported"`. A required module that is malformed -or exceeds the limits (8 MiB per module, 48 MiB aggregate, depth 64, 500,000 -values, 2 MiB per string, or 250,000 entries per table) returns -`nil, "invalid_cache"`; actionable detail is engine-logged but not exposed to -the mod. Generated Lua is decoded with the existing restricted -literal grammar and never executed. Functions, userdata, threads, metatables, -cycles, non-table roots, binary chunks, and trailing syntax cannot cross the -facade. Successful roots are decoded lazily and cached per selected version. -The API never exposes raw ROM bytes, generated source, host paths, or a mount. +An unknown version returns `nil, "unknown_version"`. `open` checks the current +completion marker, exact version-specific file inventory, source/cache +boundary, and available file sizes; it does not read or decode semantic +modules. A missing, partial, or stale cache returns `nil, "not_imported"`. + +The first content operation that needs a root reads it once, applies the +limits (8 MiB per module, 48 MiB aggregate, depth 64, 500,000 values, 2 MiB +per string, and 250,000 entries per table), decodes the restricted literal +grammar, applies canonical selected-version normalization, and caches the +detached root. Each later content or asset operation rechecks marker/file and +source-bound readiness. It also rereads the source bytes for already cached +roots; unchanged roots are not decoded again, while a changed root clears the +derived registry cache and is decoded on its next use. + +Malformed syntax, non-table roots, binary/trailing content, resource-limit +violations, and records that fail the public schema are therefore discovered +on first access rather than during `open`. The triggering `get` returns nil, +`has` returns false, or `each` returns no rows; the whole internal view is +invalidated, and a subsequent `open` against the unchanged source returns +`nil, "invalid_cache"`. Actionable detail is engine-logged but not exposed to +the mod. Generated Lua is never executed. Functions, userdata, threads, +metatables, and cycles cannot cross the facade. The API never exposes raw ROM +bytes, generated source, host paths, or a mount. ## Migration and compatibility @@ -106,7 +123,9 @@ The completion marker and per-version required-file rules live in the pure, injected `CacheContract` shared by the importer and dataset service. It also defines source-tree behavior. Neither consumer mutates `CacheFs.prefix` while checking readiness. Every `open` revalidates the contract and required module -shapes; a stale/remove/reimport transition evicts the previous semantic view. +inventory without semantic decoding. Every view operation rechecks that +readiness before serving cached state; a stale/remove/reimport transition +evicts the previous semantic view. ## Verification @@ -117,8 +136,12 @@ shapes; a stale/remove/reimport transition evicts the previous semantic view. version-prefixed generated assets, traversal rejection, stable failure reasons, and stale-marker rejection. - It also proves missing/empty/partial/stale/remove/reimport behavior, hostile - generated-source rejection, canonical Gen 1/Yellow/Gold hydration, and the - approved Kanto+ Gold records and assets. + generated-source and malformed-record rejection, canonical Gen + 1/Yellow/Gold hydration, active Red/Blue/Yellow isolation while reading Gold, + and the approved Kanto+ Gold records and assets. +- `tests/engine/dataset_views_lazy_validation.lua` counts semantic reads and + decoder calls to prove `open` decodes nothing, unused roots stay unread, and + repeated access does not re-decode an unchanged cached root. - `tests/modkit/cases/dataset_views_nontermination.lua` proves generated code is rejected rather than executed; `tests/engine/generated_data_decoder_test.lua` proves every decoder resource bound. diff --git a/src/mods/DatasetViews.lua b/src/mods/DatasetViews.lua index eb0364b9..253338d0 100644 --- a/src/mods/DatasetViews.lua +++ b/src/mods/DatasetViews.lua @@ -47,10 +47,6 @@ local GEN2_ROOTS = { gen2Constants = "constants", gen2Landmarks = "landmarks", } -local function decode(source) - return SaveSerializer.decode(source, DECODE_LIMITS) -end - local function resolvePath(root, suffix) local node = root for key in suffix:gmatch("[^.]+") do @@ -130,15 +126,15 @@ local function isDataOnly(value, state, depth) return true end -function DatasetViews.new(fs, engineRequire) +function DatasetViews.new(fs, engineRequire, decoder) assert(fs and fs.read and fs.getInfo, "DatasetViews.new requires a readable filesystem") return setmetatable({ fs = fs, engineRequire = engineRequire or require, - datasets = {} }, DatasetViews) + decoder = decoder or SaveSerializer.decode, datasets = {} }, DatasetViews) end -function DatasetViews:_validate(version, inspected) - local sources, aggregate = {}, 0 +function DatasetViews:_preflight(version, inspected) + local paths, aggregate = {}, 0 local modules = CacheContract.semanticModules(version) for _, name in ipairs(CacheContract.optionalSemanticModules(version)) do local path = inspected.prefix .. "data/generated/" .. name .. ".lua" @@ -151,31 +147,80 @@ function DatasetViews:_validate(version, inspected) if size and size > DECODE_LIMITS.maxBytes then return nil, name .. ": size limit" end aggregate = aggregate + (size or 0) if aggregate > MAX_AGGREGATE_BYTES then return nil, "aggregate size limit" end + paths[name] = path end - aggregate = 0 - for _, name in ipairs(modules) do - local path = inspected.prefix .. "data/generated/" .. name .. ".lua" - local source = self.fs.read(path) - if type(source) ~= "string" then return nil, "missing " .. name end - aggregate = aggregate + #source - if aggregate > MAX_AGGREGATE_BYTES then return nil, "aggregate size limit" end - local value, err = decode(source) - if type(value) ~= "table" then - return nil, name .. ": " .. tostring(err or "non-table root") + return { paths = paths, key = table.concat(modules, "\n") } +end + +local function resetInternal(view) + view.moduleCache, view.data, view.registries = {}, nil, nil +end + +function DatasetViews:_reject(view, moduleName, source, detail) + view.invalid = { module = moduleName, source = source, detail = detail } + view.data, view.registries = nil, nil + Logger.warn("dataset %s cache rejected: %s", view.version, tostring(detail)) + return nil +end + +function DatasetViews:_ready(view) + if view.invalid or view.unavailable then return false end + local inspected, _, detail = CacheContract.inspect(view.version, self.fs, { + allowSource = true, semantic = true, + }) + if not inspected then + view.unavailable = true + if self.datasets[view.version] == view then self.datasets[view.version] = nil end + Logger.warn("dataset %s unavailable: %s", view.version, detail) + return false + end + local plan, invalid = self:_preflight(view.version, inspected) + if not plan then + self:_reject(view, nil, nil, invalid) + return false + end + if view.prefix ~= inspected.prefix or view.plan.key ~= plan.key then + view.prefix, view.plan = inspected.prefix, plan + resetInternal(view) + else + view.plan = plan + end + for name, cached in pairs(view.moduleCache) do + local source = self.fs.read(plan.paths[name]) + if type(source) ~= "string" then + self:_reject(view, name, source, name .. ": unreadable generated module") + return false + end + if source ~= cached.source then + resetInternal(view) + break end - sources[name] = source end - return sources + return not view.invalid end function DatasetViews:_module(view, root) local moduleName = view.modules[root] if not moduleName then return nil end - local source = view.sources[moduleName] - if source == nil then return nil end local cached = view.moduleCache[moduleName] - if cached and cached.source == source then return cached.value end - local value = assert(decode(source)) + if cached then return cached.value end + local path = view.plan.paths[moduleName] + if not path then return nil end + local source = self.fs.read(path) + if type(source) ~= "string" then + return self:_reject(view, moduleName, source, + moduleName .. ": unreadable generated module") + end + local aggregate = #source + for _, loaded in pairs(view.moduleCache) do aggregate = aggregate + #loaded.source end + if aggregate > MAX_AGGREGATE_BYTES then + return self:_reject(view, moduleName, source, "aggregate size limit") + end + local value, err = self.decoder(source, DECODE_LIMITS) + if type(value) ~= "table" then + return self:_reject(view, moduleName, source, + moduleName .. ": " .. tostring(err or "non-table root")) + end view.moduleCache[moduleName] = { source = source, value = value } return value end @@ -191,6 +236,7 @@ function DatasetViews:_data(view) end, }) DatasetHydration.apply(data, view.version, self.engineRequire) + if view.invalid then error(view.invalid.detail, 0) end view.data = data return data end @@ -217,10 +263,37 @@ function DatasetViews:_registries(view) end function DatasetViews:_registry(view, name) - local registry = self:_registries(view)[name] + local service = self + local function registryForRead() + if not service:_ready(view) then return nil end + local ok, registries = pcall(service._registries, service, view) + if not ok then + if not view.invalid then service:_reject(view, nil, nil, registries) end + return nil + end + if view.invalid then return nil end + return registries[name] + end + local function validate(registry, id, value) + if value == nil then return true end + local ok, detail = Schemas.check(registry.spec, registry.name, id, + value, "override") + if ok then return true end + local target = registry.spec.target + or Schemas.targetFor(registry.name, registry.spec, view.generation) + local root = target and target:match("^[^%.]+") + local moduleName = root and view.modules[root] + if root == "gen2HeldItems" then moduleName = "items" end + local cached = moduleName and view.moduleCache[moduleName] + service:_reject(view, moduleName, cached and cached.source, + "invalid " .. registry.name .. " record: " .. detail) + return false + end local function rawAt(id) + local registry = registryForRead() local value = registry and registry:get(id) - if value == nil or not isDataOnly(value) then return nil end + if value == nil or not validate(registry, id, value) + or not isDataOnly(value) then return nil end return value end local function valueAt(id) @@ -239,9 +312,16 @@ function DatasetViews:_registry(view, name) end, each = function() local ids = {} + local registry = registryForRead() if registry then for id, value in registry:each() do - if type(id) == "string" and isDataOnly(value) then ids[#ids + 1] = id end + if not validate(registry, id, value) then + ids = {} + break + end + if type(id) == "string" and isDataOnly(value) then + ids[#ids + 1] = id + end end end table.sort(ids) @@ -259,9 +339,13 @@ end function DatasetViews:_assets(view) local service = self local assets = {} - function assets:path(path) return view.prefix .. assetRelative(path) end + function assets:path(path) + if not service:_ready(view) then return nil end + return view.prefix .. assetRelative(path) + end function assets:info(path) local full = self:path(path) + if not full then return nil end local info = service.fs.getInfo and service.fs.getInfo(full, "file") if not info or (info.type and info.type ~= "file") then return nil end local out = { type = "file" } @@ -283,43 +367,36 @@ function DatasetViews:open(version) Logger.warn("dataset %s unavailable: %s", version, detail) return nil, reason end - local sources, invalid = self:_validate(version, inspected) - if not sources then - self.datasets[version] = nil + local plan, invalid = self:_preflight(version, inspected) + if not plan then Logger.warn("dataset %s cache rejected: %s", version, invalid) return nil, "invalid_cache" end local internal = self.datasets[version] + if internal and internal.invalid then + local bad = internal.invalid + local path = bad.module and plan.paths[bad.module] + local source = path and self.fs.read(path) + if source == bad.source then return nil, "invalid_cache" end + internal = nil + self.datasets[version] = nil + end local changed = not internal or internal.prefix ~= inspected.prefix - if internal and not changed then - for name, source in pairs(internal.sources) do - if sources[name] ~= source then changed = true; break end - end - end - if internal and not changed then - for name, source in pairs(sources) do - if internal.sources[name] ~= source then changed = true; break end - end - end + or internal.plan.key ~= plan.key if changed then + if internal then internal.unavailable = true end internal = { version = version, generation = GameVersion.generation(version), prefix = inspected.prefix, + plan = plan, modules = GameVersion.generation(version) == 2 and GEN2_ROOTS or GEN1_ROOTS, - moduleCache = {}, sources = sources, + moduleCache = {}, } - local ok, buildError = pcall(function() internal.registries = self:_registries(internal) end) - if not ok then - Logger.warn("dataset %s semantic hydration failed: %s", version, - tostring(buildError)) - self.datasets[version] = nil - return nil, "invalid_cache" - end self.datasets[version] = internal else - internal.sources = sources + internal.plan = plan end local view = { version = version, generation = internal.generation, content = {} } diff --git a/src/mods/Schemas.lua b/src/mods/Schemas.lua index f07c03c9..b3fbf90d 100644 --- a/src/mods/Schemas.lua +++ b/src/mods/Schemas.lua @@ -757,6 +757,32 @@ end local R = {} Schemas.REGISTRIES = R +-- Some generated Gen 2 modules keep extractor metadata beside their public +-- record maps. These callbacks are the registry normalization boundary: the +-- metadata remains available to engine consumers through Data, but is not an +-- id a mod can read or overwrite through the record registry. +local function recordMapExcept(...) + local excluded = {} + for index = 1, select("#", ...) do excluded[select(index, ...)] = true end + return function(base, id) + if excluded[id] then return nil end + return base[id] + end, function(base) + local ids = {} + for id in pairs(base) do + if not excluded[id] then ids[#ids + 1] = id end + end + return ids + end +end + +local pokemonGen2BaseAt, pokemonGen2BaseIds = + recordMapExcept("growthRates", "tmhmMoves") +local movesGen2BaseAt, movesGen2BaseIds = + recordMapExcept("generation", "source") +local itemsGen2BaseAt, itemsGen2BaseIds = + recordMapExcept("generation", "source", "pockets") + -- ------- shared Gen 2 leaves -- -- The ROM name spaces Gold's tables key by. They are enums rather than @@ -779,6 +805,7 @@ local gen2PaletteRow = f.list(gen2Color) R.pokemon = { semantics = "record", target = "pokemon", + gen2BaseAt = pokemonGen2BaseAt, gen2BaseIds = pokemonGen2BaseIds, fields = { id = f.str, name = f.str, dex = f.int(1), index = f.opt(f.int(0, 255)), @@ -870,6 +897,7 @@ R.pokemon = { R.moves = { semantics = "record", target = "moves", + gen2BaseAt = movesGen2BaseAt, gen2BaseIds = movesGen2BaseIds, fields = { id = f.str, name = f.str, index = f.opt(f.int(0, 255)), @@ -894,6 +922,7 @@ R.moves = { R.items = { semantics = "record", target = "items", + gen2BaseAt = itemsGen2BaseAt, gen2BaseIds = itemsGen2BaseIds, fields = { id = f.str, name = f.str, index = f.opt(f.int(0, 255)), diff --git a/tests/engine/dataset_views_lazy_validation.lua b/tests/engine/dataset_views_lazy_validation.lua new file mode 100644 index 00000000..fa9ff9d0 --- /dev/null +++ b/tests/engine/dataset_views_lazy_validation.lua @@ -0,0 +1,46 @@ +-- Imported semantic modules are opened without eager reads and are decoded +-- once, on first use, through the same public facade mods receive. +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") +local Fixture = require("tests.modkit.dataset_view_fixture") +local DatasetViews = require("src.mods.DatasetViews") +local GameVersion = require("src.core.GameVersion") +local SaveSerializer = require("src.core.SaveSerializer") + +local files = {} +Fixture.cache(files, "gold") +local fs = T.sdk.memfs(files) +local rawRead = fs.read +local reads = {} +fs.read = function(path) + reads[path] = (reads[path] or 0) + 1 + return rawRead(path) +end +local decodes = 0 +local service = DatasetViews.new(fs, require, function(source, limits) + decodes = decodes + 1 + return SaveSerializer.decode(source, limits) +end) +local pokemonPath = GameVersion.cachePrefix("gold") + .. "data/generated/pokemon.lua" +local movesPath = GameVersion.cachePrefix("gold") + .. "data/generated/moves.lua" + +local view, reason = service:open("gold") +T.eq(reason, nil, "marker-valid Gold dataset opens") +T.check(view ~= nil, "open returns the lazy view") +T.eq(reads[pokemonPath] or 0, 0, "open does not read an unused root") +T.eq(reads[movesPath] or 0, 0, "open does not read another unused root") +T.eq(decodes, 0, "open decodes no semantic root") + +local first = view and view.content.pokemon:get("FIXMON") +T.eq(first and first.name, "FIXMON", "first access decodes a valid root") +T.check(decodes > 0, "first root access performs bounded decoding") +T.eq(reads[movesPath] or 0, 0, "pokemon access leaves moves unused") +local afterFirst = decodes +local second = view and view.content.pokemon:get("FIXMON") +T.eq(second and second.name, "FIXMON", "repeated access remains available") +T.eq(decodes, afterFirst, "repeated access does not re-decode cached roots") + +T.finish("dataset_views_lazy_validation") diff --git a/tests/modkit/cases/dataset_views.lua b/tests/modkit/cases/dataset_views.lua index d655fdae..be672b46 100644 --- a/tests/modkit/cases/dataset_views.lua +++ b/tests/modkit/cases/dataset_views.lua @@ -65,6 +65,13 @@ for _, id in ipairs({ "IRON_TAIL", "METAL_CLAW", "STEEL_WING", "RAIN_DANCE", out.moves[id] = gold.content.moves:has(id) end out.steel = gold.content.type_chart:has("STEEL") +out.metadataHidden = not gold.content.pokemon:has("growthRates") + and not gold.content.pokemon:has("tmhmMoves") + and not gold.content.moves:has("generation") + and not gold.content.moves:has("source") + and not gold.content.items:has("generation") + and not gold.content.items:has("source") + and not gold.content.items:has("pockets") out.executableBuiltinHidden = gold.content.statuses:get("sleep") == nil out.assetDirectory = gold.assets:info("assets/generated/battle/front") out.assetRejects = {} @@ -113,6 +120,8 @@ for _, id in ipairs(Fixture.CONTINUATIONS) do end for _, id in ipairs(Fixture.MOVES) do T.eq(out.moves[id], true, "Gold exposes " .. id) end T.eq(out.steel, true, "Gold exposes Steel") +T.eq(out.metadataHidden, true, + "Gold extractor metadata stays outside record registry id spaces") local registryCount = 0 for _ in pairs(Schemas.REGISTRIES) do registryCount = registryCount + 1 end for _ in pairs(Schemas.ALIASES) do registryCount = registryCount + 1 end @@ -206,16 +215,22 @@ local transitionRead = transitionFs.read local transitionMarkers = 0 local reimportedPokemon = require("src.import.LuaWriter").encode({ FIXMON = { id = "FIXMON", name = "REIMPORTED", dex = 1, + types = {}, catchRate = 45, baseExp = 64, growthRate = "MEDIUM_FAST", + baseStats = { hp = 45, attack = 49, defense = 49, speed = 45, + special = 65 }, + level1Moves = {}, tmhm = {}, learnset = {}, evolutions = {}, spriteFront = "assets/generated/battle/front/fixmon.png", spriteBack = "assets/generated/battle/back/fixmon.png", frontSize = 5 }, }) transitionFs.read = function(path) if path == GameVersion.cachePrefix("red") .. "rom-cache.complete" then transitionMarkers = transitionMarkers + 1 - if transitionMarkers == 2 then return nil end + -- open and the first facade read each recheck readiness; remove the cache + -- for the next open, then make the following open the reimport. + if transitionMarkers == 3 then return nil end end local body = transitionRead(path) - if transitionMarkers >= 3 + if transitionMarkers >= 4 and path == GameVersion.cachePrefix("red") .. "data/generated/pokemon.lua" then return reimportedPokemon end @@ -255,7 +270,12 @@ local mod = ... local out = {} for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver" }) do local view, reason = mod.datasets:open(version) - out[version] = { view ~= nil, reason } + local value = view and view.content.pokemon:get("BAD") + local reopened, reopenedReason = mod.datasets:open(version) + out[version] = { + first = view ~= nil, firstReason = reason, value = value, + reopened = reopened ~= nil, reason = reopenedReason, + } end mod.exports.result = out ]]) @@ -272,8 +292,9 @@ local hostileRun = T.sdk.loadMods(hostilePaths, { }) local hostileOut = hostileRun.loader.exports.hostile_probe.result for _, row in ipairs(hostile) do - T.same(hostileOut[row[1]], { false, "invalid_cache" }, - row[1] .. " hostile generated source fails closed") + T.same(hostileOut[row[1]], { + first = true, reopened = false, reason = "invalid_cache", + }, row[1] .. " hostile generated source fails closed on first root access") end T.eq(debug.gethook and debug.gethook(), beforeHook, "dataset decoding preserves the caller debug hook") @@ -283,6 +304,91 @@ if debug.sethook then end hostileRun.release() +-- A syntactically valid semantic root with a primitive record must fail closed +-- whichever public read verb encounters it first. +local malformedFiles = {} +for _, version in ipairs({ "red", "blue", "yellow" }) do + Fixture.cache(malformedFiles, version, { + pokemon = "return { FIXMON = 7 }", + }) +end +local malformedMod = Fixture.addMod(malformedFiles, "malformed_probe", [[ +local mod = ... +local out = {} +local getView = assert(mod.datasets:open("red")) +out.get = getView.content.pokemon:get("FIXMON") +local getAgain, getReason = mod.datasets:open("red") +out.getAgain = { getAgain ~= nil, getReason } + +local hasView = assert(mod.datasets:open("blue")) +out.has = hasView.content.pokemon:has("FIXMON") +local hasAgain, hasReason = mod.datasets:open("blue") +out.hasAgain = { hasAgain ~= nil, hasReason } + +local eachView = assert(mod.datasets:open("yellow")) +for id, value in eachView.content.pokemon:each() do + out.each = { id, value } + break +end +local eachAgain, eachReason = mod.datasets:open("yellow") +out.eachAgain = { eachAgain ~= nil, eachReason } +mod.exports.result = out +]]) +local malformedRun = T.sdk.loadMods({ malformedMod }, { + fs = T.sdk.memfs(malformedFiles), data = { pokemon = {} }, generation = 1, +}) +T.eq(#malformedRun.errors, 0, "malformed-record probe stays sandboxed") +local malformedOut = malformedRun.loader.exports.malformed_probe.result +T.eq(malformedOut.get, nil, "get never exposes a malformed semantic record") +T.eq(malformedOut.has, false, "has never affirms a malformed semantic record") +T.eq(malformedOut.each, nil, "each never exposes a malformed semantic record") +T.same(malformedOut.getAgain, { false, "invalid_cache" }, + "get invalidates the malformed dataset") +T.same(malformedOut.hasAgain, { false, "invalid_cache" }, + "has invalidates the malformed dataset") +T.same(malformedOut.eachAgain, { false, "invalid_cache" }, + "each invalidates the malformed dataset") +malformedRun.release() + +-- The same Gold view stays independent while each Gen 1 version is the +-- actual active runtime version and cache namespace, not just a fixture label. +for _, activeVersion in ipairs({ "red", "blue", "yellow" }) do + local matrixFiles = {} + Fixture.cache(matrixFiles, "gold") + local matrixMod = Fixture.addMod(matrixFiles, + "active_" .. activeVersion .. "_gold_probe", [[ +local mod = ... +local gold = assert(mod.datasets:open("gold")) +mod.exports.result = { + species = gold.content.pokemon:get("FIXMON").name, + foresight = gold.content.type_chart:get("NORMAL>GHOST").multiplier, + held = gold.content.held_items:get("LEFTOVERS").heldEffect, +} +]]) + GameVersion.set(activeVersion) + CacheFs.prefix = GameVersion.cachePrefix(activeVersion) + local activeData = { + pokemon = { ACTIVE = { id = "ACTIVE_" .. activeVersion, + nested = { version = activeVersion } } }, + } + local matrixRun = T.sdk.loadMods({ matrixMod }, { + fs = T.sdk.memfs(matrixFiles), data = activeData, generation = 1, + }) + T.eq(#matrixRun.errors, 0, activeVersion .. "-active Gold probe loads") + T.same(matrixRun.loader.exports["active_" .. activeVersion .. "_gold_probe"].result, + { species = "FIXMON", foresight = 0, held = "HELD_LEFTOVERS" }, + activeVersion .. "-active runtime sees canonical Gold semantics") + T.eq(GameVersion.get(), activeVersion, + activeVersion .. " remains the active GameVersion") + T.eq(CacheFs.prefix, GameVersion.cachePrefix(activeVersion), + activeVersion .. " remains the active cache namespace") + T.eq(activeData.pokemon.ACTIVE.nested.version, activeVersion, + activeVersion .. " active Data remains unchanged") + T.eq(activeData.pokemon.FIXMON, nil, + activeVersion .. " active Data receives no Gold record") + matrixRun.release() +end + GameVersion.set(originalVersion) CacheFs.prefix = originalPrefix T.finish("dataset_views") diff --git a/tests/modkit/cases/dataset_views_nontermination.lua b/tests/modkit/cases/dataset_views_nontermination.lua index e895f06a..3a0da2e6 100644 --- a/tests/modkit/cases/dataset_views_nontermination.lua +++ b/tests/modkit/cases/dataset_views_nontermination.lua @@ -10,12 +10,18 @@ Fixture.cache(files, "red", { local modPath = Fixture.addMod(files, "nontermination_probe", [[ local mod = ... local view, reason = mod.datasets:open("red") -mod.exports.result = { view ~= nil, reason } +local value = view and view.content.pokemon:get("FIXMON") +local reopened, reopenedReason = mod.datasets:open("red") +mod.exports.result = { + first = view ~= nil, firstReason = reason, value = value, + reopened = reopened ~= nil, reason = reopenedReason, +} ]]) local run = T.sdk.loadMods({ modPath }, { fs = T.sdk.memfs(files), data = { pokemon = {} }, generation = 1, }) T.same(run.loader.exports.nontermination_probe.result, - { false, "invalid_cache" }, "generated code is never executed") + { first = true, reopened = false, reason = "invalid_cache" }, + "generated code is never executed and invalidates on first root access") run.release() T.finish("dataset_views_nontermination") diff --git a/tests/modkit/dataset_view_fixture.lua b/tests/modkit/dataset_view_fixture.lua index 5d2e80bf..fe1472bc 100644 --- a/tests/modkit/dataset_view_fixture.lua +++ b/tests/modkit/dataset_view_fixture.lua @@ -30,10 +30,23 @@ local MOVES = { local function species(id, dex, generation) local row = { id = id, name = id, dex = dex, + types = {}, catchRate = 45, baseExp = 64, growthRate = "MEDIUM_FAST", + tmhm = {}, evolutions = {}, spriteFront = "assets/generated/battle/front/" .. id:lower() .. ".png", spriteBack = "assets/generated/battle/back/" .. id:lower() .. ".png", } - if generation == 2 then row.picSize = 5 else row.frontSize = 5 end + if generation == 2 then + row.baseStats = { hp = 45, attack = 49, defense = 49, speed = 45, + specialAttack = 65, specialDefense = 65 } + row.levelMoves = {} + row.picSize = 5 + else + row.baseStats = { hp = 45, attack = 49, defense = 49, speed = 45, + special = 65 } + row.level1Moves = {} + row.learnset = {} + row.frontSize = 5 + end return row end @@ -47,6 +60,10 @@ local function defaults(version) types = {}, } if generation == 2 then + pokemon.growthRates = {} + pokemon.tmhmMoves = {} + moves.generation, moves.source = 2, "fixture moves" + items.generation, items.source, items.pockets = 2, "fixture items", {} for index, id in ipairs(CONTINUATIONS) do pokemon[id] = species(id, 168 + index, generation) end From 38d515f547063cb2fb413486506a2299217ec5c2 Mon Sep 17 00:00:00 2001 From: MaxTomahawk Date: Mon, 24 Aug 2026 10:10:23 +0200 Subject: [PATCH 4/6] fix(mod-api): reserve generated metadata ids --- docs/modding.md | 4 +- docs/rfcs/0015-imported-dataset-views.md | 5 +- src/mods/Registry.lua | 7 ++ src/mods/Schemas.lua | 19 ++-- tests/engine/gen2_reserved_metadata_ids.lua | 112 ++++++++++++++++++++ 5 files changed, 137 insertions(+), 10 deletions(-) create mode 100644 tests/engine/gen2_reserved_metadata_ids.lua diff --git a/docs/modding.md b/docs/modding.md index 2845f9f2..952f876f 100644 --- a/docs/modding.md +++ b/docs/modding.md @@ -228,7 +228,9 @@ and data-only record shapes as `mod.content`, but only `get`, `has`, and `each`. Returned records are detached copies and cannot mutate either dataset. Every generated base record passes the selected generation's existing public schema before it is returned; extractor metadata beside record maps stays out -of the registry id space. A malformed record makes `get` return nil, `has` +of the registry id space and is reserved against `register`, `override`, +`patch`, and `remove` writes through the active registry. A malformed record +makes `get` return nil, `has` return false, and `each` return no rows, and invalidates that dataset view. Records containing functions, userdata, threads, metatables, or cycles are not exposed. Each open call receives an independent facade, so one mod cannot diff --git a/docs/rfcs/0015-imported-dataset-views.md b/docs/rfcs/0015-imported-dataset-views.md index 0bf7d9d9..dd25b1f9 100644 --- a/docs/rfcs/0015-imported-dataset-views.md +++ b/docs/rfcs/0015-imported-dataset-views.md @@ -71,8 +71,9 @@ the selected version's generation routing and the engine's existing `Schemas`, `Registry`, and `Builtins` normalization, so structured sources such as type matchups retain the same public ids used by the active `mod.content` facade and extractor metadata beside record maps is not exposed -as a record id. Every generated base record is checked with that selected -generation's existing public schema before it can cross `get`, `has`, or +as a record id or writable through the active registry's mutation verbs. Every +generated base record is checked with that selected generation's existing +public schema before it can cross `get`, `has`, or `each`; there is no dataset-specific duplicate schema. No register, patch, override, or remove verb is exposed. Each call returns an independent facade over the cached internal dataset, so facade mutation cannot cross mod diff --git a/src/mods/Registry.lua b/src/mods/Registry.lua index 3c7ff4bb..a98e0ca6 100644 --- a/src/mods/Registry.lua +++ b/src/mods/Registry.lua @@ -12,6 +12,12 @@ Registry.__index = Registry -- exposed to mods as mod.DELETE: a patch value that unsets a field Registry.DELETE = Merge.DELETE +local function assertWritable(self, id) + if self.spec.reservedIds and self.spec.reservedIds[id] then + error(("%s id is reserved engine metadata: %s"):format(self.name, id)) + end +end + -- spec comes from Schemas.REGISTRIES[name]; bare Registry.new(name) keeps -- the v1 record behavior for standalone use in tests and tools function Registry.new(name, spec) @@ -33,6 +39,7 @@ local function append(self, id, op, value, owner) error(self.name .. ": content is frozen after load") end assert(type(id) == "string" and id ~= "", self.name .. " id is required") + assertWritable(self, id) local list = self.ops[id] if not list then list = {} diff --git a/src/mods/Schemas.lua b/src/mods/Schemas.lua index b3fbf90d..2b6e8d38 100644 --- a/src/mods/Schemas.lua +++ b/src/mods/Schemas.lua @@ -676,9 +676,10 @@ end -- -- So beside `value` / `fields` / `keys` / `keyValue` a spec may carry -- `gen2Value` / `gen2Fields` / `gen2Keys` / `gen2KeyValue`, and beside --- `semantics` / `extra` / `write` / `baseAt` / `baseIds` / `example` / --- `notes` the matching `gen2*`. Absent means "the Gen 1 shape is right here --- too", which is the common case and why most registries carry none of this. +-- `semantics` / `extra` / `write` / `baseAt` / `baseIds` / `reservedIds` / +-- `example` / `notes` the matching `gen2*`. Absent means "the Gen 1 shape is +-- right here too", which is the common case and why most registries carry +-- none of this. -- The registry NAME, the verbs and (wherever the id space allows it) the ids -- stay shared, exactly as the routing table keeps them shared. -- @@ -700,6 +701,7 @@ local GEN2_SHAPE = { gen2KeyValue = "keyValue", gen2Extra = "extra", gen2Semantics = "semantics", gen2Write = "write", gen2BaseAt = "baseAt", gen2BaseIds = "baseIds", + gen2ReservedIds = "reservedIds", gen2Example = "example", gen2Notes = "notes", } @@ -773,14 +775,14 @@ local function recordMapExcept(...) if not excluded[id] then ids[#ids + 1] = id end end return ids - end + end, excluded end -local pokemonGen2BaseAt, pokemonGen2BaseIds = +local pokemonGen2BaseAt, pokemonGen2BaseIds, pokemonGen2ReservedIds = recordMapExcept("growthRates", "tmhmMoves") -local movesGen2BaseAt, movesGen2BaseIds = +local movesGen2BaseAt, movesGen2BaseIds, movesGen2ReservedIds = recordMapExcept("generation", "source") -local itemsGen2BaseAt, itemsGen2BaseIds = +local itemsGen2BaseAt, itemsGen2BaseIds, itemsGen2ReservedIds = recordMapExcept("generation", "source", "pockets") -- ------- shared Gen 2 leaves @@ -806,6 +808,7 @@ local gen2PaletteRow = f.list(gen2Color) R.pokemon = { semantics = "record", target = "pokemon", gen2BaseAt = pokemonGen2BaseAt, gen2BaseIds = pokemonGen2BaseIds, + gen2ReservedIds = pokemonGen2ReservedIds, fields = { id = f.str, name = f.str, dex = f.int(1), index = f.opt(f.int(0, 255)), @@ -898,6 +901,7 @@ R.pokemon = { R.moves = { semantics = "record", target = "moves", gen2BaseAt = movesGen2BaseAt, gen2BaseIds = movesGen2BaseIds, + gen2ReservedIds = movesGen2ReservedIds, fields = { id = f.str, name = f.str, index = f.opt(f.int(0, 255)), @@ -923,6 +927,7 @@ R.moves = { R.items = { semantics = "record", target = "items", gen2BaseAt = itemsGen2BaseAt, gen2BaseIds = itemsGen2BaseIds, + gen2ReservedIds = itemsGen2ReservedIds, fields = { id = f.str, name = f.str, index = f.opt(f.int(0, 255)), diff --git a/tests/engine/gen2_reserved_metadata_ids.lua b/tests/engine/gen2_reserved_metadata_ids.lua new file mode 100644 index 00000000..88495e3a --- /dev/null +++ b/tests/engine/gen2_reserved_metadata_ids.lua @@ -0,0 +1,112 @@ +-- Extractor metadata hidden from record id spaces remains engine-owned: mods +-- cannot claim, update, or remove it through the public content registries. +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") +local Mon = require("src.battle.gen2.Mon") + +local function goldData() + local data = T.fixtures.fresh() + data.pokemon.growthRates = { + GROWTH_MEDIUM_FAST = { + numerator = 1, denominator = 1, squared = 0, linear = 0, constant = 0, + }, + } + data.pokemon.tmhmMoves = { "FIX_TACKLE", "FIX_CUT" } + data.moves.generation, data.moves.source = 2, "ROM:Moves" + data.items.generation, data.items.source = 2, "ROM:Items" + data.items.pockets = { "ITEM", "BALL", "KEY_ITEM", "TM_HM" } + return data +end + +local function assertMetadata(data, refs, label) + T.eq(data.pokemon.growthRates, refs.growthRates, + label .. ": growth-rate table identity is preserved") + T.eq(data.pokemon.tmhmMoves, refs.tmhmMoves, + label .. ": TM/HM order identity is preserved") + T.eq(data.moves.generation, 2, label .. ": move generation is preserved") + T.eq(data.moves.source, "ROM:Moves", label .. ": move source is preserved") + T.eq(data.items.generation, 2, label .. ": item generation is preserved") + T.eq(data.items.source, "ROM:Items", label .. ": item source is preserved") + T.eq(data.items.pockets, refs.pockets, + label .. ": item pocket order identity is preserved") + T.eq(Mon.experienceForLevel( + Mon.growthFor(data, "GROWTH_MEDIUM_FAST"), 10), 1000, + label .. ": active Gold growth behavior is preserved") +end + +-- A no-mod load characterizes the unchanged active-boot behavior. +do + local data = goldData() + local refs = { + growthRates = data.pokemon.growthRates, + tmhmMoves = data.pokemon.tmhmMoves, + pockets = data.items.pockets, + } + local run = T.sdk.loadNone({ data = data, generation = 2 }) + T.eq(#run.errors, 0, "no-mod Gold load remains clean") + assertMetadata(run.data, refs, "no-mod Gold") + run.release() +end + +local files, paths = {}, {} +local attempts = { + register_growth_rates = [[ +local mod = ... +mod.content.pokemon:register("growthRates", { + id = "growthRates", name = "CLAIMED", dex = 999, + types = {}, + baseStats = { hp = 1, attack = 1, defense = 1, speed = 1, + specialAttack = 1, specialDefense = 1 }, + catchRate = 1, baseExp = 1, growthRate = "GROWTH_MEDIUM_FAST", + levelMoves = {}, evolutions = {}, + spriteFront = "assets/generated/battle/front/claimed.png", + spriteBack = "assets/generated/battle/back/claimed.png", picSize = 5, +}) +]], + override_move_generation = [[ +local mod = ... +mod.content.moves:override("generation", { + id = "generation", name = "CLAIMED", type = "NORMAL", + power = 1, accuracy = 100, pp = 1, effect = "NO_ADDITIONAL_EFFECT", +}) +]], + patch_item_pockets = [[ +local mod = ... +mod.content.items:patch("pockets", { price = 999 }) +]], + remove_tmhm_moves = [[ +local mod = ... +mod.content.pokemon:remove("tmhmMoves") +]], +} + +for id, body in pairs(attempts) do + files["mods/" .. id .. "/manifest.json"] = ([[{ + "id": %q, "name": %q, "version": "1.0.0", "entry": "main.lua", + "api": 2, "gen2compat": true + }]]):format(id, id) + files["mods/" .. id .. "/main.lua"] = body + paths[#paths + 1] = "mods/" .. id +end +table.sort(paths) + +local data = goldData() +local refs = { + growthRates = data.pokemon.growthRates, + tmhmMoves = data.pokemon.tmhmMoves, + pockets = data.items.pockets, +} +local run = T.sdk.loadMods(paths, { + fs = T.sdk.memfs(files), data = data, generation = 2, +}) +for id in pairs(attempts) do + local mod = run.loader.mods[id] + T.eq(mod and mod.state, "failed", id .. " is rejected") + T.check(mod and tostring(mod.failure):match("reserved") ~= nil, + id .. " reports the reserved metadata boundary") +end +assertMetadata(run.data, refs, "rejected metadata writes") +run.release() + +T.finish("gen2_reserved_metadata_ids") From cec19db5e00f0d5d01f57ff7175d6a199aeedccb Mon Sep 17 00:00:00 2001 From: MaxTomahawk Date: Mon, 24 Aug 2026 10:19:33 +0200 Subject: [PATCH 5/6] test(mod-api): cover reserved metadata replace --- tests/engine/gen2_reserved_metadata_ids.lua | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/engine/gen2_reserved_metadata_ids.lua b/tests/engine/gen2_reserved_metadata_ids.lua index 88495e3a..78e35d04 100644 --- a/tests/engine/gen2_reserved_metadata_ids.lua +++ b/tests/engine/gen2_reserved_metadata_ids.lua @@ -63,6 +63,19 @@ mod.content.pokemon:register("growthRates", { spriteFront = "assets/generated/battle/front/claimed.png", spriteBack = "assets/generated/battle/back/claimed.png", picSize = 5, }) +]], + replace_growth_rates = [[ +local mod = ... +mod.content.pokemon:register("growthRates", { + id = "growthRates", name = "REPLACED", dex = 999, + types = {}, + baseStats = { hp = 1, attack = 1, defense = 1, speed = 1, + specialAttack = 1, specialDefense = 1 }, + catchRate = 1, baseExp = 1, growthRate = "GROWTH_MEDIUM_FAST", + levelMoves = {}, evolutions = {}, + spriteFront = "assets/generated/battle/front/replaced.png", + spriteBack = "assets/generated/battle/back/replaced.png", picSize = 5, +}, { replace = true }) ]], override_move_generation = [[ local mod = ... From 68fc01dd1cedcd064debf3264bd95f5d091be467 Mon Sep 17 00:00:00 2001 From: MaxTomahawk Date: Mon, 24 Aug 2026 10:25:37 +0200 Subject: [PATCH 6/6] fix(mod-api): silence optional dataset absence --- src/mods/DatasetViews.lua | 3 +- tests/engine/dataset_views_warning_policy.lua | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 tests/engine/dataset_views_warning_policy.lua diff --git a/src/mods/DatasetViews.lua b/src/mods/DatasetViews.lua index 253338d0..ef44d6e2 100644 --- a/src/mods/DatasetViews.lua +++ b/src/mods/DatasetViews.lua @@ -359,12 +359,11 @@ function DatasetViews:open(version) if type(version) ~= "string" or not GameVersion.VERSIONS[version] then return nil, "unknown_version" end - local inspected, reason, detail = CacheContract.inspect(version, self.fs, { + local inspected, reason = CacheContract.inspect(version, self.fs, { allowSource = true, semantic = true, }) if not inspected then self.datasets[version] = nil - Logger.warn("dataset %s unavailable: %s", version, detail) return nil, reason end local plan, invalid = self:_preflight(version, inspected) diff --git a/tests/engine/dataset_views_warning_policy.lua b/tests/engine/dataset_views_warning_policy.lua new file mode 100644 index 00000000..66fc92b2 --- /dev/null +++ b/tests/engine/dataset_views_warning_policy.lua @@ -0,0 +1,56 @@ +-- Optional dataset absence is a handled API result, while a previously valid +-- view disappearing and malformed cache content remain actionable warnings. +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") +local Fixture = require("tests.modkit.dataset_view_fixture") +local DatasetViews = require("src.mods.DatasetViews") +local GameVersion = require("src.core.GameVersion") +local Logger = require("src.core.Logger") + +local previousWarn = Logger.warn +local warnings = {} +Logger.warn = function(fmt, ...) + warnings[#warnings + 1] = select("#", ...) > 0 + and string.format(fmt, ...) or fmt +end + +local ok, err = xpcall(function() + local unavailable = DatasetViews.new(T.sdk.memfs({})) + local unknown, unknownReason = unavailable:open("missing-version") + T.eq(unknown, nil, "unknown version has no view") + T.eq(unknownReason, "unknown_version", "unknown version returns its reason") + local absent, absentReason = unavailable:open("gold") + T.eq(absent, nil, "missing optional Gold dataset has no view") + T.eq(absentReason, "not_imported", "missing optional Gold returns its reason") + T.eq(#warnings, 0, "unknown and initially absent datasets do not warn") + + local files = {} + Fixture.cache(files, "gold") + local service = DatasetViews.new(T.sdk.memfs(files)) + local view = assert(service:open("gold")) + files[GameVersion.cachePrefix("gold") .. "rom-cache.complete"] = nil + T.eq(view.assets:path("assets/generated/missing.png"), nil, + "a stale view closes when its imported dataset disappears") + T.eq(#warnings, 1, "a previously valid view disappearing still warns") + T.check(warnings[1]:find("dataset gold unavailable", 1, true) ~= nil, + "the stale-view warning identifies the unavailable dataset") + + warnings = {} + local malformedFiles = {} + Fixture.cache(malformedFiles, "gold", { pokemon = "not generated data" }) + local malformed = assert(DatasetViews.new(T.sdk.memfs(malformedFiles)):open("gold")) + T.eq(#warnings, 0, "lazy open does not warn before malformed data is read") + T.eq(malformed.content.pokemon:get("FIXMON"), nil, + "malformed generated data fails closed") + T.eq(#warnings, 1, "malformed generated data still warns") + T.check(warnings[1]:find("dataset gold cache rejected", 1, true) ~= nil, + "the malformed-cache warning identifies cache rejection") + T.check(warnings[1]:find("pokemon:", 1, true) ~= nil, + "the malformed-cache warning keeps actionable module detail") +end, debug.traceback) + +Logger.warn = previousWarn +if not ok then error(err, 0) end + +T.finish("dataset_views_warning_policy")