diff --git a/docs/modding.md b/docs/modding.md index 3d1329bf..55f8ee82 100644 --- a/docs/modding.md +++ b/docs/modding.md @@ -200,6 +200,60 @@ the adapter's own coverage table: python3 tools/modkit.py gen2check mods/my_mod ``` +## Imported version datasets + +A mod can inspect semantic content from another game the player has already +imported without switching the active game or reaching into engine cache +internals: + +```lua +local gold, reason = mod.datasets:open("gold") +if not gold then + -- reason is "unknown_version", "not_imported", or "invalid_cache" + return +end + +local chikorita = gold.content.pokemon:get("CHIKORITA") +local normalVsGhost = gold.content.type_chart:get("NORMAL>GHOST") +local spritePath = gold.assets:path(chikorita.spriteFront) + +for id, record in gold.content.pokemon:each() do + -- ids are returned in deterministic lexical order +end +``` + +`view.version` and `view.generation` identify the selected dataset. +`view.content` exposes the same registry names, aliases, generation routing, +and data-only record shapes as `mod.content`, but only `get`, `has`, and +`each`. Returned records are detached copies and cannot mutate either dataset. +Every generated base record passes the selected generation's existing public +schema before it is returned; extractor metadata beside record maps stays out +of the registry id space and is reserved against `register`, `override`, +`patch`, and `remove` writes through the active registry. A malformed record +makes `get` return nil, `has` +return false, and `each` return no rows, and invalidates that dataset view. +Records containing functions, userdata, threads, metatables, or cycles are not +exposed. Each open call receives an independent facade, so one mod cannot +replace another mod view method. Canonical boot shaping is included, such as +Gen 1 defaults and Yellow corrections, and Gold's Foresight matchup rows and +derived `held_items`. + +`open` checks the completion marker, exact version-specific file inventory, +source/cache boundary, and file-size bounds without reading or decoding the +semantic modules. A root is read, bounded-decoded, normalized, and cached only +when a content operation first needs it. Each later view operation rechecks +readiness and the source bytes behind already cached roots; unchanged roots are +not decoded again. Missing, partial, and stale imports return +`nil, "not_imported"`. Malformed syntax, a resource-limit violation, or a +record that fails the public schema is discovered on first root access and +fails that operation closed; a later `open` against the same source returns +`nil, "invalid_cache"`. Generated modules use a bounded literal-only grammar +and are never executed. No raw ROM bytes or generated source are exposed. +`view.assets:path(relative)` and +`view.assets:info(relative)` accept only `assets/generated/...` paths and +keep them under the selected version cache prefix. The API never changes +`mod.game`, the active `Data` table, `GameVersion`, or cache mount state. + ## Editing maps in Tiled Maps are data, not assets, so they can be authored in a real map editor and diff --git a/docs/modding/reference/registries.md b/docs/modding/reference/registries.md index 929299f7..96693e63 100644 --- a/docs/modding/reference/registries.md +++ b/docs/modding/reference/registries.md @@ -342,6 +342,7 @@ accepted and merged as-is. | `source` | string | | `swarmGrass` | map of string -> {map?, rates, slots} | | `swarmWater` | map of string -> {map?, rate, slots} | +| `timeFishGroups` | map of string | integer 0..255 -> {day, nite} | | `treeSets` | map of string -> {common, rare} | | `trees` | map of string -> string | | `water` | map of string -> {map?, rate, slots} | @@ -509,6 +510,26 @@ mod.content.item_effects:register("MOON_FLUTE", { use = fn, field = true }) mod.content.items:patch("POTION", { price = 100 }) ``` +### On Gold (Gen 2) + +- semantics: `record` +- target: `Data.items` + +The record differs; the registry name, the verbs and the id space +do not. + +| field | type | required | +|---|---|---| +| `ball` | balls id | no | +| `effect` | item_effects id | no | +| `id` | string | yes | +| `index` | integer 0..255 | no | +| `machine` | {kind, move, number} | no | +| `name` | string | yes | +| `needsTarget` | boolean | no | +| `price` | integer >= 0 | yes | +| `tossable` | boolean | no | + ## landmarks - semantics: `record` @@ -598,7 +619,7 @@ mod.content.map_songs:override("PALLET_TOWN", "Music_Routes1") | `id` | string | yes | | `index` | integer >= 0 | no | | `label` | string | no | -| `objects` | list of any value | no | +| `objects` | list of {pokemon?, ...} | no | | `palette` | string | no | | `signs` | list of any value | no | | `tileset` | tilesets id | yes | @@ -673,6 +694,34 @@ mod.content.move_effects:register("DRAIN_PP_EFFECT", { kind = "primary", run = f mod.content.moves:patch("BLIZZARD", { accuracy = 70 }) ``` +### On Gold (Gen 2) + +- semantics: `record` +- target: `Data.moves` + +The record differs; the registry name, the verbs and the id space +do not. + +| field | type | required | +|---|---|---| +| `accuracy` | integer 0..100 | yes | +| `anim` | any value | no | +| `category` | one of "physical" | "special" | "status" | no | +| `chargeText` | string | no | +| `counterable` | boolean | no | +| `effect` | move_effects id | yes | +| `fixedDamage` | integer >= 1 | function | no | +| `highCrit` | boolean | no | +| `id` | string | yes | +| `index` | integer 0..255 | no | +| `multiHit` | integer >= 1 | list of integer >= 1 | no | +| `name` | string | yes | +| `power` | integer 0..255 | yes | +| `pp` | integer 0..64 | yes | +| `priority` | integer -7..7 | no | +| `semiInvulnerable` | boolean | no | +| `type` | type_chart id | yes | + ## music - semantics: `record` diff --git a/docs/rfcs/0015-imported-dataset-views.md b/docs/rfcs/0015-imported-dataset-views.md new file mode 100644 index 00000000..829c8a8f --- /dev/null +++ b/docs/rfcs/0015-imported-dataset-views.md @@ -0,0 +1,154 @@ +# RFC 0015: Read-only imported dataset views + +## Status + +Proposed. + +## Motivation + +A cross-version content mod can read only the active merged dataset through +`mod.content`. Even when the player has already imported another supported +game, a mod cannot inspect that version's semantic species, moves, items, type +chart, or generated asset namespace. The available alternatives are private: +mutating `CacheFs.prefix`, mounting another cache over the active one, loading +generated Lua directly, or asking for a second raw-ROM import. They compose +poorly with the active game, expose unstable import layout, and make safe +read-only use impossible from the sandbox. + +The concrete consumer is **Adaptive Trainers**, whose approved Phase G Kanto+ +sidecar must derive nine Kanto-line continuations, Steel/type and move +definitions, and generated sprites from the player's existing verified Gold +cache while Red, Blue, or Yellow remains active. `mod.content` exposes only the +active R/B/Y dataset; `mod.imports` can read only separately declared raw mod +imports; and `mod.cache` is mod-private generated output. None can inspect the +launcher-owned Gold semantic dataset without a second ROM import and mod-side +ROM interpretation. The engine API remains generic and contains no Adaptive +Trainers policy. + +## Decision and plan extended + +This implements **D-AT-004: optional Kanto+ content consumes an +active-independent semantic dataset view**. The consuming design is tracked in +the Adaptive Trainers implementation plan, +[`docs/superpowers/plans/2026-08-14-adaptive-trainers.md`](https://github.com/MaxTomahawk/gen1recomp-adaptive-trainers/blob/main/docs/superpowers/plans/2026-08-14-adaptive-trainers.md), +Task 8. The delta is a generic, additive public API and contains no trainer +pools, scaling, boss identities, version-mixing rules, or Adaptive Trainers +policy. + +## Exact API delta + +Every sandboxed mod receives: + +```lua +local view, reason = mod.datasets:open("gold") +``` + +A known version with the current completed import marker returns a read-only view: + +```lua +view = { + version = "gold", + generation = 2, + content = { + pokemon = { + get = function(self, id) end, + has = function(self, id) end, + each = function(self) end, + }, + -- every public registry name and alias + }, + assets = { + path = function(self, generatedPath) end, + info = function(self, generatedPath) end, + }, +} +``` + +`get` returns a bounded detached data-only copy or nil. `has` reports data-only +semantic presence. +`each` returns ids in lexical order and detached values. The registries use +the selected version's generation routing and the engine's existing +`Schemas`, `Registry`, and `Builtins` normalization, so structured sources +such as type matchups retain the same public ids used by the active +`mod.content` facade and extractor metadata beside record maps is not exposed +as a record id or writable through the active registry's mutation verbs. Every +generated base record is checked with that selected generation's existing +public schema before it can cross `get`, `has`, or +`each`; there is no dataset-specific duplicate schema. No register, patch, +override, or remove verb is exposed. Each call returns an independent facade +over the cached internal dataset, so facade mutation cannot cross mod +boundaries. + +`assets:path` returns the selected cache-prefixed virtual path. +`assets:info` returns sanitized `type` and optional `size` metadata. Both +accept only relative paths below `assets/generated/`, reject control +characters, absolute paths, backslashes, and traversal, and expose no byte +reader. + +An unknown version returns `nil, "unknown_version"`. `open` checks the current +completion marker, exact version-specific file inventory, source/cache +boundary, and available file sizes; it does not read or decode semantic +modules. A missing, partial, or stale cache returns `nil, "not_imported"`. + +The first content operation that needs a root reads it once, applies the +limits (8 MiB per module, 48 MiB aggregate, depth 64, 500,000 values, 2 MiB +per string, and 250,000 entries per table), decodes the restricted literal +grammar, applies canonical selected-version normalization, and caches the +detached root. Each later content or asset operation rechecks marker/file and +source-bound readiness. It also rereads the source bytes for already cached +roots; unchanged roots are not decoded again, while a changed root clears the +derived registry cache and is decoded on its next use. + +Malformed syntax, non-table roots, binary/trailing content, resource-limit +violations, and records that fail the public schema are therefore discovered +on first access rather than during `open`. The triggering `get` returns nil, +`has` returns false, or `each` returns no rows; the whole internal view is +invalidated, and a subsequent `open` against the unchanged source returns +`nil, "invalid_cache"`. Actionable detail is engine-logged but not exposed to +the mod. Generated Lua is never executed. Functions, userdata, threads, +metatables, and cycles cannot cross the facade. The API never exposes raw ROM +bytes, generated source, host paths, or a mount. + +## Migration and compatibility + +Existing mods change nothing. `mod.datasets` is additive and requires no +permission. The service is allocated lazily on the first explicit +`mod.datasets:open` call. A boot with no mods, or with mods that do not call +it, performs no cross-version cache reads. + +Opening a view does not change `GameVersion`, `CacheFs.prefix`, the active +`Data` table, PhysFS mounts, save state, or the selected game's behavior. +Red, Blue, Yellow, Gold, Silver, and Crystal keep their existing active data paths. + +The completion marker and per-version required-file rules live in the pure, +injected `CacheContract` shared by the importer and dataset service. It also +defines source-tree behavior. Neither consumer mutates `CacheFs.prefix` while +checking readiness. Every `open` revalidates the contract and required module +inventory without semantic decoding. Every view operation rechecks that +readiness before serving cached state; a stale/remove/reimport transition +evicts the previous semantic view. + +## Verification + +- `tests/modkit/cases/dataset_views.lua` loads sandboxed fixture mods through + the public API and covers Red, Blue, Yellow, Gold, Silver, and Crystal independently. +- The test proves semantic registry normalization, deterministic iteration, + detached records, read-only facades, cross-mod facade isolation, + version-prefixed generated assets, + traversal rejection, stable failure reasons, and stale-marker rejection. +- It also proves missing/empty/partial/stale/remove/reimport behavior, hostile + generated-source and malformed-record rejection, canonical Gen + 1/Yellow/Gold hydration, active Red/Blue/Yellow isolation while reading Gold, + and the approved Kanto+ Gold records and assets. +- `tests/engine/dataset_views_lazy_validation.lua` counts semantic reads and + decoder calls to prove `open` decodes nothing, unused roots stay unread, and + repeated access does not re-decode an unchanged cached root. +- `tests/modkit/cases/dataset_views_nontermination.lua` proves generated code + is rejected rather than executed; `tests/engine/generated_data_decoder_test.lua` + proves every decoder resource bound. +- `tests/engine/dataset_views_no_mod_parity.lua` is the separate guarded no-mod + parity suite and proves the service stays unallocated with zero cache reads. + +## Deprecation etiquette + +Nothing is removed, renamed, superseded, or deprecated. diff --git a/src/core/Data.lua b/src/core/Data.lua index 31d7932a..935739a9 100644 --- a/src/core/Data.lua +++ b/src/core/Data.lua @@ -82,8 +82,9 @@ local function copy(value) return out end -function Data:applyVersionedFieldData() - if require("src.core.GameVersion").isYellow() then +function Data:applyVersionedFieldData(version) + version = version or require("src.core.GameVersion").get() + if version == "yellow" then self.field.trades = copy(YELLOW_TRADES) -- The old man's catch demo is a RATTATA in Yellow -- (scripts/ViridianCity.asm ViridianCityOldManStartCatchTrainingScript @@ -106,7 +107,8 @@ end -- Fills only what the cache is missing, so an importer that learns to -- stamp one of these keys silently takes over from the engine. -function Data:seedDefaults() +function Data:seedDefaults(version) + version = version or require("src.core.GameVersion").get() local constants = self.constants or {} self.constants = constants self.field = self.field or {} @@ -131,7 +133,7 @@ function Data:seedDefaults() if constants.dexDigits == nil then constants.dexDigits = math.max(3, #tostring(constants.dexSize)) end - self:applyVersionedFieldData() + Data.applyVersionedFieldData(self, version) local boot = self.field.boot if boot == nil then boot = {} @@ -144,7 +146,7 @@ function Data:seedDefaults() -- only the un-overridden default flips, so a total conversion that set -- field.boot.screens.splash keeps its choice on any version. if boot.screens.splash == BOOT_DEFAULTS.screens.splash - and require("src.core.GameVersion").isYellow() then + and version == "yellow" then boot.screens.splash = "YellowIntro" end -- the naming screen presets the importer already extracts but nothing @@ -163,11 +165,11 @@ function Data:seedDefaults() -- extractor never writes headers for them. Seed the EVENT_BEAT_* / -- after-battle rows so Blaine's SetEventRange deactivation and talk -- after-text work like the other gyms (scripts/CinnabarGym.asm). - self:seedCinnabarGymTrainerHeaders() + Data.seedCinnabarGymTrainerHeaders(self) -- #197: the Fighting Dojo Karate Master is text_asm, so the extractor -- writes no header for him -- seed one so he engages on sight and has -- his defeat / re-talk lines (same idea as the Cinnabar seed above). - self:seedFightingDojoKarateMaster() + Data.seedFightingDojoKarateMaster(self) -- #189: 1F cabin door order vs rooms map (survey zoom) require("src.world.SsAnneLayout").apply(self.maps) end diff --git a/src/core/DatasetHydration.lua b/src/core/DatasetHydration.lua new file mode 100644 index 00000000..003cc3bf --- /dev/null +++ b/src/core/DatasetHydration.lua @@ -0,0 +1,29 @@ +-- Canonical pure shaping shared by the active boot and inactive dataset views. +-- The caller supplies both the data table and selected version; no global +-- GameVersion, cache prefix, mount, or active Data table is changed. + +local GameVersion = require("src.core.GameVersion") + +local DatasetHydration = {} + +function DatasetHydration.applyGen2(data, moduleLoader) + local chart = data.type_chart or {} + chart.matchups = chart.matchups or {} + for _, row in ipairs(chart.foresightMatchups or {}) do + chart.matchups[#chart.matchups + 1] = row + end + data.type_chart = chart + data.gen2HeldItems = + (moduleLoader or require)("src.core.gen2.ItemEffects").heldItemsFrom(data.items) + return data +end + +function DatasetHydration.apply(data, version, moduleLoader) + if GameVersion.generation(version) == 2 then + return DatasetHydration.applyGen2(data, moduleLoader) + end + require("src.core.Data").seedDefaults(data, version) + return data +end + +return DatasetHydration diff --git a/src/core/SaveSerializer.lua b/src/core/SaveSerializer.lua index 9f821f67..7adbae86 100644 --- a/src/core/SaveSerializer.lua +++ b/src/core/SaveSerializer.lua @@ -70,9 +70,27 @@ local function fail(state, why) error(("parse error at byte %d: %s"):format(state.pos, why), 0) end +local function limit(state, name) + return state.limits and state.limits[name] +end + +local function bumpNode(state) + state.nodes = state.nodes + 1 + local maximum = limit(state, "maxNodes") + if maximum and state.nodes > maximum then fail(state, "too many values") end +end + local function skip(state) - local _, last = state.src:find("^[ \t\r\n]*", state.pos) - state.pos = last + 1 + while true do + local _, last = state.src:find("^[ \t\r\n]*", state.pos) + state.pos = last + 1 + if not (state.limits and state.limits.allowComments + and state.src:sub(state.pos, state.pos + 1) == "--") then + return + end + local newline = state.src:find("\n", state.pos + 2, true) + state.pos = newline and newline + 1 or (#state.src + 1) + end end local function peek(state) @@ -90,7 +108,10 @@ local function readString(state) fail(state, "unterminated string") elseif c == '"' then state.pos = i + 1 - return table.concat(out) + local value = table.concat(out) + local maximum = limit(state, "maxStringBytes") + if maximum and #value > maximum then fail(state, "string too long") end + return value elseif c == "\\" then local nxt = src:sub(i + 1, i + 1) if nxt:match("%d") then @@ -136,10 +157,14 @@ end local readValue local function readTable(state) + bumpNode(state) state.depth = state.depth + 1 - if state.depth > MAX_DEPTH then fail(state, "table nesting too deep") end + if state.depth > (limit(state, "maxDepth") or MAX_DEPTH) then + fail(state, "table nesting too deep") + end state.pos = state.pos + 1 local out = {} + local entries, nextArray = 0, 1 skip(state) if peek(state) == "}" then state.pos = state.pos + 1 @@ -148,7 +173,7 @@ local function readTable(state) end while true do skip(state) - local key + local key, value local c = peek(state) if c == "[" then state.pos = state.pos + 1 @@ -156,15 +181,38 @@ local function readTable(state) skip(state) if peek(state) ~= "]" then fail(state, "expected ]") end state.pos = state.pos + 1 + skip(state) + if peek(state) ~= "=" then fail(state, "expected =") end + state.pos = state.pos + 1 + value = readValue(state) elseif c:match("[%a_]") then - key = readIdent(state) + local start = state.pos + local ident = readIdent(state) + skip(state) + if peek(state) == "=" then + key = ident + state.pos = state.pos + 1 + value = readValue(state) + elseif state.limits and state.limits.allowArray then + state.pos = start + key = nextArray + value = readValue(state) + else + fail(state, "expected =") + end + elseif state.limits and state.limits.allowArray then + key = nextArray + value = readValue(state) else fail(state, "expected key") end - skip(state) - if peek(state) ~= "=" then fail(state, "expected =") end - state.pos = state.pos + 1 - out[key] = readValue(state) + if key == nil then fail(state, "nil table key") end + if out[key] ~= nil then fail(state, "duplicate table key") end + entries = entries + 1 + local maximum = limit(state, "maxTableEntries") + if maximum and entries > maximum then fail(state, "too many table entries") end + out[key] = value + if type(key) == "number" and key == nextArray then nextArray = nextArray + 1 end skip(state) local sep = peek(state) if sep == "," then @@ -189,25 +237,30 @@ readValue = function(state) skip(state) local c = peek(state) if c == '"' then + bumpNode(state) return readString(state) elseif c == "{" then return readTable(state) elseif c:match("[%a_]") then -- the only bare words in the grammar are the boolean literals local word = readIdent(state) - if word == "true" then return true end - if word == "false" then return false end + if word == "true" then bumpNode(state); return true end + if word == "false" then bumpNode(state); return false end state.pos = state.pos - #word fail(state, "unexpected name '" .. word .. "'") elseif c:match("[%-%d%.]") then + bumpNode(state) return readNumber(state) end fail(state, c == "" and "unexpected end of input" or "unexpected character") end -function SaveSerializer.decode(str) +function SaveSerializer.decode(str, limits) if type(str) ~= "string" then return nil, "save must be a string" end - local state = { src = str, pos = 1, depth = 0 } + if limits and limits.maxBytes and #str > limits.maxBytes then + return nil, ("input is %d bytes (max %d)"):format(#str, limits.maxBytes) + end + local state = { src = str, pos = 1, depth = 0, nodes = 0, limits = limits } local ok, result = pcall(function() skip(state) local word = state.src:match("^[%a_][%w_]*", state.pos) @@ -219,7 +272,9 @@ function SaveSerializer.decode(str) return value end) if not ok then return nil, result end - if type(result) ~= "table" then return nil, "save root must be a table" end + if type(result) ~= "table" then + return nil, (limits and limits.rootName or "save") .. " root must be a table" + end return result end diff --git a/src/import/CacheContract.lua b/src/import/CacheContract.lua index 46261077..3e4c46b4 100644 --- a/src/import/CacheContract.lua +++ b/src/import/CacheContract.lua @@ -138,12 +138,70 @@ CacheContract.VERSION_REQUIRED_FILES_OVERRIDE = { CacheContract.VERSION_REQUIRED_FILES_OVERRIDE.silver = CacheContract.VERSION_REQUIRED_FILES_OVERRIDE.gold +local SEMANTIC_MODULES = { + [1] = { + "constants", "maps", "tilesets", "text", "text_pointers", + "trainer_headers", "font", "sprites", "pokemon", "moves", "items", + "type_chart", "trainers", "encounters", "field", "battle_anims", + }, + [2] = { + "pokemon", "moves", "items", "type_chart", "audio", "font", "maps", + "tilesets", "text", "trainers", "encounters", "sprites", "palettes", + "icons", "battle_anims", "constants", "landmarks", + }, +} + +local OPTIONAL_SEMANTIC_MODULES = { + [1] = { "audio", "palettes", "icons" }, + [2] = {}, +} + +local function copy(values) + local out = {} + for index, value in ipairs(values or {}) do out[index] = value end + return out +end + function CacheContract.requiredFilesFor(version) local override = CacheContract.VERSION_REQUIRED_FILES_OVERRIDE[version] if override then return override, true end return CacheContract.REQUIRED_FILES, false end +-- A detached, version-specific inventory for read-only consumers. Semantic +-- consumers additionally require every generated module that backs the public +-- registry surface; optional semantic roots are discovered lazily. +function CacheContract.requiredFiles(version, semantic) + local base, isOverride = CacheContract.requiredFilesFor(version) + local files = copy(base) + if not isOverride then + for _, path in ipairs(CacheContract.VERSION_REQUIRED_FILES[version] or {}) do + files[#files + 1] = path + end + end + if semantic then + for _, name in ipairs(SEMANTIC_MODULES[GameVersion.generation(version)] or {}) do + files[#files + 1] = "data/generated/" .. name .. ".lua" + end + end + local seen, out = {}, {} + for _, path in ipairs(files) do + if not seen[path] then + seen[path] = true + out[#out + 1] = path + end + end + return out +end + +function CacheContract.semanticModules(version) + return copy(SEMANTIC_MODULES[GameVersion.generation(version)]) +end + +function CacheContract.optionalSemanticModules(version) + return copy(OPTIONAL_SEMANTIC_MODULES[GameVersion.generation(version)]) +end + function CacheContract.formatFor(version) return CacheContract.VERSION_FORMAT[version] or CacheContract.FORMAT end @@ -265,4 +323,59 @@ function CacheContract.sourceTreeHasData(version) return true end +-- Inspect another version without mutating CacheFs.prefix, GameVersion, mounts, +-- or the active Data table. The injected filesystem addresses exact paths. +local function readAt(fs, path) + if fs.readAt then return fs.readAt(path) end + if fs.read then return fs.read(path) end + return nil +end + +local function isFileAt(fs, path) + if fs.getInfo then + local info = fs.getInfo(path, "file") + return info ~= nil and (info.type == nil or info.type == "file") + end + if fs.existsAt then return fs.existsAt(path) == true end + return false +end + +local function hasExactFiles(version, fs, prefix, semantic) + for _, path in ipairs(CacheContract.requiredFiles(version, semantic)) do + if not isFileAt(fs, prefix .. path) then return false, path end + end + return true +end + +local function sourceReady(version, fs, semantic) + if not (fs.getInfo and fs.getRealDirectory and fs.getSource) then return nil end + local prefix = version == "red" and "" or GameVersion.cachePrefix(version) + local complete = hasExactFiles(version, fs, prefix, semantic) + if not complete then return nil end + local first = prefix .. CacheContract.requiredFiles(version, semantic)[1] + if fs.getRealDirectory(first) ~= fs.getSource() then return nil end + return { kind = "source", prefix = prefix } +end + +function CacheContract.inspect(version, fs, opts) + opts = opts or {} + if not (GameVersion.VERSIONS[version] and fs) then + return nil, "not_imported", "unsupported cache inspection" + end + if opts.allowSource then + local source = sourceReady(version, fs, opts.semantic) + if source then return source end + end + local prefix = GameVersion.cachePrefix(version) + local marker = readAt(fs, prefix .. CacheContract.MARKER_PATH) + if not CacheContract.markerMatches(version, marker) then + return nil, "not_imported", "completion marker is missing or stale" + end + local complete, missing = hasExactFiles(version, fs, prefix, opts.semantic) + if not complete then + return nil, "not_imported", "required cache file is missing: " .. tostring(missing) + end + return { kind = "cache", prefix = prefix, marker = marker } +end + return CacheContract diff --git a/src/import/CacheFs.lua b/src/import/CacheFs.lua index 4b86d5cb..deedb517 100644 --- a/src/import/CacheFs.lua +++ b/src/import/CacheFs.lua @@ -324,9 +324,9 @@ function CacheFs.openWrite(rel) return file end --- read cache-relative `rel`; returns the bytes or nil -function CacheFs.read(rel) - rel = withPrefix(rel) +-- Read an exact version-qualified path without consulting CacheFs.prefix. +-- Readiness checks use this to inspect another version without global state. +function CacheFs.readAt(rel) local root = CacheFs.root() if root then local f = io.open(realPath(root, rel), "rb") @@ -341,6 +341,11 @@ function CacheFs.read(rel) return love.filesystem.read(rel) end +-- read cache-relative `rel`; returns the bytes or nil +function CacheFs.read(rel) + return CacheFs.readAt(withPrefix(rel)) +end + -- Read cache-relative `rel` for the active GameVersion when PhysFS may hide -- prefixed Blue/Yellow/Gold trees (fused NX mount hole). Same order Data:load -- already used: active version prefix with CacheFs.prefix cleared, then @@ -386,8 +391,7 @@ function CacheFs.loadActive(rel) end -- does cache-relative `rel` exist as a file? -function CacheFs.exists(rel) - rel = withPrefix(rel) +function CacheFs.existsAt(rel) local root = CacheFs.root() if root then local f = io.open(realPath(root, rel), "rb") @@ -398,6 +402,11 @@ function CacheFs.exists(rel) return love.filesystem.getInfo(rel, "file") ~= nil end + +function CacheFs.exists(rel) + return CacheFs.existsAt(withPrefix(rel)) +end + -- remove a single cache-relative file function CacheFs.remove(rel) rel = withPrefix(rel) diff --git a/src/mods/Builtins.lua b/src/mods/Builtins.lua index 66695a17..b5cd4143 100644 --- a/src/mods/Builtins.lua +++ b/src/mods/Builtins.lua @@ -163,8 +163,8 @@ end -- a module the build dropped disables its registry rather than the game: -- the consumer still reads its own table, so vanilla keeps working -local function load(path) - local ok, module = pcall(require, path) +local function load(path, moduleLoader) + local ok, module = pcall(moduleLoader or require, path) if ok then return module end Logger.warn("builtin registrations skipped for %s (%s)", path, tostring(module)) return nil @@ -187,14 +187,14 @@ local function isolate(registry) }, { __index = registry }) end -function Builtins.install(content, data, generation) +function Builtins.install(content, data, generation, moduleLoader) for _, entry in ipairs(registrantsFor(generation)) do local registry = content[entry.name] and isolate(content[entry.name]) if registry then if entry.install then local modules, complete = {}, true for i, path in ipairs(entry.modules) do - modules[i] = load(path) + modules[i] = load(path, moduleLoader) if modules[i] == nil then complete = false end end -- data is the fourth argument, not the second, so the existing @@ -204,7 +204,7 @@ function Builtins.install(content, data, generation) entry.install(registry, modules, Builtins.OWNER, data) end else - local module = load(entry.from) + local module = load(entry.from, moduleLoader) -- entry.fn names the entry point for a module that owns more than one -- registry (Gold's Battle owns statuses and move_effects); the default -- is the registerInto every single-registry module exposes diff --git a/src/mods/DatasetViews.lua b/src/mods/DatasetViews.lua new file mode 100644 index 00000000..ef44d6e2 --- /dev/null +++ b/src/mods/DatasetViews.lua @@ -0,0 +1,412 @@ +-- Read-only, bounded views over version-scoped generated datasets. + +local CacheContract = require("src.import.CacheContract") +local DatasetHydration = require("src.core.DatasetHydration") +local GameVersion = require("src.core.GameVersion") +local Logger = require("src.core.Logger") +local SaveSerializer = require("src.core.SaveSerializer") +local Builtins = require("src.mods.Builtins") +local Registry = require("src.mods.Registry") +local Schemas = require("src.mods.Schemas") + +local DatasetViews = {} +DatasetViews.__index = DatasetViews + +-- A generated module is ROM-sized data, never an arbitrary program. These +-- caps bound both one malicious file and the aggregate work one open performs. +local DECODE_LIMITS = { + allowArray = true, + allowComments = true, + maxBytes = 8 * 1024 * 1024, + maxDepth = 64, + maxNodes = 500000, + maxStringBytes = 2 * 1024 * 1024, + maxTableEntries = 250000, + rootName = "generated data", +} +local MAX_AGGREGATE_BYTES = 48 * 1024 * 1024 + +local GEN1_ROOTS = { + constants = "constants", maps = "maps", tilesets = "tilesets", + text = "text", text_pointers = "text_pointers", + trainer_headers = "trainer_headers", font = "font", sprites = "sprites", + pokemon = "pokemon", moves = "moves", items = "items", + type_chart = "type_chart", trainers = "trainers", + encounters = "encounters", field = "field", + battle_anims = "battle_anims", audio = "audio", + palettes = "palettes", icons = "icons", +} + +local GEN2_ROOTS = { + pokemon = "pokemon", moves = "moves", items = "items", + type_chart = "type_chart", audio = "audio", font = "font", + gen2Maps = "maps", gen2Tilesets = "tilesets", gen2Text = "text", + gen2Trainers = "trainers", gen2Encounters = "encounters", + gen2Sprites = "sprites", gen2Palettes = "palettes", + gen2Icons = "icons", gen2BattleAnims = "battle_anims", + gen2Constants = "constants", gen2Landmarks = "landmarks", +} + +local function resolvePath(root, suffix) + local node = root + for key in suffix:gmatch("[^.]+") do + if type(node) ~= "table" then return nil end + node = node[key] + end + return node +end + +local function assetRelative(path) + if type(path) ~= "string" or path == "" then + error("dataset asset path is required", 3) + end + if path:find("\\", 1, true) or path:sub(1, 1) == "/" + or path:find("[%z\1-\31]") then + error("dataset asset path must be a generated relative path", 3) + end + if path:sub(1, 17) ~= "assets/generated/" then + error("dataset assets are limited to assets/generated/", 3) + end + for segment in path:gmatch("[^/]+") do + if segment == "." or segment == ".." then + error("dataset asset path may not traverse directories", 3) + end + end + return path +end + +local function copyData(value, state, depth) + state = state or { seen = {}, nodes = 0 } + state.nodes = state.nodes + 1 + if state.nodes > DECODE_LIMITS.maxNodes then return nil, false end + local kind = type(value) + if kind == "nil" or kind == "boolean" or kind == "number" then return value end + if kind == "string" then + if #value > DECODE_LIMITS.maxStringBytes then return nil, false end + return value + end + if kind ~= "table" or getmetatable(value) ~= nil then return nil, false end + depth = (depth or 0) + 1 + if depth > DECODE_LIMITS.maxDepth or state.seen[value] then return nil, false end + state.seen[value] = true + local out, entries = {}, 0 + for key, child in pairs(value) do + entries = entries + 1 + if entries > DECODE_LIMITS.maxTableEntries then return nil, false end + local keyCopy, keyOk = copyData(key, state, depth) + local childCopy, childOk = copyData(child, state, depth) + if keyOk == false or childOk == false or keyCopy == nil then return nil, false end + out[keyCopy] = childCopy + end + state.seen[value] = nil + return out, true +end + +local function isDataOnly(value, state, depth) + state = state or { seen = {}, nodes = 0 } + state.nodes = state.nodes + 1 + if state.nodes > DECODE_LIMITS.maxNodes then return false end + local kind = type(value) + if kind == "nil" or kind == "boolean" or kind == "number" then return true end + if kind == "string" then return #value <= DECODE_LIMITS.maxStringBytes end + if kind ~= "table" or getmetatable(value) ~= nil then return false end + depth = (depth or 0) + 1 + if depth > DECODE_LIMITS.maxDepth or state.seen[value] then return false end + state.seen[value] = true + local entries = 0 + for key, child in pairs(value) do + entries = entries + 1 + if entries > DECODE_LIMITS.maxTableEntries + or not isDataOnly(key, state, depth) + or not isDataOnly(child, state, depth) then + return false + end + end + state.seen[value] = nil + return true +end + +function DatasetViews.new(fs, engineRequire, decoder) + assert(fs and fs.read and fs.getInfo, + "DatasetViews.new requires a readable filesystem") + return setmetatable({ fs = fs, engineRequire = engineRequire or require, + decoder = decoder or SaveSerializer.decode, datasets = {} }, DatasetViews) +end + +function DatasetViews:_preflight(version, inspected) + local paths, aggregate = {}, 0 + local modules = CacheContract.semanticModules(version) + for _, name in ipairs(CacheContract.optionalSemanticModules(version)) do + local path = inspected.prefix .. "data/generated/" .. name .. ".lua" + if self.fs.getInfo(path, "file") then modules[#modules + 1] = name end + end + for _, name in ipairs(modules) do + local path = inspected.prefix .. "data/generated/" .. name .. ".lua" + local info = self.fs.getInfo(path, "file") + local size = info and info.size + if size and size > DECODE_LIMITS.maxBytes then return nil, name .. ": size limit" end + aggregate = aggregate + (size or 0) + if aggregate > MAX_AGGREGATE_BYTES then return nil, "aggregate size limit" end + paths[name] = path + end + return { paths = paths, key = table.concat(modules, "\n") } +end + +local function resetInternal(view) + view.moduleCache, view.data, view.registries = {}, nil, nil +end + +function DatasetViews:_reject(view, moduleName, source, detail) + view.invalid = { module = moduleName, source = source, detail = detail } + view.data, view.registries = nil, nil + Logger.warn("dataset %s cache rejected: %s", view.version, tostring(detail)) + return nil +end + +function DatasetViews:_ready(view) + if view.invalid or view.unavailable then return false end + local inspected, _, detail = CacheContract.inspect(view.version, self.fs, { + allowSource = true, semantic = true, + }) + if not inspected then + view.unavailable = true + if self.datasets[view.version] == view then self.datasets[view.version] = nil end + Logger.warn("dataset %s unavailable: %s", view.version, detail) + return false + end + local plan, invalid = self:_preflight(view.version, inspected) + if not plan then + self:_reject(view, nil, nil, invalid) + return false + end + if view.prefix ~= inspected.prefix or view.plan.key ~= plan.key then + view.prefix, view.plan = inspected.prefix, plan + resetInternal(view) + else + view.plan = plan + end + for name, cached in pairs(view.moduleCache) do + local source = self.fs.read(plan.paths[name]) + if type(source) ~= "string" then + self:_reject(view, name, source, name .. ": unreadable generated module") + return false + end + if source ~= cached.source then + resetInternal(view) + break + end + end + return not view.invalid +end + +function DatasetViews:_module(view, root) + local moduleName = view.modules[root] + if not moduleName then return nil end + local cached = view.moduleCache[moduleName] + if cached then return cached.value end + local path = view.plan.paths[moduleName] + if not path then return nil end + local source = self.fs.read(path) + if type(source) ~= "string" then + return self:_reject(view, moduleName, source, + moduleName .. ": unreadable generated module") + end + local aggregate = #source + for _, loaded in pairs(view.moduleCache) do aggregate = aggregate + #loaded.source end + if aggregate > MAX_AGGREGATE_BYTES then + return self:_reject(view, moduleName, source, "aggregate size limit") + end + local value, err = self.decoder(source, DECODE_LIMITS) + if type(value) ~= "table" then + return self:_reject(view, moduleName, source, + moduleName .. ": " .. tostring(err or "non-table root")) + end + view.moduleCache[moduleName] = { source = source, value = value } + return value +end + +function DatasetViews:_data(view) + if view.data then return view.data end + local data = {} + setmetatable(data, { + __index = function(target, root) + local value = self:_module(view, root) + if value ~= nil then rawset(target, root, value) end + return value + end, + }) + DatasetHydration.apply(data, view.version, self.engineRequire) + if view.invalid then error(view.invalid.detail, 0) end + view.data = data + return data +end + +local function registryBase(data, target) + return function() return resolvePath(data, target) end +end + +function DatasetViews:_registries(view) + if view.registries then return view.registries end + local data = self:_data(view) + local registries = {} + for name, catalogSpec in pairs(Schemas.REGISTRIES) do + local spec = Schemas.shapeFor(name, catalogSpec, view.generation) + local registry = Registry.new(name, spec) + local target = Schemas.targetFor(name, catalogSpec, view.generation) + if target then registry.base = registryBase(data, target) end + registries[name] = registry + end + Builtins.install(registries, data, view.generation, self.engineRequire) + for _, registry in pairs(registries) do registry:freeze() end + view.registries = registries + return registries +end + +function DatasetViews:_registry(view, name) + local service = self + local function registryForRead() + if not service:_ready(view) then return nil end + local ok, registries = pcall(service._registries, service, view) + if not ok then + if not view.invalid then service:_reject(view, nil, nil, registries) end + return nil + end + if view.invalid then return nil end + return registries[name] + end + local function validate(registry, id, value) + if value == nil then return true end + local ok, detail = Schemas.check(registry.spec, registry.name, id, + value, "override") + if ok then return true end + local target = registry.spec.target + or Schemas.targetFor(registry.name, registry.spec, view.generation) + local root = target and target:match("^[^%.]+") + local moduleName = root and view.modules[root] + if root == "gen2HeldItems" then moduleName = "items" end + local cached = moduleName and view.moduleCache[moduleName] + service:_reject(view, moduleName, cached and cached.source, + "invalid " .. registry.name .. " record: " .. detail) + return false + end + local function rawAt(id) + local registry = registryForRead() + local value = registry and registry:get(id) + if value == nil or not validate(registry, id, value) + or not isDataOnly(value) then return nil end + return value + end + local function valueAt(id) + local value = rawAt(id) + if value == nil then return nil end + return (copyData(value)) + end + return { + get = function(_, id) + if type(id) ~= "string" or id == "" then return nil end + return valueAt(id) + end, + has = function(_, id) + if type(id) ~= "string" or id == "" then return false end + return rawAt(id) ~= nil + end, + each = function() + local ids = {} + local registry = registryForRead() + if registry then + for id, value in registry:each() do + if not validate(registry, id, value) then + ids = {} + break + end + if type(id) == "string" and isDataOnly(value) then + ids[#ids + 1] = id + end + end + end + table.sort(ids) + local index = 0 + return function() + index = index + 1 + local id = ids[index] + if id == nil then return nil end + return id, valueAt(id) + end + end, + } +end + +function DatasetViews:_assets(view) + local service = self + local assets = {} + function assets:path(path) + if not service:_ready(view) then return nil end + return view.prefix .. assetRelative(path) + end + function assets:info(path) + local full = self:path(path) + if not full then return nil end + local info = service.fs.getInfo and service.fs.getInfo(full, "file") + if not info or (info.type and info.type ~= "file") then return nil end + local out = { type = "file" } + if info.size ~= nil then out.size = info.size end + return out + end + return assets +end + +function DatasetViews:open(version) + if type(version) ~= "string" or not GameVersion.VERSIONS[version] then + return nil, "unknown_version" + end + local inspected, reason = CacheContract.inspect(version, self.fs, { + allowSource = true, semantic = true, + }) + if not inspected then + self.datasets[version] = nil + return nil, reason + end + local plan, invalid = self:_preflight(version, inspected) + if not plan then + Logger.warn("dataset %s cache rejected: %s", version, invalid) + return nil, "invalid_cache" + end + + local internal = self.datasets[version] + if internal and internal.invalid then + local bad = internal.invalid + local path = bad.module and plan.paths[bad.module] + local source = path and self.fs.read(path) + if source == bad.source then return nil, "invalid_cache" end + internal = nil + self.datasets[version] = nil + end + local changed = not internal or internal.prefix ~= inspected.prefix + or internal.plan.key ~= plan.key + if changed then + if internal then internal.unavailable = true end + internal = { + version = version, + generation = GameVersion.generation(version), + prefix = inspected.prefix, + plan = plan, + modules = GameVersion.generation(version) == 2 and GEN2_ROOTS or GEN1_ROOTS, + moduleCache = {}, + } + self.datasets[version] = internal + else + internal.plan = plan + end + + local view = { version = version, generation = internal.generation, content = {} } + for name in pairs(Schemas.REGISTRIES) do + view.content[name] = self:_registry(internal, name) + end + for alias, canonical in pairs(Schemas.ALIASES) do + view.content[alias] = view.content[canonical] + end + view.assets = self:_assets(internal) + return view +end + +return DatasetViews diff --git a/src/mods/Loader.lua b/src/mods/Loader.lua index 5b24bcd7..ab10fb2e 100644 --- a/src/mods/Loader.lua +++ b/src/mods/Loader.lua @@ -1154,6 +1154,15 @@ function Loader:_api(mod) developer = loader.dev == true, -- a deep copy: what a mod does to its own view never reaches the loader manifest = Merge.deepCopy(mod.manifest), + datasets = { + open = function(_, version) + if not loader.datasetViews then + local DatasetViews = engineRequire("src.mods.DatasetViews") + loader.datasetViews = DatasetViews.new(loader.fs, engineRequire) + end + return loader.datasetViews:open(version) + end, + }, content = {}, exports = {}, DELETE = Registry.DELETE, diff --git a/src/mods/Registry.lua b/src/mods/Registry.lua index 3c7ff4bb..a98e0ca6 100644 --- a/src/mods/Registry.lua +++ b/src/mods/Registry.lua @@ -12,6 +12,12 @@ Registry.__index = Registry -- exposed to mods as mod.DELETE: a patch value that unsets a field Registry.DELETE = Merge.DELETE +local function assertWritable(self, id) + if self.spec.reservedIds and self.spec.reservedIds[id] then + error(("%s id is reserved engine metadata: %s"):format(self.name, id)) + end +end + -- spec comes from Schemas.REGISTRIES[name]; bare Registry.new(name) keeps -- the v1 record behavior for standalone use in tests and tools function Registry.new(name, spec) @@ -33,6 +39,7 @@ local function append(self, id, op, value, owner) error(self.name .. ": content is frozen after load") end assert(type(id) == "string" and id ~= "", self.name .. " id is required") + assertWritable(self, id) local list = self.ops[id] if not list then list = {} diff --git a/src/mods/Schemas.lua b/src/mods/Schemas.lua index f07c03c9..2b6e8d38 100644 --- a/src/mods/Schemas.lua +++ b/src/mods/Schemas.lua @@ -676,9 +676,10 @@ end -- -- So beside `value` / `fields` / `keys` / `keyValue` a spec may carry -- `gen2Value` / `gen2Fields` / `gen2Keys` / `gen2KeyValue`, and beside --- `semantics` / `extra` / `write` / `baseAt` / `baseIds` / `example` / --- `notes` the matching `gen2*`. Absent means "the Gen 1 shape is right here --- too", which is the common case and why most registries carry none of this. +-- `semantics` / `extra` / `write` / `baseAt` / `baseIds` / `reservedIds` / +-- `example` / `notes` the matching `gen2*`. Absent means "the Gen 1 shape is +-- right here too", which is the common case and why most registries carry +-- none of this. -- The registry NAME, the verbs and (wherever the id space allows it) the ids -- stay shared, exactly as the routing table keeps them shared. -- @@ -700,6 +701,7 @@ local GEN2_SHAPE = { gen2KeyValue = "keyValue", gen2Extra = "extra", gen2Semantics = "semantics", gen2Write = "write", gen2BaseAt = "baseAt", gen2BaseIds = "baseIds", + gen2ReservedIds = "reservedIds", gen2Example = "example", gen2Notes = "notes", } @@ -757,6 +759,32 @@ end local R = {} Schemas.REGISTRIES = R +-- Some generated Gen 2 modules keep extractor metadata beside their public +-- record maps. These callbacks are the registry normalization boundary: the +-- metadata remains available to engine consumers through Data, but is not an +-- id a mod can read or overwrite through the record registry. +local function recordMapExcept(...) + local excluded = {} + for index = 1, select("#", ...) do excluded[select(index, ...)] = true end + return function(base, id) + if excluded[id] then return nil end + return base[id] + end, function(base) + local ids = {} + for id in pairs(base) do + if not excluded[id] then ids[#ids + 1] = id end + end + return ids + end, excluded +end + +local pokemonGen2BaseAt, pokemonGen2BaseIds, pokemonGen2ReservedIds = + recordMapExcept("growthRates", "tmhmMoves") +local movesGen2BaseAt, movesGen2BaseIds, movesGen2ReservedIds = + recordMapExcept("generation", "source") +local itemsGen2BaseAt, itemsGen2BaseIds, itemsGen2ReservedIds = + recordMapExcept("generation", "source", "pockets") + -- ------- shared Gen 2 leaves -- -- The ROM name spaces Gold's tables key by. They are enums rather than @@ -779,6 +807,8 @@ local gen2PaletteRow = f.list(gen2Color) R.pokemon = { semantics = "record", target = "pokemon", + gen2BaseAt = pokemonGen2BaseAt, gen2BaseIds = pokemonGen2BaseIds, + gen2ReservedIds = pokemonGen2ReservedIds, fields = { id = f.str, name = f.str, dex = f.int(1), index = f.opt(f.int(0, 255)), @@ -870,6 +900,8 @@ R.pokemon = { R.moves = { semantics = "record", target = "moves", + gen2BaseAt = movesGen2BaseAt, gen2BaseIds = movesGen2BaseIds, + gen2ReservedIds = movesGen2ReservedIds, fields = { id = f.str, name = f.str, index = f.opt(f.int(0, 255)), @@ -894,6 +926,8 @@ R.moves = { R.items = { semantics = "record", target = "items", + gen2BaseAt = itemsGen2BaseAt, gen2BaseIds = itemsGen2BaseIds, + gen2ReservedIds = itemsGen2ReservedIds, fields = { id = f.str, name = f.str, index = f.opt(f.int(0, 255)), diff --git a/tests/engine/dataset_views_lazy_validation.lua b/tests/engine/dataset_views_lazy_validation.lua new file mode 100644 index 00000000..fa9ff9d0 --- /dev/null +++ b/tests/engine/dataset_views_lazy_validation.lua @@ -0,0 +1,46 @@ +-- Imported semantic modules are opened without eager reads and are decoded +-- once, on first use, through the same public facade mods receive. +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") +local Fixture = require("tests.modkit.dataset_view_fixture") +local DatasetViews = require("src.mods.DatasetViews") +local GameVersion = require("src.core.GameVersion") +local SaveSerializer = require("src.core.SaveSerializer") + +local files = {} +Fixture.cache(files, "gold") +local fs = T.sdk.memfs(files) +local rawRead = fs.read +local reads = {} +fs.read = function(path) + reads[path] = (reads[path] or 0) + 1 + return rawRead(path) +end +local decodes = 0 +local service = DatasetViews.new(fs, require, function(source, limits) + decodes = decodes + 1 + return SaveSerializer.decode(source, limits) +end) +local pokemonPath = GameVersion.cachePrefix("gold") + .. "data/generated/pokemon.lua" +local movesPath = GameVersion.cachePrefix("gold") + .. "data/generated/moves.lua" + +local view, reason = service:open("gold") +T.eq(reason, nil, "marker-valid Gold dataset opens") +T.check(view ~= nil, "open returns the lazy view") +T.eq(reads[pokemonPath] or 0, 0, "open does not read an unused root") +T.eq(reads[movesPath] or 0, 0, "open does not read another unused root") +T.eq(decodes, 0, "open decodes no semantic root") + +local first = view and view.content.pokemon:get("FIXMON") +T.eq(first and first.name, "FIXMON", "first access decodes a valid root") +T.check(decodes > 0, "first root access performs bounded decoding") +T.eq(reads[movesPath] or 0, 0, "pokemon access leaves moves unused") +local afterFirst = decodes +local second = view and view.content.pokemon:get("FIXMON") +T.eq(second and second.name, "FIXMON", "repeated access remains available") +T.eq(decodes, afterFirst, "repeated access does not re-decode cached roots") + +T.finish("dataset_views_lazy_validation") diff --git a/tests/engine/dataset_views_no_mod_parity.lua b/tests/engine/dataset_views_no_mod_parity.lua new file mode 100644 index 00000000..1b7a4cbe --- /dev/null +++ b/tests/engine/dataset_views_no_mod_parity.lua @@ -0,0 +1,22 @@ +-- Dedicated Route B no-mod parity: the additive facade remains cold. +package.path = "./?.lua;./?/init.lua;" .. package.path +local T = require("tests.modkit") + +local reads = 0 +local fs = T.sdk.memfs({}) +local read = fs.read +fs.read = function(path) + if path:match("^[a-z]+/data/generated/") + or path:match("^[a-z]+/rom%-cache%.complete$") then + reads = reads + 1 + end + return read(path) +end +local data = { pokemon = { ACTIVE = { id = "ACTIVE", nested = { n = 1 } } } } +local run = T.sdk.loadNone({ fs = fs, data = data, generation = 1 }) +T.eq(#run.errors, 0, "no-mod load remains clean") +T.eq(run.loader.datasetViews, nil, "no-mod load does not allocate dataset service") +T.eq(reads, 0, "no-mod load performs no imported dataset reads") +T.eq(data.pokemon.ACTIVE.nested.n, 1, "no-mod data remains unchanged") +run.release() +T.finish("dataset_views_no_mod_parity") diff --git a/tests/engine/dataset_views_warning_policy.lua b/tests/engine/dataset_views_warning_policy.lua new file mode 100644 index 00000000..66fc92b2 --- /dev/null +++ b/tests/engine/dataset_views_warning_policy.lua @@ -0,0 +1,56 @@ +-- Optional dataset absence is a handled API result, while a previously valid +-- view disappearing and malformed cache content remain actionable warnings. +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") +local Fixture = require("tests.modkit.dataset_view_fixture") +local DatasetViews = require("src.mods.DatasetViews") +local GameVersion = require("src.core.GameVersion") +local Logger = require("src.core.Logger") + +local previousWarn = Logger.warn +local warnings = {} +Logger.warn = function(fmt, ...) + warnings[#warnings + 1] = select("#", ...) > 0 + and string.format(fmt, ...) or fmt +end + +local ok, err = xpcall(function() + local unavailable = DatasetViews.new(T.sdk.memfs({})) + local unknown, unknownReason = unavailable:open("missing-version") + T.eq(unknown, nil, "unknown version has no view") + T.eq(unknownReason, "unknown_version", "unknown version returns its reason") + local absent, absentReason = unavailable:open("gold") + T.eq(absent, nil, "missing optional Gold dataset has no view") + T.eq(absentReason, "not_imported", "missing optional Gold returns its reason") + T.eq(#warnings, 0, "unknown and initially absent datasets do not warn") + + local files = {} + Fixture.cache(files, "gold") + local service = DatasetViews.new(T.sdk.memfs(files)) + local view = assert(service:open("gold")) + files[GameVersion.cachePrefix("gold") .. "rom-cache.complete"] = nil + T.eq(view.assets:path("assets/generated/missing.png"), nil, + "a stale view closes when its imported dataset disappears") + T.eq(#warnings, 1, "a previously valid view disappearing still warns") + T.check(warnings[1]:find("dataset gold unavailable", 1, true) ~= nil, + "the stale-view warning identifies the unavailable dataset") + + warnings = {} + local malformedFiles = {} + Fixture.cache(malformedFiles, "gold", { pokemon = "not generated data" }) + local malformed = assert(DatasetViews.new(T.sdk.memfs(malformedFiles)):open("gold")) + T.eq(#warnings, 0, "lazy open does not warn before malformed data is read") + T.eq(malformed.content.pokemon:get("FIXMON"), nil, + "malformed generated data fails closed") + T.eq(#warnings, 1, "malformed generated data still warns") + T.check(warnings[1]:find("dataset gold cache rejected", 1, true) ~= nil, + "the malformed-cache warning identifies cache rejection") + T.check(warnings[1]:find("pokemon:", 1, true) ~= nil, + "the malformed-cache warning keeps actionable module detail") +end, debug.traceback) + +Logger.warn = previousWarn +if not ok then error(err, 0) end + +T.finish("dataset_views_warning_policy") diff --git a/tests/engine/gen2_reserved_metadata_ids.lua b/tests/engine/gen2_reserved_metadata_ids.lua new file mode 100644 index 00000000..78e35d04 --- /dev/null +++ b/tests/engine/gen2_reserved_metadata_ids.lua @@ -0,0 +1,125 @@ +-- Extractor metadata hidden from record id spaces remains engine-owned: mods +-- cannot claim, update, or remove it through the public content registries. +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") +local Mon = require("src.battle.gen2.Mon") + +local function goldData() + local data = T.fixtures.fresh() + data.pokemon.growthRates = { + GROWTH_MEDIUM_FAST = { + numerator = 1, denominator = 1, squared = 0, linear = 0, constant = 0, + }, + } + data.pokemon.tmhmMoves = { "FIX_TACKLE", "FIX_CUT" } + data.moves.generation, data.moves.source = 2, "ROM:Moves" + data.items.generation, data.items.source = 2, "ROM:Items" + data.items.pockets = { "ITEM", "BALL", "KEY_ITEM", "TM_HM" } + return data +end + +local function assertMetadata(data, refs, label) + T.eq(data.pokemon.growthRates, refs.growthRates, + label .. ": growth-rate table identity is preserved") + T.eq(data.pokemon.tmhmMoves, refs.tmhmMoves, + label .. ": TM/HM order identity is preserved") + T.eq(data.moves.generation, 2, label .. ": move generation is preserved") + T.eq(data.moves.source, "ROM:Moves", label .. ": move source is preserved") + T.eq(data.items.generation, 2, label .. ": item generation is preserved") + T.eq(data.items.source, "ROM:Items", label .. ": item source is preserved") + T.eq(data.items.pockets, refs.pockets, + label .. ": item pocket order identity is preserved") + T.eq(Mon.experienceForLevel( + Mon.growthFor(data, "GROWTH_MEDIUM_FAST"), 10), 1000, + label .. ": active Gold growth behavior is preserved") +end + +-- A no-mod load characterizes the unchanged active-boot behavior. +do + local data = goldData() + local refs = { + growthRates = data.pokemon.growthRates, + tmhmMoves = data.pokemon.tmhmMoves, + pockets = data.items.pockets, + } + local run = T.sdk.loadNone({ data = data, generation = 2 }) + T.eq(#run.errors, 0, "no-mod Gold load remains clean") + assertMetadata(run.data, refs, "no-mod Gold") + run.release() +end + +local files, paths = {}, {} +local attempts = { + register_growth_rates = [[ +local mod = ... +mod.content.pokemon:register("growthRates", { + id = "growthRates", name = "CLAIMED", dex = 999, + types = {}, + baseStats = { hp = 1, attack = 1, defense = 1, speed = 1, + specialAttack = 1, specialDefense = 1 }, + catchRate = 1, baseExp = 1, growthRate = "GROWTH_MEDIUM_FAST", + levelMoves = {}, evolutions = {}, + spriteFront = "assets/generated/battle/front/claimed.png", + spriteBack = "assets/generated/battle/back/claimed.png", picSize = 5, +}) +]], + replace_growth_rates = [[ +local mod = ... +mod.content.pokemon:register("growthRates", { + id = "growthRates", name = "REPLACED", dex = 999, + types = {}, + baseStats = { hp = 1, attack = 1, defense = 1, speed = 1, + specialAttack = 1, specialDefense = 1 }, + catchRate = 1, baseExp = 1, growthRate = "GROWTH_MEDIUM_FAST", + levelMoves = {}, evolutions = {}, + spriteFront = "assets/generated/battle/front/replaced.png", + spriteBack = "assets/generated/battle/back/replaced.png", picSize = 5, +}, { replace = true }) +]], + override_move_generation = [[ +local mod = ... +mod.content.moves:override("generation", { + id = "generation", name = "CLAIMED", type = "NORMAL", + power = 1, accuracy = 100, pp = 1, effect = "NO_ADDITIONAL_EFFECT", +}) +]], + patch_item_pockets = [[ +local mod = ... +mod.content.items:patch("pockets", { price = 999 }) +]], + remove_tmhm_moves = [[ +local mod = ... +mod.content.pokemon:remove("tmhmMoves") +]], +} + +for id, body in pairs(attempts) do + files["mods/" .. id .. "/manifest.json"] = ([[{ + "id": %q, "name": %q, "version": "1.0.0", "entry": "main.lua", + "api": 2, "gen2compat": true + }]]):format(id, id) + files["mods/" .. id .. "/main.lua"] = body + paths[#paths + 1] = "mods/" .. id +end +table.sort(paths) + +local data = goldData() +local refs = { + growthRates = data.pokemon.growthRates, + tmhmMoves = data.pokemon.tmhmMoves, + pockets = data.items.pockets, +} +local run = T.sdk.loadMods(paths, { + fs = T.sdk.memfs(files), data = data, generation = 2, +}) +for id in pairs(attempts) do + local mod = run.loader.mods[id] + T.eq(mod and mod.state, "failed", id .. " is rejected") + T.check(mod and tostring(mod.failure):match("reserved") ~= nil, + id .. " reports the reserved metadata boundary") +end +assertMetadata(run.data, refs, "rejected metadata writes") +run.release() + +T.finish("gen2_reserved_metadata_ids") diff --git a/tests/engine/generated_data_decoder_test.lua b/tests/engine/generated_data_decoder_test.lua new file mode 100644 index 00000000..dd0672f3 --- /dev/null +++ b/tests/engine/generated_data_decoder_test.lua @@ -0,0 +1,37 @@ +-- Restricted generated-data grammar and resource limits. +package.path = "./?.lua;./?/init.lua;" .. package.path +local T = require("tests.modkit") +local SaveSerializer = require("src.core.SaveSerializer") + +local limits = { + allowArray = true, allowComments = true, + maxBytes = 256, maxDepth = 4, maxNodes = 7, + maxStringBytes = 8, maxTableEntries = 5, rootName = "generated data", +} + +local value, err = SaveSerializer.decode( + '-- Generated by tools/build_data.py. DO NOT EDIT.\n' + .. 'return { rows = { "A", "B" }, [3] = true }\n', limits) +T.same(value, { rows = { "A", "B" }, [3] = true }, + "LuaWriter array and keyed-table grammar decodes as data") +T.eq(err, nil, "valid generated data has no error") + +local invalid = { + { "return function() end", "function literal" }, + { "owned = true; return {}", "global assignment" }, + { "return {}; print('x')", "trailing executable syntax" }, + { string.char(27) .. "Lua", "binary chunk" }, + { "return " .. string.rep(" ", 260) .. "{}", "source-size limit" }, + { 'return { a = { b = { c = { d = { e = {} } } } } }', "depth limit" }, + { 'return { a = "123456789" }', "string limit" }, + { 'return { 1, 2, 3, 4, 5, 6 }', "table-entry limit" }, + { 'return { a = 1, b = 2, c = 3, d = 4, e = { f = 6, g = 7 } }', + "node limit" }, + { "return 7", "non-table root" }, +} +for _, row in ipairs(invalid) do + local got = SaveSerializer.decode(row[1], limits) + T.eq(got, nil, row[2] .. " fails closed") +end + +T.finish("generated_data_decoder") diff --git a/tests/modkit/cases/dataset_views.lua b/tests/modkit/cases/dataset_views.lua new file mode 100644 index 00000000..3384f8f2 --- /dev/null +++ b/tests/modkit/cases/dataset_views.lua @@ -0,0 +1,394 @@ +-- Sandboxed public-API coverage for imported semantic dataset views. +package.path = "./?.lua;./?/init.lua;" .. package.path + +local T = require("tests.modkit") +local Fixture = require("tests.modkit.dataset_view_fixture") +local GameVersion = require("src.core.GameVersion") +local CacheFs = require("src.import.CacheFs") +local Schemas = require("src.mods.Schemas") + +local originalVersion = GameVersion.get() +local originalPrefix = CacheFs.prefix +GameVersion.set("red") + +local files = {} +for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver", "crystal" }) do + Fixture.cache(files, version) +end +local probe = Fixture.addMod(files, "dataset_probe", [[ +local mod = ... +local out = {} +for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver", "crystal" }) do + local view, reason = mod.datasets:open(version) + local mon = view and view.content.pokemon:get("FIXMON") + if mon then mon.name = "MUTATED" end + local ids = {} + if view then + for id, value in view.content.pokemon:each() do + ids[#ids + 1] = id + if value then value.name = "EACH_MUTATED" end + end + end + out[version] = { + reason = reason, generation = view and view.generation, + name = view and view.content.pokemon:get("FIXMON").name, + has = view and view.content.pokemon:has("FIXMON"), ids = ids, + aliasScripts = view and view.content.scripts == view.content.map_scripts, + aliasUi = view and view.content.ui == view.content.screens, + readOnly = view and view.content.pokemon.register == nil + and view.content.pokemon.patch == nil and view.content.pokemon.remove == nil, + } +end +local yellow = assert(mod.datasets:open("yellow")) +out.yellowOldMan = yellow.content.field:get("oldManBattle") +out.yellowBoot = yellow.content.field:get("boot") +local red = assert(mod.datasets:open("red")) +out.redBoot = red.content.field:get("boot") +out.redConstants = red.content.constants:get("dexSize") +local gold = assert(mod.datasets:open("gold")) +out.registryCount = 0 +for _, registry in pairs(gold.content) do + if type(registry) == "table" then out.registryCount = out.registryCount + 1 end +end +out.foresight = gold.content.type_chart:get("NORMAL>GHOST") +out.held = gold.content.held_items:get("LEFTOVERS") +out.continuations, out.moves, out.assets = {}, {}, {} +for _, id in ipairs({ "CROBAT", "BELLOSSOM", "POLITOED", "SLOWKING", + "STEELIX", "SCIZOR", "KINGDRA", "PORYGON2", "BLISSEY" }) do + local row = gold.content.pokemon:get(id) + out.continuations[id] = row and row.dex + out.assets[id] = row and { + gold.assets:info(row.spriteFront), gold.assets:info(row.spriteBack) } +end +for _, id in ipairs({ "IRON_TAIL", "METAL_CLAW", "STEEL_WING", "RAIN_DANCE", + "SUNNY_DAY", "SANDSTORM", "SLUDGE_BOMB", "SHADOW_BALL" }) do + out.moves[id] = gold.content.moves:has(id) +end +out.steel = gold.content.type_chart:has("STEEL") +out.metadataHidden = not gold.content.pokemon:has("growthRates") + and not gold.content.pokemon:has("tmhmMoves") + and not gold.content.moves:has("generation") + and not gold.content.moves:has("source") + and not gold.content.items:has("generation") + and not gold.content.items:has("source") + and not gold.content.items:has("pockets") +out.executableBuiltinHidden = gold.content.statuses:get("sleep") == nil +out.assetDirectory = gold.assets:info("assets/generated/battle/front") +out.assetRejects = {} +for _, path in ipairs({ "/assets/generated/x", "assets\\generated\\x", + "assets/generated/../x", "assets/generated/./x", "other/x", + "assets/generated/x\1" }) do + out.assetRejects[#out.assetRejects + 1] = pcall(gold.assets.path, gold.assets, path) +end +local unknown, unknownReason = mod.datasets:open("missing-version") +out.unknown = { unknown ~= nil, unknownReason } +mod.exports.result = out +]]) + +local active = { pokemon = { ACTIVE = { id = "ACTIVE", nested = { n = 1 } } } } +local run = T.sdk.loadMods({ probe }, { + fs = T.sdk.memfs(files), data = active, generation = 1, +}) +T.eq(#run.errors, 0, "dataset public probe loads clean") +local out = run.loader.exports.dataset_probe.result +for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver", "crystal" }) do + T.eq(out[version].reason, nil, version .. " opens") + T.eq(out[version].generation, + GameVersion.generation(version), + version .. " reports selected generation") + T.eq(out[version].name, "FIXMON", version .. " get/each values are detached") + T.eq(out[version].has, true, version .. " has reads without exposing a copy") + T.eq(out[version].aliasScripts, true, version .. " scripts alias is canonical") + T.eq(out[version].aliasUi, true, version .. " ui alias is canonical") + T.eq(out[version].readOnly, true, version .. " registry has no write verbs") +end +T.eq(out.redConstants, 1, "Red derives dexSize through canonical defaults") +T.eq(out.redBoot and out.redBoot.screens and out.redBoot.screens.splash, + "IntroMovie", "Red uses canonical boot defaults") +T.eq(out.yellowOldMan and out.yellowOldMan.species, "RATTATA", + "Yellow applies canonical correction") +T.eq(out.yellowBoot and out.yellowBoot.screens and out.yellowBoot.screens.splash, + "YellowIntro", + "Yellow uses its canonical splash") +T.eq(out.foresight and out.foresight.multiplier, 0, + "Gold appends Foresight matchup rows") +T.eq(out.held and out.held.heldEffect, "HELD_LEFTOVERS", "Gold derives held_items") +for _, id in ipairs(Fixture.CONTINUATIONS) do + T.check(out.continuations[id] ~= nil, "Gold exposes " .. id) + T.same(out.assets[id], { { type = "file" }, { type = "file" } }, + "Gold exposes namespaced front/back assets for " .. id) +end +for _, id in ipairs(Fixture.MOVES) do T.eq(out.moves[id], true, "Gold exposes " .. id) end +T.eq(out.steel, true, "Gold exposes Steel") +T.eq(out.metadataHidden, true, + "Gold extractor metadata stays outside record registry id spaces") +local registryCount = 0 +for _ in pairs(Schemas.REGISTRIES) do registryCount = registryCount + 1 end +for _ in pairs(Schemas.ALIASES) do registryCount = registryCount + 1 end +T.eq(out.registryCount, registryCount, + "view exposes every canonical registry and alias") +T.eq(out.executableBuiltinHidden, true, + "engine records containing closures do not cross the data-only facade") +T.eq(out.assetDirectory, nil, "asset info does not expose directories") +for index, accepted in ipairs(out.assetRejects) do + T.eq(accepted, false, "asset rejection " .. index) +end +T.same(out.unknown, { false, "unknown_version" }, "unknown version fails stably") +T.eq(active.pokemon.ACTIVE.nested.n, 1, "active Data stays unchanged") +T.eq(active.pokemon.FIXMON, nil, "inactive records do not merge into active Data") +T.eq(GameVersion.get(), "red", "active GameVersion stays unchanged") +T.eq(CacheFs.prefix, originalPrefix, "CacheFs prefix stays unchanged") +run.release() + +-- Facade replacement and nested record mutation do not cross mod boundaries. +local isolation = {} +Fixture.cache(isolation, "red") +local mutator = Fixture.addMod(isolation, "dataset_mutator", [[ +local mod = ... +local view = assert(mod.datasets:open("red")) +view.content.pokemon.get = function() return { name = "POISONED" } end +local row = view.content.pokemon:get("FIXMON") +if row then row.name = "CHANGED" end +]]) +local observer = Fixture.addMod(isolation, "dataset_observer", [[ +local mod = ... +local view = assert(mod.datasets:open("red")) +mod.exports.name = view.content.pokemon:get("FIXMON").name +]]) +isolation["mods/dataset_observer/manifest.json"] = [[{ + "id": "dataset_observer", "name": "dataset_observer", "version": "1.0.0", + "entry": "main.lua", "api": 2, "games": ["all"], + "dependencies": ["dataset_mutator"] +}]] +local isolationRun = T.sdk.loadMods({ mutator, observer }, { + fs = T.sdk.memfs(isolation), data = { pokemon = {} }, generation = 1, +}) +T.eq(isolationRun.loader.exports.dataset_observer.name, "FIXMON", + "facade and record mutation cannot cross mods") +isolationRun.release() + +-- Marker-only, empty, partial, and stale-after-first-open caches fail closed. +local readiness = {} +local emptyPrefix = GameVersion.cachePrefix("red") +readiness[emptyPrefix .. "rom-cache.complete"] = + "rom-cache-v10:" .. GameVersion.info("red").sha1 +Fixture.cache(readiness, "blue") +readiness[GameVersion.cachePrefix("blue") .. "data/generated/moves.lua"] = nil +Fixture.cache(readiness, "gold") +local staleReads = 0 +local readinessFs = T.sdk.memfs(readiness) +local rawRead = readinessFs.read +readinessFs.read = function(path) + if path == GameVersion.cachePrefix("gold") .. "rom-cache.complete" then + staleReads = staleReads + 1 + if staleReads > 1 then return "rom-cache-v9:stale" end + end + return rawRead(path) +end +local readinessMod = Fixture.addMod(readiness, "readiness_probe", [[ +local mod = ... +local out = {} +for _, version in ipairs({ "red", "blue" }) do + local view, reason = mod.datasets:open(version) + out[version] = { view ~= nil, reason } +end +local first, firstReason = mod.datasets:open("gold") +local second, secondReason = mod.datasets:open("gold") +out.gold = { first ~= nil, firstReason, second ~= nil, secondReason } +mod.exports.result = out +]]) +local readinessRun = T.sdk.loadMods({ readinessMod }, { + fs = readinessFs, data = { pokemon = {} }, generation = 1, +}) +local readyOut = readinessRun.loader.exports.readiness_probe.result +T.same(readyOut.red, { false, "not_imported" }, "marker-only empty cache fails") +T.same(readyOut.blue, { false, "not_imported" }, "partial cache fails") +T.same(readyOut.gold, { true, nil, false, "not_imported" }, + "cached view is invalidated when marker becomes stale") +readinessRun.release() + +-- Removal followed by a fresh import cannot resurrect the pre-removal view. +local transition = {} +Fixture.cache(transition, "red") +local transitionFs = T.sdk.memfs(transition) +local transitionRead = transitionFs.read +local transitionMarkers = 0 +local reimportedPokemon = require("src.import.LuaWriter").encode({ + FIXMON = { id = "FIXMON", name = "REIMPORTED", dex = 1, + types = {}, catchRate = 45, baseExp = 64, growthRate = "MEDIUM_FAST", + baseStats = { hp = 45, attack = 49, defense = 49, speed = 45, + special = 65 }, + level1Moves = {}, tmhm = {}, learnset = {}, evolutions = {}, + spriteFront = "assets/generated/battle/front/fixmon.png", + spriteBack = "assets/generated/battle/back/fixmon.png", frontSize = 5 }, +}) +transitionFs.read = function(path) + if path == GameVersion.cachePrefix("red") .. "rom-cache.complete" then + transitionMarkers = transitionMarkers + 1 + -- open and the first facade read each recheck readiness; remove the cache + -- for the next open, then make the following open the reimport. + if transitionMarkers == 3 then return nil end + end + local body = transitionRead(path) + if transitionMarkers >= 4 + and path == GameVersion.cachePrefix("red") .. "data/generated/pokemon.lua" then + return reimportedPokemon + end + return body +end +local transitionMod = Fixture.addMod(transition, "transition_probe", [[ +local mod = ... +local out = {} +for index = 1, 3 do + local view, reason = mod.datasets:open("red") + out[index] = { view and view.content.pokemon:get("FIXMON").name, reason } +end +mod.exports.result = out +]]) +local transitionRun = T.sdk.loadMods({ transitionMod }, { + fs = transitionFs, data = { pokemon = {} }, generation = 1, +}) +T.same(transitionRun.loader.exports.transition_probe.result, { + { "FIXMON" }, { nil, "not_imported" }, { "REIMPORTED" }, +}, "remove and reimport transition rebuilds the semantic view") +transitionRun.release() + +-- Every hostile source is rejected as data, with one stable public reason. +local hostile = { + { "red", "return { BAD = function() return 1 end }" }, + { "blue", "owned = true; return {}" }, + { "yellow", "return 7" }, + { "gold", string.char(27) .. "Lua" }, + { "silver", "return {}; while true do end" }, +} +local hostileFiles, hostilePaths = {}, {} +for _, row in ipairs(hostile) do + Fixture.cache(hostileFiles, row[1], { pokemon = row[2] }) +end +local hostileMod = Fixture.addMod(hostileFiles, "hostile_probe", [[ +local mod = ... +local out = {} +for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver", "crystal" }) do + local view, reason = mod.datasets:open(version) + local value = view and view.content.pokemon:get("BAD") + local reopened, reopenedReason = mod.datasets:open(version) + out[version] = { + first = view ~= nil, firstReason = reason, value = value, + reopened = reopened ~= nil, reason = reopenedReason, + } +end +mod.exports.result = out +]]) +hostilePaths[1] = hostileMod +local previousHook, previousMask, previousCount +local beforeHook +if debug.gethook and debug.sethook then + previousHook, previousMask, previousCount = debug.gethook() + beforeHook = function() end + debug.sethook(beforeHook, "", 1000) +end +local hostileRun = T.sdk.loadMods(hostilePaths, { + fs = T.sdk.memfs(hostileFiles), data = { pokemon = {} }, generation = 1, +}) +local hostileOut = hostileRun.loader.exports.hostile_probe.result +for _, row in ipairs(hostile) do + T.same(hostileOut[row[1]], { + first = true, reopened = false, reason = "invalid_cache", + }, row[1] .. " hostile generated source fails closed on first root access") +end +T.eq(debug.gethook and debug.gethook(), beforeHook, + "dataset decoding preserves the caller debug hook") +if debug.sethook then + if previousHook then debug.sethook(previousHook, previousMask, previousCount) + else debug.sethook() end +end +hostileRun.release() + +-- A syntactically valid semantic root with a primitive record must fail closed +-- whichever public read verb encounters it first. +local malformedFiles = {} +for _, version in ipairs({ "red", "blue", "yellow" }) do + Fixture.cache(malformedFiles, version, { + pokemon = "return { FIXMON = 7 }", + }) +end +local malformedMod = Fixture.addMod(malformedFiles, "malformed_probe", [[ +local mod = ... +local out = {} +local getView = assert(mod.datasets:open("red")) +out.get = getView.content.pokemon:get("FIXMON") +local getAgain, getReason = mod.datasets:open("red") +out.getAgain = { getAgain ~= nil, getReason } + +local hasView = assert(mod.datasets:open("blue")) +out.has = hasView.content.pokemon:has("FIXMON") +local hasAgain, hasReason = mod.datasets:open("blue") +out.hasAgain = { hasAgain ~= nil, hasReason } + +local eachView = assert(mod.datasets:open("yellow")) +for id, value in eachView.content.pokemon:each() do + out.each = { id, value } + break +end +local eachAgain, eachReason = mod.datasets:open("yellow") +out.eachAgain = { eachAgain ~= nil, eachReason } +mod.exports.result = out +]]) +local malformedRun = T.sdk.loadMods({ malformedMod }, { + fs = T.sdk.memfs(malformedFiles), data = { pokemon = {} }, generation = 1, +}) +T.eq(#malformedRun.errors, 0, "malformed-record probe stays sandboxed") +local malformedOut = malformedRun.loader.exports.malformed_probe.result +T.eq(malformedOut.get, nil, "get never exposes a malformed semantic record") +T.eq(malformedOut.has, false, "has never affirms a malformed semantic record") +T.eq(malformedOut.each, nil, "each never exposes a malformed semantic record") +T.same(malformedOut.getAgain, { false, "invalid_cache" }, + "get invalidates the malformed dataset") +T.same(malformedOut.hasAgain, { false, "invalid_cache" }, + "has invalidates the malformed dataset") +T.same(malformedOut.eachAgain, { false, "invalid_cache" }, + "each invalidates the malformed dataset") +malformedRun.release() + +-- The same Gold view stays independent while each Gen 1 version is the +-- actual active runtime version and cache namespace, not just a fixture label. +for _, activeVersion in ipairs({ "red", "blue", "yellow" }) do + local matrixFiles = {} + Fixture.cache(matrixFiles, "gold") + local matrixMod = Fixture.addMod(matrixFiles, + "active_" .. activeVersion .. "_gold_probe", [[ +local mod = ... +local gold = assert(mod.datasets:open("gold")) +mod.exports.result = { + species = gold.content.pokemon:get("FIXMON").name, + foresight = gold.content.type_chart:get("NORMAL>GHOST").multiplier, + held = gold.content.held_items:get("LEFTOVERS").heldEffect, +} +]]) + GameVersion.set(activeVersion) + CacheFs.prefix = GameVersion.cachePrefix(activeVersion) + local activeData = { + pokemon = { ACTIVE = { id = "ACTIVE_" .. activeVersion, + nested = { version = activeVersion } } }, + } + local matrixRun = T.sdk.loadMods({ matrixMod }, { + fs = T.sdk.memfs(matrixFiles), data = activeData, generation = 1, + }) + T.eq(#matrixRun.errors, 0, activeVersion .. "-active Gold probe loads") + T.same(matrixRun.loader.exports["active_" .. activeVersion .. "_gold_probe"].result, + { species = "FIXMON", foresight = 0, held = "HELD_LEFTOVERS" }, + activeVersion .. "-active runtime sees canonical Gold semantics") + T.eq(GameVersion.get(), activeVersion, + activeVersion .. " remains the active GameVersion") + T.eq(CacheFs.prefix, GameVersion.cachePrefix(activeVersion), + activeVersion .. " remains the active cache namespace") + T.eq(activeData.pokemon.ACTIVE.nested.version, activeVersion, + activeVersion .. " active Data remains unchanged") + T.eq(activeData.pokemon.FIXMON, nil, + activeVersion .. " active Data receives no Gold record") + matrixRun.release() +end + +GameVersion.set(originalVersion) +CacheFs.prefix = originalPrefix +T.finish("dataset_views") diff --git a/tests/modkit/cases/dataset_views_nontermination.lua b/tests/modkit/cases/dataset_views_nontermination.lua new file mode 100644 index 00000000..3a0da2e6 --- /dev/null +++ b/tests/modkit/cases/dataset_views_nontermination.lua @@ -0,0 +1,27 @@ +-- A generated chunk that would loop if executed must be rejected as syntax. +package.path = "./?.lua;./?/init.lua;" .. package.path +local T = require("tests.modkit") +local Fixture = require("tests.modkit.dataset_view_fixture") + +local files = {} +Fixture.cache(files, "red", { + pokemon = "while true do end; return {}", +}) +local modPath = Fixture.addMod(files, "nontermination_probe", [[ +local mod = ... +local view, reason = mod.datasets:open("red") +local value = view and view.content.pokemon:get("FIXMON") +local reopened, reopenedReason = mod.datasets:open("red") +mod.exports.result = { + first = view ~= nil, firstReason = reason, value = value, + reopened = reopened ~= nil, reason = reopenedReason, +} +]]) +local run = T.sdk.loadMods({ modPath }, { + fs = T.sdk.memfs(files), data = { pokemon = {} }, generation = 1, +}) +T.same(run.loader.exports.nontermination_probe.result, + { first = true, reopened = false, reason = "invalid_cache" }, + "generated code is never executed and invalidates on first root access") +run.release() +T.finish("dataset_views_nontermination") diff --git a/tests/modkit/dataset_view_fixture.lua b/tests/modkit/dataset_view_fixture.lua new file mode 100644 index 00000000..9d75e2e2 --- /dev/null +++ b/tests/modkit/dataset_view_fixture.lua @@ -0,0 +1,138 @@ +local GameVersion = require("src.core.GameVersion") +local LuaWriter = require("src.import.LuaWriter") +local CacheContract = require("src.import.CacheContract") + +local Fixture = {} + +local GEN1_MODULES = { + "constants", "maps", "tilesets", "text", "text_pointers", + "trainer_headers", "font", "sprites", "pokemon", "moves", "items", + "type_chart", "trainers", "encounters", "field", "battle_anims", +} +local GEN1_OPTIONAL_MODULES = { "audio", "palettes", "icons" } + +local GEN2_MODULES = { + "pokemon", "moves", "items", "type_chart", "audio", "font", "maps", + "tilesets", "text", "trainers", "encounters", "sprites", "palettes", + "icons", "battle_anims", "constants", "landmarks", +} + +local CONTINUATIONS = { + "CROBAT", "BELLOSSOM", "POLITOED", "SLOWKING", "STEELIX", + "SCIZOR", "KINGDRA", "PORYGON2", "BLISSEY", +} + +local MOVES = { + "IRON_TAIL", "METAL_CLAW", "STEEL_WING", "RAIN_DANCE", "SUNNY_DAY", + "SANDSTORM", "SLUDGE_BOMB", "SHADOW_BALL", +} + +local function species(id, dex, generation) + local row = { + id = id, name = id, dex = dex, + types = {}, catchRate = 45, baseExp = 64, growthRate = "MEDIUM_FAST", + tmhm = {}, evolutions = {}, + spriteFront = "assets/generated/battle/front/" .. id:lower() .. ".png", + spriteBack = "assets/generated/battle/back/" .. id:lower() .. ".png", + } + if generation == 2 then + row.baseStats = { hp = 45, attack = 49, defense = 49, speed = 45, + specialAttack = 65, specialDefense = 65 } + row.levelMoves = {} + row.picSize = 5 + else + row.baseStats = { hp = 45, attack = 49, defense = 49, speed = 45, + special = 65 } + row.level1Moves = {} + row.learnset = {} + row.frontSize = 5 + end + return row +end + +local function defaults(version) + local generation = GameVersion.generation(version) + local pokemon = { FIXMON = species("FIXMON", 1, generation) } + local moves = {} + local items = {} + local typeChart = { + matchups = { { attacker = "NORMAL", defender = "ROCK", multiplier = 5 } }, + types = {}, + } + if generation == 2 then + pokemon.growthRates = {} + pokemon.tmhmMoves = {} + moves.generation, moves.source = 2, "fixture moves" + items.generation, items.source, items.pockets = 2, "fixture items", {} + for index, id in ipairs(CONTINUATIONS) do + pokemon[id] = species(id, 168 + index, generation) + end + for index, id in ipairs(MOVES) do + moves[id] = { id = id, name = id, index = index, type = "STEEL", + power = 50, accuracy = 100, pp = 15, effect = "NO_ADDITIONAL_EFFECT" } + end + typeChart.types.STEEL = { name = "STEEL", category = "physical", index = 9 } + typeChart.foresightMatchups = { + { attacker = "NORMAL", defender = "GHOST", multiplier = 0 }, + } + items.LEFTOVERS = { id = "LEFTOVERS", name = "LEFTOVERS", price = 0, + heldEffect = "HELD_LEFTOVERS", heldParameter = 0 } + end + return { + constants = {}, maps = {}, tilesets = {}, text = {}, text_pointers = {}, + trainer_headers = {}, font = {}, sprites = {}, pokemon = pokemon, + moves = moves, items = items, type_chart = typeChart, trainers = {}, + encounters = {}, field = { oakSpeech = {} }, battle_anims = {}, + audio = {}, palettes = {}, icons = {}, landmarks = {}, + } +end + +function Fixture.cache(files, version, overrides) + local prefix = GameVersion.cachePrefix(version) + local generation = GameVersion.generation(version) + local values = defaults(version) + for name, value in pairs(overrides or {}) do values[name] = value end + files[prefix .. "rom-cache.complete"] = + CacheContract.markerFor(version) + for _, name in ipairs(generation == 2 and GEN2_MODULES or GEN1_MODULES) do + local value = values[name] + files[prefix .. "data/generated/" .. name .. ".lua"] = + type(value) == "string" and value or LuaWriter.encode(value or {}) + end + if generation == 1 then + for _, name in ipairs(GEN1_OPTIONAL_MODULES) do + files[prefix .. "data/generated/" .. name .. ".lua"] = + LuaWriter.encode(values[name] or {}) + end + end + for _, path in ipairs(CacheContract.requiredFiles(version)) do + if files[prefix .. path] == nil then + files[prefix .. path] = path:match("%.lua$") and LuaWriter.encode({}) or "fixture" + end + end + if type(values.pokemon) == "table" then + for _, row in pairs(values.pokemon) do + if type(row) == "table" and row.spriteFront then + files[prefix .. row.spriteFront] = "front" + files[prefix .. row.spriteBack] = "back" + end + end + end + return files +end + +function Fixture.addMod(files, id, body) + files["mods/" .. id .. "/manifest.json"] = ([[{ + "id": %q, "name": %q, "version": "1.0.0", "entry": "main.lua", + "api": 2, "games": ["all"] + }]]):format(id, id) + files["mods/" .. id .. "/main.lua"] = body + return "mods/" .. id +end + +Fixture.GEN1_MODULES = GEN1_MODULES +Fixture.GEN2_MODULES = GEN2_MODULES +Fixture.CONTINUATIONS = CONTINUATIONS +Fixture.MOVES = MOVES + +return Fixture