fix(mod-api): validate dataset roots lazily

docs/modding.md

@@ -226,18 +226,28 @@ end
 `view.content` exposes the same registry names, aliases, generation routing,
 and data-only record shapes as `mod.content`, but only `get`, `has`, and
 `each`. Returned records are detached copies and cannot mutate either dataset.
+Every generated base record passes the selected generation's existing public
+schema before it is returned; extractor metadata beside record maps stays out
+of the registry id space. A malformed record makes `get` return nil, `has`
+return false, and `each` return no rows, and invalidates that dataset view.
 Records containing functions, userdata, threads, metatables, or cycles are not
 exposed. Each open call receives an independent facade, so one mod cannot
 replace another mod view method. Canonical boot shaping is included, such as
 Gen 1 defaults and Yellow corrections, and Gold's Foresight matchup rows and
 derived `held_items`.
 
-The marker and every required generated module for the selected version are
-rechecked on each open. Missing, partial, and stale imports return
-`nil, "not_imported"`; malformed or resource-limit-breaking generated data
-returns `nil, "invalid_cache"`. Generated modules are decoded with a bounded
-literal-only grammar and are never executed. No raw ROM bytes or generated
-source are exposed. `view.assets:path(relative)` and
+`open` checks the completion marker, exact version-specific file inventory,
+source/cache boundary, and file-size bounds without reading or decoding the
+semantic modules. A root is read, bounded-decoded, normalized, and cached only
+when a content operation first needs it. Each later view operation rechecks
+readiness and the source bytes behind already cached roots; unchanged roots are
+not decoded again. Missing, partial, and stale imports return
+`nil, "not_imported"`. Malformed syntax, a resource-limit violation, or a
+record that fails the public schema is discovered on first root access and
+fails that operation closed; a later `open` against the same source returns
+`nil, "invalid_cache"`. Generated modules use a bounded literal-only grammar
+and are never executed. No raw ROM bytes or generated source are exposed.
+`view.assets:path(relative)` and
 `view.assets:info(relative)` accept only `assets/generated/...` paths and
 keep them under the selected version cache prefix. The API never changes
 `mod.game`, the active `Data` table, `GameVersion`, or cache mount state.

docs/modding/reference/registries.md

@@ -510,6 +510,26 @@ mod.content.item_effects:register("MOON_FLUTE", { use = fn, field = true })
 mod.content.items:patch("POTION", { price = 100 })
 ```
 
+### On Gold (Gen 2)
+
+- semantics: `record`
+- target: `Data.items`
+
+The record differs; the registry name, the verbs and the id space
+do not.
+
+| field | type | required |
+|---|---|---|
+| `ball` | balls id | no |
+| `effect` | item_effects id | no |
+| `id` | string | yes |
+| `index` | integer 0..255 | no |
+| `machine` | {kind, move, number} | no |
+| `name` | string | yes |
+| `needsTarget` | boolean | no |
+| `price` | integer >= 0 | yes |
+| `tossable` | boolean | no |
+
 ## landmarks
 
 - semantics: `record`
@@ -674,6 +694,34 @@ mod.content.move_effects:register("DRAIN_PP_EFFECT", { kind = "primary", run = f
 mod.content.moves:patch("BLIZZARD", { accuracy = 70 })
 ```
 
+### On Gold (Gen 2)
+
+- semantics: `record`
+- target: `Data.moves`
+
+The record differs; the registry name, the verbs and the id space
+do not.
+
+| field | type | required |
+|---|---|---|
+| `accuracy` | integer 0..100 | yes |
+| `anim` | any value | no |
+| `category` | one of "physical" | "special" | "status" | no |
+| `chargeText` | string | no |
+| `counterable` | boolean | no |
+| `effect` | move_effects id | yes |
+| `fixedDamage` | integer >= 1 | function | no |
+| `highCrit` | boolean | no |
+| `id` | string | yes |
+| `index` | integer 0..255 | no |
+| `multiHit` | integer >= 1 | list of integer >= 1 | no |
+| `name` | string | yes |
+| `power` | integer 0..255 | yes |
+| `pp` | integer 0..64 | yes |
+| `priority` | integer -7..7 | no |
+| `semiInvulnerable` | boolean | no |
+| `type` | type_chart id | yes |
+
 ## music
 
 - semantics: `record`

docs/rfcs/0015-imported-dataset-views.md

@@ -70,9 +70,13 @@ semantic presence.
 the selected version's generation routing and the engine's existing
 `Schemas`, `Registry`, and `Builtins` normalization, so structured sources
 such as type matchups retain the same public ids used by the active
-`mod.content` facade. No register, patch, override, or remove verb is exposed. Each call returns an
-independent facade over the cached internal dataset, so facade mutation cannot cross
-mod boundaries.
+`mod.content` facade and extractor metadata beside record maps is not exposed
+as a record id. Every generated base record is checked with that selected
+generation's existing public schema before it can cross `get`, `has`, or
+`each`; there is no dataset-specific duplicate schema. No register, patch,
+override, or remove verb is exposed. Each call returns an independent facade
+over the cached internal dataset, so facade mutation cannot cross mod
+boundaries.
 
 `assets:path` returns the selected cache-prefixed virtual path.
 `assets:info` returns sanitized `type` and optional `size` metadata. Both
@@ -80,16 +84,29 @@ accept only relative paths below `assets/generated/`, reject control
 characters, absolute paths, backslashes, and traversal, and expose no byte
 reader.
 
-An unknown version returns `nil, "unknown_version"`. A missing, partial, or
-stale cache returns `nil, "not_imported"`. A required module that is malformed
-or exceeds the limits (8 MiB per module, 48 MiB aggregate, depth 64, 500,000
-values, 2 MiB per string, or 250,000 entries per table) returns
-`nil, "invalid_cache"`; actionable detail is engine-logged but not exposed to
-the mod. Generated Lua is decoded with the existing restricted
-literal grammar and never executed. Functions, userdata, threads, metatables,
-cycles, non-table roots, binary chunks, and trailing syntax cannot cross the
-facade. Successful roots are decoded lazily and cached per selected version.
-The API never exposes raw ROM bytes, generated source, host paths, or a mount.
+An unknown version returns `nil, "unknown_version"`. `open` checks the current
+completion marker, exact version-specific file inventory, source/cache
+boundary, and available file sizes; it does not read or decode semantic
+modules. A missing, partial, or stale cache returns `nil, "not_imported"`.
+
+The first content operation that needs a root reads it once, applies the
+limits (8 MiB per module, 48 MiB aggregate, depth 64, 500,000 values, 2 MiB
+per string, and 250,000 entries per table), decodes the restricted literal
+grammar, applies canonical selected-version normalization, and caches the
+detached root. Each later content or asset operation rechecks marker/file and
+source-bound readiness. It also rereads the source bytes for already cached
+roots; unchanged roots are not decoded again, while a changed root clears the
+derived registry cache and is decoded on its next use.
+
+Malformed syntax, non-table roots, binary/trailing content, resource-limit
+violations, and records that fail the public schema are therefore discovered
+on first access rather than during `open`. The triggering `get` returns nil,
+`has` returns false, or `each` returns no rows; the whole internal view is
+invalidated, and a subsequent `open` against the unchanged source returns
+`nil, "invalid_cache"`. Actionable detail is engine-logged but not exposed to
+the mod. Generated Lua is never executed. Functions, userdata, threads,
+metatables, and cycles cannot cross the facade. The API never exposes raw ROM
+bytes, generated source, host paths, or a mount.
 
 ## Migration and compatibility
 
@@ -106,7 +123,9 @@ The completion marker and per-version required-file rules live in the pure,
 injected `CacheContract` shared by the importer and dataset service. It also
 defines source-tree behavior. Neither consumer mutates `CacheFs.prefix` while
 checking readiness. Every `open` revalidates the contract and required module
-shapes; a stale/remove/reimport transition evicts the previous semantic view.
+inventory without semantic decoding. Every view operation rechecks that
+readiness before serving cached state; a stale/remove/reimport transition
+evicts the previous semantic view.
 
 ## Verification
 
@@ -117,8 +136,12 @@ shapes; a stale/remove/reimport transition evicts the previous semantic view.
   version-prefixed generated assets,
   traversal rejection, stable failure reasons, and stale-marker rejection.
 - It also proves missing/empty/partial/stale/remove/reimport behavior, hostile
-  generated-source rejection, canonical Gen 1/Yellow/Gold hydration, and the
-  approved Kanto+ Gold records and assets.
+  generated-source and malformed-record rejection, canonical Gen
+  1/Yellow/Gold hydration, active Red/Blue/Yellow isolation while reading Gold,
+  and the approved Kanto+ Gold records and assets.
+- `tests/engine/dataset_views_lazy_validation.lua` counts semantic reads and
+  decoder calls to prove `open` decodes nothing, unused roots stay unread, and
+  repeated access does not re-decode an unchanged cached root.
 - `tests/modkit/cases/dataset_views_nontermination.lua` proves generated code
   is rejected rather than executed; `tests/engine/generated_data_decoder_test.lua`
   proves every decoder resource bound.

src/mods/DatasetViews.lua

@@ -47,10 +47,6 @@ local GEN2_ROOTS = {
   gen2Constants = "constants", gen2Landmarks = "landmarks",
 }
 
-local function decode(source)
-  return SaveSerializer.decode(source, DECODE_LIMITS)
-end
-
 local function resolvePath(root, suffix)
   local node = root
   for key in suffix:gmatch("[^.]+") do
@@ -130,15 +126,15 @@ local function isDataOnly(value, state, depth)
   return true
 end
 
-function DatasetViews.new(fs, engineRequire)
+function DatasetViews.new(fs, engineRequire, decoder)
   assert(fs and fs.read and fs.getInfo,
     "DatasetViews.new requires a readable filesystem")
   return setmetatable({ fs = fs, engineRequire = engineRequire or require,
-    datasets = {} }, DatasetViews)
+    decoder = decoder or SaveSerializer.decode, datasets = {} }, DatasetViews)
 end
 
-function DatasetViews:_validate(version, inspected)
-  local sources, aggregate = {}, 0
+function DatasetViews:_preflight(version, inspected)
+  local paths, aggregate = {}, 0
   local modules = CacheContract.semanticModules(version)
   for _, name in ipairs(CacheContract.optionalSemanticModules(version)) do
     local path = inspected.prefix .. "data/generated/" .. name .. ".lua"
@@ -151,31 +147,80 @@ function DatasetViews:_validate(version, inspected)
     if size and size > DECODE_LIMITS.maxBytes then return nil, name .. ": size limit" end
     aggregate = aggregate + (size or 0)
     if aggregate > MAX_AGGREGATE_BYTES then return nil, "aggregate size limit" end
+    paths[name] = path
   end
-  aggregate = 0
-  for _, name in ipairs(modules) do
-    local path = inspected.prefix .. "data/generated/" .. name .. ".lua"
-    local source = self.fs.read(path)
-    if type(source) ~= "string" then return nil, "missing " .. name end
-    aggregate = aggregate + #source
-    if aggregate > MAX_AGGREGATE_BYTES then return nil, "aggregate size limit" end
-    local value, err = decode(source)
-    if type(value) ~= "table" then
-      return nil, name .. ": " .. tostring(err or "non-table root")
+  return { paths = paths, key = table.concat(modules, "\n") }
 end
-    sources[name] = source
+
+local function resetInternal(view)
+  view.moduleCache, view.data, view.registries = {}, nil, nil
 end
-  return sources
+
+function DatasetViews:_reject(view, moduleName, source, detail)
+  view.invalid = { module = moduleName, source = source, detail = detail }
+  view.data, view.registries = nil, nil
+  Logger.warn("dataset %s cache rejected: %s", view.version, tostring(detail))
+  return nil
+end
+
+function DatasetViews:_ready(view)
+  if view.invalid or view.unavailable then return false end
+  local inspected, _, detail = CacheContract.inspect(view.version, self.fs, {
+    allowSource = true, semantic = true,
+  })
+  if not inspected then
+    view.unavailable = true
+    if self.datasets[view.version] == view then self.datasets[view.version] = nil end
+    Logger.warn("dataset %s unavailable: %s", view.version, detail)
+    return false
+  end
+  local plan, invalid = self:_preflight(view.version, inspected)
+  if not plan then
+    self:_reject(view, nil, nil, invalid)
+    return false
+  end
+  if view.prefix ~= inspected.prefix or view.plan.key ~= plan.key then
+    view.prefix, view.plan = inspected.prefix, plan
+    resetInternal(view)
+  else
+    view.plan = plan
+  end
+  for name, cached in pairs(view.moduleCache) do
+    local source = self.fs.read(plan.paths[name])
+    if type(source) ~= "string" then
+      self:_reject(view, name, source, name .. ": unreadable generated module")
+      return false
+    end
+    if source ~= cached.source then
+      resetInternal(view)
+      break
+    end
+  end
+  return not view.invalid
 end
 
 function DatasetViews:_module(view, root)
   local moduleName = view.modules[root]
   if not moduleName then return nil end
-  local source = view.sources[moduleName]
-  if source == nil then return nil end
   local cached = view.moduleCache[moduleName]
-  if cached and cached.source == source then return cached.value end
-  local value = assert(decode(source))
+  if cached then return cached.value end
+  local path = view.plan.paths[moduleName]
+  if not path then return nil end
+  local source = self.fs.read(path)
+  if type(source) ~= "string" then
+    return self:_reject(view, moduleName, source,
+      moduleName .. ": unreadable generated module")
+  end
+  local aggregate = #source
+  for _, loaded in pairs(view.moduleCache) do aggregate = aggregate + #loaded.source end
+  if aggregate > MAX_AGGREGATE_BYTES then
+    return self:_reject(view, moduleName, source, "aggregate size limit")
+  end
+  local value, err = self.decoder(source, DECODE_LIMITS)
+  if type(value) ~= "table" then
+    return self:_reject(view, moduleName, source,
+      moduleName .. ": " .. tostring(err or "non-table root"))
+  end
   view.moduleCache[moduleName] = { source = source, value = value }
   return value
 end
@@ -191,6 +236,7 @@ function DatasetViews:_data(view)
     end,
   })
   DatasetHydration.apply(data, view.version, self.engineRequire)
+  if view.invalid then error(view.invalid.detail, 0) end
   view.data = data
   return data
 end
@@ -217,10 +263,37 @@ function DatasetViews:_registries(view)
 end
 
 function DatasetViews:_registry(view, name)
-  local registry = self:_registries(view)[name]
+  local service = self
+  local function registryForRead()
+    if not service:_ready(view) then return nil end
+    local ok, registries = pcall(service._registries, service, view)
+    if not ok then
+      if not view.invalid then service:_reject(view, nil, nil, registries) end
+      return nil
+    end
+    if view.invalid then return nil end
+    return registries[name]
+  end
+  local function validate(registry, id, value)
+    if value == nil then return true end
+    local ok, detail = Schemas.check(registry.spec, registry.name, id,
+      value, "override")
+    if ok then return true end
+    local target = registry.spec.target
+      or Schemas.targetFor(registry.name, registry.spec, view.generation)
+    local root = target and target:match("^[^%.]+")
+    local moduleName = root and view.modules[root]
+    if root == "gen2HeldItems" then moduleName = "items" end
+    local cached = moduleName and view.moduleCache[moduleName]
+    service:_reject(view, moduleName, cached and cached.source,
+      "invalid " .. registry.name .. " record: " .. detail)
+    return false
+  end
   local function rawAt(id)
+    local registry = registryForRead()
     local value = registry and registry:get(id)
-    if value == nil or not isDataOnly(value) then return nil end
+    if value == nil or not validate(registry, id, value)
+        or not isDataOnly(value) then return nil end
     return value
   end
   local function valueAt(id)
@@ -239,9 +312,16 @@ function DatasetViews:_registry(view, name)
     end,
     each = function()
       local ids = {}
+      local registry = registryForRead()
       if registry then
         for id, value in registry:each() do
-          if type(id) == "string" and isDataOnly(value) then ids[#ids + 1] = id end
+          if not validate(registry, id, value) then
+            ids = {}
+            break
+          end
+          if type(id) == "string" and isDataOnly(value) then
+            ids[#ids + 1] = id
+          end
         end
       end
       table.sort(ids)
@@ -259,9 +339,13 @@ end
 function DatasetViews:_assets(view)
   local service = self
   local assets = {}
-  function assets:path(path) return view.prefix .. assetRelative(path) end
+  function assets:path(path)
+    if not service:_ready(view) then return nil end
+    return view.prefix .. assetRelative(path)
+  end
   function assets:info(path)
     local full = self:path(path)
+    if not full then return nil end
     local info = service.fs.getInfo and service.fs.getInfo(full, "file")
     if not info or (info.type and info.type ~= "file") then return nil end
     local out = { type = "file" }
@@ -283,43 +367,36 @@ function DatasetViews:open(version)
     Logger.warn("dataset %s unavailable: %s", version, detail)
     return nil, reason
   end
-  local sources, invalid = self:_validate(version, inspected)
-  if not sources then
-    self.datasets[version] = nil
+  local plan, invalid = self:_preflight(version, inspected)
+  if not plan then
     Logger.warn("dataset %s cache rejected: %s", version, invalid)
     return nil, "invalid_cache"
   end
 
   local internal = self.datasets[version]
+  if internal and internal.invalid then
+    local bad = internal.invalid
+    local path = bad.module and plan.paths[bad.module]
+    local source = path and self.fs.read(path)
+    if source == bad.source then return nil, "invalid_cache" end
+    internal = nil
+    self.datasets[version] = nil
+  end
   local changed = not internal or internal.prefix ~= inspected.prefix
-  if internal and not changed then
-    for name, source in pairs(internal.sources) do
-      if sources[name] ~= source then changed = true; break end
-    end
-  end
-  if internal and not changed then
-    for name, source in pairs(sources) do
-      if internal.sources[name] ~= source then changed = true; break end
-    end
-  end
+    or internal.plan.key ~= plan.key
   if changed then
+    if internal then internal.unavailable = true end
     internal = {
       version = version,
       generation = GameVersion.generation(version),
       prefix = inspected.prefix,
+      plan = plan,
       modules = GameVersion.generation(version) == 2 and GEN2_ROOTS or GEN1_ROOTS,
-      moduleCache = {}, sources = sources,
+      moduleCache = {},
     }
-    local ok, buildError = pcall(function() internal.registries = self:_registries(internal) end)
-    if not ok then
-      Logger.warn("dataset %s semantic hydration failed: %s", version,
-        tostring(buildError))
-      self.datasets[version] = nil
-      return nil, "invalid_cache"
-    end
     self.datasets[version] = internal
   else
-    internal.sources = sources
+    internal.plan = plan
   end
 
   local view = { version = version, generation = internal.generation, content = {} }

src/mods/Schemas.lua

@@ -757,6 +757,32 @@ end
 local R = {}
 Schemas.REGISTRIES = R
 
+-- Some generated Gen 2 modules keep extractor metadata beside their public
+-- record maps. These callbacks are the registry normalization boundary: the
+-- metadata remains available to engine consumers through Data, but is not an
+-- id a mod can read or overwrite through the record registry.
+local function recordMapExcept(...)
+  local excluded = {}
+  for index = 1, select("#", ...) do excluded[select(index, ...)] = true end
+  return function(base, id)
+    if excluded[id] then return nil end
+    return base[id]
+  end, function(base)
+    local ids = {}
+    for id in pairs(base) do
+      if not excluded[id] then ids[#ids + 1] = id end
+    end
+    return ids
+  end
+end
+
+local pokemonGen2BaseAt, pokemonGen2BaseIds =
+  recordMapExcept("growthRates", "tmhmMoves")
+local movesGen2BaseAt, movesGen2BaseIds =
+  recordMapExcept("generation", "source")
+local itemsGen2BaseAt, itemsGen2BaseIds =
+  recordMapExcept("generation", "source", "pockets")
+
 -- ------- shared Gen 2 leaves
 --
 -- The ROM name spaces Gold's tables key by.  They are enums rather than
@@ -779,6 +805,7 @@ local gen2PaletteRow = f.list(gen2Color)
 
 R.pokemon = {
   semantics = "record", target = "pokemon",
+  gen2BaseAt = pokemonGen2BaseAt, gen2BaseIds = pokemonGen2BaseIds,
   fields = {
     id = f.str, name = f.str, dex = f.int(1),
     index = f.opt(f.int(0, 255)),
@@ -870,6 +897,7 @@ R.pokemon = {
 
 R.moves = {
   semantics = "record", target = "moves",
+  gen2BaseAt = movesGen2BaseAt, gen2BaseIds = movesGen2BaseIds,
   fields = {
     id = f.str, name = f.str,
     index = f.opt(f.int(0, 255)),
@@ -894,6 +922,7 @@ R.moves = {
 
 R.items = {
   semantics = "record", target = "items",
+  gen2BaseAt = itemsGen2BaseAt, gen2BaseIds = itemsGen2BaseIds,
   fields = {
     id = f.str, name = f.str,
     index = f.opt(f.int(0, 255)),

tests/engine/dataset_views_lazy_validation.lua

@@ -0,0 +1,46 @@
+-- Imported semantic modules are opened without eager reads and are decoded
+-- once, on first use, through the same public facade mods receive.
+package.path = "./?.lua;./?/init.lua;" .. package.path
+
+local T = require("tests.modkit")
+local Fixture = require("tests.modkit.dataset_view_fixture")
+local DatasetViews = require("src.mods.DatasetViews")
+local GameVersion = require("src.core.GameVersion")
+local SaveSerializer = require("src.core.SaveSerializer")
+
+local files = {}
+Fixture.cache(files, "gold")
+local fs = T.sdk.memfs(files)
+local rawRead = fs.read
+local reads = {}
+fs.read = function(path)
+  reads[path] = (reads[path] or 0) + 1
+  return rawRead(path)
+end
+local decodes = 0
+local service = DatasetViews.new(fs, require, function(source, limits)
+  decodes = decodes + 1
+  return SaveSerializer.decode(source, limits)
+end)
+local pokemonPath = GameVersion.cachePrefix("gold")
+  .. "data/generated/pokemon.lua"
+local movesPath = GameVersion.cachePrefix("gold")
+  .. "data/generated/moves.lua"
+
+local view, reason = service:open("gold")
+T.eq(reason, nil, "marker-valid Gold dataset opens")
+T.check(view ~= nil, "open returns the lazy view")
+T.eq(reads[pokemonPath] or 0, 0, "open does not read an unused root")
+T.eq(reads[movesPath] or 0, 0, "open does not read another unused root")
+T.eq(decodes, 0, "open decodes no semantic root")
+
+local first = view and view.content.pokemon:get("FIXMON")
+T.eq(first and first.name, "FIXMON", "first access decodes a valid root")
+T.check(decodes > 0, "first root access performs bounded decoding")
+T.eq(reads[movesPath] or 0, 0, "pokemon access leaves moves unused")
+local afterFirst = decodes
+local second = view and view.content.pokemon:get("FIXMON")
+T.eq(second and second.name, "FIXMON", "repeated access remains available")
+T.eq(decodes, afterFirst, "repeated access does not re-decode cached roots")
+
+T.finish("dataset_views_lazy_validation")

tests/modkit/cases/dataset_views.lua

@@ -65,6 +65,13 @@ for _, id in ipairs({ "IRON_TAIL", "METAL_CLAW", "STEEL_WING", "RAIN_DANCE",
   out.moves[id] = gold.content.moves:has(id)
 end
 out.steel = gold.content.type_chart:has("STEEL")
+out.metadataHidden = not gold.content.pokemon:has("growthRates")
+  and not gold.content.pokemon:has("tmhmMoves")
+  and not gold.content.moves:has("generation")
+  and not gold.content.moves:has("source")
+  and not gold.content.items:has("generation")
+  and not gold.content.items:has("source")
+  and not gold.content.items:has("pockets")
 out.executableBuiltinHidden = gold.content.statuses:get("sleep") == nil
 out.assetDirectory = gold.assets:info("assets/generated/battle/front")
 out.assetRejects = {}
@@ -113,6 +120,8 @@ for _, id in ipairs(Fixture.CONTINUATIONS) do
 end
 for _, id in ipairs(Fixture.MOVES) do T.eq(out.moves[id], true, "Gold exposes " .. id) end
 T.eq(out.steel, true, "Gold exposes Steel")
+T.eq(out.metadataHidden, true,
+  "Gold extractor metadata stays outside record registry id spaces")
 local registryCount = 0
 for _ in pairs(Schemas.REGISTRIES) do registryCount = registryCount + 1 end
 for _ in pairs(Schemas.ALIASES) do registryCount = registryCount + 1 end
@@ -206,16 +215,22 @@ local transitionRead = transitionFs.read
 local transitionMarkers = 0
 local reimportedPokemon = require("src.import.LuaWriter").encode({
   FIXMON = { id = "FIXMON", name = "REIMPORTED", dex = 1,
+    types = {}, catchRate = 45, baseExp = 64, growthRate = "MEDIUM_FAST",
+    baseStats = { hp = 45, attack = 49, defense = 49, speed = 45,
+      special = 65 },
+    level1Moves = {}, tmhm = {}, learnset = {}, evolutions = {},
     spriteFront = "assets/generated/battle/front/fixmon.png",
     spriteBack = "assets/generated/battle/back/fixmon.png", frontSize = 5 },
 })
 transitionFs.read = function(path)
   if path == GameVersion.cachePrefix("red") .. "rom-cache.complete" then
     transitionMarkers = transitionMarkers + 1
-    if transitionMarkers == 2 then return nil end
+    -- open and the first facade read each recheck readiness; remove the cache
+    -- for the next open, then make the following open the reimport.
+    if transitionMarkers == 3 then return nil end
   end
   local body = transitionRead(path)
-  if transitionMarkers >= 3
+  if transitionMarkers >= 4
       and path == GameVersion.cachePrefix("red") .. "data/generated/pokemon.lua" then
     return reimportedPokemon
   end
@@ -255,7 +270,12 @@ local mod = ...
 local out = {}
 for _, version in ipairs({ "red", "blue", "yellow", "gold", "silver" }) do
   local view, reason = mod.datasets:open(version)
-  out[version] = { view ~= nil, reason }
+  local value = view and view.content.pokemon:get("BAD")
+  local reopened, reopenedReason = mod.datasets:open(version)
+  out[version] = {
+    first = view ~= nil, firstReason = reason, value = value,
+    reopened = reopened ~= nil, reason = reopenedReason,
+  }
 end
 mod.exports.result = out
 ]])
@@ -272,8 +292,9 @@ local hostileRun = T.sdk.loadMods(hostilePaths, {
 })
 local hostileOut = hostileRun.loader.exports.hostile_probe.result
 for _, row in ipairs(hostile) do
-  T.same(hostileOut[row[1]], { false, "invalid_cache" },
-    row[1] .. " hostile generated source fails closed")
+  T.same(hostileOut[row[1]], {
+    first = true, reopened = false, reason = "invalid_cache",
+  }, row[1] .. " hostile generated source fails closed on first root access")
 end
 T.eq(debug.gethook and debug.gethook(), beforeHook,
   "dataset decoding preserves the caller debug hook")
@@ -283,6 +304,91 @@ if debug.sethook then
 end
 hostileRun.release()
 
+-- A syntactically valid semantic root with a primitive record must fail closed
+-- whichever public read verb encounters it first.
+local malformedFiles = {}
+for _, version in ipairs({ "red", "blue", "yellow" }) do
+  Fixture.cache(malformedFiles, version, {
+    pokemon = "return { FIXMON = 7 }",
+  })
+end
+local malformedMod = Fixture.addMod(malformedFiles, "malformed_probe", [[
+local mod = ...
+local out = {}
+local getView = assert(mod.datasets:open("red"))
+out.get = getView.content.pokemon:get("FIXMON")
+local getAgain, getReason = mod.datasets:open("red")
+out.getAgain = { getAgain ~= nil, getReason }
+
+local hasView = assert(mod.datasets:open("blue"))
+out.has = hasView.content.pokemon:has("FIXMON")
+local hasAgain, hasReason = mod.datasets:open("blue")
+out.hasAgain = { hasAgain ~= nil, hasReason }
+
+local eachView = assert(mod.datasets:open("yellow"))
+for id, value in eachView.content.pokemon:each() do
+  out.each = { id, value }
+  break
+end
+local eachAgain, eachReason = mod.datasets:open("yellow")
+out.eachAgain = { eachAgain ~= nil, eachReason }
+mod.exports.result = out
+]])
+local malformedRun = T.sdk.loadMods({ malformedMod }, {
+  fs = T.sdk.memfs(malformedFiles), data = { pokemon = {} }, generation = 1,
+})
+T.eq(#malformedRun.errors, 0, "malformed-record probe stays sandboxed")
+local malformedOut = malformedRun.loader.exports.malformed_probe.result
+T.eq(malformedOut.get, nil, "get never exposes a malformed semantic record")
+T.eq(malformedOut.has, false, "has never affirms a malformed semantic record")
+T.eq(malformedOut.each, nil, "each never exposes a malformed semantic record")
+T.same(malformedOut.getAgain, { false, "invalid_cache" },
+  "get invalidates the malformed dataset")
+T.same(malformedOut.hasAgain, { false, "invalid_cache" },
+  "has invalidates the malformed dataset")
+T.same(malformedOut.eachAgain, { false, "invalid_cache" },
+  "each invalidates the malformed dataset")
+malformedRun.release()
+
+-- The same Gold view stays independent while each Gen 1 version is the
+-- actual active runtime version and cache namespace, not just a fixture label.
+for _, activeVersion in ipairs({ "red", "blue", "yellow" }) do
+  local matrixFiles = {}
+  Fixture.cache(matrixFiles, "gold")
+  local matrixMod = Fixture.addMod(matrixFiles,
+    "active_" .. activeVersion .. "_gold_probe", [[
+local mod = ...
+local gold = assert(mod.datasets:open("gold"))
+mod.exports.result = {
+  species = gold.content.pokemon:get("FIXMON").name,
+  foresight = gold.content.type_chart:get("NORMAL>GHOST").multiplier,
+  held = gold.content.held_items:get("LEFTOVERS").heldEffect,
+}
+]])
+  GameVersion.set(activeVersion)
+  CacheFs.prefix = GameVersion.cachePrefix(activeVersion)
+  local activeData = {
+    pokemon = { ACTIVE = { id = "ACTIVE_" .. activeVersion,
+      nested = { version = activeVersion } } },
+  }
+  local matrixRun = T.sdk.loadMods({ matrixMod }, {
+    fs = T.sdk.memfs(matrixFiles), data = activeData, generation = 1,
+  })
+  T.eq(#matrixRun.errors, 0, activeVersion .. "-active Gold probe loads")
+  T.same(matrixRun.loader.exports["active_" .. activeVersion .. "_gold_probe"].result,
+    { species = "FIXMON", foresight = 0, held = "HELD_LEFTOVERS" },
+    activeVersion .. "-active runtime sees canonical Gold semantics")
+  T.eq(GameVersion.get(), activeVersion,
+    activeVersion .. " remains the active GameVersion")
+  T.eq(CacheFs.prefix, GameVersion.cachePrefix(activeVersion),
+    activeVersion .. " remains the active cache namespace")
+  T.eq(activeData.pokemon.ACTIVE.nested.version, activeVersion,
+    activeVersion .. " active Data remains unchanged")
+  T.eq(activeData.pokemon.FIXMON, nil,
+    activeVersion .. " active Data receives no Gold record")
+  matrixRun.release()
+end
+
 GameVersion.set(originalVersion)
 CacheFs.prefix = originalPrefix
 T.finish("dataset_views")

tests/modkit/cases/dataset_views_nontermination.lua

@@ -10,12 +10,18 @@ Fixture.cache(files, "red", {
 local modPath = Fixture.addMod(files, "nontermination_probe", [[
 local mod = ...
 local view, reason = mod.datasets:open("red")
-mod.exports.result = { view ~= nil, reason }
+local value = view and view.content.pokemon:get("FIXMON")
+local reopened, reopenedReason = mod.datasets:open("red")
+mod.exports.result = {
+  first = view ~= nil, firstReason = reason, value = value,
+  reopened = reopened ~= nil, reason = reopenedReason,
+}
 ]])
 local run = T.sdk.loadMods({ modPath }, {
   fs = T.sdk.memfs(files), data = { pokemon = {} }, generation = 1,
 })
 T.same(run.loader.exports.nontermination_probe.result,
-  { false, "invalid_cache" }, "generated code is never executed")
+  { first = true, reopened = false, reason = "invalid_cache" },
+  "generated code is never executed and invalidates on first root access")
 run.release()
 T.finish("dataset_views_nontermination")

tests/modkit/dataset_view_fixture.lua

@@ -30,10 +30,23 @@ local MOVES = {
 local function species(id, dex, generation)
   local row = {
     id = id, name = id, dex = dex,
+    types = {}, catchRate = 45, baseExp = 64, growthRate = "MEDIUM_FAST",
+    tmhm = {}, evolutions = {},
     spriteFront = "assets/generated/battle/front/" .. id:lower() .. ".png",
     spriteBack = "assets/generated/battle/back/" .. id:lower() .. ".png",
   }
-  if generation == 2 then row.picSize = 5 else row.frontSize = 5 end
+  if generation == 2 then
+    row.baseStats = { hp = 45, attack = 49, defense = 49, speed = 45,
+      specialAttack = 65, specialDefense = 65 }
+    row.levelMoves = {}
+    row.picSize = 5
+  else
+    row.baseStats = { hp = 45, attack = 49, defense = 49, speed = 45,
+      special = 65 }
+    row.level1Moves = {}
+    row.learnset = {}
+    row.frontSize = 5
+  end
   return row
 end
 
@@ -47,6 +60,10 @@ local function defaults(version)
     types = {},
   }
   if generation == 2 then
+    pokemon.growthRates = {}
+    pokemon.tmhmMoves = {}
+    moves.generation, moves.source = 2, "fixture moves"
+    items.generation, items.source, items.pockets = 2, "fixture items", {}
     for index, id in ipairs(CONTINUATIONS) do
       pokemon[id] = species(id, 168 + index, generation)
     end