diff --git a/scripts/test.sh b/scripts/test.sh index 18bb960e..fe1a3120 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -22,6 +22,7 @@ set -uo pipefail cd "$(dirname "$0")/.." LUA=${LUA:-luajit} +LUA54=${LUA54:-lua5.4} BLESS=0 QUICK=0 SHOTS=${WITH_SHOTS:-0} @@ -134,6 +135,16 @@ if [ -f data/generated/maps.lua ]; then run_tier "T3 save editor: wheel scrolling" "$LUA" tests/save_editor_wheel_bug595_test.lua run_tier "T3 save editor: pad / NX input" "$LUA" tests/save_editor_pad_input_test.lua run_tier "T5 link (loopback lockstep)" "$LUA" tests/run_link_tests.lua + # The oversize-save vendor oracle (tests/save_oversize_vendor_test.lua) + # cross-checks the launcher's footer-truncation import against the + # INDEPENDENT PKHeX-derived gen1lib codec, which cannot run under luajit + # (native 5.3+ operators). Needs a stock Lua 5.3/5.4; skip when absent. + if command -v "$LUA54" >/dev/null 2>&1; then + run_tier "T3 save oversize vendor oracle" "$LUA54" tests/save_oversize_vendor_test.lua + else + echo "" + echo "-- T3 save oversize vendor oracle: skipped (no '$LUA54' on PATH; set LUA54=...)" + fi fi else echo "" diff --git a/src/import/LauncherView.lua b/src/import/LauncherView.lua index dc25188f..7be70f26 100644 --- a/src/import/LauncherView.lua +++ b/src/import/LauncherView.lua @@ -1505,6 +1505,8 @@ local function buildConfirmModal(imp, m) imp:_confirmModUpdate(c.id, c.release) elseif c.kind == "enableAll" then imp:_setAllMods(true, true) + elseif c.kind == "importOversize" then + imp:_importSave(c.version, c.source, true) else imp:_toggleMod(c.id, true) end diff --git a/src/import/RomImporter.lua b/src/import/RomImporter.lua index 0ecbfc2a..059c8135 100644 --- a/src/import/RomImporter.lua +++ b/src/import/RomImporter.lua @@ -1564,7 +1564,7 @@ end -- the target tab forward so the notice (and, on success, the new active slot) -- is visible. Requires the ROM to be imported first, since a save is only -- playable with its game's data present. -function RomImporter:_importSave(version, source) +function RomImporter:_importSave(version, source, force) if self.workState == "working" then return end if GameVersion.VERSIONS[self.tab] or self.tab == "mods" then self.tab = version @@ -1574,15 +1574,35 @@ function RomImporter:_importSave(version, source) .. GameVersion.info(version).displayName .. " ROM before importing a save." } return end - local ok, res = require("src.import.SaveFileIO").importToSlot(source, version) + local ok, res, info = require("src.import.SaveFileIO").importToSlot(source, version, force) if ok then self:_refreshSlots(version) self.activeSlot[version] = res self.slotScroll[version] = math.huge -- pin the new row on screen (clamped in draw) self.saveNotice[version] = { ok = true, text = "Imported save into " .. tostring(res) .. "." } - else - self.saveNotice[version] = { ok = false, text = tostring(res) } + return end + if res == nil and info and info.needsConfirm then + -- A .sav larger than 32 KB whose first 32768 bytes checksum: the surplus + -- is almost certainly an emulator RTC footer, so ask before truncating. + -- The yes arm re-enters with force=true; cancel leaves the file untouched. + self._modConfirm = { + kind = "importOversize", + version = version, + source = source, + title = "Oversized save file", + lines = { + ("This save is %d bytes; a cartridge save is exactly %d bytes (32 KB).") + :format(info.size, 32768), + "It may come from a ROM that saved the battery image with an emulator.", + "The extra bytes would be discarded.", + "Import it anyway?", + }, + yesLabel = "Import anyway", + } + return + end + self.saveNotice[version] = { ok = false, text = tostring(res) } end -- "Import save" button: open a native .sav picker and import the pick. diff --git a/src/import/SaveFileIO.lua b/src/import/SaveFileIO.lua index 42d45014..c9a149f5 100644 --- a/src/import/SaveFileIO.lua +++ b/src/import/SaveFileIO.lua @@ -64,18 +64,34 @@ local function readSource(source) return nil, "could not read the save file: " .. tostring(openErr) end --- importToSlot(source, version) -> ok, slotIdOrErr --- source: an absolute path, a LOVE DroppedFile, or raw 32768 bytes. On success +-- importToSlot(source, version, force) -> ok, slotIdOrErr | (false, nil, info) +-- source: an absolute path, a LOVE DroppedFile, or raw bytes. On success -- registers a new slot for the version, writes the imported save into it, makes -- it the active slot, and returns true + the new slot id. On any failure --- returns false + a friendly message. -function SaveFileIO.importToSlot(source, version) +-- returns false + a friendly message. force only matters for a file LARGER +-- than 32768 bytes whose first 32768 bytes carry a valid main-data checksum +-- (i.e. a cartridge save padded with an emulator RTC footer): without force +-- this returns false, nil, { needsConfirm = true, size = #bytes } so the +-- launcher can ask the player before truncating; with force the extra bytes +-- are dropped and the 32768-byte save imports. +function SaveFileIO.importToSlot(source, version, force) version = version or GameVersion.get() local bytes, readErr = readSource(source) if not bytes then return false, readErr end if #bytes ~= SAVE_SIZE then - return false, ("A save file must be %d bytes (32 KB); this one is %d.") - :format(SAVE_SIZE, #bytes) + local check = SaveConvert.mainChecksumValid(bytes) + if check == nil then + return false, ("A save file must be %d bytes (32 KB); this one is %d.") + :format(SAVE_SIZE, #bytes) + end + if check == false then + return false, "save data checksum invalid (main data checksum mismatch)" + end + if #bytes > SAVE_SIZE and not force then + return false, nil, { needsConfirm = true, size = #bytes } + end + bytes = #bytes > SAVE_SIZE and bytes:sub(1, SAVE_SIZE) + or (bytes .. string.rep("\0", SAVE_SIZE - #bytes)) end -- 3rd arg: the crosswalk has to come from THIS game's ROM cache. The -- launcher imports before the cache is mounted on the un-prefixed paths, so diff --git a/src/save_convert/GenSave.lua b/src/save_convert/GenSave.lua index 8c226292..9f1c3274 100644 --- a/src/save_convert/GenSave.lua +++ b/src/save_convert/GenSave.lua @@ -190,6 +190,18 @@ local function checksum(bytes, from, to) return bit.band(bit.bnot(sum), 0xFF) end +-- Main-data checksum gate used before an import policy is decided. Returns +-- nil when the buffer is too short to even carry the stored checksum byte +-- (offset O.mainChecksum, the last byte of wMainData), false on a mismatch, +-- true when it matches. Works on any length >= O.mainChecksum + 1, so a +-- caller can classify a truncated or footer-padded file without a full +-- decode -- the checksummed region (0x2598..0x3522) always sits entirely +-- inside the first 0x3524 bytes of a save. +function GenSave.mainChecksumValid(bytes) + if #bytes < O.mainChecksum + 1 then return nil end + return checksum(bytes, O.checksumStart, O.checksumEnd) == u8(bytes, O.mainChecksum) +end + -- flag_array packs LSB-first within each byte (bit 0 of byte 0 = index 0). -- This is pokered's runtime FlagAction convention (home/predef macros): it -- takes flag number N, addresses byte N/8, and builds the mask by rotating diff --git a/src/save_convert/SaveConvert.lua b/src/save_convert/SaveConvert.lua index ef2547e3..f6ca12d8 100644 --- a/src/save_convert/SaveConvert.lua +++ b/src/save_convert/SaveConvert.lua @@ -25,6 +25,7 @@ local GenSave = require("src.save_convert.GenSave") local SaveConvert = {} SaveConvert.SAVE_SIZE = GenSave.SAVE_SIZE +SaveConvert.mainChecksumValid = GenSave.mainChecksumValid -- ------------------------------------------------------------------ -- Crosswalk data loading (cached). Mirrors src/core/Data.lua: prefer diff --git a/tests/engine/save_file_io_tests.lua b/tests/engine/save_file_io_tests.lua index 90a7fd07..d452c1f5 100644 --- a/tests/engine/save_file_io_tests.lua +++ b/tests/engine/save_file_io_tests.lua @@ -204,6 +204,112 @@ do check(type(erre) == "string", "the empty-export failure carries a message") end +-- ---------------------------------------------- oversize / truncated policy +-- A .sav LARGER than 32768 bytes whose first 32768 bytes carry a valid +-- main-data checksum is a cartridge save with a trailing emulator RTC footer +-- (VBA appends 44/48 bytes -- bgb.bircd.org/rtcsave.html). Without force the +-- import must NOT happen silently: it returns (false, nil, {needsConfirm}) +-- so the launcher can ask. With force the surplus is dropped. A file +-- SHORTER than 32768 is refused unless its checksum region is intact, in +-- which case it imports zero-padded (a truncated box region, not a loss). + +-- DroppedFile-shaped source for arbitrary bytes (readSource disambiguates a +-- raw string of length != 32768 as a path, so tests hand a file object). +local function fileSource(bytes) + return { + _bytes = bytes, + open = function() return true end, + getSize = function(self) return #self._bytes end, + read = function(self) return self._bytes end, + close = function() return true end, + } +end + +-- A realistic 44-byte VBA MBC3 RTC footer (4 dwords, 16-byte latched copies, +-- 8-byte unix timestamp, 4-byte unix timestamp -- bgb.bircd.org/rtcsave.html). +local function rtcFooter() + local parts = {} + local function pushLe(v) + parts[#parts + 1] = string.char(v % 256, math.floor(v / 256) % 256, + math.floor(v / 65536) % 256, math.floor(v / 16777216) % 256) + end + pushLe(27) -- days + pushLe(29) -- hours + pushLe(11) -- minutes + pushLe(200) -- seconds + parts[#parts + 1] = string.rep("\0", 16) -- latched RTC copies + parts[#parts + 1] = string.rep("\0", 8) -- 64-bit unix timestamp + pushLe(0x669A00BF) -- 32-bit unix timestamp (2024-07-09) + return table.concat(parts) +end + +do + local oversize = syntheticSave("OVS") .. rtcFooter() + eq(#oversize, 32768 + 44, "the oversize fixture is 32812 bytes like the real VBA save") + + local files = fresh() + local ok, res, info = SaveFileIO.importToSlot(fileSource(oversize), "red") + eq(ok, false, "an oversize valid save is not imported without confirmation") + eq(res, nil, "the oversize result carries no error string") + check(info ~= nil and info.needsConfirm == true, "the oversize result requests confirmation") + eq(info and info.size, #oversize, "the confirmation carries the actual file size") + eq(#SaveData.listSlots("red"), 0, "no slot is created before confirmation") + + -- forcing the import truncates the footer away + local fok, slotId = SaveFileIO.importToSlot(fileSource(oversize), "red", true) + eq(fok, true, "force imports the truncated save") + local loaded = SaveData.load("red") + eq(loaded and loaded.player.name, "OVS", "the forced import keeps the player name") + eq(SaveData.activeSlot("red"), slotId, "the forced import becomes active") + + local eok, path = SaveFileIO.exportActiveSlot("red") + eq(eok, true, "the forced import exports") + local rel = path:gsub("^/fake/save/", "") + local outBytes = files[rel] + eq(outBytes and #outBytes, GenSave.SAVE_SIZE, + "the export of a forced import is exactly 32768 bytes (footer dropped)") + check(outBytes and mainChecksumValid(outBytes), + "the forced-import export carries a valid main-data checksum") +end + +do + -- oversize but corrupt: flip a byte inside the checksummed region + local bad = syntheticSave("BAD") .. rtcFooter() + bad = bad:sub(1, OFF.money) + .. string.char((bad:byte(OFF.money + 1) + 1) % 256) + .. bad:sub(OFF.money + 2) + fresh() + local ok, res = SaveFileIO.importToSlot(fileSource(bad), "red") + eq(ok, false, "an oversize file with a bad checksum is rejected") + check(type(res) == "string" and res:find("checksum", 1, true) ~= nil, + "the oversize bad-checksum error mentions the checksum") + eq(#SaveData.listSlots("red"), 0, "no slot is created for a bad oversize file") +end + +do + -- truncated to 14000 bytes: >= 13572 keeps the whole checksum region and the + -- stored checksum byte intact, so the checksum validates and the save imports + -- with the missing tail (box banks) zero-filled. + local truncated = syntheticSave("SHORT"):sub(1, 14000) + check(truncated:len() >= OFF.mainChecksum + 1, + "the truncated fixture still carries the stored checksum byte") + fresh() + local ok, slotId = SaveFileIO.importToSlot(fileSource(truncated), "red") + eq(ok, true, "a truncated file with a valid checksum imports zero-padded") + local loaded = SaveData.load("red") + eq(loaded and loaded.player.name, "SHORT", "the truncated import keeps the player name") + eq(#SaveData.listSlots("red"), 1, "the truncated import creates a slot") + eq(SaveData.activeSlot("red"), slotId, "the truncated import becomes active") + + -- truncated but too short to even carry the checksum byte -> refused + local short = truncated:sub(1, OFF.mainChecksum) + local sok, serr = SaveFileIO.importToSlot(fileSource(short), "red") + eq(sok, false, "a file too short to hold a checksum byte is refused") + check(type(serr) == "string" and serr:find("32", 1, true) ~= nil, + "the too-short error names the required size") + eq(#SaveData.listSlots("red"), 1, "the too-short refusal creates no new slot") +end + -- ---------------------------------------------- fixture-gated real save do diff --git a/tests/save_oversize_vendor_test.lua b/tests/save_oversize_vendor_test.lua new file mode 100644 index 00000000..c1c0452a --- /dev/null +++ b/tests/save_oversize_vendor_test.lua @@ -0,0 +1,174 @@ +-- Independent-oracle test for the oversize-save import path in +-- src/import/SaveFileIO.lua (importToSlot force/truncate when a .sav exceeds +-- 32768 bytes with a valid main-data checksum -- i.e. a cartridge save padded +-- with an emulator RTC footer). +-- +-- The fixture is built by the VENDOR codec (tools/save_convert/vendor/ +-- gen1lib.lua, a PKHeX-derived Gen1 .sav<->JSON codec): the bytes GenSave +-- later imports were never produced by GenSave. The post-truncation export is +-- then re-parsed by that SAME vendor codec -- a second source independent of +-- GenSave -- confirming the forced truncation drops only the footer. +-- +-- Runs under stock Lua 5.3/5.4/5.5 (gen1lib needs native bitwise operators and +-- cannot even be parsed by LuaJIT); GenSave gets a `bit` shim backed by those +-- operators, exactly like tools/save_convert/crosscheck.lua. +-- lua tests/save_oversize_vendor_test.lua +-- +-- The luajit side of this policy lives in save_file_io_tests.lua; this file is +-- the out-of-band vendor oracle (see save_convert_tests.lua for the same +-- split). It lives OUTSIDE tests/engine/ on purpose: tier_runner globs that +-- directory under luajit, which cannot parse gen1lib. scripts/test.sh runs it +-- as its own lua5.4 tier when that interpreter is available. + +package.path = "./?.lua;./?/init.lua;" .. package.path + +-- `bit` shim backed by native Lua 5.3+ operators (crosscheck.lua's). +if not pcall(require, "bit") then + package.preload["bit"] = function() + local M = {} + function M.band(a, ...) local r = a; for _, v in ipairs({...}) do r = r & v end; return r & 0xFFFFFFFF end + function M.bor(a, ...) local r = a; for _, v in ipairs({...}) do r = r | v end; return r & 0xFFFFFFFF end + function M.bxor(a, ...) local r = a; for _, v in ipairs({...}) do r = r ~ v end; return r & 0xFFFFFFFF end + function M.bnot(a) return (~a) & 0xFFFFFFFF end + function M.lshift(a, n) return (a << n) & 0xFFFFFFFF end + function M.rshift(a, n) return (a & 0xFFFFFFFF) >> n end + return M + end +end + +local T = require("tests.harness") +local check, eq = T.check, T.eq +love = love or require("tests.love_stub") + +local GenSave = require("src.save_convert.GenSave") +local SaveConvert = require("src.save_convert.SaveConvert") +local SaveData = require("src.core.SaveData") +local GameVersion = require("src.core.GameVersion") +local SaveFileIO = require("src.import.SaveFileIO") + +local gen1 = dofile("tools/save_convert/vendor/gen1lib.lua") + +-- The vendor fixture needs no data/generated for BUILDING, but the import +-- (SaveConvert.importSav -> GenSave.decode) needs the crosswalk tables, so +-- skip cleanly on a checkout that never imported a ROM (like the luajit suite). +local loadPokemon = loadfile("data/generated/pokemon.lua") +if not loadPokemon then + print("save_oversize_vendor skipped (needs data/generated/ for GenSave codec)") + os.exit(0) +end + +local realFS = love.filesystem + +-- Same love.filesystem stub as save_file_io_tests.lua: keyed by full path with +-- the export surface SaveFileIO reaches (createDirectory/getSaveDirectory). +local function memfs(files) + return { + files = files, + write = function(path, content) files[path] = content return true end, + read = function(path) return files[path] end, + remove = function(path) files[path] = nil return true end, + getInfo = function(path) + if files[path] then return { type = "file" } end + local prefix = path .. "/" + for key in pairs(files) do + if key:sub(1, #prefix) == prefix then return { type = "directory" } end + end + return nil + end, + createDirectory = function() return true end, + getSaveDirectory = function() return "/fake/save" end, + } +end + +local function fresh() + local files = {} + love.filesystem = memfs(files) + SaveData.resetSlotState() + GameVersion.set("red") + return files +end + +-- DroppedFile-shaped source (readSource treats a raw string of length != 32768 +-- as a path, so hand a file object). +local function fileSource(bytes) + return { + _bytes = bytes, + open = function() return true end, + getSize = function(self) return #self._bytes end, + read = function(self) return self._bytes end, + close = function() return true end, + } +end + +-- A realistic 44-byte VBA MBC3 RTC footer (bgb.bircd.org/rtcsave.html). +local function rtcFooter() + local parts = {} + local function pushLe(v) + parts[#parts + 1] = string.char(v % 256, math.floor(v / 256) % 256, + math.floor(v / 65536) % 256, math.floor(v / 16777216) % 256) + end + pushLe(27); pushLe(29); pushLe(11); pushLe(200) + parts[#parts + 1] = string.rep("\0", 16) + parts[#parts + 1] = string.rep("\0", 8) + pushLe(0x669A00BF) + return table.concat(parts) +end + +-- Build a 32768-byte save ENTIRELY through the vendor codec: zero base buffer, +-- trainer/party/box fields written by gen1lib, checksum recomputed by +-- gen1lib. GenSave had no part in producing these bytes. +local function vendorSave() + local data = { + raw_base64 = gen1.base64_encode(string.rep("\0", GenSave.SAVE_SIZE)), + trainer = { + name = "VENDOR", id = 12345, rival_name = "BLUE", + money = 4321, coins = 0, badges = 0, options = 0, starter = 0, + pikachu_friendship = 0, pikachu_beach_score = 0, + }, + current_box = 1, + party = {}, + boxes = {}, + } + return gen1.build_save(data) +end + +-- ---------------------------------------------- vendor-built oversize round trip + +do + local base = vendorSave() + eq(#base, GenSave.SAVE_SIZE, "the vendor codec builds a 32768-byte save") + eq(SaveConvert.mainChecksumValid(base), true, + "the vendor-built save carries a valid main-data checksum") + + local oversize = base .. rtcFooter() + eq(#oversize, 32768 + 44, "the oversize fixture is 32812 bytes") + + local files = fresh() + -- the save the project has never seen imports cleanly once force truncates + local ok, slotId = SaveFileIO.importToSlot(fileSource(oversize), "red", true) + eq(ok, true, "the vendor-built oversize save imports with force") + local loaded = SaveData.load("red") + eq(loaded and loaded.player.name, "VENDOR", "the imported save keeps the vendor-written name") + eq(loaded and loaded.money, 4321, "the imported save keeps the vendor-written money") + + local eok, path = SaveFileIO.exportActiveSlot("red") + eq(eok, true, "the forced import exports") + local rel = path:gsub("^/fake/save/", "") + local outBytes = files[rel] + eq(outBytes and #outBytes, GenSave.SAVE_SIZE, "the export is exactly 32768 bytes") + + -- INDEPENDENT ORACLE: the vendor codec re-parses the project's export. If + -- the truncation had damaged the save, or GenSave's codec self-consistently + -- corrupted it, parse_save would disagree here. + local outBuf = gen1.string_to_bytes(outBytes) + local parsed = gen1.parse_save(outBuf) + eq(parsed.trainer.name, "VENDOR", "vendor parse of the export: name intact") + eq(parsed.trainer.money, 4321, "vendor parse of the export: money intact") + eq(parsed.trainer.id, 12345, "vendor parse of the export: trainer id intact") + eq(#parsed.party, 0, "vendor parse of the export: empty party preserved") + eq(#parsed.boxes, 12, "vendor parse of the export: 12 boxes present") +end + +love.filesystem = realFS + +T.finish("save_oversize_vendor")