seal transparent back sprites

lib/BattlePics.lua

@@ -57,6 +57,26 @@
 -- and they come back untouched because that is what their own shape says, not
 -- because they were special-cased.
 --
+-- The drain/mouth cut is for a pic STANDING ON THE MAP, where a mouth is a
+-- real hole with real ground behind it. A pic PINNED TO THE MENU has no such
+-- hole to be: under BACK SPRITES the player's mon is drawn in the GB's own
+-- slot with its feet flush on the text box (BattleState.backPlacement pins
+-- row 96), so the only thing under its lowest row is white box. Nothing can
+-- reach it from below, whatever the opening's width, and the caller says so
+-- by asking for a SEALED BOTTOM -- for which the rule stops being a heuristic
+-- and becomes exact: paper is whatever the background cannot walk to from the
+-- left, the right or the top.
+--
+-- That is the difference between a Pikachu that reads as a mon and one that
+-- reads as wireframe. The pale-bodied back pics -- Pikachu, Seel, Dewgong,
+-- Chansey, Jigglypuff -- are drawn as OUTLINES: everything inside the ink is
+-- shade 0 and every one of them is keyed away, so the figure is a rim with the
+-- arena showing through it. Each one also has a wide opening along its bottom,
+-- which the drain cut correctly reads as a mouth and the sealed bottom
+-- correctly does not. Twelve of this game's 151 back pics turn on it; the
+-- other 139 come back byte-identical either way, because they had nothing
+-- under them the flood was getting in through.
+--
 -- The silhouette is untouched, so the mon still cuts cleanly against the
 -- world; only its insides stop being see-through.
 --
@@ -71,14 +91,23 @@ local V = ...
 
 local BattlePics = {}
 
--- Cached by the image the engine handed over. Weak keys, so a pic that goes
--- out of scope takes its filled twin with it rather than pinning a texture
--- for the session.
-local cache = setmetatable({}, { __mode = "k" })
+-- Cached by the image the engine handed over, one table per bottom rule --
+-- the same pic answers differently sealed and unsealed, and a single table
+-- would hand the wrong twin back to whichever caller asked second. Weak keys,
+-- so a pic that goes out of scope takes its filled twin with it rather than
+-- pinning a texture for the session.
+local function newCache()
+  return {
+    [false] = setmetatable({}, { __mode = "k" }),
+    [true]  = setmetatable({}, { __mode = "k" }),
+  }
+end
+local cache = newCache()
 
--- What an enclosed hole is filled with. White, because white is what the
--- battle field was: this restores the pixel the artist drew and the engine
--- then keyed away, it does not invent a new one.
+-- What an enclosed hole is filled with when the pic itself offers nothing
+-- better. White, because white is what the battle field was: this restores the
+-- pixel the artist drew and the engine then keyed away, it does not invent a
+-- new one.
 BattlePics.FILL = { 1, 1, 1, 1 }
 
 -- Anything at or under this alpha counts as keyed-out rather than drawn.
@@ -154,6 +183,42 @@ local function inkBounds(data, w, h)
   return x0, y0, x1, y1
 end
 
+-- The colour the keyed-away shade would have had: the LIGHTEST colour still
+-- standing in the pic.
+--
+-- Pure white is only the right answer while the pic is still grays, and by the
+-- time it reaches here it usually is not. picImage hands a pic over AFTER the
+-- bake -- a species SGB colour, a BGP fade mid-animation, PAL_BLACK for the
+-- whole screen while the blackout text is up -- and shade 0 travels with the
+-- rest. A white belly inside a blacked-out mon would be the one lit thing on a
+-- dark screen; inside a warm-palette mon it would be a cold patch the artist
+-- never drew.
+--
+-- So the paper is read off the pic rather than assumed, which needs shade 0 to
+-- have survived somewhere in it. It always has: every one of this game's 151
+-- back pics keeps at least one opaque shade-0 pixel -- a highlight down a
+-- cheek, the white of an eye -- because only the shade-0 pixels the decoder
+-- could reach were keyed. So what comes back is the baked shade 0 itself, not
+-- an approximation of it, and it tracks every palette the engine picks without
+-- being told which one that was.
+--
+-- Ranked by channel sum, which orders four DMG shades exactly: a palette maps
+-- all three channels monotonically, so lightest by sum is lightest full stop.
+local function paperColor(data, x0, y0, x1, y1)
+  local best, pr, pg, pb = -1, nil, nil, nil
+  for y = y0, y1 do
+    for x = x0, x1 do
+      local r, g, b, a = data:getPixel(x, y)
+      if a > CUT then
+        local lum = r + g + b
+        if lum > best then best, pr, pg, pb = lum, r, g, b end
+      end
+    end
+  end
+  if best < 0 then return nil end
+  return pr, pg, pb
+end
+
 -- The widest opening along the bottom of a figure that still counts as a drain
 -- rather than a mouth. See the header for the measurements either side of it.
 BattlePics.DRAIN = 6
@@ -161,7 +226,9 @@ BattlePics.DRAIN = 6
 -- Mark every transparent pixel the BACKGROUND can reach, flooding inward from
 -- the edges of the artwork's box: the left, the right and the top whole, and
 -- along the bottom only those openings wide enough to be background rather
--- than the underside of a figure the drawing ran out of.
+-- than the underside of a figure the drawing ran out of -- or none of them at
+-- all, for a pic whose feet are on the text box and which therefore has
+-- nothing behind its lowest row to let in.
 --
 -- Confined to the box as well as seeded from it, so the empty frame under a
 -- short pic cannot walk around a sealed drain and come back up through it.
@@ -169,7 +236,7 @@ BattlePics.DRAIN = 6
 -- An explicit stack rather than recursion: a 56x56 pic is three thousand
 -- pixels and a keyed-out background is most of them, which is a deeper call
 -- chain than is worth risking for no gain.
-local function markOutside(data, w, h, x0, y0, x1, y1)
+local function markOutside(data, w, h, x0, y0, x1, y1, sealBottom)
   local outside = {}
   local stack, top = {}, 0
   local function clear(x, y)
@@ -191,17 +258,21 @@ local function markOutside(data, w, h, x0, y0, x1, y1)
     push(x1, y)
   end
   -- the bottom, run by run: a wide one is the gap between two legs and lets
-  -- the world through, a narrow one is where a belly ran out and is sealed
-  local x = x0
-  while x <= x1 do
-    if clear(x, y1) then
-      local from = x
-      while x <= x1 and clear(x, y1) do x = x + 1 end
-      if (x - from) > BattlePics.DRAIN then
-        for k = from, x - 1 do push(k, y1) end
+  -- the world through, a narrow one is where a belly ran out and is sealed.
+  -- Skipped whole for a pic on the box, where even the widest of them has
+  -- white paper behind it rather than arena.
+  if not sealBottom then
+    local x = x0
+    while x <= x1 do
+      if clear(x, y1) then
+        local from = x
+        while x <= x1 and clear(x, y1) do x = x + 1 end
+        if (x - from) > BattlePics.DRAIN then
+          for k = from, x - 1 do push(k, y1) end
+        end
+      else
+        x = x + 1
       end
-    else
-      x = x + 1
     end
   end
   while top > 0 do
@@ -219,9 +290,15 @@ end
 -- The pic with its enclosed holes filled, or the pic itself when that could
 -- not be done (no pixel access, a driver that refused the readback). Never
 -- nil for a non-nil argument: a caller must always have something to draw.
-function BattlePics.filled(img)
+--
+-- sealBottom for a pic pinned to the text box rather than standing on the map:
+-- see the header. A caller that does not say defaults to the map, which is
+-- where all but one of this mod's pics are.
+function BattlePics.filled(img, sealBottom)
   if not img then return img end
-  local hit = cache[img]
+  sealBottom = sealBottom and true or false
+  local slot = cache[sealBottom]
+  local hit = slot[img]
   if hit ~= nil then return hit or img end
 
   local made = nil
@@ -231,8 +308,12 @@ function BattlePics.filled(img)
     local w, h = data:getDimensions()
     local x0, y0, x1, y1 = inkBounds(data, w, h)
     if not x0 then return end          -- a pic with nothing drawn in it
-    local outside = markOutside(data, w, h, x0, y0, x1, y1)
+    local outside = markOutside(data, w, h, x0, y0, x1, y1, sealBottom)
     local fill = BattlePics.FILL
+    local pr, pg, pb = paperColor(data, x0, y0, x1, y1)
+    local fr = pr or fill[1]
+    local fg = pg or fill[2]
+    local fb = pb or fill[3]
     local changed = false
     -- only inside the box: everything beyond it is frame the artist never
     -- reached, and filling that would put the mon in a white rectangle
@@ -242,7 +323,7 @@ function BattlePics.filled(img)
         if not outside[row + x] then
           local _, _, _, a = data:getPixel(x, y)
           if a <= CUT then
-            data:setPixel(x, y, fill[1], fill[2], fill[3], fill[4])
+            data:setPixel(x, y, fr, fg, fb, fill[4])
             changed = true
           end
         end
@@ -255,12 +336,12 @@ function BattlePics.filled(img)
     made = out
   end)
 
-  cache[img] = (ok and made) or false
+  slot[img] = (ok and made) or false
   return made or img
 end
 
 function BattlePics.invalidate()
-  cache = setmetatable({}, { __mode = "k" })
+  cache = newCache()
 end
 
 return BattlePics

lib/OverworldBattle.lua

@@ -104,6 +104,31 @@ function OverworldBattle.backPinned()
   return OverworldBattle.backSetting:get() and true or false
 end
 
+-- Whether a pic is the one drawn in the GB's own slot with its feet on the
+-- text box, rather than geometry standing out on the map.
+--
+-- Exactly the player's side under BACK SPRITES -- its mon, or the trainer back
+-- that holds the slot until "Go!" -- because that is the only pic this mod
+-- ever leaves flat (see drawPicsLayer below). The foe is a billboard on its
+-- tile whichever mode is on, and with the mode off the player's side is one
+-- too, so both of those keep the open bottom that lets the arena through a
+-- stride. What the answer buys is in BattlePics: a pic on the box has nothing
+-- behind its lowest row, so its bottom edge seals.
+-- Read by TRUTHINESS rather than against nil, because sideTexture blanks the
+-- side it is not rendering by setting the field to FALSE (see OFF) and holds
+-- it that way for the whole render -- during which the pic layer runs, and
+-- picImage asks this. A nil test passes a `false` straight through to the
+-- index below, and the error comes out of sideTexture into the pcall that
+-- calls it: the foe's billboard is dropped for the frame and the Pokemon
+-- simply is not there.
+function OverworldBattle.pinnedPic(battle, img)
+  if not (battle and img) then return false end
+  if not OverworldBattle.backPinned() then return false end
+  if img == battle.playerBackPic then return true end
+  local player = battle.player
+  return (player and img == player.sprite) and true or false
+end
+
 -- ------- both mons face you
 --
 -- Standing on a map, seen from in front, a Pokemon showing you its BACK is
@@ -831,11 +856,17 @@ function OverworldBattle.install()
   -- behind it. There is a world back there now, so they are filled here
   -- instead -- see BattlePics, which puts the paper back without touching
   -- the silhouette.
+  --
+  -- The pinned pic is told that its feet are on the box, which is what lets
+  -- the pale-bodied back sprites be filled at all: their bellies leak out
+  -- through an opening too wide to read as a drain, and only the box under
+  -- them settles that it is not a hole. Passed the pre-bake image, because
+  -- that is the one the battle holds a reference to.
   local innerPic = BattleState.picImage
   function BattleState:picImage(img)
     local out = innerPic(self, img)
     if not OverworldBattle.shot() then return out end
-    return BattlePics.filled(out)
+    return BattlePics.filled(out, OverworldBattle.pinnedPic(self, img))
   end
 
   -- While a billboard texture is being rendered both pics are put in the same