Patch file at boot to get their size

src/fs.rs

@@ -83,8 +83,6 @@ impl CachedFilesystem {
 
     /// Use the file information that was generated during file discovery to fill out a GlobalFilesystem struct
     pub fn make_from_promise(discovery: DiscoveryResult) -> CachedFilesystem {
-        let arc = resource::arc();
-
         // Load the default config, which we will then join with the other configs
         let mut config = match ModConfig::from_json(DEFAULT_CONFIG) {
             Ok(cfg) => cfg,
@@ -105,13 +103,10 @@ impl CachedFilesystem {
         let mut hashed_paths: HashMap<Hash40, PathBuf> = HashMap::new();
         let mut hashed_sizes: HashMap<Hash40, usize> = HashMap::new();
 
-        for hash in modfs.handlers().bound_hashes() {
-            if let Ok(data) = arc.get_file_data_from_hash(hash, config::region()) {
-                let base_size = data.decomp_size as usize;
-                let size = modfs.handlers().patched_size_for_hash(hash, base_size).unwrap_or(base_size);
-                hashed_paths.insert(hash, get_path_from_hash(hash));
-                hashed_sizes.insert(hash, size);
-            }
+        // Patch every handler-bound file once now so the exact result size is what gets written into the data.arc
+        for (&hash, &size) in modfs.measure_patched_sizes() {
+            hashed_paths.insert(hash, get_path_from_hash(hash));
+            hashed_sizes.insert(hash, size);
         }
 
         for dep in nus3audio_deps {

src/modfs/handlers/bgm.rs

@@ -30,10 +30,6 @@ impl FileHandler for BgmHandler {
         &["bgm_property.bin"]
     }
 
-    fn patched_size(&self, _hash: Hash40, base_size: usize) -> usize {
-        base_size * 30
-    }
-
     fn discover(&mut self, _ctx: &mut DiscoveryContext, full_path: &Path, local: &Path, _size: usize) -> Option<Hash40> {
         let (base_local, _region) = super::strip_regional(local);
 

src/modfs/handlers/motionlist.rs

@@ -118,8 +118,4 @@ impl FileHandler for MotionListHandler {
             .map_err(|e| ModFsError::Handler(format!("failed to write patched motion_list: {:?}", e)))?;
         Ok(writer.into_inner())
     }
-
-    fn patched_size(&self, _hash: Hash40, base_size: usize) -> usize {
-        base_size * 2
-    }
 }

src/modfs/handlers/msbt.rs

@@ -178,10 +178,6 @@ impl FileHandler for MsbtHandler {
             .map_err(|e| ModFsError::Handler(format!("failed to write patched msbt: {:?}", e)))?;
         Ok(cursor.into_inner())
     }
-
-    fn patched_size(&self, _hash: Hash40, base_size: usize) -> usize {
-        base_size * 2
-    }
 }
 
 fn to_text_data(text_type: &TextType) -> Vec<u8> {

src/modfs/handlers/nus3audio.rs

@@ -11,15 +11,9 @@ use crate::{
     modfs::{registry::FileHandler, DiscoveryContext, ModFsError},
 };
 
-/// Slack added per patch file on top of its size, to cover section headers, alignment and the JUNK padding the
-/// writer emits when the merged file is laid out again.
-const PATCH_SLACK: usize = 0x100;
-
 #[derive(Default)]
 pub struct Nus3audioHandler {
     patches: HashMap<Hash40, Vec<PathBuf>>,
-    /// Sum of the on-disk sizes of every patch bound to a target, used to bound the merged file's size
-    patch_sizes: HashMap<Hash40, usize>,
 }
 
 impl FileHandler for Nus3audioHandler {
@@ -35,13 +29,12 @@ impl FileHandler for Nus3audioHandler {
         &["patch3audio"]
     }
 
-    fn discover(&mut self, _ctx: &mut DiscoveryContext, full_path: &Path, local: &Path, size: usize) -> Option<Hash40> {
+    fn discover(&mut self, _ctx: &mut DiscoveryContext, full_path: &Path, local: &Path, _size: usize) -> Option<Hash40> {
         let base_local = local.with_extension("nus3audio");
         let (base_local, _region) = super::strip_regional(&base_local);
 
         let hash = super::try_smash_hash(&base_local)?;
         self.patches.entry(hash).or_default().push(full_path.to_path_buf());
-        *self.patch_sizes.entry(hash).or_default() += size + PATCH_SLACK;
 
         if let Some(s) = local.to_str() {
             hashes::add(s);
@@ -52,17 +45,6 @@ impl FileHandler for Nus3audioHandler {
         Some(hash)
     }
 
-    fn patched_size(&self, hash: Hash40, base_size: usize) -> usize {
-        match self.patch_sizes.get(&hash) {
-            // Only allocates about as much as all of the audio patches will take up.
-            // This may not even be the final patched size as auto patches can
-            // replace audio instead of append it.
-            Some(extra) => (base_size + extra).next_multiple_of(0x1000),
-            // Fallback in case the above fails I guess
-            None => base_size * 2,
-        }
-    }
-
     fn apply(&self, hash: Hash40, bytes: Vec<u8>) -> Result<Vec<u8>, ModFsError> {
         let Some(patches) = self.patches.get(&hash) else {
             return Ok(bytes);

src/modfs/handlers/prc.rs

@@ -86,9 +86,4 @@ impl FileHandler for PrcHandler {
             .map_err(|e| ModFsError::Handler(format!("failed to write patched prc: {:?}", e)))?;
         Ok(writer.into_inner())
     }
-
-    fn patched_size(&self, _hash: Hash40, base_size: usize) -> usize {
-        base_size * 2
-    }
-
 }

src/modfs/mod.rs

@@ -1,4 +1,5 @@
 use std::{
+    collections::HashMap,
     io::Read,
     path::{Path, PathBuf},
 };
@@ -71,6 +72,8 @@ pub struct ModFs {
     handlers: HandlerRegistry,
     conflict_mode: ConflictMode,
     dir_index: std::sync::RwLock<Option<DirIndex>>,
+    /// Exact size of every handler-patched file, measured by running its handler chain once at boot
+    patched_sizes: HashMap<Hash40, usize>,
 }
 
 struct DirIndex {
@@ -114,6 +117,7 @@ impl ModFs {
             handlers,
             conflict_mode: ConflictMode::default(),
             dir_index: std::sync::RwLock::new(None),
+            patched_sizes: HashMap::new(),
         }
     }
 
@@ -187,12 +191,65 @@ impl ModFs {
             if let Some(s) = self.virt.max_size(hash) {
                 return Some(s);
             }
-            let base = self.patch.get(path).map(|e| e.size)?;
-            return Some(self.handlers.patched_size_for_hash(hash, base).unwrap_or(base));
+            if let Some(&s) = self.patched_sizes.get(&hash) {
+                return Some(s);
+            }
         }
         self.patch.get(path).map(|e| e.size)
     }
 
+    /// Runs the handler chain for every bound hash once and records the exact size of the result. That size is what
+    /// gets written into the data.arc, so the game allocates exactly what the patched file needs instead of a guessed
+    /// multiple of the vanilla size. The patched bytes are dropped right away and regenerated when the file loads.
+    ///
+    /// Files whose base can't be read (not in the arc, unreadable patch entry) are skipped. If a handler fails, the
+    /// load will fail the same way later, so the unpatched size is recorded for it.
+    pub fn measure_patched_sizes(&mut self) -> &HashMap<Hash40, usize> {
+        let bound: Vec<Hash40> = self.handlers.bound_hashes().collect();
+        let mut sizes = HashMap::with_capacity(bound.len());
+
+        for hash in bound {
+            let base = match self.read_unpatched(hash) {
+                Ok(bytes) => bytes,
+                Err(e) => {
+                    debug!(
+                        "modfs: not measuring '{}' ({:#x}), the base file could not be read: {:?}",
+                        crate::hashes::find(hash),
+                        hash.0,
+                        e
+                    );
+                    continue;
+                },
+            };
+            let base_len = base.len();
+            let size = match self.handlers.apply_chain(hash, base) {
+                Ok(patched) => patched.len(),
+                Err(e) => {
+                    warn!(
+                        "modfs: failed to patch '{}' ({:#x}) while measuring its size, keeping the unpatched size {:#x}: {:?}",
+                        crate::hashes::find(hash),
+                        hash.0,
+                        base_len,
+                        e
+                    );
+                    base_len
+                },
+            };
+            debug!(
+                "modfs: '{}' ({:#x}) patches to {:#x} bytes (base {:#x})",
+                crate::hashes::find(hash),
+                hash.0,
+                size,
+                base_len
+            );
+            sizes.insert(hash, size);
+        }
+
+        info!(target: "std", "modfs: measured exact sizes for {} patched files", sizes.len());
+        self.patched_sizes = sizes;
+        &self.patched_sizes
+    }
+
     pub fn read_dir<'a>(&'a self, path: &Path) -> Vec<PathBuf> {
         self.with_dir_index(|index| index.children.get(path).cloned().unwrap_or_default())
     }
@@ -291,13 +348,18 @@ impl ModFs {
     }
 
     pub fn read_base(&self, hash: Hash40) -> Result<Vec<u8>, ModFsError> {
-        let bytes = if let Some((local, entry)) = self.patch.entry_for_hash(hash) {
-            std::fs::read(entry.full_path(local))?
+        let bytes = self.read_unpatched(hash)?;
+        self.handlers.apply_chain(hash, bytes)
+    }
+
+    /// Reads the file a handler chain starts from: the mod's own file if one replaces it, otherwise the vanilla one
+    fn read_unpatched(&self, hash: Hash40) -> Result<Vec<u8>, ModFsError> {
+        if let Some((local, entry)) = self.patch.entry_for_hash(hash) {
+            Ok(std::fs::read(entry.full_path(local))?)
         } else {
             let arc = crate::resource::arc();
-            arc.get_file_contents(hash, config::region())?
-        };
-        self.handlers.apply_chain(hash, bytes)
+            Ok(arc.get_file_contents(hash, config::region())?)
+        }
     }
 
     pub fn resolve_stream_path(&self, hash: Hash40) -> Option<(std::path::PathBuf, usize)> {

src/modfs/registry.rs

@@ -31,10 +31,6 @@ pub trait FileHandler: Send + Sync + 'static {
         Ok(bytes)
     }
 
-    fn patched_size(&self, _hash: Hash40, base_size: usize) -> usize {
-        base_size * 10
-    }
-
     fn finalize(&self, _config: &mut ModConfig) {}
 }
 
@@ -108,14 +104,6 @@ impl HandlerRegistry {
         self.handlers_for_hash(hash).iter().any(|id| self.handlers[id.0].patches_load())
     }
 
-    /// Returns the largest buffer size any handler bound to `hash` needs, or `None` if no handler is bound.
-    pub fn patched_size_for_hash(&self, hash: Hash40, base_size: usize) -> Option<usize> {
-        self.handlers_for_hash(hash)
-            .iter()
-            .map(|id| self.handlers[id.0].patched_size(hash, base_size))
-            .max()
-    }
-
     pub fn apply_chain(&self, hash: Hash40, mut bytes: Vec<u8>) -> Result<Vec<u8>, ModFsError> {
         for id in self.handlers_for_hash(hash) {
             bytes = self.handlers[id.0].apply(hash, bytes)?;