From c280157d06bf35f5b3b9c1b4deb85bd61f76d2aa Mon Sep 17 00:00:00 2001 From: misieur Date: Thu, 11 Jun 2026 23:09:59 +0200 Subject: [PATCH] Enhance multithreading and add num_threads option --- java/rust/src/lib.rs | 13 +- .../main/java/dev/misieur/packobf/Native.java | 4 +- .../java/dev/misieur/packobf/PackOBF.java | 3 +- .../dev/misieur/packobf/options/Options.java | 4 +- .../packobf/progress/OptimizingProgress.java | 33 ++ .../misieur/packobf/progress/Progress.java | 7 +- packobf/src/file_parser.rs | 8 +- packobf/src/lib.rs | 284 +++++++++++------- packobf/src/options.rs | 5 + packobf_cli/src/main.rs | 51 +++- packobf_gui/src/main.rs | 9 + 11 files changed, 294 insertions(+), 127 deletions(-) create mode 100644 java/src/main/java/dev/misieur/packobf/progress/OptimizingProgress.java diff --git a/java/rust/src/lib.rs b/java/rust/src/lib.rs index 60625b9..023ee15 100644 --- a/java/rust/src/lib.rs +++ b/java/rust/src/lib.rs @@ -59,6 +59,10 @@ pub extern "system" fn Java_dev_misieur_packobf_Native_optimizeZip<'caller>( corrupt_png_files: env .get_field(&options, jni_str!("corruptPngFiles"), jni_sig!("Z"))? .z()?, + num_threads: Some( + env.get_field(&options, jni_str!("numThreads"), jni_sig!("I"))? + .i()? as usize, + ), } }; @@ -119,12 +123,17 @@ pub extern "system" fn Java_dev_misieur_packobf_Native_optimizeZip<'caller>( total: t, } => (1, c as i32, t as i32, None), Progress::Parsing { current: s } => (2, 0, 0, Some(s)), - Progress::Building { + Progress::Optimizing { current: s, index: i, total: t, } => (3, i as i32, t as i32, Some(s)), - Progress::Done => (4, 0, 0, None), + Progress::Building { + current: s, + index: i, + total: t, + } => (4, i as i32, t as i32, Some(s)), + Progress::Done => (5, 0, 0, None), }; let _ = env.with_local_frame(16, |env| { diff --git a/java/src/main/java/dev/misieur/packobf/Native.java b/java/src/main/java/dev/misieur/packobf/Native.java index adc421b..2d8479a 100644 --- a/java/src/main/java/dev/misieur/packobf/Native.java +++ b/java/src/main/java/dev/misieur/packobf/Native.java @@ -14,12 +14,13 @@ private Native() { static class Options { - public Options(int compression, int shaderCompression, boolean renameFiles, boolean blockUnzipping, boolean corruptPngFiles) { + public Options(int compression, int shaderCompression, boolean renameFiles, boolean blockUnzipping, boolean corruptPngFiles, int numThreads) { this.compression = compression; this.shaderCompression = shaderCompression; this.renameFiles = renameFiles; this.blockUnzipping = blockUnzipping; this.corruptPngFiles = corruptPngFiles; + this.numThreads = numThreads; } public int compression; @@ -27,6 +28,7 @@ public Options(int compression, int shaderCompression, boolean renameFiles, bool public boolean renameFiles; public boolean blockUnzipping; public boolean corruptPngFiles; + public int numThreads; } interface LogCallback { diff --git a/java/src/main/java/dev/misieur/packobf/PackOBF.java b/java/src/main/java/dev/misieur/packobf/PackOBF.java index 1aebf9b..cee4a19 100644 --- a/java/src/main/java/dev/misieur/packobf/PackOBF.java +++ b/java/src/main/java/dev/misieur/packobf/PackOBF.java @@ -42,7 +42,8 @@ public static byte[] optimizeZip( options.shaderCompression().value, options.renameFiles(), options.blockUnzipping(), - options.corruptPngFiles() + options.corruptPngFiles(), + options.numThreads() != null ? options.numThreads() : 0 ), (level, message) -> logCallback.onLog(switch (level) { case 0 -> LogLevel.INFO; diff --git a/java/src/main/java/dev/misieur/packobf/options/Options.java b/java/src/main/java/dev/misieur/packobf/options/Options.java index e01c400..fbab4b3 100644 --- a/java/src/main/java/dev/misieur/packobf/options/Options.java +++ b/java/src/main/java/dev/misieur/packobf/options/Options.java @@ -1,6 +1,8 @@ package dev.misieur.packobf.options; -public record Options(Compression compression, ShaderCompression shaderCompression, boolean renameFiles, boolean blockUnzipping, boolean corruptPngFiles) { +import dev.misieur.packobf.annotations.Nullable; + +public record Options(Compression compression, ShaderCompression shaderCompression, boolean renameFiles, boolean blockUnzipping, boolean corruptPngFiles, @Nullable Integer numThreads) { public static Options simplest() { return new Options( Compression.SIMPLEST, diff --git a/java/src/main/java/dev/misieur/packobf/progress/OptimizingProgress.java b/java/src/main/java/dev/misieur/packobf/progress/OptimizingProgress.java new file mode 100644 index 0000000..224c5d2 --- /dev/null +++ b/java/src/main/java/dev/misieur/packobf/progress/OptimizingProgress.java @@ -0,0 +1,33 @@ +package dev.misieur.packobf.progress; + +public final class OptimizingProgress extends Progress { + /** + * The total number of files to optimize + */ + private final int total; + /** + * The current file that PackOBF started to optimize + */ + private final Current current; + + public OptimizingProgress(int total, Current current) { + this.total = total; + this.current = current; + } + + public int total() { + return total; + } + + public Current current() { + return current; + } + + @Override + public State state() { + return State.OPTIMIZING; + } + + public record Current(String name, int index) { + } +} diff --git a/java/src/main/java/dev/misieur/packobf/progress/Progress.java b/java/src/main/java/dev/misieur/packobf/progress/Progress.java index 369fbeb..a394751 100644 --- a/java/src/main/java/dev/misieur/packobf/progress/Progress.java +++ b/java/src/main/java/dev/misieur/packobf/progress/Progress.java @@ -1,6 +1,6 @@ package dev.misieur.packobf.progress; -public abstract sealed class Progress permits BuildingProgress, DoneProgress, IdleProgress, ParsingProgress, ReadingZipProgress { +public abstract sealed class Progress permits IdleProgress, ReadingZipProgress, ParsingProgress, OptimizingProgress, BuildingProgress, DoneProgress { private State state; public abstract State state(); @@ -9,8 +9,9 @@ public enum State { IDLE(0), READING_ZIP(1), PARSING(2), - BUILDING(3), - DONE(4); + OPTIMIZING(3), + BUILDING(4), + DONE(5); private final int value; diff --git a/packobf/src/file_parser.rs b/packobf/src/file_parser.rs index d1b6c19..0865651 100644 --- a/packobf/src/file_parser.rs +++ b/packobf/src/file_parser.rs @@ -16,6 +16,7 @@ use crate::{get_type, parse_path, LogMessage, Progress}; use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; use std::str::FromStr; use std::sync::Arc; +use rayon::ThreadPool; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::watch::Sender; use crate::resource_pack::files::unknowntexture::UnknownTexture; @@ -25,9 +26,12 @@ pub fn parse_resource_pack_files( entries: &mut Vec<(String, Vec)>, progress: Sender, pack: Arc, + thread_pool: &ThreadPool ) { - entries.par_iter_mut().for_each(move |(name, content)| { - parse_resource_pack_file(logger, &progress, &pack, name, content); + thread_pool.install(|| { + entries.par_iter_mut().for_each(move |(name, content)| { + parse_resource_pack_file(logger, &progress, &pack, name, content); + }); }); } diff --git a/packobf/src/lib.rs b/packobf/src/lib.rs index 61c1156..89d948d 100644 --- a/packobf/src/lib.rs +++ b/packobf/src/lib.rs @@ -13,7 +13,7 @@ pub mod utils; use crate::cache::Cache; use crate::optimized_zip_writer::OptimizedZipWriter; -use crate::options::Options; +use crate::options::{Options, ShaderCompression}; use crate::resource_pack::files::atlas::Atlas; use crate::resource_pack::files::blockstate::Blockstate; use crate::resource_pack::files::font::Font; @@ -25,19 +25,21 @@ use crate::resource_pack::files::shader::Shader; use crate::resource_pack::files::sound::Sound; use crate::resource_pack::files::sound_definitions::SoundDefinitions; use crate::resource_pack::files::texture::Texture; +use crate::resource_pack::files::unknowntexture::UnknownTexture; use crate::resource_pack::identifier::Identifier; use crate::resource_pack::mapping; use crate::resource_pack::mapping::{IdUsageCounter, Mapping}; use crate::resource_pack::pack::ResourcePack; use crate::LogLevel::Info; use rayon::prelude::*; -use std::io::{Cursor, Error, Read}; +use rayon::{ThreadPool, ThreadPoolBuilder}; +use std::error::Error; +use std::io::{Cursor, Read}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::watch::Sender; use zip::ZipArchive; -use crate::resource_pack::files::unknowntexture::UnknownTexture; pub fn process_zip( input_bytes: Vec, @@ -45,7 +47,7 @@ pub fn process_zip( progress: Sender, logger: &UnboundedSender, cache_file: &Option, -) -> zip::result::ZipResult> { +) -> Result, Box> { let _ = progress.send(Progress::Idle); #[cfg(feature = "profiling")] profiler::profiler::PROFILER.store(Arc::new(profiler::profiler::Profiler::new())); @@ -80,11 +82,16 @@ pub fn process_zip( let id_usage_counter = IdUsageCounter::default(); mapping::set_id_usage_counter(id_usage_counter); + let pool = ThreadPoolBuilder::new() + .num_threads(options.num_threads.unwrap_or(0)) + .build()?; + file_parser::parse_resource_pack_files( logger, &mut entries, progress.clone(), Arc::clone(&pack), + &pool, ); usage_checker::check_usage(logger, &pack); @@ -95,7 +102,7 @@ pub fn process_zip( } mapping::set_mappings(mapping); - let mut items = collect_files(pack); + let mut items = collect_files(pack, &pool); let total = items.len(); let mut output = Cursor::new(Vec::new()); @@ -123,18 +130,71 @@ pub fn process_zip( None }; - items.par_iter_mut().for_each(|(name, item)| { - match add_item_to_archive( - options, &progress, logger, total, &writer, &counter, &cache, name, item, - ) { - Ok(_) => {} - Err(e) => { - let _ = logger.send(LogMessage { - level: LogLevel::Error, - message: format!("Failed to add item to archive: {}", e), - }); + pool.install(|| { + let total_to_optimize = AtomicUsize::new(0); + let to_optimize: Vec<_> = items + .par_iter_mut() + .filter(|(_, item)| match item { + ResourcePackItem::Texture(_) => { + total_to_optimize.fetch_add(1, Ordering::Relaxed); + true + } + ResourcePackItem::UnknownTexture(_) => { + total_to_optimize.fetch_add(1, Ordering::Relaxed); + true + } + ResourcePackItem::Shader(_) => { + if options.shader_compression != ShaderCompression::None { + total_to_optimize.fetch_add(1, Ordering::Relaxed); + true + } else { + false + } + } + ResourcePackItem::Sound(_) => { + total_to_optimize.fetch_add(1, Ordering::Relaxed); + true + } + _ => false, + }) + .collect(); + + let total_to_optimize = to_optimize.len(); + to_optimize.into_par_iter().for_each(|(name, item)| { + let _ = progress.send(Progress::Optimizing { + current: name.to_string(), + index: counter.fetch_add(1, Ordering::Relaxed), + total: total_to_optimize, + }); + match item { + ResourcePackItem::Texture(x) => { + x.unknown_texture.optimize(options, logger, &cache); + } + ResourcePackItem::UnknownTexture(x) => { + x.optimize(options, logger, &cache); + } + ResourcePackItem::Shader(x) => { + x.optimize(options, logger); + } + ResourcePackItem::Sound(x) => { + x.optimize(logger, &cache); + } + _ => {} } - } + }); + items.par_iter_mut().for_each(|(name, item)| { + match add_item_to_archive( + options, &progress, logger, total, &writer, &counter, &cache, name, item, + ) { + Ok(_) => {} + Err(e) => { + let _ = logger.send(LogMessage { + level: LogLevel::Error, + message: format!("Failed to add item to archive: {}", e), + }); + } + } + }); }); writer.finish()?; @@ -161,7 +221,7 @@ fn add_item_to_archive( cache: &Option, name: &mut String, item: &mut ResourcePackItem, -) -> Result<(), Error> { +) -> Result<(), Box> { let _ = progress.send(Progress::Building { current: name.to_string(), index: counter.fetch_add(1, Ordering::Relaxed), @@ -185,10 +245,12 @@ fn add_item_to_archive( } } match item { - ResourcePackItem::Texture(o) => { - o.optimize(options, logger, cache); - writer.add_file(name.as_str(), o.unknown_texture.bytes.as_slice(), options, cache) - } + ResourcePackItem::Texture(o) => writer.add_file( + name.as_str(), + o.unknown_texture.bytes.as_slice(), + options, + cache, + ), ResourcePackItem::Shader(o) => { o.optimize(options, logger); writer.add_file(name.as_str(), o.content.as_bytes(), options, cache) @@ -215,7 +277,6 @@ fn add_item_to_archive( writer.add_file(name.as_str(), o.to_string().as_bytes(), options, cache) } ResourcePackItem::Sound(o) => { - o.optimize(logger, cache); writer.add_file(name.as_str(), o.bytes.as_slice(), options, cache) } ResourcePackItem::SoundDefinitions(o) => { @@ -225,99 +286,103 @@ fn add_item_to_archive( writer.add_file(name.as_str(), o.to_string().as_bytes(), options, cache) } ResourcePackItem::UnknownTexture(o) => { - o.optimize(options, logger, cache); writer.add_file(name.as_str(), o.bytes.as_slice(), options, cache) } }?; Ok(()) } -fn collect_files(pack: Arc) -> Vec<(String, ResourcePackItem)> { +fn collect_files( + pack: Arc, + thread_pool: &ThreadPool, +) -> Vec<(String, ResourcePackItem)> { profile_scope!("collect_files"); - let texture_iter = pack.textures.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Texture(kv.value().clone()), - ) - }); - let unknown_texture_iter = pack.unknown_textures.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::UnknownTexture(kv.value().clone()), - ) - }); - let shader_iter = pack.shaders.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Shader(kv.value().clone()), - ) - }); - let model_iter = pack.models.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Model(kv.value().clone()), - ) - }); - let json_iter = pack - .json_files - .par_iter() - .map(|kv| (kv.key().clone(), ResourcePackItem::Json(kv.value().clone()))); - let unknown_iter = pack.unknown_files.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Unknown(kv.value().clone()), - ) - }); - let blockstate_iter = pack.blockstates.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::BlockStateDefinition(kv.value().clone()), - ) - }); - let font_iter = pack.fonts.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::FontDefinition(kv.value().clone()), - ) - }); - let item_iter = pack.items.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::ItemDefinition(kv.value().clone()), - ) - }); - let sound_iter = pack.sounds.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Sound(kv.value().clone()), - ) - }); - let sound_definitions_iter = pack.sound_definitions.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::SoundDefinitions(kv.value().clone()), - ) - }); - let atlas_iter = pack.atlases.par_iter().map(|kv| { - ( - kv.key().clone(), - ResourcePackItem::Atlas(kv.value().clone()), - ) - }); + thread_pool.install(|| { + let texture_iter = pack.textures.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Texture(kv.value().clone()), + ) + }); + let unknown_texture_iter = pack.unknown_textures.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::UnknownTexture(kv.value().clone()), + ) + }); + let shader_iter = pack.shaders.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Shader(kv.value().clone()), + ) + }); + let model_iter = pack.models.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Model(kv.value().clone()), + ) + }); + let json_iter = pack + .json_files + .par_iter() + .map(|kv| (kv.key().clone(), ResourcePackItem::Json(kv.value().clone()))); + let unknown_iter = pack.unknown_files.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Unknown(kv.value().clone()), + ) + }); + let blockstate_iter = pack.blockstates.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::BlockStateDefinition(kv.value().clone()), + ) + }); + let font_iter = pack.fonts.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::FontDefinition(kv.value().clone()), + ) + }); + let item_iter = pack.items.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::ItemDefinition(kv.value().clone()), + ) + }); + let sound_iter = pack.sounds.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Sound(kv.value().clone()), + ) + }); + let sound_definitions_iter = pack.sound_definitions.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::SoundDefinitions(kv.value().clone()), + ) + }); + let atlas_iter = pack.atlases.par_iter().map(|kv| { + ( + kv.key().clone(), + ResourcePackItem::Atlas(kv.value().clone()), + ) + }); - texture_iter - .chain(unknown_texture_iter) - .chain(shader_iter) - .chain(model_iter) - .chain(json_iter) - .chain(unknown_iter) - .chain(blockstate_iter) - .chain(font_iter) - .chain(item_iter) - .chain(sound_iter) - .chain(sound_definitions_iter) - .chain(atlas_iter) - .collect() + texture_iter + .chain(unknown_texture_iter) + .chain(shader_iter) + .chain(model_iter) + .chain(json_iter) + .chain(unknown_iter) + .chain(blockstate_iter) + .chain(font_iter) + .chain(item_iter) + .chain(sound_iter) + .chain(sound_definitions_iter) + .chain(atlas_iter) + .collect() + }) } #[derive(Clone, Debug)] @@ -330,6 +395,11 @@ pub enum Progress { Parsing { current: String, }, + Optimizing { + current: String, + index: usize, + total: usize, + }, Building { current: String, index: usize, diff --git a/packobf/src/options.rs b/packobf/src/options.rs index f5687cc..ea61f99 100644 --- a/packobf/src/options.rs +++ b/packobf/src/options.rs @@ -15,6 +15,8 @@ pub struct Options { pub block_unzipping: bool, #[arg(long)] pub corrupt_png_files: bool, + #[arg(long)] + pub num_threads: Option, } #[derive(ValueEnum, Clone, Debug)] @@ -32,6 +34,7 @@ impl Options { rename_files: false, block_unzipping: false, corrupt_png_files: false, + num_threads: None, } } @@ -42,6 +45,7 @@ impl Options { rename_files: false, block_unzipping: false, corrupt_png_files: false, + num_threads: None, } } @@ -52,6 +56,7 @@ impl Options { rename_files: true, block_unzipping: true, corrupt_png_files: true, + num_threads: None, } } diff --git a/packobf_cli/src/main.rs b/packobf_cli/src/main.rs index fc66dd9..ee8e0a7 100644 --- a/packobf_cli/src/main.rs +++ b/packobf_cli/src/main.rs @@ -32,6 +32,7 @@ struct Args { static LOOKING_GLASS: Emoji<'_, '_> = Emoji("🔍 ", ""); static TRUCK: Emoji<'_, '_> = Emoji("🚚 ", ""); static CLIP: Emoji<'_, '_> = Emoji("🔗 ", ""); +static OPTIMIZING: Emoji<'_, '_> = Emoji("🚀 ", ""); static BUILDING: Emoji<'_, '_> = Emoji("⚒️ ", ""); static SPARKLE: Emoji<'_, '_> = Emoji("✨ ", ":-)"); static CHECK: Emoji<'_, '_> = Emoji("✅ ", "OK "); @@ -77,7 +78,7 @@ pub async fn run_progress_loop( let global_started = Instant::now(); let mut stage_started = Instant::now(); let mut current_pb: Option = None; - // 0: Idle, 1: Reading, 2: Parsing, 4: Building + // 1: Idle, 2: Reading, 3: Parsing, 4: Optimizing, 5: Building let mut current_stage: u8 = 0; let clear_current = |pb: &mut Option| { @@ -136,7 +137,7 @@ pub async fn run_progress_loop( if current_stage != 1 { println!( "{} {} Initializing...", - style("[1/4]").bold().dim(), + style("[1/5]").bold().dim(), LOOKING_GLASS ); current_stage = 1; @@ -150,7 +151,7 @@ pub async fn run_progress_loop( clear_current(&mut current_pb); println!( "{} {} Reading archive...", - style("[2/4]").bold().dim(), + style("[2/5]").bold().dim(), TRUCK ); current_stage = 2; @@ -160,7 +161,7 @@ pub async fn run_progress_loop( let pb = current_pb.get_or_insert_with(|| { let p = ProgressBar::new(total as u64); p.set_style(bar_style.clone()); - p.set_prefix("[2/4]"); + p.set_prefix("[2/5]"); p }); if pb.position() < current as u64 { @@ -176,12 +177,12 @@ pub async fn run_progress_loop( clear_current(&mut current_pb); println!( "{} {} Parsing resource files...", - style("[3/4]").bold().dim(), + style("[3/5]").bold().dim(), CLIP ); let pb = ProgressBar::new_spinner(); pb.set_style(spinner_style.clone()); - pb.set_prefix("[3/4]"); + pb.set_prefix("[3/5]"); current_pb = Some(pb); current_stage = 3; stage_started = Instant::now(); @@ -193,7 +194,7 @@ pub async fn run_progress_loop( } } - Progress::Building { + Progress::Optimizing { current, index, total, @@ -201,20 +202,50 @@ pub async fn run_progress_loop( if current_stage != 4 { print_finished_stage(&mut current_pb, "Parsing Complete", stage_started); + clear_current(&mut current_pb); + println!( + "{} {} Optimizing resource pack...", + style("[4/5]").bold().dim(), + OPTIMIZING + ); + current_stage = 4; + stage_started = Instant::now(); + } + + let pb = current_pb.get_or_insert_with(|| { + let p = ProgressBar::new(total as u64); + p.set_style(bar_style.clone()); + p.set_prefix("[4/5]"); + p + }); + if pb.position() < index as u64 { + pb.set_position(index as u64); + } + pb.set_message(format!("File: {}", current)); + } + + Progress::Building { + current, + index, + total, + } => { + if current_stage != 5 { + print_finished_stage(&mut current_pb, "Optimizing Complete", stage_started); + clear_current(&mut current_pb); println!( "{} {} Building resource pack...", - style("[4/4]").bold().dim(), + style("[5/5]").bold().dim(), BUILDING ); - current_stage = 4; + current_stage = 5; stage_started = Instant::now(); } let pb = current_pb.get_or_insert_with(|| { let p = ProgressBar::new(total as u64); p.set_style(bar_style.clone()); - p.set_prefix("[4/4]"); + p.set_prefix("[5/5]"); p }); if pb.position() < index as u64 { diff --git a/packobf_gui/src/main.rs b/packobf_gui/src/main.rs index 6a6ddb0..d4729e0 100644 --- a/packobf_gui/src/main.rs +++ b/packobf_gui/src/main.rs @@ -521,6 +521,7 @@ async fn run_packobf(path: String, cache: Option, state: Arc, state: Arc { + format!("Optimizing ({}/{}) {}", index, total, current) + } + Progress::Building { current, index,