diff --git a/src/bit_writer.rs b/src/bit_writer.rs index 8c99196..94b9134 100644 --- a/src/bit_writer.rs +++ b/src/bit_writer.rs @@ -90,7 +90,6 @@ fn write_roundtrip() { b.pad(0, &mut data_buffer); b.flush_whole_bytes(&mut data_buffer); - let len = data_buffer.len() as i64; let mut cursor = std::io::Cursor::new(data_buffer); let mut reader = BitReader::new(&mut cursor); diff --git a/src/huffman_calc.rs b/src/huffman_calc.rs index 0f1a9a2..0940257 100644 --- a/src/huffman_calc.rs +++ b/src/huffman_calc.rs @@ -414,11 +414,8 @@ mod calc_zlib { if overflow > 0 { // redistribute the bit lengths to remove the overflow - let mut bits = max_bits; - while overflow > 0 { - bits -= 1; - + let mut bits = max_bits - 1; while bl_count[bits] == 0 { bits -= 1; } @@ -431,7 +428,7 @@ mod calc_zlib { } // now reassign the bitlengths to the nodes (since we already have them in the right order) - bits = max_bits; + let mut bits = max_bits; for node in nodes.iter() { if let HuffTree::Leaf(idx) = node.tree { while bl_count[bits] == 0 { @@ -488,18 +485,24 @@ mod calc_zlib { } #[test] - fn roundtrip_huffman_code() { + fn roundtrip_huffman_code_simple() { test_result(&[0, 1, 2, 4, 8, 16, 32], 7, &[0, 5, 5, 4, 3, 2, 1]); + } + // requires overflow treatment + #[test] + fn roundtrip_huffman_code_overflow() { test_result( &[ 1, 0, 1, 1, 5, 10, 9, 18, 29, 59, 91, 28, 11, 1, 2, 0, 12, 1, 0, ], 7, - &[7, 0, 7, 7, 5, 5, 5, 5, 3, 2, 2, 3, 5, 7, 6, 0, 5, 6], + &[7, 0, 7, 7, 6, 5, 5, 4, 3, 2, 2, 3, 5, 7, 7, 0, 5, 7], ); + } - // requires overflow treatment + #[test] + fn roundtrip_huffman_code_normal() { test_result( &[ 1, 0, 1, 0, 9, 9, 8, 19, 36, 70, 73, 34, 5, 6, 0, 0, 11, 1, 0, diff --git a/src/huffman_encoding.rs b/src/huffman_encoding.rs index 07f0a6d..826d8ba 100644 --- a/src/huffman_encoding.rs +++ b/src/huffman_encoding.rs @@ -372,7 +372,6 @@ fn roundtrip_huffman_bitreadwrite() { bit_writer.write(0x1234, 16, &mut data_buffer); bit_writer.pad(0, &mut data_buffer); - let data_buffer_size = data_buffer.len(); let mut reader = Cursor::new(&data_buffer); let mut bit_reader = BitReader::new(&mut reader); diff --git a/tests/end_to_end.rs b/tests/end_to_end.rs index b8a6c6f..ee44268 100644 --- a/tests/end_to_end.rs +++ b/tests/end_to_end.rs @@ -52,6 +52,12 @@ fn test_sample1() { fn verifyresult(compressed_data: &[u8]) { let result = decompress_deflate_stream(compressed_data, true).unwrap(); let recomp = recompress_deflate_stream(&result.plain_text, &result.cabac_encoded).unwrap(); + + println!( + "compressed size: {}, cabac: {}", + compressed_data.len(), + result.cabac_encoded.len() + ); assert_eq!(compressed_data, recomp); } @@ -76,8 +82,9 @@ fn test_file(filename: &str) { level, ); + assert_eq!(err, 0, "shouldn't fail zlib compression"); + output.set_len(output_size as usize); - println!("output size: {}, err = {}", output.len(), err); } let minusheader = &output[2..output.len() - 4]; diff --git a/util/Cargo.toml b/util/Cargo.toml index 711d8ac..f51a80d 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -20,7 +20,7 @@ default-boxed = "0.2" crc32fast = "1.3" libz-sys = "1.1" flate2 = "1.0" -preflate_rs = { path = ".." } +preflate-rs = { path = ".." } walkdir = "2.3" zstd = "0.13.0" diff --git a/util/src/main.rs b/util/src/main.rs index 8c74f43..1a8641f 100644 --- a/util/src/main.rs +++ b/util/src/main.rs @@ -32,7 +32,7 @@ fn main() { if results.len() > 0 { for result in results { totalzstd -= result.compressed_size as u64; - totalzstd += result.zstd as u64; + totalzstd += result.zstd + result.cabac_length as u64; println!( " Found compressed data {:?} s={} c={} zstd={} cabac={}", result.signature, @@ -43,8 +43,7 @@ fn main() { ); } } - - println!("total seen ratio {}", totalzstd as f64 / totalseen as f64); } } + println!("total seen ratio {}", totalzstd as f64 / totalseen as f64); } diff --git a/util/src/search_signature.rs b/util/src/search_signature.rs index 9acc0aa..a8a3537 100644 --- a/util/src/search_signature.rs +++ b/util/src/search_signature.rs @@ -1,12 +1,9 @@ use std::{ collections::HashMap, - io::{BufReader, Cursor, Read, Seek, SeekFrom, Write}, + io::{Cursor, Read, Seek, SeekFrom, Write}, }; -use byteorder::{LittleEndian, ReadBytesExt}; - use anyhow; -use flate2::read::DeflateDecoder; use preflate_rs::decompress_deflate_stream; use crate::zip_structs::{ @@ -37,7 +34,7 @@ pub fn add_location( compressed: &[u8], ) { let ret = decompress_deflate_stream(compressed, true); - if let Err(e) = ret + if let Err(_e) = ret { //println!("Error decompressing {:?} {:?} at {}", signature, e, start); return; diff --git a/util/src/zip_structs.rs b/util/src/zip_structs.rs index fc70e23..ae9744f 100644 --- a/util/src/zip_structs.rs +++ b/util/src/zip_structs.rs @@ -2,16 +2,7 @@ use byteorder::{LittleEndian, ReadBytesExt}; use std::io::{Read, Seek, SeekFrom}; pub const ZIP_LOCAL_FILE_HEADER_SIGNATURE: u32 = 0x04034b50; -pub const ZIP_CENTRAL_DIRECTORY_SIGNATURE: u32 = 0x02014b50; -pub const ZIP_LOCAL_FILE_HEADER_SIZE_IN_BYTES: u32 = 30; -pub const GENERAL_BIT_HAS_DATA_DESCRIPTOR: u16 = 0x0008; pub const ZIP64_EXTENDED_INFORMATION_TYPE_TAG: u16 = 0x0001; -//pub const ZIP64_EXTENDED_INFORMATION_SIZE_IN_BYTES: u32 = 28; -pub const ZIP_CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE: u32 = 0x02014b50; -pub const ZIP_CENTRAL_DIRECTORY_FILE_HEADER_SIZE_IN_BYTES: u32 = 46; -pub const ZIP_END_OF_CENTRAL_DIRECTORY_RECORD_SIGNATURE: u32 = 0x06054b50; -//pub const ZIP_END_OF_CENTRAL_DIRECTORY_RECORD_SIZE_IN_BYTES: u32 = 22; -pub const ZIP64_END_OF_CENTRAL_DIRECTORY_RECORD_SIGNATURE: u32 = 0x06064b50; #[derive(Clone)] pub struct ZipLocalFileHeader { @@ -65,10 +56,6 @@ impl ZipLocalFileHeader { self.extra_field_length = binary_reader.read_u16::()?; Ok(()) } - - pub fn fhas_data_descriptor(&self) -> bool { - (self.general_purpose_bit_flag & GENERAL_BIT_HAS_DATA_DESCRIPTOR) != 0 - } } pub struct ZipExtendedInformationHeader { @@ -248,126 +235,3 @@ impl Zip64ExtendedInformation { Ok(()) } } - -#[derive(Clone)] -pub struct ZipCentralDirectoryFileHeader { - pub central_file_header_signature: u32, - pub version_made_by: u16, - pub version_needed_to_extract: u16, - pub general_purpose_bit_flag: u16, - pub compression_method: u16, - pub last_mod_file_time: u16, - pub last_mod_file_date: u16, - pub crc32: u32, - pub compressed_size: u64, // 4 bytes (may be set to 64bit value in Zip64 case) - pub uncompressed_size: u64, // 4 bytes (may be set to 64bit value in Zip64 case) - pub file_name_length: u16, - pub extra_field_length: u16, - pub file_comment_length: u16, - pub disk_number_start: u32, // 2 bytes (may be set to 32bit value in Zip64 case) - pub internal_file_attributes: u16, - pub external_file_attributes: u32, - pub relative_offset_of_local_header: u64, // 4 bytes (may be set to 64bit value in Zip64 case) -} - -impl ZipCentralDirectoryFileHeader { - pub fn create_and_load(binary_reader: &mut R) -> anyhow::Result { - let mut zip_ext_info_header = Self::new(); - zip_ext_info_header.load(binary_reader)?; - Ok(zip_ext_info_header) - } - - fn new() -> Self { - ZipCentralDirectoryFileHeader { - central_file_header_signature: 0, - version_made_by: 0, - version_needed_to_extract: 0, - general_purpose_bit_flag: 0, - compression_method: 0, - last_mod_file_time: 0, - last_mod_file_date: 0, - crc32: 0, - compressed_size: 0, - uncompressed_size: 0, - file_name_length: 0, - extra_field_length: 0, - file_comment_length: 0, - disk_number_start: 0, - internal_file_attributes: 0, - external_file_attributes: 0, - relative_offset_of_local_header: 0, - } - } - - fn load(&mut self, binary_reader: &mut R) -> anyhow::Result<()> { - self.central_file_header_signature = binary_reader.read_u32::()?; - self.version_made_by = binary_reader.read_u16::()?; - self.version_needed_to_extract = binary_reader.read_u16::()?; - self.general_purpose_bit_flag = binary_reader.read_u16::()?; - self.compression_method = binary_reader.read_u16::()?; - self.last_mod_file_time = binary_reader.read_u16::()?; - self.last_mod_file_date = binary_reader.read_u16::()?; - self.crc32 = binary_reader.read_u32::()?; - self.compressed_size = binary_reader.read_u32::()? as u64; - self.uncompressed_size = binary_reader.read_u32::()? as u64; - self.file_name_length = binary_reader.read_u16::()?; - self.extra_field_length = binary_reader.read_u16::()?; - self.file_comment_length = binary_reader.read_u16::()?; - self.disk_number_start = binary_reader.read_u16::()? as u32; - self.internal_file_attributes = binary_reader.read_u16::()?; - self.external_file_attributes = binary_reader.read_u32::()?; - self.relative_offset_of_local_header = binary_reader.read_u32::()? as u64; - - Ok(()) - } -} - -pub struct ZipEndOfCentralDirectoryRecord { - pub end_of_central_dir_signature: u32, - pub number_of_this_disk: u16, - pub number_of_the_disk_with_the_start_of_the_central_directory: u16, - pub total_number_of_entries_in_the_central_directory_on_this_disk: u16, - pub total_number_of_entries_in_the_central_directory: u16, - pub size_of_the_central_directory: u32, - pub offset_of_start_of_central_directory_with_respect_to_the_starting_disk_number: u32, - // 2 bytes .ZIP file comment (variable size) - pub zipfile_comment_length: u16, -} - -impl ZipEndOfCentralDirectoryRecord { - pub fn create_and_load(binary_reader: &mut R) -> anyhow::Result { - let mut zip_ext_info_header = Self::new(); - zip_ext_info_header.load(binary_reader)?; - Ok(zip_ext_info_header) - } - - fn new() -> Self { - ZipEndOfCentralDirectoryRecord { - end_of_central_dir_signature: 0, - number_of_this_disk: 0, - number_of_the_disk_with_the_start_of_the_central_directory: 0, - total_number_of_entries_in_the_central_directory_on_this_disk: 0, - total_number_of_entries_in_the_central_directory: 0, - size_of_the_central_directory: 0, - offset_of_start_of_central_directory_with_respect_to_the_starting_disk_number: 0, - zipfile_comment_length: 0, - } - } - - fn load(&mut self, binary_reader: &mut R) -> anyhow::Result<()> { - self.end_of_central_dir_signature = binary_reader.read_u32::()?; - self.number_of_this_disk = binary_reader.read_u16::()?; - self.number_of_the_disk_with_the_start_of_the_central_directory = - binary_reader.read_u16::()?; - self.total_number_of_entries_in_the_central_directory_on_this_disk = - binary_reader.read_u16::()?; - self.total_number_of_entries_in_the_central_directory = - binary_reader.read_u16::()?; - self.size_of_the_central_directory = binary_reader.read_u32::()?; - self.offset_of_start_of_central_directory_with_respect_to_the_starting_disk_number = - binary_reader.read_u32::()?; - self.zipfile_comment_length = binary_reader.read_u16::()?; - - Ok(()) - } -}