fix png compression

src/bit_writer.rs

@@ -90,7 +90,6 @@ fn write_roundtrip() {
     b.pad(0, &mut data_buffer);
     b.flush_whole_bytes(&mut data_buffer);
 
-    let len = data_buffer.len() as i64;
     let mut cursor = std::io::Cursor::new(data_buffer);
     let mut reader = BitReader::new(&mut cursor);
 

src/huffman_calc.rs

@@ -414,11 +414,8 @@ mod calc_zlib {
 
         if overflow > 0 {
             // redistribute the bit lengths to remove the overflow
-            let mut bits = max_bits;
-
             while overflow > 0 {
-                bits -= 1;
-
+                let mut bits = max_bits - 1;
                 while bl_count[bits] == 0 {
                     bits -= 1;
                 }
@@ -431,7 +428,7 @@ mod calc_zlib {
             }
 
             // now reassign the bitlengths to the nodes (since we already have them in the right order)
-            bits = max_bits;
+            let mut bits = max_bits;
             for node in nodes.iter() {
                 if let HuffTree::Leaf(idx) = node.tree {
                     while bl_count[bits] == 0 {
@@ -488,18 +485,24 @@ mod calc_zlib {
     }
 
     #[test]
-    fn roundtrip_huffman_code() {
+    fn roundtrip_huffman_code_simple() {
         test_result(&[0, 1, 2, 4, 8, 16, 32], 7, &[0, 5, 5, 4, 3, 2, 1]);
+    }
 
+    // requires overflow treatment
+    #[test]
+    fn roundtrip_huffman_code_overflow() {
         test_result(
             &[
                 1, 0, 1, 1, 5, 10, 9, 18, 29, 59, 91, 28, 11, 1, 2, 0, 12, 1, 0,
             ],
             7,
-            &[7, 0, 7, 7, 5, 5, 5, 5, 3, 2, 2, 3, 5, 7, 6, 0, 5, 6],
+            &[7, 0, 7, 7, 6, 5, 5, 4, 3, 2, 2, 3, 5, 7, 7, 0, 5, 7],
         );
+    }
 
-        // requires overflow treatment
+    #[test]
+    fn roundtrip_huffman_code_normal() {
         test_result(
             &[
                 1, 0, 1, 0, 9, 9, 8, 19, 36, 70, 73, 34, 5, 6, 0, 0, 11, 1, 0,

src/huffman_encoding.rs

@@ -372,7 +372,6 @@ fn roundtrip_huffman_bitreadwrite() {
     bit_writer.write(0x1234, 16, &mut data_buffer);
     bit_writer.pad(0, &mut data_buffer);
 
-    let data_buffer_size = data_buffer.len();
     let mut reader = Cursor::new(&data_buffer);
     let mut bit_reader = BitReader::new(&mut reader);
 

tests/end_to_end.rs

@@ -52,6 +52,12 @@ fn test_sample1() {
 fn verifyresult(compressed_data: &[u8]) {
     let result = decompress_deflate_stream(compressed_data, true).unwrap();
     let recomp = recompress_deflate_stream(&result.plain_text, &result.cabac_encoded).unwrap();
+
+    println!(
+        "compressed size: {}, cabac: {}",
+        compressed_data.len(),
+        result.cabac_encoded.len()
+    );
     assert_eq!(compressed_data, recomp);
 }
 
@@ -76,8 +82,9 @@ fn test_file(filename: &str) {
                 level,
             );
 
+            assert_eq!(err, 0, "shouldn't fail zlib compression");
+
             output.set_len(output_size as usize);
-            println!("output size: {}, err = {}", output.len(), err);
         }
 
         let minusheader = &output[2..output.len() - 4];

util/Cargo.toml

@@ -20,7 +20,7 @@ default-boxed = "0.2"
 crc32fast = "1.3"
 libz-sys = "1.1"
 flate2 = "1.0"
-preflate_rs = { path = ".." }
+preflate-rs = { path = ".." }
 walkdir = "2.3"
 zstd = "0.13.0"
 

util/src/main.rs

@@ -32,7 +32,7 @@ fn main() {
             if results.len() > 0 {
                 for result in results {
                     totalzstd -= result.compressed_size as u64;
-                    totalzstd += result.zstd as u64;
+                    totalzstd += result.zstd + result.cabac_length as u64;
                     println!(
                         "  Found compressed data {:?} s={} c={} zstd={} cabac={}",
                         result.signature,
@@ -43,8 +43,7 @@ fn main() {
                     );
                 }
             }
-
-            println!("total seen ratio {}", totalzstd as f64 / totalseen as f64);
         }
     }
+    println!("total seen ratio {}", totalzstd as f64 / totalseen as f64);
 }

util/src/search_signature.rs

@@ -1,12 +1,9 @@
 use std::{
     collections::HashMap,
-    io::{BufReader, Cursor, Read, Seek, SeekFrom, Write},
+    io::{Cursor, Read, Seek, SeekFrom, Write},
 };
 
-use byteorder::{LittleEndian, ReadBytesExt};
-
 use anyhow;
-use flate2::read::DeflateDecoder;
 use preflate_rs::decompress_deflate_stream;
 
 use crate::zip_structs::{
@@ -37,7 +34,7 @@ pub fn add_location(
     compressed: &[u8],
 ) {
     let ret = decompress_deflate_stream(compressed, true);
-    if let Err(e) = ret
+    if let Err(_e) = ret
     {
         //println!("Error decompressing {:?} {:?} at {}", signature, e, start);
         return;

util/src/zip_structs.rs

@@ -2,16 +2,7 @@ use byteorder::{LittleEndian, ReadBytesExt};
 use std::io::{Read, Seek, SeekFrom};
 
 pub const ZIP_LOCAL_FILE_HEADER_SIGNATURE: u32 = 0x04034b50;
-pub const ZIP_CENTRAL_DIRECTORY_SIGNATURE: u32 = 0x02014b50;
-pub const ZIP_LOCAL_FILE_HEADER_SIZE_IN_BYTES: u32 = 30;
-pub const GENERAL_BIT_HAS_DATA_DESCRIPTOR: u16 = 0x0008;
 pub const ZIP64_EXTENDED_INFORMATION_TYPE_TAG: u16 = 0x0001;
-//pub const ZIP64_EXTENDED_INFORMATION_SIZE_IN_BYTES: u32 = 28;
-pub const ZIP_CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE: u32 = 0x02014b50;
-pub const ZIP_CENTRAL_DIRECTORY_FILE_HEADER_SIZE_IN_BYTES: u32 = 46;
-pub const ZIP_END_OF_CENTRAL_DIRECTORY_RECORD_SIGNATURE: u32 = 0x06054b50;
-//pub const ZIP_END_OF_CENTRAL_DIRECTORY_RECORD_SIZE_IN_BYTES: u32 = 22;
-pub const ZIP64_END_OF_CENTRAL_DIRECTORY_RECORD_SIGNATURE: u32 = 0x06064b50;
 
 #[derive(Clone)]
 pub struct ZipLocalFileHeader {
@@ -65,10 +56,6 @@ impl ZipLocalFileHeader {
         self.extra_field_length = binary_reader.read_u16::<LittleEndian>()?;
         Ok(())
     }
-
-    pub fn fhas_data_descriptor(&self) -> bool {
-        (self.general_purpose_bit_flag & GENERAL_BIT_HAS_DATA_DESCRIPTOR) != 0
-    }
 }
 
 pub struct ZipExtendedInformationHeader {
@@ -248,126 +235,3 @@ impl Zip64ExtendedInformation {
         Ok(())
     }
 }
-
-#[derive(Clone)]
-pub struct ZipCentralDirectoryFileHeader {
-    pub central_file_header_signature: u32,
-    pub version_made_by: u16,
-    pub version_needed_to_extract: u16,
-    pub general_purpose_bit_flag: u16,
-    pub compression_method: u16,
-    pub last_mod_file_time: u16,
-    pub last_mod_file_date: u16,
-    pub crc32: u32,
-    pub compressed_size: u64, // 4 bytes (may be set to 64bit value in Zip64 case)
-    pub uncompressed_size: u64, // 4 bytes (may be set to 64bit value in Zip64 case)
-    pub file_name_length: u16,
-    pub extra_field_length: u16,
-    pub file_comment_length: u16,
-    pub disk_number_start: u32, // 2 bytes (may be set to 32bit value in Zip64 case)
-    pub internal_file_attributes: u16,
-    pub external_file_attributes: u32,
-    pub relative_offset_of_local_header: u64, // 4 bytes (may be set to 64bit value in Zip64 case)
-}
-
-impl ZipCentralDirectoryFileHeader {
-    pub fn create_and_load<R: Read>(binary_reader: &mut R) -> anyhow::Result<Self> {
-        let mut zip_ext_info_header = Self::new();
-        zip_ext_info_header.load(binary_reader)?;
-        Ok(zip_ext_info_header)
-    }
-
-    fn new() -> Self {
-        ZipCentralDirectoryFileHeader {
-            central_file_header_signature: 0,
-            version_made_by: 0,
-            version_needed_to_extract: 0,
-            general_purpose_bit_flag: 0,
-            compression_method: 0,
-            last_mod_file_time: 0,
-            last_mod_file_date: 0,
-            crc32: 0,
-            compressed_size: 0,
-            uncompressed_size: 0,
-            file_name_length: 0,
-            extra_field_length: 0,
-            file_comment_length: 0,
-            disk_number_start: 0,
-            internal_file_attributes: 0,
-            external_file_attributes: 0,
-            relative_offset_of_local_header: 0,
-        }
-    }
-
-    fn load<R: Read>(&mut self, binary_reader: &mut R) -> anyhow::Result<()> {
-        self.central_file_header_signature = binary_reader.read_u32::<LittleEndian>()?;
-        self.version_made_by = binary_reader.read_u16::<LittleEndian>()?;
-        self.version_needed_to_extract = binary_reader.read_u16::<LittleEndian>()?;
-        self.general_purpose_bit_flag = binary_reader.read_u16::<LittleEndian>()?;
-        self.compression_method = binary_reader.read_u16::<LittleEndian>()?;
-        self.last_mod_file_time = binary_reader.read_u16::<LittleEndian>()?;
-        self.last_mod_file_date = binary_reader.read_u16::<LittleEndian>()?;
-        self.crc32 = binary_reader.read_u32::<LittleEndian>()?;
-        self.compressed_size = binary_reader.read_u32::<LittleEndian>()? as u64;
-        self.uncompressed_size = binary_reader.read_u32::<LittleEndian>()? as u64;
-        self.file_name_length = binary_reader.read_u16::<LittleEndian>()?;
-        self.extra_field_length = binary_reader.read_u16::<LittleEndian>()?;
-        self.file_comment_length = binary_reader.read_u16::<LittleEndian>()?;
-        self.disk_number_start = binary_reader.read_u16::<LittleEndian>()? as u32;
-        self.internal_file_attributes = binary_reader.read_u16::<LittleEndian>()?;
-        self.external_file_attributes = binary_reader.read_u32::<LittleEndian>()?;
-        self.relative_offset_of_local_header = binary_reader.read_u32::<LittleEndian>()? as u64;
-
-        Ok(())
-    }
-}
-
-pub struct ZipEndOfCentralDirectoryRecord {
-    pub end_of_central_dir_signature: u32,
-    pub number_of_this_disk: u16,
-    pub number_of_the_disk_with_the_start_of_the_central_directory: u16,
-    pub total_number_of_entries_in_the_central_directory_on_this_disk: u16,
-    pub total_number_of_entries_in_the_central_directory: u16,
-    pub size_of_the_central_directory: u32,
-    pub offset_of_start_of_central_directory_with_respect_to_the_starting_disk_number: u32,
-    // 2 bytes .ZIP file comment (variable size)
-    pub zipfile_comment_length: u16,
-}
-
-impl ZipEndOfCentralDirectoryRecord {
-    pub fn create_and_load<R: Read>(binary_reader: &mut R) -> anyhow::Result<Self> {
-        let mut zip_ext_info_header = Self::new();
-        zip_ext_info_header.load(binary_reader)?;
-        Ok(zip_ext_info_header)
-    }
-
-    fn new() -> Self {
-        ZipEndOfCentralDirectoryRecord {
-            end_of_central_dir_signature: 0,
-            number_of_this_disk: 0,
-            number_of_the_disk_with_the_start_of_the_central_directory: 0,
-            total_number_of_entries_in_the_central_directory_on_this_disk: 0,
-            total_number_of_entries_in_the_central_directory: 0,
-            size_of_the_central_directory: 0,
-            offset_of_start_of_central_directory_with_respect_to_the_starting_disk_number: 0,
-            zipfile_comment_length: 0,
-        }
-    }
-
-    fn load<R: Read>(&mut self, binary_reader: &mut R) -> anyhow::Result<()> {
-        self.end_of_central_dir_signature = binary_reader.read_u32::<LittleEndian>()?;
-        self.number_of_this_disk = binary_reader.read_u16::<LittleEndian>()?;
-        self.number_of_the_disk_with_the_start_of_the_central_directory =
-            binary_reader.read_u16::<LittleEndian>()?;
-        self.total_number_of_entries_in_the_central_directory_on_this_disk =
-            binary_reader.read_u16::<LittleEndian>()?;
-        self.total_number_of_entries_in_the_central_directory =
-            binary_reader.read_u16::<LittleEndian>()?;
-        self.size_of_the_central_directory = binary_reader.read_u32::<LittleEndian>()?;
-        self.offset_of_start_of_central_directory_with_respect_to_the_starting_disk_number =
-            binary_reader.read_u32::<LittleEndian>()?;
-        self.zipfile_comment_length = binary_reader.read_u16::<LittleEndian>()?;
-
-        Ok(())
-    }
-}