From 45b7a575d7de1ccb1ec6e012372a84996866feb9 Mon Sep 17 00:00:00 2001 From: JoergDF Date: Fri, 5 Jun 2026 11:34:00 +0200 Subject: [PATCH 01/16] Initial working version of archiving. --- Cargo.lock | 29 +++++ Cargo.toml | 1 + src/archive.rs | 262 ++++++++++++++++++++++++++++++++++++++++++++++ src/common_io.rs | 38 +++++-- src/decryption.rs | 23 ++-- src/encryption.rs | 26 +++-- src/lib.rs | 1 + src/main.rs | 21 ++-- 8 files changed, 369 insertions(+), 32 deletions(-) create mode 100644 src/archive.rs diff --git a/Cargo.lock b/Cargo.lock index 8d07d31..a5ff856 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,6 +317,7 @@ dependencies = [ "sha2", "sha3", "typenum", + "walkdir", ] [[package]] @@ -722,6 +723,15 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "secrecy" version = "0.10.3" @@ -870,6 +880,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -928,6 +948,15 @@ dependencies = [ "semver", ] +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index ae27c42..9e8117f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ secrecy = "0.10.3" sha2 = "0.11.0" sha3 = "0.12.0" typenum = "1.19.0" +walkdir = "2.5.0" [profile.release] lto = "thin" diff --git a/src/archive.rs b/src/archive.rs new file mode 100644 index 0000000..a44d147 --- /dev/null +++ b/src/archive.rs @@ -0,0 +1,262 @@ +use std::fs::{self, File, FileTimes}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::mem::size_of; +use std::time::{Duration, UNIX_EPOCH}; +use walkdir::{WalkDir, IntoIter}; + +use crate::{CHUNK_SIZE, Result}; +use crate::common_io::{ReadChunk, WriteFiles}; + +const TYPE_FILE: u8 = 0; +const TYPE_DIRECTORY: u8 = 1; +const ARCHIVE_HEADER_LENGTH_SIZE: usize = 2; + + +pub struct ArchiveRead { + walk_dir: IntoIter, + f_in: Option, + data_size: u64, + buf_out: Vec, + no_more_files: bool, +} + +impl ArchiveRead { + pub fn new(f_in_path: PathBuf) -> Self { + let walk_dir = WalkDir::new(&f_in_path).into_iter(); + let buf_out = Vec::with_capacity(CHUNK_SIZE * 3); + + Self { walk_dir, f_in: None, data_size: 0, buf_out, no_more_files: false } + } + + fn get_next_file(&mut self) -> Result<(Option>, Option, bool)> { + let mut archive_header = vec![]; + // header size, place holder + archive_header.extend([0u8; ARCHIVE_HEADER_LENGTH_SIZE]); + + if let Some(entry) = self.walk_dir.next() { + if let Ok(entry) = entry { + if entry.file_type().is_file() || entry.file_type().is_dir() { + if entry.file_type().is_file() { + // type: file + archive_header.push(TYPE_FILE); + } else { + // type: directory + archive_header.push(TYPE_DIRECTORY); + } + + // path length and path (including filename) + let path_string = entry.path().to_string_lossy(); + let path_len: u16 = path_string.len().try_into()?; + archive_header.extend(path_len.to_le_bytes()); + archive_header.extend(path_string.as_bytes()); + + // last access time + let time_accessed = entry.metadata()?.accessed()?.duration_since(UNIX_EPOCH)?.as_secs(); + archive_header.extend(time_accessed.to_le_bytes()); + // last modification time + let time_modified = entry.metadata()?.modified()?.duration_since(UNIX_EPOCH)?.as_secs(); + archive_header.extend(time_modified.to_le_bytes()); + + if entry.file_type().is_file() { + // file size + let file_size = entry.metadata()?.len(); + archive_header.extend(file_size.to_le_bytes()); + } + + // header size + let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from(archive_header.len() - 2)?.to_le_bytes(); + archive_header[0] = header_size[0]; + archive_header[1] = header_size[1]; + + //println!("{:?}", path_string); + //println!("{:x?}", archive_header); + + let mut path = None; + if entry.file_type().is_file() { + path = Some(entry.into_path()); + } + Ok( (Some(archive_header), path, false) ) + } else if entry.file_type().is_symlink() { + // fixme: get target path? file or directory (important for windows, also: user needs to be admin)? + eprintln!("Ignored Symlink: {}", entry.path().display()); + // if entry.path_is_symlink() { + // let target_path = std::fs::read_link(entry.path())?; + // } + Ok((None, None, false)) + } else { + eprintln!("Ignored entry: {:?}", entry); + Ok((None, None, false)) + } + } else { + // file/directory could not be accessed + eprintln!("Error entry: {:?}", entry); // fixme: check if files/directories that could not be accessed are printed + Ok((None, None, false)) + } + } else { + // no more entries + Ok((None, None, true)) + } + } +} + +impl ReadChunk for ArchiveRead { + fn read_chunk(&mut self) -> Result<(Vec, bool)> { + + while !self.no_more_files { + // buffer 2 chunks, hence read ahead 1 chunk to detect the final file and final-chunk flag can be set on the last chunk + if self.buf_out.len() > 2 * CHUNK_SIZE { + let chunk = self.buf_out.drain(..CHUNK_SIZE).collect(); + return Ok((chunk, false)); + } + + if self.data_size == 0 { + let vec_in; + let filepath; + (vec_in, filepath, self.no_more_files) = self.get_next_file()?; + + if vec_in.is_none() && filepath.is_none() { + // entry ignored + continue; + } + + if let Some(filepath) = filepath { + if let Ok(f_in) = File::open(&filepath) { + self.f_in = Some(f_in); + self.data_size = self.f_in.as_ref().unwrap().metadata()?.len(); + } else { + eprintln!("Could not open - skipped: {}", filepath.display()); + continue; + } + } + + // if file could not be opened, its admin data should be skipped, + // therefore save the admin data after the file handling, + // but for directories it is required + if let Some(vec_in) = vec_in { + self.buf_out.extend(vec_in); + } + } else { + let buf_len = CHUNK_SIZE.min(self.data_size.try_into()?); + let mut buf_read = vec![0u8; buf_len]; + + self.f_in.as_ref().unwrap().read_exact(&mut buf_read)?; + self.data_size -= u64::try_from(buf_len)?; + + self.buf_out.extend(buf_read); + } + } + + if self.buf_out.len() > CHUNK_SIZE { + let chunk = self.buf_out.drain(..CHUNK_SIZE).collect(); + return Ok((chunk, false)); + } + + // set final chunk flag + Ok((self.buf_out.clone(), true)) + } +} + + +#[derive(Default)] +pub struct ArchiveWrite { + f_out: Option, + buf_out: Vec, + header_length: Option, + file_size: u64, + file_times: FileTimes, + file_path: PathBuf, +} + +impl ArchiveWrite { + pub fn new() -> Self { + let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); + Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), file_path: PathBuf::new() } + } +} + +impl WriteFiles for ArchiveWrite { + fn write_files(&mut self, buf_in: &[u8]) -> Result<()> { + self.buf_out.extend(buf_in); + + while !self.buf_out.is_empty() { + if let Some(mut f_out) = self.f_out.as_ref() { + // write to file + if (self.buf_out.len() as u64) < self.file_size { + f_out.write_all(&self.buf_out)?; + self.file_size -= self.buf_out.len() as u64; + self.buf_out.clear(); + } else { + let file_data: Vec = self.buf_out.drain(..usize::try_from(self.file_size)?).collect(); + f_out.write_all(&file_data)?; + // set file times after all data has been written + if f_out.set_times(self.file_times).is_err() { + eprintln!("Could not set timestamps for file {}", self.file_path.display()); + } + self.file_size = 0; + self.f_out = None; + } + } else if let Some(header_length) = self.header_length { + if self.buf_out.len() >= header_length { + // get header + let header: Vec = self.buf_out.drain(..header_length).collect(); + + // type + let file_type = header[0]; + + // path length + let mut s = 1; + let mut e = s + size_of::(); + let path_len = u16::from_le_bytes(header[s..e].try_into()?); + // path + s = e; e += usize::from(path_len); + let path_bytes = &header[s..e]; + let path_str = str::from_utf8(path_bytes)?; + let entry_path = Path::new(path_str); + + // access time + s = e; e += size_of::(); + let time_accessed_seconds = u64::from_le_bytes( header[s..e].try_into()? ); + let time_accessed = UNIX_EPOCH + Duration::from_secs(time_accessed_seconds); + // modification time + s = e; e += size_of::(); + let time_modified_seconds = u64::from_le_bytes( header[s..e].try_into()? ); + let time_modified = UNIX_EPOCH + Duration::from_secs(time_modified_seconds); + self.file_times = FileTimes::new() + .set_accessed(time_accessed) + .set_modified(time_modified); + + //println!("{:?}", entry_path); + + // create type + if file_type == TYPE_DIRECTORY { + fs::create_dir_all(entry_path)?; + // set timestamps of directory + if !File::open(entry_path).is_ok_and(|dir| dir.set_times(self.file_times).is_ok()) { + eprintln!("Could not set timestamps for directory {}", entry_path.display()); + } + } else if file_type == TYPE_FILE { + self.file_path = entry_path.to_path_buf(); + self.f_out = Some( File::create(&self.file_path)? ); + // file size + s = e; e += size_of::(); + self.file_size = u64::from_le_bytes( header[s..e].try_into()? ); + } else { + return Err(format!("Archive contains unknown file type: {file_type}").into()); + } + + self.header_length = None; + } else { + break; // not enough data + } + } else if self.buf_out.len() >= ARCHIVE_HEADER_LENGTH_SIZE { + // get header length + let header_length_bytes: Vec = self.buf_out.drain(..ARCHIVE_HEADER_LENGTH_SIZE).collect(); + self.header_length = Some( u16::from_le_bytes(header_length_bytes.try_into().unwrap()).into() ); + } else { + break; // not enough data + } + } + Ok(()) + } +} diff --git a/src/common_io.rs b/src/common_io.rs index c906af7..e1bca8a 100644 --- a/src/common_io.rs +++ b/src/common_io.rs @@ -7,7 +7,16 @@ use secrecy::SecretSlice; use std::collections::HashMap; -use crate::{Result, SPLIT_ENC_FILE_EXT}; +use crate::{AES_NONCE_SIZE, AES_TAG_SIZE, CHA_NONCE_SIZE, CHA_TAG_SIZE, Result, SPLIT_ENC_FILE_EXT}; + + +pub trait ReadChunk { + fn read_chunk(&mut self) -> Result<(Vec, bool)>; +} + +pub trait WriteFiles { + fn write_files(&mut self, buf_in: &[u8]) -> Result<()>; +} /// Struct for file input pub struct ReadInput { @@ -38,7 +47,7 @@ impl ReadInput { /// # Returns /// - `Ok(Self)` on success /// - `Err(...)` when an I/O or conversion error occurs - pub fn new(f_in_path: PathBuf, chunk_size: usize, f_in_header_size: usize) -> Result { + pub fn new(f_in_path: PathBuf, chunk_size: usize, f_in_header_size: u64) -> Result { let f_in = File::open(&f_in_path)?; let mut f_in_total_size = 0; @@ -57,8 +66,13 @@ impl ReadInput { f_in_total_size = f_in.metadata()?.len(); } + if f_in_header_size != 0 + && f_in_total_size < f_in_header_size + (CHA_NONCE_SIZE + CHA_TAG_SIZE + AES_NONCE_SIZE + AES_TAG_SIZE) as u64 { + return Err("File cannot be decoded".into()); + } + // header was already read, therefore remaining file size must be reduced by the header's size - let f_in_total_size_remaining = f_in_total_size - u64::try_from(f_in_header_size)?; + let f_in_total_size_remaining = f_in_total_size - f_in_header_size; Ok( Self { f_in, f_in_path, f_in_total_size_remaining, chunk_size, f_in_split, split_index: 0, f_in_read_count: 0 } ) } @@ -130,13 +144,15 @@ impl ReadInput { Ok(()) } +} +impl ReadChunk for ReadInput { /// Read a single chunk from the logical file(s) and indicate whether it is the final chunk. /// /// # Returns /// - `Ok((buf_in, final_chunk))` contains the read chunk buffer and if it is the final chunk /// - `Err(...)` when an I/O or conversion error occurs - pub fn read_chunk(&mut self) -> Result<(Vec, bool)> { + fn read_chunk(&mut self) -> Result<(Vec, bool)> { let (read_size, final_chunk) = self.input_sizes()?; let mut buf_in = vec![0u8; read_size]; self.read_files(&mut buf_in)?; @@ -174,7 +190,9 @@ impl WriteOutput { let f_out = File::create(&f_out_path)?; Ok( Self { f_out, f_out_path, f_out_split, split_index: 0, f_out_write_count: 0 } ) } +} +impl WriteFiles for WriteOutput { /// Writes the provided buffer across one or more output files according to the configured splits. /// /// - If `self.f_out_split` is empty: append the entire buffer to the current output file. @@ -184,7 +202,7 @@ impl WriteOutput { /// # Returns /// - `Ok(())` on success /// - `Err(...)` when an I/O or conversion error occurs - pub fn write_files(&mut self, buf: &[u8]) -> Result<()> { + fn write_files(&mut self, buf: &[u8]) -> Result<()> { if self.f_out_split.is_empty() { self.f_out.write_all(buf)?; } else { @@ -256,8 +274,8 @@ impl CryptIo { Receiver<(Vec, u32, bool)>, Sender<(Vec, u32)>, usize) -> Vec>>, - mut read_input: ReadInput, - mut write_output: WriteOutput, + read_input: &mut (impl ReadChunk + ?Sized), + mut write_output: Box, ) -> Result<()> { let cpu_count = num_cpus::get(); @@ -364,20 +382,20 @@ mod tests { // only first split let mut buf = [0u8; 1000]; let f_in_path = PathBuf::from("test_split_in.c00"); - let mut ri = ReadInput::new(f_in_path.clone(), 0, HEADER_SIZE).unwrap(); + let mut ri = ReadInput::new(f_in_path.clone(), 0, HEADER_SIZE as u64).unwrap(); ri.read_files(&mut buf).unwrap(); assert_eq!(buf, data0); // all splits let mut buf = [0u8; 3028]; - let mut ri = ReadInput::new(f_in_path.clone(), 0, HEADER_SIZE).unwrap(); + let mut ri = ReadInput::new(f_in_path.clone(), 0, HEADER_SIZE as u64).unwrap(); ri.read_files(&mut buf).unwrap(); assert_eq!(buf[..], [&data0[..], &data1[..], &data2[..]].concat()); // 2 reads let mut buf0 = [0u8; 4]; let mut buf1 = [0u8; 2000]; - let mut ri = ReadInput::new(f_in_path.clone(), 0, HEADER_SIZE).unwrap(); + let mut ri = ReadInput::new(f_in_path.clone(), 0, HEADER_SIZE as u64).unwrap(); ri.read_files(&mut buf0).unwrap(); assert_eq!(buf0[..], data0[..4]); ri.read_files(&mut buf1).unwrap(); diff --git a/src/decryption.rs b/src/decryption.rs index 7e5b792..5c045c1 100644 --- a/src/decryption.rs +++ b/src/decryption.rs @@ -12,7 +12,8 @@ use crossbeam_channel::{bounded, Sender, Receiver}; use crate::{Result, KEY_SIZE, CHA_NONCE_SIZE, AES_NONCE_SIZE, CHUNK_SIZE, COMPRESS_LENGTH_SIZE, ENCRYPTED_FILE_EXT, SPLIT_ENC_FILE_EXT, CHA_TAG_SIZE, AES_TAG_SIZE, HEADER_SIZE, FILE_FORMAT_VERSION}; use crate::common::{get_pass_bytes, key_derivation}; -use crate::common_io::{CryptIo, ReadInput, WriteOutput}; +use crate::common_io::{CryptIo, ReadInput, WriteFiles, WriteOutput}; +use crate::archive::ArchiveWrite; /// Handles file decryption operations using dual-layer decryption and decompression. @@ -316,6 +317,10 @@ impl Decryption { /// - `Ok(())` on successful decryption /// - `Err` if file operations, password handling, or decryption fails pub fn decrypt(filepath_in: &Path, keyfilepath: Option<&PathBuf>) -> Result<()> { + if filepath_in.is_dir() { + return Err("Cannot decrypt a directory".into()); + } + let mut filepath_out = filepath_in.to_path_buf(); if filepath_in.extension() == Some(std::ffi::OsStr::new(ENCRYPTED_FILE_EXT)) || filepath_in.extension() == Some(std::ffi::OsStr::new(SPLIT_ENC_FILE_EXT)) { @@ -329,12 +334,9 @@ impl Decryption { let mut read_input = ReadInput::new( filepath_in.to_path_buf(), CHUNK_SIZE + CHA_NONCE_SIZE + CHA_TAG_SIZE + AES_NONCE_SIZE + AES_TAG_SIZE, - HEADER_SIZE + HEADER_SIZE as u64 )?; - // set write parameters and create output file - let write_output = WriteOutput::new(filepath_out, vec![])?; - // Read file header let mut header = [0u8; HEADER_SIZE]; read_input.read_files(&mut header)?; @@ -357,7 +359,16 @@ impl Decryption { let (key_cha, key_aes) = Self::derive_keys(salt_cha, salt_aes, &key)?; let compress = (file_format & 0x01) != 0; - CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::decrypt_pipe, read_input, write_output)?; + let archive = (file_format & 0x02) != 0; + + let write_output: Box = if archive { + Box::new( ArchiveWrite::new() ) + } else { + // set write parameters and create output file + Box::new( WriteOutput::new(filepath_out, vec![])? ) + }; + + CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::decrypt_pipe, &mut read_input, write_output)?; Ok(()) } diff --git a/src/encryption.rs b/src/encryption.rs index 58d5056..fec562a 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -16,8 +16,8 @@ use std::collections::HashMap; use crate::{Result, SALT_SIZE, KEY_SIZE, CHA_NONCE_SIZE, CHA_TAG_SIZE, AES_NONCE_SIZE, AES_TAG_SIZE, CHUNK_SIZE, COMPRESS_LENGTH_SIZE, ENCRYPTED_FILE_EXT, SPLIT_ENC_FILE_EXT, HEADER_SIZE, FILE_FORMAT_VERSION}; use crate::common::{get_pass_bytes, key_derivation}; -use crate::common_io::{CryptIo, ReadInput, WriteOutput}; - +use crate::common_io::{CryptIo, ReadInput, WriteOutput, ReadChunk, WriteFiles}; +use crate::archive::ArchiveRead; /// Handles file encryption operations using dual-layer encryption and compression. /// @@ -363,6 +363,14 @@ impl Encryption { filepath_out.add_extension(SPLIT_ENC_FILE_EXT); } + let build_archive = filepath_in.is_dir(); + + let mut read_input: Box = if build_archive { + Box::new( ArchiveRead::new(filepath_in.to_path_buf()) ) + } else { + Box::new( ReadInput::new(filepath_in.to_path_buf(), CHUNK_SIZE, 0)? ) + }; + let (salt_pw, key) = Self::hash_password(keyfilepath)?; let (salt_cha, key_cha, salt_aes, key_aes) = Self::derive_keys(&key)?; @@ -371,26 +379,24 @@ impl Encryption { // 0 version of file format // 1 info about file format // bit 0: compression on(1)/off(0) + // bit 1: archive // 2..33 32-byte-salt of password hash // 34..65 32-byte-salt of cha key derivation // 66..97 32-byte-salt of aes key derivation let mut header = Vec::with_capacity(HEADER_SIZE); header.push(FILE_FORMAT_VERSION); - header.push(u8::from(compress)); + header.push(u8::from(compress) | (u8::from(build_archive) << 1)); header.extend(salt_pw); header.extend(salt_cha); header.extend(salt_aes); - // set read parameters and open input file - let read_input = ReadInput::new(filepath_in.to_path_buf(), CHUNK_SIZE, 0)?; - // set write parameters and create output file - let mut write_output = WriteOutput::new(filepath_out, split)?; + let mut write_output = Box::new( WriteOutput::new(filepath_out, split)? ); // write header write_output.write_files(&header)?; - CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::encrypt_pipe, read_input, write_output)?; + CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::encrypt_pipe, read_input.as_mut(), write_output)?; Ok(()) } @@ -734,7 +740,9 @@ mod tests { // input file does not exist assert!(Encryption::encrypt(&PathBuf::from("test_miss"), None, false, vec![]).is_err()); - assert!(Decryption::decrypt(&PathBuf::from("test_miss"), None).is_err()); + assert!(Decryption::decrypt(&PathBuf::from("test_miss.cce"), None).is_err()); + assert!(!fs::exists("test_miss").unwrap()); + assert!(!fs::exists("test_miss.cce").unwrap()); // with compression fs::write(&filepath_in, &data).unwrap(); diff --git a/src/lib.rs b/src/lib.rs index d5020f3..eae2fb3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ pub mod common; pub mod common_io; pub mod encryption; pub mod decryption; +pub mod archive; pub const FILE_FORMAT_VERSION: u8 = 4; diff --git a/src/main.rs b/src/main.rs index 93381c3..1f9ccff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,8 +9,8 @@ use cryptcrypt::Result; #[derive(Parser)] #[command(version, about, verbatim_doc_comment, long_about = None)] -/// Program for encryption and decryption of a file. -/// If no option is given, file is encrypted. +/// Program for encryption and decryption of file or directory. +/// If no option is given, input is encrypted. A directory as input causes the build of an encrypted archive. /// With option -s the encrypted output is split into files with extensions .c00, .c01, .c02, ... /// If a file ending on .c00 is decrypted, the whole split series will be read. struct Args { @@ -31,8 +31,9 @@ struct Args { value_parser = |s: &str| { let cfg = Config::new().with_binary(); cfg.parse_size(s) })] split: Vec, - /// File that should be encrypted or decrypted - file: PathBuf, + /// File that should be encrypted or decrypted. + /// If a directory is given, all its files and sub-directories are concatenated and encrypted. + file_or_dir: PathBuf, } /// Main entry point for the cryptcrypt application. @@ -46,7 +47,7 @@ fn main() -> ExitCode { match result { Ok(()) => ExitCode::SUCCESS, Err(e) => { - eprintln!("{}", e); + eprintln!("Error: {}", e); ExitCode::FAILURE } } @@ -58,8 +59,14 @@ fn main() -> ExitCode { /// based on the provided flags. fn run() -> Result<()> { let args = Args::parse(); - - let filepath = args.file.canonicalize()?; + + let filepath = if args.file_or_dir.is_dir() { + // a directory should be used as is (relative or absolute), therefore do not canonicalize, which results in an absolute path + args.file_or_dir + } else { + args.file_or_dir.canonicalize()? + }; + let keyfilepath = args.keyfile.map(|path| path.canonicalize()).transpose()?; if args.decrypt { From 6dc526d9905b7bc0f0dcace8f77dfc9a49e5d811 Mon Sep 17 00:00:00 2001 From: JoergDF Date: Tue, 9 Jun 2026 11:51:39 +0200 Subject: [PATCH 02/16] Extend archived elements and compatibility between windows and unix - conversion of window paths to unix paths - add symlinks - add permissions for unix - refactor archive code --- Cargo.lock | 7 + Cargo.toml | 3 +- src/archive.rs | 344 ++++++++++++++++++++++++++++++---------------- src/encryption.rs | 2 +- 4 files changed, 237 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5ff856..4aad253 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,6 +316,7 @@ dependencies = [ "secrecy", "sha2", "sha3", + "typed-path", "typenum", "walkdir", ] @@ -840,6 +841,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.0" diff --git a/Cargo.toml b/Cargo.toml index 9e8117f..5e00ae3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ rpassword = "7.4.0" secrecy = "0.10.3" sha2 = "0.11.0" sha3 = "0.12.0" +typed-path = "0.12.3" typenum = "1.19.0" walkdir = "2.5.0" @@ -27,4 +28,4 @@ lto = "thin" codegen-units = 1 [lints.rust] -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] } \ No newline at end of file +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] } diff --git a/src/archive.rs b/src/archive.rs index a44d147..b9a5017 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -1,6 +1,8 @@ use std::fs::{self, File, FileTimes}; use std::io::{Read, Write}; +use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; +use typed_path::Utf8WindowsPath; use std::mem::size_of; use std::time::{Duration, UNIX_EPOCH}; use walkdir::{WalkDir, IntoIter}; @@ -8,8 +10,13 @@ use walkdir::{WalkDir, IntoIter}; use crate::{CHUNK_SIZE, Result}; use crate::common_io::{ReadChunk, WriteFiles}; -const TYPE_FILE: u8 = 0; -const TYPE_DIRECTORY: u8 = 1; + +const TYPE_FILE: u8 = 0x00; +const TYPE_DIRECTORY: u8 = 0x01; +const TYPE_SYMLINK_FILE: u8 = 0x02; +const TYPE_SYMLINK_DIR: u8 = 0x03; +const TYPE_UNIX: u8 = 0x00; +const TYPE_WINDOWS: u8 = 0x10; const ARCHIVE_HEADER_LENGTH_SIZE: usize = 2; @@ -22,80 +29,110 @@ pub struct ArchiveRead { } impl ArchiveRead { - pub fn new(f_in_path: PathBuf) -> Self { - let walk_dir = WalkDir::new(&f_in_path).into_iter(); + pub fn new(f_in_path: &Path) -> Self { + // in a directory: list files first, then sub-directories + let walk_dir = WalkDir::new(f_in_path).sort_by_key(|x| x.file_type().is_dir()).into_iter(); let buf_out = Vec::with_capacity(CHUNK_SIZE * 3); Self { walk_dir, f_in: None, data_size: 0, buf_out, no_more_files: false } } - fn get_next_file(&mut self) -> Result<(Option>, Option, bool)> { - let mut archive_header = vec![]; - // header size, place holder - archive_header.extend([0u8; ARCHIVE_HEADER_LENGTH_SIZE]); - - if let Some(entry) = self.walk_dir.next() { - if let Ok(entry) = entry { - if entry.file_type().is_file() || entry.file_type().is_dir() { - if entry.file_type().is_file() { - // type: file - archive_header.push(TYPE_FILE); - } else { - // type: directory - archive_header.push(TYPE_DIRECTORY); - } + fn build_archive_header(entry: &walkdir::DirEntry) -> Result<(Vec, Option)> { + // archive header initialized with place holder for header size + let mut archive_header = vec![0u8; ARCHIVE_HEADER_LENGTH_SIZE]; + + let mut entry_type = if entry.file_type().is_file() { + TYPE_FILE + } else if entry.file_type().is_dir() { + TYPE_DIRECTORY + } else if entry.file_type().is_symlink() { + // whether a symlink is a file or a directory is only relevant for windows (when creating them there) + if entry.path().is_dir() { + TYPE_SYMLINK_DIR + } else { + // if target of symlink does not exist (hence it can't be evaluated + // whether it is a file or a directory), type file is used + TYPE_SYMLINK_FILE + } + } else { + return Err(format!("Ignored unsupported file type for archive: {}", entry.path().display()).into()); + }; - // path length and path (including filename) - let path_string = entry.path().to_string_lossy(); - let path_len: u16 = path_string.len().try_into()?; - archive_header.extend(path_len.to_le_bytes()); - archive_header.extend(path_string.as_bytes()); - - // last access time - let time_accessed = entry.metadata()?.accessed()?.duration_since(UNIX_EPOCH)?.as_secs(); - archive_header.extend(time_accessed.to_le_bytes()); - // last modification time - let time_modified = entry.metadata()?.modified()?.duration_since(UNIX_EPOCH)?.as_secs(); - archive_header.extend(time_modified.to_le_bytes()); - - if entry.file_type().is_file() { - // file size - let file_size = entry.metadata()?.len(); - archive_header.extend(file_size.to_le_bytes()); - } + if cfg!(windows) { + entry_type |= TYPE_WINDOWS; + } else { + entry_type |= TYPE_UNIX; + } + archive_header.push(entry_type); - // header size - let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from(archive_header.len() - 2)?.to_le_bytes(); - archive_header[0] = header_size[0]; - archive_header[1] = header_size[1]; + // path length and path (including filename) + let path_string = if entry.file_type().is_dir() { + entry.path().to_string_lossy() + } else { + entry.file_name().to_string_lossy() + }; + let path_len: u16 = path_string.len().try_into()?; + archive_header.extend(path_len.to_le_bytes()); + archive_header.extend(path_string.as_bytes()); + + // last access time + let time_accessed = entry.metadata()?.accessed()?.duration_since(UNIX_EPOCH)?.as_secs(); + archive_header.extend(time_accessed.to_le_bytes()); + // last modification time + let time_modified = entry.metadata()?.modified()?.duration_since(UNIX_EPOCH)?.as_secs(); + archive_header.extend(time_modified.to_le_bytes()); + + if entry.file_type().is_file() { + // file size + let file_size = entry.metadata()?.len(); + archive_header.extend(file_size.to_le_bytes()); + + } else if entry.file_type().is_symlink() { + // target path of symlink + let target_path = fs::read_link(entry.path())?; + let target_path_string = target_path.to_string_lossy(); + let target_path_len: u16 = target_path_string.len().try_into()?; + archive_header.extend(target_path_len.to_le_bytes()); + archive_header.extend(target_path_string.as_bytes()); + } - //println!("{:?}", path_string); - //println!("{:x?}", archive_header); + // permissions + let mut perm: u16 = 0; + if cfg!(unix) && (entry.file_type().is_file() || entry.file_type().is_dir()) { + use std::os::unix::fs::PermissionsExt; + let permission_mode = entry.metadata()?.permissions().mode(); + // use 12 least significant bits + perm = (permission_mode & 0x0FFF) as u16; + } + archive_header.extend(perm.to_le_bytes()); + + // header size + let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from(archive_header.len() - 2)?.to_le_bytes(); + archive_header[0] = header_size[0]; + archive_header[1] = header_size[1]; + + let mut path = None; + if entry.file_type().is_file() { + path = Some(entry.clone().into_path()); + } + + Ok((archive_header, path)) + } - let mut path = None; - if entry.file_type().is_file() { - path = Some(entry.into_path()); + fn get_next_archive_item(&mut self) -> Result<(Option>, Option)> { + if let Some(entry) = self.walk_dir.next() { + match &entry { + Ok(entry) => { + match Self::build_archive_header(entry) { + Ok((archive_header, filepath)) => Ok((Some(archive_header), filepath)), + Err(e) => Err(format!("Skipped entry {} Error: {e}", entry.path().display()).into()) } - Ok( (Some(archive_header), path, false) ) - } else if entry.file_type().is_symlink() { - // fixme: get target path? file or directory (important for windows, also: user needs to be admin)? - eprintln!("Ignored Symlink: {}", entry.path().display()); - // if entry.path_is_symlink() { - // let target_path = std::fs::read_link(entry.path())?; - // } - Ok((None, None, false)) - } else { - eprintln!("Ignored entry: {:?}", entry); - Ok((None, None, false)) - } - } else { - // file/directory could not be accessed - eprintln!("Error entry: {:?}", entry); // fixme: check if files/directories that could not be accessed are printed - Ok((None, None, false)) + }, + Err(e) => Err(format!("Skipped entry {entry:?} Error: {e}").into()) } } else { // no more entries - Ok((None, None, true)) + Ok((None, None)) } } } @@ -111,19 +148,21 @@ impl ReadChunk for ArchiveRead { } if self.data_size == 0 { - let vec_in; + let archive_header; let filepath; - (vec_in, filepath, self.no_more_files) = self.get_next_file()?; - if vec_in.is_none() && filepath.is_none() { - // entry ignored - continue; + match self.get_next_archive_item() { + Ok(values) => (archive_header, filepath) = values, + Err(e) => { + eprintln!("{e}"); + continue; + } } - if let Some(filepath) = filepath { - if let Ok(f_in) = File::open(&filepath) { + if let Some(filepath) = &filepath { + if let Ok(f_in) = File::open(filepath) { + self.data_size = f_in.metadata()?.len(); self.f_in = Some(f_in); - self.data_size = self.f_in.as_ref().unwrap().metadata()?.len(); } else { eprintln!("Could not open - skipped: {}", filepath.display()); continue; @@ -133,8 +172,10 @@ impl ReadChunk for ArchiveRead { // if file could not be opened, its admin data should be skipped, // therefore save the admin data after the file handling, // but for directories it is required - if let Some(vec_in) = vec_in { - self.buf_out.extend(vec_in); + if let Some(hdr) = &archive_header { + self.buf_out.extend(hdr); + } else { + self.no_more_files = true; } } else { let buf_len = CHUNK_SIZE.min(self.data_size.try_into()?); @@ -166,12 +207,123 @@ pub struct ArchiveWrite { file_size: u64, file_times: FileTimes, file_path: PathBuf, + dir_path: PathBuf, } impl ArchiveWrite { pub fn new() -> Self { let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); - Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), file_path: PathBuf::new() } + Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), + file_path: PathBuf::new(), dir_path: PathBuf::new() } + } + + fn get_path_from_header(header: &[u8], current_end_index: usize, created_on_os_type: u8) -> Result<(String, usize)> { + // new start index of header field + let mut s = current_end_index; + // new end index of header field + let mut e = current_end_index + size_of::(); + + // path length + let path_len = u16::from_le_bytes(header[s..e].try_into()?); + // path + s = e; e += usize::from(path_len); + let path_bytes = &header[s..e]; + let path_str = str::from_utf8(path_bytes)?; + // convert Windows path to unix path, it on unix (windows can handle unix path) + let entry_path = if cfg!(unix) && created_on_os_type == TYPE_WINDOWS { + Utf8WindowsPath::new(path_str).with_unix_encoding().to_string() + } else { + path_str.to_string() + }; + + Ok((entry_path, e)) + } + + fn eval_header(&mut self, header: &[u8]) -> Result<()> { + // type + let file_type = header[0] & 0x0F; + let created_on_os_type = header[0] & 0xF0; + + let mut s ; + let mut e = 1; + + // entry's path + let entry_path; + (entry_path, e) = Self::get_path_from_header(header, e, created_on_os_type)?; + + // access time + s = e; e += size_of::(); + let time_accessed_seconds = u64::from_le_bytes( header[s..e].try_into()? ); + let time_accessed = UNIX_EPOCH + Duration::from_secs(time_accessed_seconds); + // modification time + s = e; e += size_of::(); + let time_modified_seconds = u64::from_le_bytes( header[s..e].try_into()? ); + let time_modified = UNIX_EPOCH + Duration::from_secs(time_modified_seconds); + self.file_times = FileTimes::new() + .set_accessed(time_accessed) + .set_modified(time_modified); + + // create type + if file_type == TYPE_DIRECTORY { + fs::create_dir_all(&entry_path)?; + self.dir_path = PathBuf::from(&entry_path); + // set timestamps of directory + if !File::open(&entry_path).is_ok_and(|dir| dir.set_times(self.file_times).is_ok()) { + eprintln!("Could not set timestamps for directory {}", entry_path); + } + } else if file_type == TYPE_FILE { + self.file_path = self.dir_path.join(&entry_path); + self.f_out = Some( File::create(&self.file_path)? ); + + // file size + s = e; e += size_of::(); + self.file_size = u64::from_le_bytes( header[s..e].try_into()? ); + } else if file_type == TYPE_SYMLINK_FILE || file_type == TYPE_SYMLINK_DIR { + // symlink's target path + let target_path; + (target_path, e) = Self::get_path_from_header(header, e, created_on_os_type)?; + + let sym_path = self.dir_path.join(&entry_path); + + // create symlink + // remove it, if it already exists, otherwise symlink can't be created + let _ = fs::remove_file(&sym_path); + #[cfg(unix)] + { + std::os::unix::fs::symlink(target_path, &sym_path)?; + } + #[cfg(windows)] + { + if file_type == TYPE_SYMLINK_FILE { + std::os::windows::fs::symlink_file(&target_path, &sym_path)?; + } + if file_type == TYPE_SYMLINK_DIR { + std::os::windows::fs::symlink_dir(&target_path, &sym_path)?; + } + } + } else { + return Err(format!("Archive contains unknown file type: {file_type}").into()); + } + + // permissions + // if this is a unix system and the archive was created on a unix system, set permission mode + if cfg!(unix) && created_on_os_type == TYPE_UNIX && (file_type == TYPE_DIRECTORY || file_type == TYPE_FILE) { + s = e; e += size_of::(); + let perm = u16::from_le_bytes( header[s..e].try_into()? ); + + let path = if file_type == TYPE_FILE { + &self.file_path + } else { + &self.dir_path + }; + let fd = File::open(path)?; + let mut permissions = fd.metadata()?.permissions(); + let mode_masked = permissions.mode() & 0xFFFF_F000; + permissions.set_mode(mode_masked | u32::from(perm & 0x0FFF)); + fd.set_permissions(permissions)?; + } + + Ok(()) } } @@ -201,49 +353,7 @@ impl WriteFiles for ArchiveWrite { // get header let header: Vec = self.buf_out.drain(..header_length).collect(); - // type - let file_type = header[0]; - - // path length - let mut s = 1; - let mut e = s + size_of::(); - let path_len = u16::from_le_bytes(header[s..e].try_into()?); - // path - s = e; e += usize::from(path_len); - let path_bytes = &header[s..e]; - let path_str = str::from_utf8(path_bytes)?; - let entry_path = Path::new(path_str); - - // access time - s = e; e += size_of::(); - let time_accessed_seconds = u64::from_le_bytes( header[s..e].try_into()? ); - let time_accessed = UNIX_EPOCH + Duration::from_secs(time_accessed_seconds); - // modification time - s = e; e += size_of::(); - let time_modified_seconds = u64::from_le_bytes( header[s..e].try_into()? ); - let time_modified = UNIX_EPOCH + Duration::from_secs(time_modified_seconds); - self.file_times = FileTimes::new() - .set_accessed(time_accessed) - .set_modified(time_modified); - - //println!("{:?}", entry_path); - - // create type - if file_type == TYPE_DIRECTORY { - fs::create_dir_all(entry_path)?; - // set timestamps of directory - if !File::open(entry_path).is_ok_and(|dir| dir.set_times(self.file_times).is_ok()) { - eprintln!("Could not set timestamps for directory {}", entry_path.display()); - } - } else if file_type == TYPE_FILE { - self.file_path = entry_path.to_path_buf(); - self.f_out = Some( File::create(&self.file_path)? ); - // file size - s = e; e += size_of::(); - self.file_size = u64::from_le_bytes( header[s..e].try_into()? ); - } else { - return Err(format!("Archive contains unknown file type: {file_type}").into()); - } + self.eval_header(&header)?; self.header_length = None; } else { diff --git a/src/encryption.rs b/src/encryption.rs index fec562a..8198134 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -366,7 +366,7 @@ impl Encryption { let build_archive = filepath_in.is_dir(); let mut read_input: Box = if build_archive { - Box::new( ArchiveRead::new(filepath_in.to_path_buf()) ) + Box::new( ArchiveRead::new(filepath_in) ) } else { Box::new( ReadInput::new(filepath_in.to_path_buf(), CHUNK_SIZE, 0)? ) }; From 6a5011ea11b47970c08c39ad4047720645630a44 Mon Sep 17 00:00:00 2001 From: JoergDF Date: Thu, 11 Jun 2026 17:03:20 +0200 Subject: [PATCH 03/16] Improve error handling and a performance optimization. - tx.send errors should not cover other errors in encryption/decryption, hence ignore. - Small optimization in archiving: replace one I/O access for file length by already known value. --- src/archive.rs | 37 +++++++++++++++++++++---------------- src/common_io.rs | 14 +++++++------- src/decryption.rs | 4 ++-- src/encryption.rs | 4 ++-- 4 files changed, 32 insertions(+), 27 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index b9a5017..f28c25d 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -23,7 +23,7 @@ const ARCHIVE_HEADER_LENGTH_SIZE: usize = 2; pub struct ArchiveRead { walk_dir: IntoIter, f_in: Option, - data_size: u64, + file_size: u64, buf_out: Vec, no_more_files: bool, } @@ -34,10 +34,10 @@ impl ArchiveRead { let walk_dir = WalkDir::new(f_in_path).sort_by_key(|x| x.file_type().is_dir()).into_iter(); let buf_out = Vec::with_capacity(CHUNK_SIZE * 3); - Self { walk_dir, f_in: None, data_size: 0, buf_out, no_more_files: false } + Self { walk_dir, f_in: None, file_size: 0, buf_out, no_more_files: false } } - fn build_archive_header(entry: &walkdir::DirEntry) -> Result<(Vec, Option)> { + fn build_archive_header(entry: &walkdir::DirEntry) -> Result<(Vec, Option<(PathBuf, u64)>)> { // archive header initialized with place holder for header size let mut archive_header = vec![0u8; ARCHIVE_HEADER_LENGTH_SIZE]; @@ -75,6 +75,8 @@ impl ArchiveRead { archive_header.extend(path_len.to_le_bytes()); archive_header.extend(path_string.as_bytes()); + // println!("{}", entry.path().display()); + // last access time let time_accessed = entry.metadata()?.accessed()?.duration_since(UNIX_EPOCH)?.as_secs(); archive_header.extend(time_accessed.to_le_bytes()); @@ -82,9 +84,10 @@ impl ArchiveRead { let time_modified = entry.metadata()?.modified()?.duration_since(UNIX_EPOCH)?.as_secs(); archive_header.extend(time_modified.to_le_bytes()); + let mut file_size = 0; if entry.file_type().is_file() { // file size - let file_size = entry.metadata()?.len(); + file_size = entry.metadata()?.len(); archive_header.extend(file_size.to_le_bytes()); } else if entry.file_type().is_symlink() { @@ -111,20 +114,20 @@ impl ArchiveRead { archive_header[0] = header_size[0]; archive_header[1] = header_size[1]; - let mut path = None; + let mut filepath_and_size = None; if entry.file_type().is_file() { - path = Some(entry.clone().into_path()); + filepath_and_size = Some((entry.clone().into_path(), file_size)); } - Ok((archive_header, path)) + Ok((archive_header, filepath_and_size)) } - fn get_next_archive_item(&mut self) -> Result<(Option>, Option)> { + fn get_next_archive_item(&mut self) -> Result<(Option>, Option<(PathBuf, u64)>)> { if let Some(entry) = self.walk_dir.next() { match &entry { Ok(entry) => { match Self::build_archive_header(entry) { - Ok((archive_header, filepath)) => Ok((Some(archive_header), filepath)), + Ok((archive_header, filepath_and_size)) => Ok((Some(archive_header), filepath_and_size)), Err(e) => Err(format!("Skipped entry {} Error: {e}", entry.path().display()).into()) } }, @@ -147,21 +150,21 @@ impl ReadChunk for ArchiveRead { return Ok((chunk, false)); } - if self.data_size == 0 { + if self.file_size == 0 { let archive_header; - let filepath; + let filepath_and_size; match self.get_next_archive_item() { - Ok(values) => (archive_header, filepath) = values, + Ok(values) => (archive_header, filepath_and_size) = values, Err(e) => { eprintln!("{e}"); continue; } } - if let Some(filepath) = &filepath { + if let Some((filepath, file_size)) = &filepath_and_size { if let Ok(f_in) = File::open(filepath) { - self.data_size = f_in.metadata()?.len(); + self.file_size = *file_size; self.f_in = Some(f_in); } else { eprintln!("Could not open - skipped: {}", filepath.display()); @@ -178,11 +181,11 @@ impl ReadChunk for ArchiveRead { self.no_more_files = true; } } else { - let buf_len = CHUNK_SIZE.min(self.data_size.try_into()?); + let buf_len = CHUNK_SIZE.min(self.file_size.try_into()?); let mut buf_read = vec![0u8; buf_len]; self.f_in.as_ref().unwrap().read_exact(&mut buf_read)?; - self.data_size -= u64::try_from(buf_len)?; + self.file_size -= u64::try_from(buf_len)?; self.buf_out.extend(buf_read); } @@ -251,6 +254,8 @@ impl ArchiveWrite { let entry_path; (entry_path, e) = Self::get_path_from_header(header, e, created_on_os_type)?; + // println!("{}", entry_path); + // access time s = e; e += size_of::(); let time_accessed_seconds = u64::from_le_bytes( header[s..e].try_into()? ); diff --git a/src/common_io.rs b/src/common_io.rs index e1bca8a..7db413b 100644 --- a/src/common_io.rs +++ b/src/common_io.rs @@ -313,6 +313,13 @@ impl CryptIo { drop(tx_in); + // join and error handling of file writer thread + match writer_handle.join() { + Ok(Ok(())) => {}, + Ok(Err(e)) => return Err(e.into()), + Err(panic) => return Err(format!("Writer thread panicked: {:?}", panic).into()), + } + // join and error handling of encryption/decryption threads for ch in crypt_handles { match ch.join() { @@ -322,13 +329,6 @@ impl CryptIo { } } - // join and error handling of file writer thread - match writer_handle.join() { - Ok(Ok(())) => {}, - Ok(Err(e)) => return Err(e.into()), - Err(panic) => return Err(format!("Writer thread panicked: {:?}", panic).into()), - } - Ok(()) } } diff --git a/src/decryption.rs b/src/decryption.rs index 5c045c1..92b578b 100644 --- a/src/decryption.rs +++ b/src/decryption.rs @@ -266,7 +266,7 @@ impl Decryption { for (buf_in, chunk_count, final_chunk) in rx_in { let buf_aes = Self::aes_decrypt_buffer(&key_aes, &buf_in).map_err(|e| e.to_string())?; let buf_cha = Self::cha_decrypt_buffer(&key_cha, &buf_aes, chunk_count, final_chunk).map_err(|e| e.to_string())?; - tx_e.send((buf_cha, chunk_count)).map_err(|e| e.to_string())?; + if tx_e.send((buf_cha, chunk_count)).is_err() { break } } Ok(()) })); @@ -293,7 +293,7 @@ impl Decryption { thread_handles.push(thread::spawn( move || -> std::result::Result<(), String> { for (buf_in, chunk_count) in rx_c { let buf_zip = Self::decompress_buffer(&buf_in).map_err(|e| e.to_string())?; - tx_out.send((buf_zip, chunk_count)).map_err(|e| e.to_string())?; + if tx_out.send((buf_zip, chunk_count)).is_err() { break } } Ok(()) })); diff --git a/src/encryption.rs b/src/encryption.rs index 8198134..2c1be81 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -302,7 +302,7 @@ impl Encryption { thread_handles.push(thread::spawn( move || -> std::result::Result<(), String> { for (buf_in, chunk_count, final_chunk) in rx_in { let buf_zip = Self::compress_buffer(&buf_in).map_err(|e| e.to_string())?; - tx_c.send((buf_zip, chunk_count, final_chunk)).map_err(|e| e.to_string())?; + if tx_c.send((buf_zip, chunk_count, final_chunk)).is_err() { break } } Ok(()) })); @@ -330,7 +330,7 @@ impl Encryption { for (buf_in, chunk_count, final_chunk) in rx_in { let buf_cha = Self::cha_encrypt_buffer(&key_cha, &buf_in, chunk_count, final_chunk).map_err(|e| e.to_string())?; let buf_aes = Self::aes_encrypt_buffer(&key_aes, &buf_cha).map_err(|e| e.to_string())?; - tx_out.send((buf_aes, chunk_count)).map_err(|e| e.to_string())?; + if tx_out.send((buf_aes, chunk_count)).is_err() { break } } Ok(()) })); From 158e41e3910a7721db9d399c3442a8e141ab1c8c Mon Sep 17 00:00:00 2001 From: JoergDF Date: Tue, 23 Jun 2026 14:36:50 +0200 Subject: [PATCH 04/16] Change sequential read of files for archive to parallel read with threads. --- Cargo.lock | 11 ++ Cargo.toml | 2 + src/archive.rs | 386 ++++++++++++++++++++++++++++++---------------- src/common_io.rs | 40 +++-- src/decryption.rs | 14 +- src/encryption.rs | 23 +-- 6 files changed, 312 insertions(+), 164 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4aad253..667e9aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -307,6 +307,7 @@ dependencies = [ "chacha20poly1305", "clap", "crossbeam-channel", + "filetime", "hkdf", "num_cpus", "parse-size", @@ -388,6 +389,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "foldhash" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index 5e00ae3..fc79add 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,8 @@ bzip2 = "0.6.1" chacha20poly1305 = "0.10.1" clap = { version = "4.5.60", features = ["derive"] } crossbeam-channel = "0.5.15" +#drill-press = "0.1.2" +filetime = "0.2.29" hkdf = "0.13.0" num_cpus = "1.17.0" parse-size = "1.1.0" diff --git a/src/archive.rs b/src/archive.rs index f28c25d..4e1ce60 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -1,14 +1,20 @@ +use std::collections::HashMap; use std::fs::{self, File, FileTimes}; use std::io::{Read, Write}; +use std::mem::{self, size_of}; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; -use typed_path::Utf8WindowsPath; -use std::mem::size_of; +use std::thread; use std::time::{Duration, UNIX_EPOCH}; -use walkdir::{WalkDir, IntoIter}; +use typed_path::Utf8WindowsPath; +use walkdir::WalkDir; +use crossbeam_channel::{Receiver, Select, TryRecvError, bounded}; +use num_cpus; +use filetime; +//use drill_press::{Segments, SparseFile}; -use crate::{CHUNK_SIZE, Result}; use crate::common_io::{ReadChunk, WriteFiles}; +use crate::{CHUNK_SIZE, Result}; const TYPE_FILE: u8 = 0x00; @@ -19,29 +25,124 @@ const TYPE_UNIX: u8 = 0x00; const TYPE_WINDOWS: u8 = 0x10; const ARCHIVE_HEADER_LENGTH_SIZE: usize = 2; - pub struct ArchiveRead { - walk_dir: IntoIter, - f_in: Option, - file_size: u64, + pub thread_handles: Vec>>, + rx_out_receivers: Vec, bool)>>, + channel_index: Option, + channel_finished: Vec, buf_out: Vec, - no_more_files: bool, } impl ArchiveRead { pub fn new(f_in_path: &Path) -> Self { - // in a directory: list files first, then sub-directories - let walk_dir = WalkDir::new(f_in_path).sort_by_key(|x| x.file_type().is_dir()).into_iter(); - let buf_out = Vec::with_capacity(CHUNK_SIZE * 3); + let num_workers = num_cpus::get(); // fixme: too much cpus? -1 for walkdir, and what about bzip,crypt? + let mut thread_handles = Vec::with_capacity(num_workers + 1); + let mut rx_out_receivers = Vec::with_capacity(num_workers); + + let (tx_paths, rx_paths) = bounded(num_workers * 2); + + { + let f_in_path = f_in_path.to_path_buf(); + let tx_paths = tx_paths.clone(); + thread_handles.push(thread::spawn(move || -> std::result::Result<(), String> { + for entry in WalkDir::new(f_in_path) { + match entry { + Ok(entry) => { + let _ = tx_paths.send(entry); + } + Err(ref e) => { return Err(format!("Skipped entry {entry:?} Error: {e}")); } + } + } + Ok(()) + })); + } - Self { walk_dir, f_in: None, file_size: 0, buf_out, no_more_files: false } + drop(tx_paths); + + for _ in 0..num_workers { + let rx_paths = rx_paths.clone(); + let (tx_out, rx_out) = bounded(num_workers); + rx_out_receivers.push(rx_out); + + thread_handles.push(thread::spawn(move || -> std::result::Result<(), String> { + for entry in rx_paths { + let archive_header; + let filepath_and_size; + //let sparse_segments; //fixme + // println!("{:?}", entry); + match Self::build_archive_header(&entry) { + Ok(values) => (archive_header, filepath_and_size/* , sparse_segments */) = values, + Err(e) => { + eprintln!("Skipped entry {} - Reason: {e}", entry.path().display()); + continue; + } + } + + if let Some((filepath, mut file_size)) = filepath_and_size { + if let Ok(mut f_in) = File::open(&filepath) { + //println!("send fah {} {}", archive_header.len(), file_size); + + // send header of file + // empty files (with length 0), must set last_chunk to true + let last_chunk = file_size == 0; + let _ = tx_out.send((archive_header, last_chunk)); + + // if file_size == 0 { continue; } + // fixme + // let seg_data_size: u64 = sparse_segments.data().map(|sd| sd.end - sd.start).sum(); + // let mut data_size = if sparse_segments.is_empty() { + // file_size + // } else { + // seg_data_size + // }; + + // while data_size != 0 { + // let buf_len = CHUNK_SIZE.min(usize::try_from(data_size).map_err(|e| e.to_string())?); + // let mut buf_read = vec![0u8; buf_len]; + // } + + + // read file and send its data + while file_size != 0 { + let buf_len = CHUNK_SIZE.min(usize::try_from(file_size).map_err(|e| e.to_string())?); + let mut buf_read = vec![0u8; buf_len]; + + f_in.read_exact(&mut buf_read).map_err(|e| e.to_string())?; + file_size -= buf_len as u64; + + let last_chunk = file_size == 0; + let _ = tx_out.send((buf_read, last_chunk)); + //println!("{:?} {} {} {}", filepath, file_size, buf_len, last_chunk); + } + } else { + eprintln!("Could not open - skipped: {}", filepath.display()); + continue; + } + } else { + // send header of directory + //println!("send ah {}", archive_header.len()); + let _ = tx_out.send((archive_header, true)); + } + } + //println!("DONE"); // {}", rx_out_receivers.clone().len()); + // all entries done, send finish message + //let _ = tx_out.send((Vec::new(), true)); + + Ok(()) + })); + } + + let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); + + Self { thread_handles, rx_out_receivers, channel_index: None, channel_finished: vec![false; num_workers], buf_out } } - fn build_archive_header(entry: &walkdir::DirEntry) -> Result<(Vec, Option<(PathBuf, u64)>)> { - // archive header initialized with place holder for header size + #[allow(clippy::type_complexity)] + fn build_archive_header(entry: &walkdir::DirEntry) -> Result<(Vec, Option<(PathBuf, u64)>/* , Vec */)> { + // archive header initialized with place holder for header size let mut archive_header = vec![0u8; ARCHIVE_HEADER_LENGTH_SIZE]; - let mut entry_type = if entry.file_type().is_file() { + let entry_type = if entry.file_type().is_file() { TYPE_FILE } else if entry.file_type().is_dir() { TYPE_DIRECTORY @@ -50,27 +151,19 @@ impl ArchiveRead { if entry.path().is_dir() { TYPE_SYMLINK_DIR } else { - // if target of symlink does not exist (hence it can't be evaluated - // whether it is a file or a directory), type file is used + // if target of symlink does not exist (hence it can't be evaluated + // whether it is a file or a directory), type file is used TYPE_SYMLINK_FILE } } else { return Err(format!("Ignored unsupported file type for archive: {}", entry.path().display()).into()); }; - if cfg!(windows) { - entry_type |= TYPE_WINDOWS; - } else { - entry_type |= TYPE_UNIX; - } - archive_header.push(entry_type); + let os_type = if cfg!(unix) { TYPE_UNIX } else { TYPE_WINDOWS }; + archive_header.push(os_type | entry_type); // path length and path (including filename) - let path_string = if entry.file_type().is_dir() { - entry.path().to_string_lossy() - } else { - entry.file_name().to_string_lossy() - }; + let path_string = entry.path().to_string_lossy(); let path_len: u16 = path_string.len().try_into()?; archive_header.extend(path_len.to_le_bytes()); archive_header.extend(path_string.as_bytes()); @@ -85,12 +178,30 @@ impl ArchiveRead { archive_header.extend(time_modified.to_le_bytes()); let mut file_size = 0; - if entry.file_type().is_file() { + //let mut sparse_segments = vec![]; + if entry_type == TYPE_FILE { // file size file_size = entry.metadata()?.len(); archive_header.extend(file_size.to_le_bytes()); - - } else if entry.file_type().is_symlink() { + + // get holes of sparse files + //if file_size > 0 { // fixme + // if let Ok(mut f_in) = File::open(&entry.path()) { + // sparse_segments = f_in.scan_chunks()?; + + // archive_header.extend( u32::try_from(sparse_segments.holes().count())?.to_le_bytes() ); + + // for hole in sparse_segments.holes() { + // archive_header.extend(hole.start.to_le_bytes()); + // archive_header.extend(hole.end.to_le_bytes()); + // } + // } else { + // // could not open file, add 0 holes fixme: correct? + // archive_header.extend( 0u32.to_le_bytes() ); + // } + // println!("{:?} {:?}", sparse_segments, entry.path()); + //} + } else if entry_type == TYPE_SYMLINK_FILE || entry_type == TYPE_SYMLINK_DIR { // target path of symlink let target_path = fs::read_link(entry.path())?; let target_path_string = target_path.to_string_lossy(); @@ -101,7 +212,7 @@ impl ArchiveRead { // permissions let mut perm: u16 = 0; - if cfg!(unix) && (entry.file_type().is_file() || entry.file_type().is_dir()) { + if os_type == TYPE_UNIX && (entry_type == TYPE_FILE || entry_type == TYPE_DIRECTORY) { use std::os::unix::fs::PermissionsExt; let permission_mode = entry.metadata()?.permissions().mode(); // use 12 least significant bits @@ -115,109 +226,89 @@ impl ArchiveRead { archive_header[1] = header_size[1]; let mut filepath_and_size = None; - if entry.file_type().is_file() { + if entry_type == TYPE_FILE { filepath_and_size = Some((entry.clone().into_path(), file_size)); } - Ok((archive_header, filepath_and_size)) - } - - fn get_next_archive_item(&mut self) -> Result<(Option>, Option<(PathBuf, u64)>)> { - if let Some(entry) = self.walk_dir.next() { - match &entry { - Ok(entry) => { - match Self::build_archive_header(entry) { - Ok((archive_header, filepath_and_size)) => Ok((Some(archive_header), filepath_and_size)), - Err(e) => Err(format!("Skipped entry {} Error: {e}", entry.path().display()).into()) - } - }, - Err(e) => Err(format!("Skipped entry {entry:?} Error: {e}").into()) - } - } else { - // no more entries - Ok((None, None)) - } + Ok((archive_header, filepath_and_size/* , sparse_segments */)) } } impl ReadChunk for ArchiveRead { fn read_chunk(&mut self) -> Result<(Vec, bool)> { - - while !self.no_more_files { - // buffer 2 chunks, hence read ahead 1 chunk to detect the final file and final-chunk flag can be set on the last chunk - if self.buf_out.len() > 2 * CHUNK_SIZE { - let chunk = self.buf_out.drain(..CHUNK_SIZE).collect(); - return Ok((chunk, false)); + while !self.channel_finished.iter().all(|x| *x) && self.buf_out.len() <= CHUNK_SIZE { + // stay on same channel until last chunk of file using a blocking receive + if let Some(channel_index) = self.channel_index + && let Ok((data, last_chunk)) = self.rx_out_receivers[channel_index].recv() + { + //println!("cont recv, dat_len: {}, l {}", data.len(), last_chunk); + self.buf_out.extend(data); + if last_chunk { + self.channel_index = None; + } + continue; // check if there is already enough data in self.buf_out[] } - if self.file_size == 0 { - let archive_header; - let filepath_and_size; - - match self.get_next_archive_item() { - Ok(values) => (archive_header, filepath_and_size) = values, - Err(e) => { - eprintln!("{e}"); - continue; - } - } + let mut sel = Select::new(); + for rx in &self.rx_out_receivers { + sel.recv(rx); + } - if let Some((filepath, file_size)) = &filepath_and_size { - if let Ok(f_in) = File::open(filepath) { - self.file_size = *file_size; - self.f_in = Some(f_in); - } else { - eprintln!("Could not open - skipped: {}", filepath.display()); - continue; + let sel_rdy_idx = sel.ready(); + //println!("sel_rdy_idx {}", sel_rdy_idx); + match self.rx_out_receivers[sel_rdy_idx].try_recv() { + Ok((data, last_chunk)) => { + // println!("recv {}, dat_len: {}, l {}", sel_rdy_idx, data.len(), last_chunk); + self.buf_out.extend(data); + if !last_chunk { + self.channel_index = Some(sel_rdy_idx); } } - - // if file could not be opened, its admin data should be skipped, - // therefore save the admin data after the file handling, - // but for directories it is required - if let Some(hdr) = &archive_header { - self.buf_out.extend(hdr); - } else { - self.no_more_files = true; - } - } else { - let buf_len = CHUNK_SIZE.min(self.file_size.try_into()?); - let mut buf_read = vec![0u8; buf_len]; - - self.f_in.as_ref().unwrap().read_exact(&mut buf_read)?; - self.file_size -= u64::try_from(buf_len)?; - - self.buf_out.extend(buf_read); + Err(TryRecvError::Disconnected) => { self.channel_finished[sel_rdy_idx] = true; } + Err(TryRecvError::Empty) => {} } } if self.buf_out.len() > CHUNK_SIZE { let chunk = self.buf_out.drain(..CHUNK_SIZE).collect(); - return Ok((chunk, false)); + Ok((chunk, false)) + } else { + // set final chunk flag + Ok((self.buf_out.clone(), true)) } + } - // set final chunk flag - Ok((self.buf_out.clone(), true)) + fn join_threads(&mut self) -> Result<()> { + let thread_handles = mem::take(&mut self.thread_handles); + for th in thread_handles { + match th.join() { + Ok(Ok(())) => {} + Ok(Err(e)) => return Err(e.into()), + Err(panic) => { + return Err(format!("ArchiveRead thread panicked: {:?}", panic).into()); + } + } + } + Ok(()) } } - #[derive(Default)] pub struct ArchiveWrite { f_out: Option, buf_out: Vec, header_length: Option, - file_size: u64, + file_size: u64, file_times: FileTimes, file_path: PathBuf, - dir_path: PathBuf, + dir_times: HashMap, } impl ArchiveWrite { - pub fn new() -> Self { + pub fn new() -> Self { let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), - file_path: PathBuf::new(), dir_path: PathBuf::new() } + file_path: PathBuf::new(), dir_times: HashMap::new() } } fn get_path_from_header(header: &[u8], current_end_index: usize, created_on_os_type: u8) -> Result<(String, usize)> { @@ -226,14 +317,14 @@ impl ArchiveWrite { // new end index of header field let mut e = current_end_index + size_of::(); - // path length + // path length let path_len = u16::from_le_bytes(header[s..e].try_into()?); // path s = e; e += usize::from(path_len); let path_bytes = &header[s..e]; let path_str = str::from_utf8(path_bytes)?; - // convert Windows path to unix path, it on unix (windows can handle unix path) - let entry_path = if cfg!(unix) && created_on_os_type == TYPE_WINDOWS { + // convert Windows path to unix path, if on unix (windows can handle unix path) + let entry_path = if cfg!(unix) && created_on_os_type == TYPE_WINDOWS { Utf8WindowsPath::new(path_str).with_unix_encoding().to_string() } else { path_str.to_string() @@ -247,21 +338,22 @@ impl ArchiveWrite { let file_type = header[0] & 0x0F; let created_on_os_type = header[0] & 0xF0; - let mut s ; + let mut s; let mut e = 1; // entry's path - let entry_path; - (entry_path, e) = Self::get_path_from_header(header, e, created_on_os_type)?; + let entry_path_string; + (entry_path_string, e) = Self::get_path_from_header(header, e, created_on_os_type)?; + let entry_path = PathBuf::from(&entry_path_string); - // println!("{}", entry_path); + println!("{}", entry_path.display()); // access time - s = e; e += size_of::(); + s = e; e += size_of::(); let time_accessed_seconds = u64::from_le_bytes( header[s..e].try_into()? ); let time_accessed = UNIX_EPOCH + Duration::from_secs(time_accessed_seconds); // modification time - s = e; e += size_of::(); + s = e; e += size_of::(); let time_modified_seconds = u64::from_le_bytes( header[s..e].try_into()? ); let time_modified = UNIX_EPOCH + Duration::from_secs(time_modified_seconds); self.file_times = FileTimes::new() @@ -271,57 +363,80 @@ impl ArchiveWrite { // create type if file_type == TYPE_DIRECTORY { fs::create_dir_all(&entry_path)?; - self.dir_path = PathBuf::from(&entry_path); + println!("D: {:?} {:?}", entry_path, self.file_times); + // set timestamps of directory + // if files will be added afterwards, the directory's original timestamps need to be set again if !File::open(&entry_path).is_ok_and(|dir| dir.set_times(self.file_times).is_ok()) { - eprintln!("Could not set timestamps for directory {}", entry_path); + eprintln!("Could not set original timestamps for directory {}", entry_path.display()); } + + self.dir_times.insert(entry_path.clone(), self.file_times); + } else if file_type == TYPE_FILE { - self.file_path = self.dir_path.join(&entry_path); - self.f_out = Some( File::create(&self.file_path)? ); - + // create directory (of file), if it doesn't exists + if let Some(dir) = &entry_path.parent() && !dir.exists() { + println!("F: {:?}", dir); + fs::create_dir_all(dir)?; + } + + self.f_out = Some(File::create(&entry_path)?); + + // for error handling + self.file_path = entry_path.clone(); + // file size - s = e; e += size_of::(); + s = e; e += size_of::(); self.file_size = u64::from_le_bytes( header[s..e].try_into()? ); } else if file_type == TYPE_SYMLINK_FILE || file_type == TYPE_SYMLINK_DIR { + // create directory (of symlink), if it doesn't exists + if let Some(dir) = &entry_path.parent() && !dir.exists() { + println!("S: {:?}", dir); + fs::create_dir_all(dir)?; + } + // symlink's target path let target_path; (target_path, e) = Self::get_path_from_header(header, e, created_on_os_type)?; - let sym_path = self.dir_path.join(&entry_path); - // create symlink // remove it, if it already exists, otherwise symlink can't be created - let _ = fs::remove_file(&sym_path); + let _ = fs::remove_file(&entry_path); #[cfg(unix)] { - std::os::unix::fs::symlink(target_path, &sym_path)?; + std::os::unix::fs::symlink(&target_path, &entry_path)?; } #[cfg(windows)] { if file_type == TYPE_SYMLINK_FILE { - std::os::windows::fs::symlink_file(&target_path, &sym_path)?; + std::os::windows::fs::symlink_file(&target_path, &entry_path)?; } if file_type == TYPE_SYMLINK_DIR { - std::os::windows::fs::symlink_dir(&target_path, &sym_path)?; + std::os::windows::fs::symlink_dir(&target_path, &entry_path)?; } } + + // set timestamps of symlink + // replace with fs::set_times_nofollow() when in stable rust version + if filetime::set_symlink_file_times( + &entry_path, + filetime::FileTime::from_system_time(time_accessed), + filetime::FileTime::from_system_time(time_modified) + ).is_err() { + eprintln!("Could not set original timestamps for symlink {}", entry_path.display()); + } } else { return Err(format!("Archive contains unknown file type: {file_type}").into()); } // permissions // if this is a unix system and the archive was created on a unix system, set permission mode - if cfg!(unix) && created_on_os_type == TYPE_UNIX && (file_type == TYPE_DIRECTORY || file_type == TYPE_FILE) { + if cfg!(unix) && created_on_os_type == TYPE_UNIX && (file_type == TYPE_DIRECTORY || file_type == TYPE_FILE) + { s = e; e += size_of::(); - let perm = u16::from_le_bytes( header[s..e].try_into()? ); + let perm = u16::from_le_bytes(header[s..e].try_into()?); - let path = if file_type == TYPE_FILE { - &self.file_path - } else { - &self.dir_path - }; - let fd = File::open(path)?; + let fd = File::open(&entry_path)?; let mut permissions = fd.metadata()?.permissions(); let mode_masked = permissions.mode() & 0xFFFF_F000; permissions.set_mode(mode_masked | u32::from(perm & 0x0FFF)); @@ -348,8 +463,17 @@ impl WriteFiles for ArchiveWrite { f_out.write_all(&file_data)?; // set file times after all data has been written if f_out.set_times(self.file_times).is_err() { - eprintln!("Could not set timestamps for file {}", self.file_path.display()); + eprintln!("Could not set original timestamps for file {}", self.file_path.display()); } + // as a new file was created, the file's parent directory would get the current timestamp, + // but the original one is desired, therefore set original timestamp for the directory + if let Some(dir_path) = self.file_path.parent() + && let Some(file_times) = self.dir_times.get(dir_path) { + if !File::open(dir_path).is_ok_and(|dir| dir.set_times(*file_times).is_ok()) { + eprintln!("Could not set original timestamps for directory {}", dir_path.display()); + } + } else { /* do nothing */ } + self.file_size = 0; self.f_out = None; } diff --git a/src/common_io.rs b/src/common_io.rs index 7db413b..cd67bb1 100644 --- a/src/common_io.rs +++ b/src/common_io.rs @@ -5,13 +5,14 @@ use std::thread; use crossbeam_channel::{bounded, Sender, Receiver}; use secrecy::SecretSlice; use std::collections::HashMap; - +use num_cpus; use crate::{AES_NONCE_SIZE, AES_TAG_SIZE, CHA_NONCE_SIZE, CHA_TAG_SIZE, Result, SPLIT_ENC_FILE_EXT}; pub trait ReadChunk { fn read_chunk(&mut self) -> Result<(Vec, bool)>; + fn join_threads(&mut self) -> Result<()>; } pub trait WriteFiles { @@ -47,8 +48,8 @@ impl ReadInput { /// # Returns /// - `Ok(Self)` on success /// - `Err(...)` when an I/O or conversion error occurs - pub fn new(f_in_path: PathBuf, chunk_size: usize, f_in_header_size: u64) -> Result { - let f_in = File::open(&f_in_path)?; + pub fn new(f_in_path: &PathBuf, chunk_size: usize, f_in_header_size: u64) -> Result { + let f_in = File::open(f_in_path)?; let mut f_in_total_size = 0; let mut f_in_split = vec![]; @@ -74,7 +75,7 @@ impl ReadInput { // header was already read, therefore remaining file size must be reduced by the header's size let f_in_total_size_remaining = f_in_total_size - f_in_header_size; - Ok( Self { f_in, f_in_path, f_in_total_size_remaining, chunk_size, f_in_split, split_index: 0, f_in_read_count: 0 } ) + Ok( Self { f_in, f_in_path: f_in_path.clone(), f_in_total_size_remaining, chunk_size, f_in_split, split_index: 0, f_in_read_count: 0 } ) } /// Calculate how much need to be read for the current chunk, whether it is the final chunk @@ -159,6 +160,11 @@ impl ReadChunk for ReadInput { Ok((buf_in, final_chunk)) } + + fn join_threads(&mut self) -> Result<()> { + // do nothing + Ok(()) + } } @@ -274,8 +280,8 @@ impl CryptIo { Receiver<(Vec, u32, bool)>, Sender<(Vec, u32)>, usize) -> Vec>>, - read_input: &mut (impl ReadChunk + ?Sized), - mut write_output: Box, + mut read_input: Box, + mut write_output: Box, ) -> Result<()> { let cpu_count = num_cpus::get(); @@ -313,6 +319,9 @@ impl CryptIo { drop(tx_in); + // get errors of threads spawned in ArchiveRead::new() + read_input.join_threads()?; + // join and error handling of file writer thread match writer_handle.join() { Ok(Ok(())) => {}, @@ -328,7 +337,8 @@ impl CryptIo { Err(panic) => return Err(format!("Crypt thread panicked: {:?}", panic).into()), } } - + + Ok(()) } } @@ -346,14 +356,14 @@ mod tests { #[test] fn test_input_sizes() { - let test_file = "test_input_sizes.bin"; + let test_file = &PathBuf::from("test_input_sizes.bin"); let data = vec![0u8; 100]; fs::write(test_file, &data).unwrap(); // Case 1: chunk not final // File size 100, header 10 -> remaining 90. Chunk 50. - let mut ri = ReadInput::new(PathBuf::from(test_file), 50, 10).unwrap(); + let mut ri = ReadInput::new(test_file, 50, 10).unwrap(); let (read_size, final_chunk) = ri.input_sizes().unwrap(); assert_eq!(read_size, 50); assert!(!final_chunk); @@ -381,21 +391,21 @@ mod tests { // only first split let mut buf = [0u8; 1000]; - let f_in_path = PathBuf::from("test_split_in.c00"); - let mut ri = ReadInput::new(f_in_path.clone(), 0, HEADER_SIZE as u64).unwrap(); + let f_in_path = &PathBuf::from("test_split_in.c00"); + let mut ri = ReadInput::new(f_in_path, 0, HEADER_SIZE as u64).unwrap(); ri.read_files(&mut buf).unwrap(); assert_eq!(buf, data0); // all splits let mut buf = [0u8; 3028]; - let mut ri = ReadInput::new(f_in_path.clone(), 0, HEADER_SIZE as u64).unwrap(); + let mut ri = ReadInput::new(f_in_path, 0, HEADER_SIZE as u64).unwrap(); ri.read_files(&mut buf).unwrap(); assert_eq!(buf[..], [&data0[..], &data1[..], &data2[..]].concat()); // 2 reads let mut buf0 = [0u8; 4]; let mut buf1 = [0u8; 2000]; - let mut ri = ReadInput::new(f_in_path.clone(), 0, HEADER_SIZE as u64).unwrap(); + let mut ri = ReadInput::new(f_in_path, 0, HEADER_SIZE as u64).unwrap(); ri.read_files(&mut buf0).unwrap(); assert_eq!(buf0[..], data0[..4]); ri.read_files(&mut buf1).unwrap(); @@ -475,7 +485,7 @@ mod tests { #[test] fn test_read_chunk() { - let test_file = "test_read_chunk.bin"; + let test_file = &PathBuf::from("test_read_chunk.bin"); let data = vec![1u8; 100]; fs::write(test_file, &data).unwrap(); @@ -485,7 +495,7 @@ mod tests { // Chunk 1: 40 bytes, final=false. // Chunk 2: 40 bytes, final=false. // Chunk 3: 10 bytes, final=true. - let mut ri = ReadInput::new(PathBuf::from(test_file), 40, 10).unwrap(); + let mut ri = ReadInput::new(test_file, 40, 10).unwrap(); // Skip header let mut header = [0u8; 10]; diff --git a/src/decryption.rs b/src/decryption.rs index 92b578b..6bac3af 100644 --- a/src/decryption.rs +++ b/src/decryption.rs @@ -1,5 +1,5 @@ use std::thread; -use std::{io::Read, path::Path}; +use std::io::Read; use std::path::PathBuf; use std::collections::HashMap; use argon2::Argon2; @@ -316,12 +316,12 @@ impl Decryption { /// # Returns /// - `Ok(())` on successful decryption /// - `Err` if file operations, password handling, or decryption fails - pub fn decrypt(filepath_in: &Path, keyfilepath: Option<&PathBuf>) -> Result<()> { + pub fn decrypt(filepath_in: &PathBuf, keyfilepath: Option<&PathBuf>) -> Result<()> { if filepath_in.is_dir() { return Err("Cannot decrypt a directory".into()); } - let mut filepath_out = filepath_in.to_path_buf(); + let mut filepath_out = filepath_in.clone(); if filepath_in.extension() == Some(std::ffi::OsStr::new(ENCRYPTED_FILE_EXT)) || filepath_in.extension() == Some(std::ffi::OsStr::new(SPLIT_ENC_FILE_EXT)) { // remove encrypted-file-extension @@ -331,11 +331,11 @@ impl Decryption { } // set read parameters - let mut read_input = ReadInput::new( - filepath_in.to_path_buf(), + let mut read_input = Box:: new( ReadInput::new( + filepath_in, CHUNK_SIZE + CHA_NONCE_SIZE + CHA_TAG_SIZE + AES_NONCE_SIZE + AES_TAG_SIZE, HEADER_SIZE as u64 - )?; + )? ); // Read file header let mut header = [0u8; HEADER_SIZE]; @@ -368,7 +368,7 @@ impl Decryption { Box::new( WriteOutput::new(filepath_out, vec![])? ) }; - CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::decrypt_pipe, &mut read_input, write_output)?; + CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::decrypt_pipe, read_input, write_output)?; Ok(()) } diff --git a/src/encryption.rs b/src/encryption.rs index 2c1be81..b9aac42 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -1,5 +1,5 @@ use std::io::Read; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::thread; use argon2::Argon2; use chacha20poly1305::{XChaCha20Poly1305}; @@ -355,25 +355,26 @@ impl Encryption { /// # Returns /// - `Ok(())` on successful encryption /// - `Err` if file operations, password handling, or encryption fails - pub fn encrypt(filepath_in: &Path, keyfilepath: Option<&PathBuf>, compress: bool, split: Vec) -> Result<()> { - let mut filepath_out = filepath_in.to_path_buf(); + pub fn encrypt(filepath_in: &PathBuf, keyfilepath: Option<&PathBuf>, compress: bool, split: Vec) -> Result<()> { + let mut filepath_out = filepath_in.clone(); if split.is_empty() { filepath_out.add_extension(ENCRYPTED_FILE_EXT); } else { filepath_out.add_extension(SPLIT_ENC_FILE_EXT); } + + // ask for password, before there can be error messages of archive + let (salt_pw, key) = Self::hash_password(keyfilepath)?; + let (salt_cha, key_cha, salt_aes, key_aes) = Self::derive_keys(&key)?; let build_archive = filepath_in.is_dir(); - let mut read_input: Box = if build_archive { - Box::new( ArchiveRead::new(filepath_in) ) + let read_input: Box = if build_archive { + Box::new( ArchiveRead::new(filepath_in) ) } else { - Box::new( ReadInput::new(filepath_in.to_path_buf(), CHUNK_SIZE, 0)? ) + Box::new( ReadInput::new(filepath_in, CHUNK_SIZE, 0)? ) }; - let (salt_pw, key) = Self::hash_password(keyfilepath)?; - let (salt_cha, key_cha, salt_aes, key_aes) = Self::derive_keys(&key)?; - // file header // byte description // 0 version of file format @@ -390,13 +391,13 @@ impl Encryption { header.extend(salt_cha); header.extend(salt_aes); - // set write parameters and create output file + // set write parameters and create output file let mut write_output = Box::new( WriteOutput::new(filepath_out, split)? ); // write header write_output.write_files(&header)?; - CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::encrypt_pipe, read_input.as_mut(), write_output)?; + CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::encrypt_pipe, read_input, write_output)?; Ok(()) } From 2abf65335863a7bbc69609d07c3dce3977d5e736 Mon Sep 17 00:00:00 2001 From: JoergDF Date: Thu, 25 Jun 2026 16:53:10 +0200 Subject: [PATCH 05/16] Add support for hard links in archives. Read and save them only on unix. Write them on unix and windows. --- src/archive.rs | 217 +++++++++++++++++++++++++++++++++-------------- src/common_io.rs | 9 +- 2 files changed, 163 insertions(+), 63 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 4e1ce60..98e55e3 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -1,8 +1,6 @@ -use std::collections::HashMap; use std::fs::{self, File, FileTimes}; use std::io::{Read, Write}; use std::mem::{self, size_of}; -use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::thread; use std::time::{Duration, UNIX_EPOCH}; @@ -11,6 +9,8 @@ use walkdir::WalkDir; use crossbeam_channel::{Receiver, Select, TryRecvError, bounded}; use num_cpus; use filetime; +#[cfg(unix)] +use std::collections::HashMap; //use drill_press::{Segments, SparseFile}; use crate::common_io::{ReadChunk, WriteFiles}; @@ -21,6 +21,7 @@ const TYPE_FILE: u8 = 0x00; const TYPE_DIRECTORY: u8 = 0x01; const TYPE_SYMLINK_FILE: u8 = 0x02; const TYPE_SYMLINK_DIR: u8 = 0x03; +const TYPE_HARDLINK: u8 = 0x04; const TYPE_UNIX: u8 = 0x00; const TYPE_WINDOWS: u8 = 0x10; const ARCHIVE_HEADER_LENGTH_SIZE: usize = 2; @@ -45,10 +46,34 @@ impl ArchiveRead { let f_in_path = f_in_path.to_path_buf(); let tx_paths = tx_paths.clone(); thread_handles.push(thread::spawn(move || -> std::result::Result<(), String> { + #[cfg(unix)] + let mut hard_link_files: HashMap = HashMap::new(); + for entry in WalkDir::new(f_in_path) { match entry { Ok(entry) => { - let _ = tx_paths.send(entry); + // check if entry is a hard link + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + use walkdir::DirEntryExt; + + let mut hard_link_target: Option = None; + if entry.file_type().is_file() + && let Ok(meta) = entry.metadata() && meta.nlink() > 1 { + let file_id = entry.ino(); + if let Some(hl_target) = hard_link_files.get(&file_id) { + // entry is hard link + hard_link_target = Some(hl_target.to_owned()); + } else { + // entry is taken as original file path (i.e. target of hard link) + hard_link_files.insert(file_id, entry.clone().into_path()); + } + } + let _ = tx_paths.send((entry, hard_link_target)); + } + #[cfg(windows)] + let _ = tx_paths.send((entry, None)); } Err(ref e) => { return Err(format!("Skipped entry {entry:?} Error: {e}")); } } @@ -65,12 +90,12 @@ impl ArchiveRead { rx_out_receivers.push(rx_out); thread_handles.push(thread::spawn(move || -> std::result::Result<(), String> { - for entry in rx_paths { + for (entry, hard_link_target) in rx_paths { let archive_header; let filepath_and_size; //let sparse_segments; //fixme // println!("{:?}", entry); - match Self::build_archive_header(&entry) { + match Self::build_archive_header(&entry, &hard_link_target) { Ok(values) => (archive_header, filepath_and_size/* , sparse_segments */) = values, Err(e) => { eprintln!("Skipped entry {} - Reason: {e}", entry.path().display()); @@ -119,7 +144,7 @@ impl ArchiveRead { continue; } } else { - // send header of directory + // send header of entries without additional data //println!("send ah {}", archive_header.len()); let _ = tx_out.send((archive_header, true)); } @@ -137,12 +162,23 @@ impl ArchiveRead { Self { thread_handles, rx_out_receivers, channel_index: None, channel_finished: vec![false; num_workers], buf_out } } + fn add_path_to_header(path: &Path, archive_header: &mut Vec) -> Result<()> { + // path length and path + let path_string = path.to_string_lossy(); + let path_len: u16 = path_string.len().try_into()?; + archive_header.extend(path_len.to_le_bytes()); + archive_header.extend(path_string.as_bytes()); + Ok(()) + } + #[allow(clippy::type_complexity)] - fn build_archive_header(entry: &walkdir::DirEntry) -> Result<(Vec, Option<(PathBuf, u64)>/* , Vec */)> { + fn build_archive_header(entry: &walkdir::DirEntry, hard_link_target: &Option) -> Result<(Vec, Option<(PathBuf, u64)>/* , Vec */)> { // archive header initialized with place holder for header size let mut archive_header = vec![0u8; ARCHIVE_HEADER_LENGTH_SIZE]; - let entry_type = if entry.file_type().is_file() { + let entry_type = if hard_link_target.is_some() { + TYPE_HARDLINK + } else if entry.file_type().is_file() { TYPE_FILE } else if entry.file_type().is_dir() { TYPE_DIRECTORY @@ -163,13 +199,25 @@ impl ArchiveRead { archive_header.push(os_type | entry_type); // path length and path (including filename) - let path_string = entry.path().to_string_lossy(); - let path_len: u16 = path_string.len().try_into()?; - archive_header.extend(path_len.to_le_bytes()); - archive_header.extend(path_string.as_bytes()); + Self::add_path_to_header(entry.path(), &mut archive_header)?; // println!("{}", entry.path().display()); + if entry_type == TYPE_HARDLINK { + + // target path of hard link + let target_path = hard_link_target.as_ref().unwrap(); + Self::add_path_to_header(target_path, &mut archive_header)?; + + // header size + let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from( + archive_header.len() - ARCHIVE_HEADER_LENGTH_SIZE)?.to_le_bytes(); + archive_header[0] = header_size[0]; + archive_header[1] = header_size[1]; + + return Ok((archive_header, None)) + } + // last access time let time_accessed = entry.metadata()?.accessed()?.duration_since(UNIX_EPOCH)?.as_secs(); archive_header.extend(time_accessed.to_le_bytes()); @@ -201,27 +249,34 @@ impl ArchiveRead { // } // println!("{:?} {:?}", sparse_segments, entry.path()); //} + } else if entry_type == TYPE_SYMLINK_FILE || entry_type == TYPE_SYMLINK_DIR { // target path of symlink let target_path = fs::read_link(entry.path())?; - let target_path_string = target_path.to_string_lossy(); - let target_path_len: u16 = target_path_string.len().try_into()?; - archive_header.extend(target_path_len.to_le_bytes()); - archive_header.extend(target_path_string.as_bytes()); + Self::add_path_to_header(&target_path, &mut archive_header)?; } // permissions - let mut perm: u16 = 0; - if os_type == TYPE_UNIX && (entry_type == TYPE_FILE || entry_type == TYPE_DIRECTORY) { + #[cfg(unix)] + { use std::os::unix::fs::PermissionsExt; - let permission_mode = entry.metadata()?.permissions().mode(); - // use 12 least significant bits - perm = (permission_mode & 0x0FFF) as u16; + + let mut perm: u16 = 0; + if entry_type == TYPE_FILE || entry_type == TYPE_DIRECTORY { + let permission_mode = entry.metadata()?.permissions().mode(); + // use 12 least significant bits + perm = (permission_mode & 0x0FFF) as u16; + } + archive_header.extend(perm.to_le_bytes()); + } + #[cfg(windows)] + { + archive_header.extend(0u16.to_le_bytes()); } - archive_header.extend(perm.to_le_bytes()); // header size - let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from(archive_header.len() - 2)?.to_le_bytes(); + let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from( + archive_header.len() - ARCHIVE_HEADER_LENGTH_SIZE)?.to_le_bytes(); archive_header[0] = header_size[0]; archive_header[1] = header_size[1]; @@ -301,14 +356,25 @@ pub struct ArchiveWrite { file_size: u64, file_times: FileTimes, file_path: PathBuf, - dir_times: HashMap, + dir_times: Vec<(PathBuf, FileTimes)>, + pending_hardlinks: Vec<(PathBuf, PathBuf)>, } impl ArchiveWrite { pub fn new() -> Self { let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), - file_path: PathBuf::new(), dir_times: HashMap::new() } + file_path: PathBuf::new(), dir_times: vec![], pending_hardlinks: vec![] } + } + + fn create_parent_directory(entry_path: &Path) -> Result<()> { + // create directory, if it doesn't exists + // as parallel threads are used to create an archive, the sequence of entries differ from the sequence returned be walkdir + // hence a file, etc. might show up before its directory was created + if let Some(dir) = entry_path.parent() && !dir.exists() { + fs::create_dir_all(dir)?; + } + Ok(()) } fn get_path_from_header(header: &[u8], current_end_index: usize, created_on_os_type: u8) -> Result<(String, usize)> { @@ -346,7 +412,16 @@ impl ArchiveWrite { (entry_path_string, e) = Self::get_path_from_header(header, e, created_on_os_type)?; let entry_path = PathBuf::from(&entry_path_string); - println!("{}", entry_path.display()); + // println!("{}", entry_path.display()); + + if file_type == TYPE_HARDLINK { + // hard link's target path + let target_path; + (target_path, _) = Self::get_path_from_header(header, e, created_on_os_type)?; + + self.pending_hardlinks.push((PathBuf::from(target_path), entry_path)); + return Ok(()) + } // access time s = e; e += size_of::(); @@ -362,23 +437,16 @@ impl ArchiveWrite { // create type if file_type == TYPE_DIRECTORY { + // create directory + // it could be an empty one therefore it would not be created by other entries fs::create_dir_all(&entry_path)?; - println!("D: {:?} {:?}", entry_path, self.file_times); - - // set timestamps of directory - // if files will be added afterwards, the directory's original timestamps need to be set again - if !File::open(&entry_path).is_ok_and(|dir| dir.set_times(self.file_times).is_ok()) { - eprintln!("Could not set original timestamps for directory {}", entry_path.display()); - } - self.dir_times.insert(entry_path.clone(), self.file_times); + // save timestamps for restoring them at the end + self.dir_times.push((entry_path.clone(), self.file_times)); } else if file_type == TYPE_FILE { // create directory (of file), if it doesn't exists - if let Some(dir) = &entry_path.parent() && !dir.exists() { - println!("F: {:?}", dir); - fs::create_dir_all(dir)?; - } + Self::create_parent_directory(&entry_path)?; self.f_out = Some(File::create(&entry_path)?); @@ -388,20 +456,19 @@ impl ArchiveWrite { // file size s = e; e += size_of::(); self.file_size = u64::from_le_bytes( header[s..e].try_into()? ); + } else if file_type == TYPE_SYMLINK_FILE || file_type == TYPE_SYMLINK_DIR { // create directory (of symlink), if it doesn't exists - if let Some(dir) = &entry_path.parent() && !dir.exists() { - println!("S: {:?}", dir); - fs::create_dir_all(dir)?; - } + Self::create_parent_directory(&entry_path)?; // symlink's target path let target_path; (target_path, e) = Self::get_path_from_header(header, e, created_on_os_type)?; - // create symlink - // remove it, if it already exists, otherwise symlink can't be created + // remove symlink, if it already exists, otherwise it can't be created let _ = fs::remove_file(&entry_path); + + // create symlink #[cfg(unix)] { std::os::unix::fs::symlink(&target_path, &entry_path)?; @@ -417,7 +484,7 @@ impl ArchiveWrite { } // set timestamps of symlink - // replace with fs::set_times_nofollow() when in stable rust version + // replace with fs::set_times_nofollow() when stable rust version supports it if filetime::set_symlink_file_times( &entry_path, filetime::FileTime::from_system_time(time_accessed), @@ -431,16 +498,26 @@ impl ArchiveWrite { // permissions // if this is a unix system and the archive was created on a unix system, set permission mode - if cfg!(unix) && created_on_os_type == TYPE_UNIX && (file_type == TYPE_DIRECTORY || file_type == TYPE_FILE) + #[cfg(unix)] { - s = e; e += size_of::(); - let perm = u16::from_le_bytes(header[s..e].try_into()?); + use std::os::unix::fs::PermissionsExt; - let fd = File::open(&entry_path)?; - let mut permissions = fd.metadata()?.permissions(); - let mode_masked = permissions.mode() & 0xFFFF_F000; - permissions.set_mode(mode_masked | u32::from(perm & 0x0FFF)); - fd.set_permissions(permissions)?; + if created_on_os_type == TYPE_UNIX && (file_type == TYPE_DIRECTORY || file_type == TYPE_FILE) { + s = e; e += size_of::(); + let perm = u16::from_le_bytes(header[s..e].try_into()?); + + let fd = File::open(&entry_path)?; + let mut permissions = fd.metadata()?.permissions(); + let mode_masked = permissions.mode() & 0xFFFF_F000; + permissions.set_mode(mode_masked | u32::from(perm & 0x0FFF)); + fd.set_permissions(permissions)?; + } + } + #[cfg(windows)] + { + // keep compiler quiet + s = e; e += size_of::(); + let _perm = u16::from_le_bytes(header[s..e].try_into()?); } Ok(()) @@ -465,15 +542,6 @@ impl WriteFiles for ArchiveWrite { if f_out.set_times(self.file_times).is_err() { eprintln!("Could not set original timestamps for file {}", self.file_path.display()); } - // as a new file was created, the file's parent directory would get the current timestamp, - // but the original one is desired, therefore set original timestamp for the directory - if let Some(dir_path) = self.file_path.parent() - && let Some(file_times) = self.dir_times.get(dir_path) { - if !File::open(dir_path).is_ok_and(|dir| dir.set_times(*file_times).is_ok()) { - eprintln!("Could not set original timestamps for directory {}", dir_path.display()); - } - } else { /* do nothing */ } - self.file_size = 0; self.f_out = None; } @@ -496,6 +564,31 @@ impl WriteFiles for ArchiveWrite { break; // not enough data } } + + Ok(()) + } + + fn write_others(&self) -> Result<()> { + // create hard links, if there are any + for (target_path, entry_path) in &self.pending_hardlinks { + // create directory (of hard link), if it doesn't exists + Self::create_parent_directory(entry_path)?; + + // remove hard link, if it already exists, otherwise it can't be created + let _ = fs::remove_file(entry_path); + + fs::hard_link(target_path, entry_path)?; + } + + // set timestamps of directories + // need to be done after all elements have been created, as creation of an element + // updates timestamp of its parent directory to now + for (dir_path, dir_time) in &self.dir_times { + if !File::open(dir_path).is_ok_and(|dir| dir.set_times(*dir_time).is_ok()) { + eprintln!("Could not set original timestamps for directory {}", dir_path.display()); + } + } + Ok(()) } } diff --git a/src/common_io.rs b/src/common_io.rs index cd67bb1..f64bb18 100644 --- a/src/common_io.rs +++ b/src/common_io.rs @@ -17,6 +17,7 @@ pub trait ReadChunk { pub trait WriteFiles { fn write_files(&mut self, buf_in: &[u8]) -> Result<()>; + fn write_others(&self) -> Result<()>; } /// Struct for file input @@ -247,6 +248,11 @@ impl WriteFiles for WriteOutput { Ok(()) } + + fn write_others(&self) -> Result<()> { + // do nothing + Ok(()) + } } @@ -305,6 +311,8 @@ impl CryptIo { write_index += 1; } } + write_output.write_others().map_err(|e| e.to_string())?; + Ok(()) }); @@ -337,7 +345,6 @@ impl CryptIo { Err(panic) => return Err(format!("Crypt thread panicked: {:?}", panic).into()), } } - Ok(()) } From f76fbed8f5ebde15cb4ab7b55cc7c490454b5883 Mon Sep 17 00:00:00 2001 From: JoergDF Date: Fri, 26 Jun 2026 13:31:45 +0200 Subject: [PATCH 06/16] Fix setting of timestamp of directories on windows. --- src/archive.rs | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 98e55e3..25c2347 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -3,7 +3,7 @@ use std::io::{Read, Write}; use std::mem::{self, size_of}; use std::path::{Path, PathBuf}; use std::thread; -use std::time::{Duration, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use typed_path::Utf8WindowsPath; use walkdir::WalkDir; use crossbeam_channel::{Receiver, Select, TryRecvError, bounded}; @@ -74,6 +74,7 @@ impl ArchiveRead { } #[cfg(windows)] let _ = tx_paths.send((entry, None)); + // windows: use number_of_links() and file_index() of std::os::windows::fs::MetadataExt, when supported by stable rust } Err(ref e) => { return Err(format!("Skipped entry {entry:?} Error: {e}")); } } @@ -356,7 +357,7 @@ pub struct ArchiveWrite { file_size: u64, file_times: FileTimes, file_path: PathBuf, - dir_times: Vec<(PathBuf, FileTimes)>, + dir_times: Vec<(PathBuf, SystemTime, SystemTime)>, pending_hardlinks: Vec<(PathBuf, PathBuf)>, } @@ -442,7 +443,7 @@ impl ArchiveWrite { fs::create_dir_all(&entry_path)?; // save timestamps for restoring them at the end - self.dir_times.push((entry_path.clone(), self.file_times)); + self.dir_times.push((entry_path.clone(), time_accessed, time_modified)); } else if file_type == TYPE_FILE { // create directory (of file), if it doesn't exists @@ -485,12 +486,13 @@ impl ArchiveWrite { // set timestamps of symlink // replace with fs::set_times_nofollow() when stable rust version supports it - if filetime::set_symlink_file_times( + match filetime::set_symlink_file_times( &entry_path, filetime::FileTime::from_system_time(time_accessed), filetime::FileTime::from_system_time(time_modified) - ).is_err() { - eprintln!("Could not set original timestamps for symlink {}", entry_path.display()); + ) { + Ok(()) => {}, + Err(e) => eprintln!("Could not set original timestamps for symlink {}: {e}", entry_path.display()), } } else { return Err(format!("Archive contains unknown file type: {file_type}").into()); @@ -583,10 +585,15 @@ impl WriteFiles for ArchiveWrite { // set timestamps of directories // need to be done after all elements have been created, as creation of an element // updates timestamp of its parent directory to now - for (dir_path, dir_time) in &self.dir_times { - if !File::open(dir_path).is_ok_and(|dir| dir.set_times(*dir_time).is_ok()) { - eprintln!("Could not set original timestamps for directory {}", dir_path.display()); - } + for (dir_path, atime, mtime) in &self.dir_times { + match filetime::set_file_times( + dir_path, + filetime::FileTime::from_system_time(*atime), + filetime::FileTime::from_system_time(*mtime) + ) { + Ok(()) => {}, + Err(e) => eprintln!("Could not set original timestamps for directory {}: {e}", dir_path.display()), + }; } Ok(()) From 6eaa8b69f072cb51724036a8fc1486789fa70eee Mon Sep 17 00:00:00 2001 From: JoergDF Date: Tue, 30 Jun 2026 13:49:22 +0200 Subject: [PATCH 07/16] Add archive support for sparse files. --- Cargo.lock | 124 ++++++++++++++++++-- Cargo.toml | 3 +- src/archive.rs | 308 ++++++++++++++++++++++++++++++++++--------------- 3 files changed, 334 insertions(+), 101 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 667e9aa..2e052cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", "cpufeatures 0.2.17", ] @@ -154,6 +154,22 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.4" @@ -166,7 +182,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", "cpufeatures 0.2.17", ] @@ -177,7 +193,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -307,6 +323,7 @@ dependencies = [ "chacha20poly1305", "clap", "crossbeam-channel", + "drill-press", "filetime", "hkdf", "num_cpus", @@ -320,6 +337,7 @@ dependencies = [ "typed-path", "typenum", "walkdir", + "winapi", ] [[package]] @@ -383,22 +401,62 @@ dependencies = [ "ctutils", ] +[[package]] +name = "drill-press" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f980f20e7c746f567c2c335924468fa647ca38f23d87fdcfbd74340d02783210" +dependencies = [ + "cfg-if 0.1.10", + "errno", + "libc", + "thiserror", + "winapi", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "filetime" version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "foldhash" version = "0.1.5" @@ -421,7 +479,7 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", "wasi", ] @@ -432,7 +490,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", "r-efi", "rand_core 0.10.1", @@ -539,7 +597,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.3.0", ] @@ -629,7 +687,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.2.17", "opaque-debug", "universal-hash", @@ -807,7 +865,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.3.0", "digest 0.11.3", ] @@ -823,6 +881,12 @@ dependencies = [ "sponge-cursor", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "sponge-cursor" version = "0.1.0" @@ -852,6 +916,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "typed-path" version = "0.12.3" @@ -966,6 +1050,22 @@ dependencies = [ "semver", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -975,6 +1075,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index fc79add..58dd0f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ bzip2 = "0.6.1" chacha20poly1305 = "0.10.1" clap = { version = "4.5.60", features = ["derive"] } crossbeam-channel = "0.5.15" -#drill-press = "0.1.2" +drill-press = "0.1.2" filetime = "0.2.29" hkdf = "0.13.0" num_cpus = "1.17.0" @@ -24,6 +24,7 @@ sha3 = "0.12.0" typed-path = "0.12.3" typenum = "1.19.0" walkdir = "2.5.0" +winapi = "0.3.9" [profile.release] lto = "thin" diff --git a/src/archive.rs b/src/archive.rs index 25c2347..361e31f 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -1,5 +1,5 @@ use std::fs::{self, File, FileTimes}; -use std::io::{Read, Write}; +use std::io::{Read, Seek, Write}; use std::mem::{self, size_of}; use std::path::{Path, PathBuf}; use std::thread; @@ -11,7 +11,7 @@ use num_cpus; use filetime; #[cfg(unix)] use std::collections::HashMap; -//use drill_press::{Segments, SparseFile}; +use drill_press::{SegmentType, Segment, Segments, SparseFile}; use crate::common_io::{ReadChunk, WriteFiles}; use crate::{CHUNK_SIZE, Result}; @@ -36,7 +36,7 @@ pub struct ArchiveRead { impl ArchiveRead { pub fn new(f_in_path: &Path) -> Self { - let num_workers = num_cpus::get(); // fixme: too much cpus? -1 for walkdir, and what about bzip,crypt? + let num_workers = num_cpus::get(); let mut thread_handles = Vec::with_capacity(num_workers + 1); let mut rx_out_receivers = Vec::with_capacity(num_workers); @@ -76,7 +76,7 @@ impl ArchiveRead { let _ = tx_paths.send((entry, None)); // windows: use number_of_links() and file_index() of std::os::windows::fs::MetadataExt, when supported by stable rust } - Err(ref e) => { return Err(format!("Skipped entry {entry:?} Error: {e}")); } + Err(ref e) => { eprintln!("Skipped entry while walking directory tree - Reason: {e}"); } } } Ok(()) @@ -94,65 +94,74 @@ impl ArchiveRead { for (entry, hard_link_target) in rx_paths { let archive_header; let filepath_and_size; - //let sparse_segments; //fixme - // println!("{:?}", entry); + let sparse_segments; + match Self::build_archive_header(&entry, &hard_link_target) { - Ok(values) => (archive_header, filepath_and_size/* , sparse_segments */) = values, + Ok(values) => (archive_header, filepath_and_size, sparse_segments) = values, Err(e) => { - eprintln!("Skipped entry {} - Reason: {e}", entry.path().display()); + eprintln!("Skipped entry on building archive header for {} - Reason: {e}", entry.path().display()); continue; } } if let Some((filepath, mut file_size)) = filepath_and_size { - if let Ok(mut f_in) = File::open(&filepath) { - //println!("send fah {} {}", archive_header.len(), file_size); - - // send header of file - // empty files (with length 0), must set last_chunk to true - let last_chunk = file_size == 0; - let _ = tx_out.send((archive_header, last_chunk)); - - // if file_size == 0 { continue; } - // fixme - // let seg_data_size: u64 = sparse_segments.data().map(|sd| sd.end - sd.start).sum(); - // let mut data_size = if sparse_segments.is_empty() { - // file_size - // } else { - // seg_data_size - // }; - - // while data_size != 0 { - // let buf_len = CHUNK_SIZE.min(usize::try_from(data_size).map_err(|e| e.to_string())?); - // let mut buf_read = vec![0u8; buf_len]; - // } - + match File::open(&filepath) { + Ok(mut f_in) => { + // send header of file + // empty files (with length 0), must set last_chunk to true + let last_chunk = file_size == 0; + let _ = tx_out.send((archive_header, last_chunk)); - // read file and send its data - while file_size != 0 { - let buf_len = CHUNK_SIZE.min(usize::try_from(file_size).map_err(|e| e.to_string())?); - let mut buf_read = vec![0u8; buf_len]; - f_in.read_exact(&mut buf_read).map_err(|e| e.to_string())?; - file_size -= buf_len as u64; + fn read_data(f_in: &mut File, mut data_size: u64) -> Result<(Vec, u64)> { + let buf_len = CHUNK_SIZE.min(usize::try_from(data_size)?); + let mut buf_read = vec![0u8; buf_len]; + f_in.read_exact(&mut buf_read)?; + data_size -= buf_len as u64; + Ok((buf_read, data_size)) + } - let last_chunk = file_size == 0; - let _ = tx_out.send((buf_read, last_chunk)); - //println!("{:?} {} {} {}", filepath, file_size, buf_len, last_chunk); - } - } else { - eprintln!("Could not open - skipped: {}", filepath.display()); - continue; + if sparse_segments.is_empty() { + // read file and send its data + while file_size != 0 { + let buf_read; + (buf_read, file_size) = read_data(&mut f_in, file_size).map_err(|e| e.to_string())?; + let last_chunk = file_size == 0; + let _ = tx_out.send((buf_read, last_chunk)); + } + } else { + // read/skip segments of a sparse file and send its data + for (idx, seg) in sparse_segments.iter().enumerate() { + let last_segment = (sparse_segments.len() - 1) == idx; + if seg.is_data() { + let mut seg_size = seg.len(); + while seg_size != 0 { + let buf_read; + (buf_read, seg_size) = read_data(&mut f_in, seg_size).map_err(|e| e.to_string())?; + let last_chunk = seg_size == 0 && last_segment; + let _ = tx_out.send((buf_read, last_chunk)); + } + } else { // hole + f_in.seek_relative( + i64::try_from( seg.len() ).map_err(|e| e.to_string())? + ).map_err(|e| e.to_string())?; + if last_segment { + let _ = tx_out.send((vec![], true)); + } + } + } + } + }, + Err(e) => { + eprintln!("Skipped entry on opening file {} - Reason: {e}", filepath.display()); + continue; + }, } } else { // send header of entries without additional data - //println!("send ah {}", archive_header.len()); let _ = tx_out.send((archive_header, true)); } } - //println!("DONE"); // {}", rx_out_receivers.clone().len()); - // all entries done, send finish message - //let _ = tx_out.send((Vec::new(), true)); Ok(()) })); @@ -173,10 +182,19 @@ impl ArchiveRead { } #[allow(clippy::type_complexity)] - fn build_archive_header(entry: &walkdir::DirEntry, hard_link_target: &Option) -> Result<(Vec, Option<(PathBuf, u64)>/* , Vec */)> { + fn build_archive_header(entry: &walkdir::DirEntry, hard_link_target: &Option) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { // archive header initialized with place holder for header size let mut archive_header = vec![0u8; ARCHIVE_HEADER_LENGTH_SIZE]; + // set header size, call at end of building the header + fn set_header_size(archive_header: &mut [u8]) -> Result<()> { + let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from( + archive_header.len() - ARCHIVE_HEADER_LENGTH_SIZE)?.to_le_bytes(); + archive_header[0] = header_size[0]; + archive_header[1] = header_size[1]; + Ok(()) + } + let entry_type = if hard_link_target.is_some() { TYPE_HARDLINK } else if entry.file_type().is_file() { @@ -193,7 +211,7 @@ impl ArchiveRead { TYPE_SYMLINK_FILE } } else { - return Err(format!("Ignored unsupported file type for archive: {}", entry.path().display()).into()); + return Err("Unsupported file type".into()); }; let os_type = if cfg!(unix) { TYPE_UNIX } else { TYPE_WINDOWS }; @@ -211,12 +229,9 @@ impl ArchiveRead { Self::add_path_to_header(target_path, &mut archive_header)?; // header size - let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from( - archive_header.len() - ARCHIVE_HEADER_LENGTH_SIZE)?.to_le_bytes(); - archive_header[0] = header_size[0]; - archive_header[1] = header_size[1]; + set_header_size(&mut archive_header)?; - return Ok((archive_header, None)) + return Ok((archive_header, None, vec![])) } // last access time @@ -227,29 +242,31 @@ impl ArchiveRead { archive_header.extend(time_modified.to_le_bytes()); let mut file_size = 0; - //let mut sparse_segments = vec![]; + let mut sparse_segments = vec![]; if entry_type == TYPE_FILE { // file size file_size = entry.metadata()?.len(); archive_header.extend(file_size.to_le_bytes()); - // get holes of sparse files - //if file_size > 0 { // fixme - // if let Ok(mut f_in) = File::open(&entry.path()) { - // sparse_segments = f_in.scan_chunks()?; - - // archive_header.extend( u32::try_from(sparse_segments.holes().count())?.to_le_bytes() ); - - // for hole in sparse_segments.holes() { - // archive_header.extend(hole.start.to_le_bytes()); - // archive_header.extend(hole.end.to_le_bytes()); - // } - // } else { - // // could not open file, add 0 holes fixme: correct? - // archive_header.extend( 0u32.to_le_bytes() ); - // } - // println!("{:?} {:?}", sparse_segments, entry.path()); - //} + // sparse file + // if the files can be scanned for sparse parts, the holes are saved in the archive header + if let Ok(mut f_in) = File::open(entry.path()) + && let Ok(segs) = f_in.scan_chunks() { + sparse_segments = segs; + + // println!("{} {:?}", entry.path().display(), sparse_segments); + + // number of holes + let holes_count = sparse_segments.holes().count(); + archive_header.extend(u16::try_from( holes_count )?.to_le_bytes()); + + // start and end index of holes, if any + for hole in sparse_segments.holes() { + // start and end are of type u64 + archive_header.extend(hole.start.to_le_bytes()); + archive_header.extend(hole.end.to_le_bytes()); + } + } } else if entry_type == TYPE_SYMLINK_FILE || entry_type == TYPE_SYMLINK_DIR { // target path of symlink @@ -276,17 +293,14 @@ impl ArchiveRead { } // header size - let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from( - archive_header.len() - ARCHIVE_HEADER_LENGTH_SIZE)?.to_le_bytes(); - archive_header[0] = header_size[0]; - archive_header[1] = header_size[1]; + set_header_size(&mut archive_header)?; let mut filepath_and_size = None; if entry_type == TYPE_FILE { filepath_and_size = Some((entry.clone().into_path(), file_size)); } - Ok((archive_header, filepath_and_size/* , sparse_segments */)) + Ok((archive_header, filepath_and_size, sparse_segments)) } } @@ -297,7 +311,6 @@ impl ReadChunk for ArchiveRead { if let Some(channel_index) = self.channel_index && let Ok((data, last_chunk)) = self.rx_out_receivers[channel_index].recv() { - //println!("cont recv, dat_len: {}, l {}", data.len(), last_chunk); self.buf_out.extend(data); if last_chunk { self.channel_index = None; @@ -311,10 +324,8 @@ impl ReadChunk for ArchiveRead { } let sel_rdy_idx = sel.ready(); - //println!("sel_rdy_idx {}", sel_rdy_idx); match self.rx_out_receivers[sel_rdy_idx].try_recv() { Ok((data, last_chunk)) => { - // println!("recv {}, dat_len: {}, l {}", sel_rdy_idx, data.len(), last_chunk); self.buf_out.extend(data); if !last_chunk { self.channel_index = Some(sel_rdy_idx); @@ -359,13 +370,17 @@ pub struct ArchiveWrite { file_path: PathBuf, dir_times: Vec<(PathBuf, SystemTime, SystemTime)>, pending_hardlinks: Vec<(PathBuf, PathBuf)>, + sparse_segments: Vec, + sparse_segments_index: usize, + data_segment_size: u64, } impl ArchiveWrite { pub fn new() -> Self { let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), - file_path: PathBuf::new(), dir_times: vec![], pending_hardlinks: vec![] } + file_path: PathBuf::new(), dir_times: vec![], pending_hardlinks: vec![], sparse_segments: vec![], + sparse_segments_index: 0, data_segment_size: 0} } fn create_parent_directory(entry_path: &Path) -> Result<()> { @@ -400,6 +415,35 @@ impl ArchiveWrite { Ok((entry_path, e)) } + // On Windows set sparse flag for a sparse file + #[cfg(windows)] + fn set_files_sparse_win(file: &File)-> std::io::Result<()> { + use std::os::windows::io::AsRawHandle; + use winapi::um::ioapiset::DeviceIoControl; + use winapi::um::winioctl::FSCTL_SET_SPARSE; + + let handle = file.as_raw_handle(); + let mut bytes_returned = 0; + unsafe { + let result = DeviceIoControl( + handle as _, + FSCTL_SET_SPARSE, + std::ptr::null_mut(), + 0, + std::ptr::null_mut(), + 0, + &mut bytes_returned, + std::ptr::null_mut(), + ); + + if result == 0 { + return Err(std::io::Error::last_os_error()); + } + } + Ok(()) + } + + fn eval_header(&mut self, header: &[u8]) -> Result<()> { // type let file_type = header[0] & 0x0F; @@ -449,7 +493,8 @@ impl ArchiveWrite { // create directory (of file), if it doesn't exists Self::create_parent_directory(&entry_path)?; - self.f_out = Some(File::create(&entry_path)?); + let file = File::create(&entry_path)?; + self.f_out = Some(file.try_clone()?); // for error handling self.file_path = entry_path.clone(); @@ -458,6 +503,38 @@ impl ArchiveWrite { s = e; e += size_of::(); self.file_size = u64::from_le_bytes( header[s..e].try_into()? ); + // holes of a sparse file + s = e; e += size_of::(); + let holes_count = u16::from_le_bytes( header[s..e].try_into()? ); + + if holes_count > 0 { + // restore data- and hole-segments of sparse file + let mut data_start = 0; + for _ in 0..holes_count { + s = e; e += size_of::(); + let hole_start = u64::from_le_bytes( header[s..e].try_into()? ); + s = e; e += size_of::(); + let hole_end = u64::from_le_bytes( header[s..e].try_into()? ); + + if hole_start != 0 { + // if first segment is not a hole, add a data segment + self.sparse_segments.push( Segment { segment_type: SegmentType::Data, range: data_start..hole_start} ); + } + self.sparse_segments.push( Segment { segment_type: SegmentType::Hole, range: hole_start..hole_end } ); + data_start = hole_end; + } + // if last segment is not a hole, add a data segment + if data_start != self.file_size { + self.sparse_segments.push( Segment { segment_type: SegmentType::Data, range: data_start..self.file_size} ); + } + // println!("{:?} {:?}", entry_path.display(), self.sparse_segments); + + // Windows requires to set sparse flag for a sparse file + #[cfg(windows)] + Self::set_files_sparse_win(&file)?; + } + + } else if file_type == TYPE_SYMLINK_FILE || file_type == TYPE_SYMLINK_DIR { // create directory (of symlink), if it doesn't exists Self::create_parent_directory(&entry_path)?; @@ -530,23 +607,71 @@ impl WriteFiles for ArchiveWrite { fn write_files(&mut self, buf_in: &[u8]) -> Result<()> { self.buf_out.extend(buf_in); - while !self.buf_out.is_empty() { + loop { if let Some(mut f_out) = self.f_out.as_ref() { - // write to file - if (self.buf_out.len() as u64) < self.file_size { - f_out.write_all(&self.buf_out)?; - self.file_size -= self.buf_out.len() as u64; - self.buf_out.clear(); + + let mut write_data = |f_out_size: u64| -> Result { + let data_size = self.buf_out.len().min(f_out_size.try_into()?); + let buf_out_slice: Vec = self.buf_out.drain(..data_size).collect(); + f_out.write_all(&buf_out_slice)?; + Ok(data_size as u64) + }; + + if self.sparse_segments.is_empty() { + // write non-sparse file + let write_size = write_data(self.file_size)?; + self.file_size -= write_size; + if self.buf_out.is_empty() { + break; + } } else { - let file_data: Vec = self.buf_out.drain(..usize::try_from(self.file_size)?).collect(); - f_out.write_all(&file_data)?; + // write sparse file + let segment = &self.sparse_segments[self.sparse_segments_index]; + + match segment.segment_type { + SegmentType::Data => { + if self.data_segment_size == 0 { + self.data_segment_size = segment.len(); + } + + let write_size = write_data(self.data_segment_size)?; + self.data_segment_size -= write_size; + self.file_size -= write_size; + if self.data_segment_size == 0 { + self.sparse_segments_index += 1; + } + } + SegmentType::Hole => { + f_out.seek_relative((segment.len() - 1).try_into()?)?; + f_out.write_all(&[0])?; + f_out.drill_hole(segment.range.start, segment.range.end)?; + self.file_size -= segment.len(); + self.sparse_segments_index += 1; + } + } + + if self.file_size == 0 { + self.data_segment_size = 0; + self.sparse_segments_index = 0; + self.sparse_segments.clear(); + } + + // no data left and no hole as next segment + if self.buf_out.is_empty() + && self.sparse_segments_index < self.sparse_segments.len() + && self.sparse_segments[self.sparse_segments_index].segment_type != SegmentType::Hole { + break; + } + } + + if self.file_size == 0 { // set file times after all data has been written if f_out.set_times(self.file_times).is_err() { eprintln!("Could not set original timestamps for file {}", self.file_path.display()); } - self.file_size = 0; self.f_out = None; } + } else if let Some(header_length) = self.header_length { if self.buf_out.len() >= header_length { // get header @@ -586,6 +711,7 @@ impl WriteFiles for ArchiveWrite { // need to be done after all elements have been created, as creation of an element // updates timestamp of its parent directory to now for (dir_path, atime, mtime) in &self.dir_times { + // supports Windows and Unix match filetime::set_file_times( dir_path, filetime::FileTime::from_system_time(*atime), From 16c5e23bc8b7017359fc6c4720de5d243b8a1e89 Mon Sep 17 00:00:00 2001 From: JoergDF Date: Tue, 30 Jun 2026 16:08:29 +0200 Subject: [PATCH 08/16] Add documentation --- src/archive.rs | 198 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 172 insertions(+), 26 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 361e31f..8b2d64e 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -26,15 +26,35 @@ const TYPE_UNIX: u8 = 0x00; const TYPE_WINDOWS: u8 = 0x10; const ARCHIVE_HEADER_LENGTH_SIZE: usize = 2; +/// Handles the reading and archiving of files/directories. +/// +/// Walks the file system directory tree, processes files in parallel, +/// constructs archive headers, and serves the serialized archive stream in chunks. pub struct ArchiveRead { + /// Background worker thread handles running the directory walk and processing jobs. pub thread_handles: Vec>>, + /// Channels receiving processed archive data blocks from the parallel worker threads. rx_out_receivers: Vec, bool)>>, + /// Index of the active channel currently being read. channel_index: Option, + /// Tracks which worker threads have finished processing their tasks. channel_finished: Vec, + /// Accumulates output data to be served in uniform chunks of `CHUNK_SIZE`. buf_out: Vec, } impl ArchiveRead { + /// Initializes the archive reading process by starting parallel worker threads. + /// + /// One worker thread walks the directory tree and sends discovered file entries and hard link + /// information to a channel. Multiple worker threads then process these entries, build archive + /// headers, read file contents, and send the formatted data to receivers. + /// + /// # Arguments + /// - `f_in_path`: The root path of the directory tree to archive. + /// + /// # Returns + /// - A new `ArchiveRead` instance. pub fn new(f_in_path: &Path) -> Self { let num_workers = num_cpus::get(); let mut thread_handles = Vec::with_capacity(num_workers + 1); @@ -94,7 +114,7 @@ impl ArchiveRead { for (entry, hard_link_target) in rx_paths { let archive_header; let filepath_and_size; - let sparse_segments; + let sparse_segments; match Self::build_archive_header(&entry, &hard_link_target) { Ok(values) => (archive_header, filepath_and_size, sparse_segments) = values, @@ -112,7 +132,15 @@ impl ArchiveRead { let last_chunk = file_size == 0; let _ = tx_out.send((archive_header, last_chunk)); - + /// Helper function to read a chunk of data from a file up to the chunk limit. + /// + /// # Arguments + /// - `f_in`: File handle to read from. + /// - `data_size`: Total remaining data size to read. + /// + /// # Returns + /// - `Ok((buffer, remaining_size))` on success. + /// - `Err` on I/O or conversion error. fn read_data(f_in: &mut File, mut data_size: u64) -> Result<(Vec, u64)> { let buf_len = CHUNK_SIZE.min(usize::try_from(data_size)?); let mut buf_read = vec![0u8; buf_len]; @@ -172,8 +200,17 @@ impl ArchiveRead { Self { thread_handles, rx_out_receivers, channel_index: None, channel_finished: vec![false; num_workers], buf_out } } + /// Appends the file path length and path string to the archive header buffer. + /// + /// # Arguments + /// - `path`: The file system path to encode. + /// - `archive_header`: The mutable buffer to append the encoded path to. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` if the path length exceeds `u16` capacity. fn add_path_to_header(path: &Path, archive_header: &mut Vec) -> Result<()> { - // path length and path + // path length and path let path_string = path.to_string_lossy(); let path_len: u16 = path_string.len().try_into()?; archive_header.extend(path_len.to_le_bytes()); @@ -181,15 +218,35 @@ impl ArchiveRead { Ok(()) } + /// Builds the archive header bytes for a given file system entry. + /// + /// Creates a metadata block containing file type, path, timestamps, size, + /// sparse segments (holes), and permissions. + /// + /// # Arguments + /// - `entry`: The directory entry to construct the header for. + /// - `hard_link_target`: Optional path pointing to the target if this is a hard link. + /// + /// # Returns + /// - `Ok((archive_header, filepath_and_size, sparse_segments))` on success. + /// - `Err` if metadata retrieval or OS-specific operations fail. #[allow(clippy::type_complexity)] fn build_archive_header(entry: &walkdir::DirEntry, hard_link_target: &Option) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { // archive header initialized with place holder for header size let mut archive_header = vec![0u8; ARCHIVE_HEADER_LENGTH_SIZE]; - // set header size, call at end of building the header + /// Computes and sets the final header size at the beginning of the header buffer. + /// It is called after all other header fields have been added to the header buffer. + /// + /// # Arguments + /// - `archive_header`: The mutable slice representing the archive header. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` if the header length cannot be converted to `u16`. fn set_header_size(archive_header: &mut [u8]) -> Result<()> { - let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = u16::try_from( - archive_header.len() - ARCHIVE_HEADER_LENGTH_SIZE)?.to_le_bytes(); + let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = + u16::try_from(archive_header.len() - ARCHIVE_HEADER_LENGTH_SIZE)?.to_le_bytes(); archive_header[0] = header_size[0]; archive_header[1] = header_size[1]; Ok(()) @@ -223,7 +280,6 @@ impl ArchiveRead { // println!("{}", entry.path().display()); if entry_type == TYPE_HARDLINK { - // target path of hard link let target_path = hard_link_target.as_ref().unwrap(); Self::add_path_to_header(target_path, &mut archive_header)?; @@ -231,7 +287,7 @@ impl ArchiveRead { // header size set_header_size(&mut archive_header)?; - return Ok((archive_header, None, vec![])) + return Ok((archive_header, None, vec![])); } // last access time @@ -247,7 +303,7 @@ impl ArchiveRead { // file size file_size = entry.metadata()?.len(); archive_header.extend(file_size.to_le_bytes()); - + // sparse file // if the files can be scanned for sparse parts, the holes are saved in the archive header if let Ok(mut f_in) = File::open(entry.path()) @@ -278,7 +334,7 @@ impl ArchiveRead { #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - + let mut perm: u16 = 0; if entry_type == TYPE_FILE || entry_type == TYPE_DIRECTORY { let permission_mode = entry.metadata()?.permissions().mode(); @@ -289,7 +345,7 @@ impl ArchiveRead { } #[cfg(windows)] { - archive_header.extend(0u16.to_le_bytes()); + archive_header.extend(0u16.to_le_bytes()); } // header size @@ -305,6 +361,14 @@ impl ArchiveRead { } impl ReadChunk for ArchiveRead { + /// Reads a chunk of archived data, pulling from active worker channels. + /// + /// Polls channels from parallel workers and aggregates the data into `buf_out`. + /// Returns chunks of `CHUNK_SIZE` until all threads finish and all data is read. + /// + /// # Returns + /// - `Ok((chunk, last_chunk))` on success, where `last_chunk` is true if this is the final block. + /// - `Err` on I/O or coordination error. fn read_chunk(&mut self) -> Result<(Vec, bool)> { while !self.channel_finished.iter().all(|x| *x) && self.buf_out.len() <= CHUNK_SIZE { // stay on same channel until last chunk of file using a blocking receive @@ -345,6 +409,11 @@ impl ReadChunk for ArchiveRead { } } + /// Joins all background worker threads and propagates any execution errors. + /// + /// # Returns + /// - `Ok(())` if all threads exited successfully. + /// - `Err` if any thread failed or panicked. fn join_threads(&mut self) -> Result<()> { let thread_handles = mem::take(&mut self.thread_handles); for th in thread_handles { @@ -360,22 +429,41 @@ impl ReadChunk for ArchiveRead { } } +/// Handles extracting and writing archived entries back to the file system. +/// +/// Decodes the incoming archive stream, creating files, directories, symlinks, +/// and hard links, restoring their permissions and timestamps. #[derive(Default)] pub struct ArchiveWrite { + /// Active file handle for the entry currently being written. f_out: Option, + /// Internal buffer containing data received from the decryption stream. buf_out: Vec, + /// Length of the header currently being processed. header_length: Option, + /// Total bytes remaining to be written for the current file. file_size: u64, + /// Timestamps (accessed, modified) of the current file being written. file_times: FileTimes, + /// Path of the current file being written. file_path: PathBuf, + /// List of directories and their original timestamps to be restored after extraction completes. dir_times: Vec<(PathBuf, SystemTime, SystemTime)>, + /// List of pending hard link creations (target, link_path) to execute after extraction. pending_hardlinks: Vec<(PathBuf, PathBuf)>, + /// Scanned sparse segments (data/hole) for the current sparse file. sparse_segments: Vec, + /// Index of the current sparse segment being written. sparse_segments_index: usize, + /// Size in bytes of the current sparse data segment. data_segment_size: u64, } impl ArchiveWrite { + /// Initializes a new, empty `ArchiveWrite` instance. + /// + /// # Returns + /// - A default `ArchiveWrite` with allocated output buffer. pub fn new() -> Self { let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), @@ -383,16 +471,38 @@ impl ArchiveWrite { sparse_segments_index: 0, data_segment_size: 0} } + /// Ensures that the parent directory of the given path exists. + /// + /// Creates the parent directory recursively if it does not already exist. + /// As parallel threads are used to create an archive, the sequence of entries differ from the + /// sequence returned by walkdir, hence a file, etc. might show up before its directory was created. + /// + /// # Arguments + /// - `entry_path`: The file system path whose parent directory should be created. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` on file system creation failure. fn create_parent_directory(entry_path: &Path) -> Result<()> { - // create directory, if it doesn't exists - // as parallel threads are used to create an archive, the sequence of entries differ from the sequence returned be walkdir - // hence a file, etc. might show up before its directory was created if let Some(dir) = entry_path.parent() && !dir.exists() { fs::create_dir_all(dir)?; } Ok(()) } + /// Parses a file path from the archive header slice. + /// + /// Reads the path length, extracts the path bytes, converts Windows path separators + /// to Unix format if running on Unix, and returns the path string along with the new end index. + /// + /// # Arguments + /// - `header`: The archive header bytes. + /// - `current_end_index`: The starting index in the header to read from. + /// - `created_on_os_type`: OS type flag indicating which system the archive was created on. + /// + /// # Returns + /// - `Ok((parsed_path, next_index))` on success. + /// - `Err` on parse or UTF-8 decoding failure. fn get_path_from_header(header: &[u8], current_end_index: usize, created_on_os_type: u8) -> Result<(String, usize)> { // new start index of header field let mut s = current_end_index; @@ -415,9 +525,16 @@ impl ArchiveWrite { Ok((entry_path, e)) } - // On Windows set sparse flag for a sparse file + /// Configures a file as a sparse file on Windows. + /// + /// # Arguments + /// - `file`: Reference to the file to set as sparse. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` if the system call fails. #[cfg(windows)] - fn set_files_sparse_win(file: &File)-> std::io::Result<()> { + fn set_files_sparse_win(file: &File) -> std::io::Result<()> { use std::os::windows::io::AsRawHandle; use winapi::um::ioapiset::DeviceIoControl; use winapi::um::winioctl::FSCTL_SET_SPARSE; @@ -443,7 +560,17 @@ impl ArchiveWrite { Ok(()) } - + /// Evaluates a parsed archive header to create the corresponding file system entry. + /// + /// Handles directories, files (including sparse configuration), symlinks, and hard links. + /// Sets file size, times, and system-level permissions depending on OS. + /// + /// # Arguments + /// - `header`: The raw header bytes. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` on creation, I/O, or permission errors. fn eval_header(&mut self, header: &[u8]) -> Result<()> { // type let file_type = header[0] & 0x0F; @@ -465,7 +592,7 @@ impl ArchiveWrite { (target_path, _) = Self::get_path_from_header(header, e, created_on_os_type)?; self.pending_hardlinks.push((PathBuf::from(target_path), entry_path)); - return Ok(()) + return Ok(()); } // access time @@ -482,7 +609,7 @@ impl ArchiveWrite { // create type if file_type == TYPE_DIRECTORY { - // create directory + // create directory // it could be an empty one therefore it would not be created by other entries fs::create_dir_all(&entry_path)?; @@ -534,7 +661,6 @@ impl ArchiveWrite { Self::set_files_sparse_win(&file)?; } - } else if file_type == TYPE_SYMLINK_FILE || file_type == TYPE_SYMLINK_DIR { // create directory (of symlink), if it doesn't exists Self::create_parent_directory(&entry_path)?; @@ -560,7 +686,7 @@ impl ArchiveWrite { std::os::windows::fs::symlink_dir(&target_path, &entry_path)?; } } - + // set timestamps of symlink // replace with fs::set_times_nofollow() when stable rust version supports it match filetime::set_symlink_file_times( @@ -604,12 +730,23 @@ impl ArchiveWrite { } impl WriteFiles for ArchiveWrite { + /// Processes incoming stream data and writes it to the current active file. + /// + /// Handles headers to initialize files, parses sparse segments, and writes + /// data chunks. Performs physical file creation and metadata restoration. + /// + /// # Arguments + /// - `buf_in`: Raw input byte slice. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` on writing, seeking, or parsing failure. fn write_files(&mut self, buf_in: &[u8]) -> Result<()> { self.buf_out.extend(buf_in); loop { if let Some(mut f_out) = self.f_out.as_ref() { - + // Local helper closure to write buffered data to file let mut write_data = |f_out_size: u64| -> Result { let data_size = self.buf_out.len().min(f_out_size.try_into()?); let buf_out_slice: Vec = self.buf_out.drain(..data_size).collect(); @@ -633,7 +770,7 @@ impl WriteFiles for ArchiveWrite { if self.data_segment_size == 0 { self.data_segment_size = segment.len(); } - + let write_size = write_data(self.data_segment_size)?; self.data_segment_size -= write_size; self.file_size -= write_size; @@ -649,7 +786,7 @@ impl WriteFiles for ArchiveWrite { self.sparse_segments_index += 1; } } - + if self.file_size == 0 { self.data_segment_size = 0; self.sparse_segments_index = 0; @@ -695,6 +832,15 @@ impl WriteFiles for ArchiveWrite { Ok(()) } + /// Finalizes the extraction process by completing delayed operations. + /// + /// Creates any pending hard links and restores directory timestamps. These operations + /// are deferred until the end of extraction to prevent sub-file creations from altering parent + /// directory timestamps. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` if hard link creation or timestamp updates fail. fn write_others(&self) -> Result<()> { // create hard links, if there are any for (target_path, entry_path) in &self.pending_hardlinks { @@ -708,13 +854,13 @@ impl WriteFiles for ArchiveWrite { } // set timestamps of directories - // need to be done after all elements have been created, as creation of an element + // need to be done after all elements have been created, as creation of an element // updates timestamp of its parent directory to now for (dir_path, atime, mtime) in &self.dir_times { // supports Windows and Unix match filetime::set_file_times( dir_path, - filetime::FileTime::from_system_time(*atime), + filetime::FileTime::from_system_time(*atime), filetime::FileTime::from_system_time(*mtime) ) { Ok(()) => {}, From 476dea5cfc05666a5600949b94edf5bd1cd265e1 Mon Sep 17 00:00:00 2001 From: JoergDF Date: Sat, 4 Jul 2026 10:50:17 +0200 Subject: [PATCH 09/16] Add unit test for archive, fix sparse files. - Punch holes of sparse files at end of file writing, not when holes occur. - Update library versions --- Cargo.lock | 357 +++------------------------------- src/archive.rs | 473 +++++++++++++++++++++++++++++++++++++++++++++- src/encryption.rs | 39 ++++ 3 files changed, 532 insertions(+), 337 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e052cb..925442d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,12 +88,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "anyhow" -version = "1.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" - [[package]] name = "argon2" version = "0.5.3" @@ -112,12 +106,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "bitflags" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" - [[package]] name = "blake2" version = "0.10.6" @@ -138,9 +126,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", ] @@ -189,9 +177,9 @@ dependencies = [ [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if 1.0.4", "cpufeatures 0.3.0", @@ -264,9 +252,9 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "colorchoice" @@ -395,7 +383,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer 0.12.0", + "block-buffer 0.12.1", "const-oid", "crypto-common 0.2.2", "ctutils", @@ -414,12 +402,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - [[package]] name = "errno" version = "0.2.8" @@ -457,12 +439,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "generic-array" version = "0.14.7" @@ -486,33 +462,16 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if 1.0.4", "libc", "r-efi", "rand_core 0.10.1", - "wasip2", - "wasip3", -] - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", ] -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - [[package]] name = "heck" version = "0.5.0" @@ -545,31 +504,13 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ "typenum", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", -] - [[package]] name = "inout" version = "0.1.4" @@ -585,12 +526,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - [[package]] name = "keccak" version = "0.2.0" @@ -601,12 +536,6 @@ dependencies = [ "cpufeatures 0.3.0", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libbz2-rs-sys" version = "0.2.5" @@ -619,18 +548,6 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "log" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" - -[[package]] -name = "memchr" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" - [[package]] name = "num_cpus" version = "1.17.0" @@ -702,16 +619,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -723,9 +630,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -738,12 +645,12 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ - "chacha20 0.10.0", - "getrandom 0.4.2", + "chacha20 0.10.1", + "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -774,9 +681,9 @@ checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rpassword" -version = "7.5.2" +version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac5b223d9738ef56e0b98305410be40fa0941bf6036c56f1506751e43552d64" +checksum = "2da316a15f47e3d053de9cb2c439650bd8fa4aaeb9365f2e5f27f492ff73c196" dependencies = [ "libc", "rtoolbox", @@ -811,54 +718,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.150" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - [[package]] name = "sha2" version = "0.11.0" @@ -907,9 +766,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -944,9 +803,9 @@ checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" @@ -954,12 +813,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "universal-hash" version = "0.5.1" @@ -998,58 +851,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", -] - -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "winapi" version = "0.3.9" @@ -1169,114 +970,20 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", @@ -1285,12 +992,6 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" - -[[package]] -name = "zmij" -version = "1.0.21" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" diff --git a/src/archive.rs b/src/archive.rs index 8b2d64e..a4b141c 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -310,7 +310,7 @@ impl ArchiveRead { && let Ok(segs) = f_in.scan_chunks() { sparse_segments = segs; - // println!("{} {:?}", entry.path().display(), sparse_segments); + println!("arch: {} {:?}", entry.path().display(), sparse_segments); // number of holes let holes_count = sparse_segments.holes().count(); @@ -534,7 +534,7 @@ impl ArchiveWrite { /// - `Ok(())` on success. /// - `Err` if the system call fails. #[cfg(windows)] - fn set_files_sparse_win(file: &File) -> std::io::Result<()> { + fn set_sparse_file_on_windows(file: &File) -> std::io::Result<()> { use std::os::windows::io::AsRawHandle; use winapi::um::ioapiset::DeviceIoControl; use winapi::um::winioctl::FSCTL_SET_SPARSE; @@ -620,10 +620,10 @@ impl ArchiveWrite { // create directory (of file), if it doesn't exists Self::create_parent_directory(&entry_path)?; - let file = File::create(&entry_path)?; - self.f_out = Some(file.try_clone()?); + // create file + self.f_out = Some(File::create(&entry_path)?); - // for error handling + // for printing errors self.file_path = entry_path.clone(); // file size @@ -654,11 +654,11 @@ impl ArchiveWrite { if data_start != self.file_size { self.sparse_segments.push( Segment { segment_type: SegmentType::Data, range: data_start..self.file_size} ); } - // println!("{:?} {:?}", entry_path.display(), self.sparse_segments); + println!("unar: {:?} {:?}", entry_path.display(), self.sparse_segments); // Windows requires to set sparse flag for a sparse file #[cfg(windows)] - Self::set_files_sparse_win(&file)?; + Self::set_sparse_file_on_windows(self.f_out.as_ref().unwrap())?; } } else if file_type == TYPE_SYMLINK_FILE || file_type == TYPE_SYMLINK_DIR { @@ -770,7 +770,7 @@ impl WriteFiles for ArchiveWrite { if self.data_segment_size == 0 { self.data_segment_size = segment.len(); } - + let write_size = write_data(self.data_segment_size)?; self.data_segment_size -= write_size; self.file_size -= write_size; @@ -781,13 +781,17 @@ impl WriteFiles for ArchiveWrite { SegmentType::Hole => { f_out.seek_relative((segment.len() - 1).try_into()?)?; f_out.write_all(&[0])?; - f_out.drill_hole(segment.range.start, segment.range.end)?; self.file_size -= segment.len(); self.sparse_segments_index += 1; } } if self.file_size == 0 { + // holes must not be set before the whole file was written + for hole in self.sparse_segments.holes() { + f_out.drill_hole(hole.start, hole.end)?; + } + self.data_segment_size = 0; self.sparse_segments_index = 0; self.sparse_segments.clear(); @@ -871,3 +875,454 @@ impl WriteFiles for ArchiveWrite { Ok(()) } } + + + +// ====================================================================== +// Unit tests +// ====================================================================== + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::{self, File}; + use std::io::{Read, Seek, SeekFrom, Write}; + use std::path::{Path, PathBuf}; + use std::time::UNIX_EPOCH; + + struct TestDir { + path: PathBuf, + } + + impl TestDir { + fn new(dirname: &str) -> Self { + let path = PathBuf::from(dirname); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + Self { path } + } + } + + impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } + } + + #[test] + fn test_add_path_to_header() { + let mut buf = Vec::new(); + ArchiveRead::add_path_to_header(Path::new("hello/world.txt"), &mut buf).unwrap(); + + // Path length should be 15 (2 bytes, little-endian) + let expected_len = 15u16.to_le_bytes(); + assert_eq!(buf[0..2], expected_len); + assert_eq!(&buf[2..], b"hello/world.txt"); + } + + #[test] + fn test_create_parent_directory() { + let temp_dir = TestDir::new("test_archive_create_parent_dir"); + let nested_file = temp_dir.path.join("a/b/c/file.txt"); + assert!(!nested_file.parent().unwrap().exists()); + + ArchiveWrite::create_parent_directory(&nested_file).unwrap(); + assert!(nested_file.parent().unwrap().exists()); + + let nested_file2 = temp_dir.path.join("a/b/c/file2.txt"); + assert!(nested_file2.parent().unwrap().exists()); + ArchiveWrite::create_parent_directory(&nested_file2).unwrap(); + assert!(nested_file2.parent().unwrap().exists()); + } + + #[test] + fn test_get_path_from_header() { + // Test standard unix path decoding + let mut header = Vec::new(); + let path_str = "foo/bar/baz.txt"; + let path_len = path_str.len() as u16; + header.extend_from_slice(&path_len.to_le_bytes()); + header.extend_from_slice(path_str.as_bytes()); + + let (decoded, end_idx) = ArchiveWrite::get_path_from_header(&header, 0, TYPE_UNIX).unwrap(); + assert_eq!(decoded, "foo/bar/baz.txt"); + assert_eq!(end_idx, header.len()); + + // Test Windows path on Unix system decoding conversion + let mut header_win = Vec::new(); + let path_str_win = "foo\\bar\\baz.txt"; + let path_len_win = path_str_win.len() as u16; + header_win.extend_from_slice(&path_len_win.to_le_bytes()); + header_win.extend_from_slice(path_str_win.as_bytes()); + + let (decoded_win, end_idx_win) = ArchiveWrite::get_path_from_header(&header_win, 0, TYPE_WINDOWS).unwrap(); + if cfg!(unix) { + assert_eq!(decoded_win, "foo/bar/baz.txt"); + } else { + assert_eq!(decoded_win, "foo\\bar\\baz.txt"); + } + assert_eq!(end_idx_win, header_win.len()); + } + + #[test] + fn test_archive_read_write() { + let src_dir = TestDir::new("test_archive_src"); + + // Create standard folder and file + let file1_path = src_dir.path.join("file1.txt"); + let content1 = b"Hello from file 1!"; + fs::write(&file1_path, content1).unwrap(); + + // Create empty file + let empty_file_path = src_dir.path.join("empty.txt"); + fs::write(&empty_file_path, b"").unwrap(); + + // Create nested folder and file + let sub_dir = src_dir.path.join("subdir"); + fs::create_dir(&sub_dir).unwrap(); + let file2_path = sub_dir.join("file2.bin"); + let content2 = vec![0xAA; 5000]; + fs::write(&file2_path, &content2).unwrap(); + + // Create sparse file + let sparse_path = src_dir.path.join("sparse.bin"); + { + let mut f = File::create(&sparse_path).unwrap(); + + #[cfg(windows)] + ArchiveWrite::set_sparse_file_on_windows(&f).unwrap(); + + // Write 64KB of data at start + let start_data = vec![1u8; 65536]; + f.write_all(&start_data).unwrap(); + + // Seek to 128KB and write 64KB of data at end (grows the file to 192KB) + f.seek(SeekFrom::Start(131072)).unwrap(); + + let end_data = vec![2u8; 65536]; + f.write_all(&end_data).unwrap(); + + // Explicitly punch the hole in the middle region (64KB to 128KB) + f.drill_hole(65536, 131072).unwrap(); + } + + // Create symlink (windows and unix) and hardlink (Unix only) + let symlink_path = src_dir.path.join("link.txt"); + #[cfg(unix)] + let hardlink_path = src_dir.path.join("hardlink.txt"); + #[cfg(unix)] + { + std::os::unix::fs::symlink("file1.txt", &symlink_path).unwrap(); + fs::hard_link(&file1_path, &hardlink_path).unwrap(); + } + #[cfg(windows)] + { + std::os::windows::fs::symlink_file("file1.txt", &symlink_path).unwrap(); + } + + // Get original modified time of file1 + let meta_orig1 = fs::metadata(&file1_path).unwrap(); + let modified_orig1 = meta_orig1.modified().unwrap(); + + // Get original modified time of subdirectory + let meta_orig2 = fs::metadata(&sub_dir).unwrap(); + let modified_orig2 = meta_orig2.modified().unwrap(); + + // Perform archiving using ArchiveRead + let mut reader = ArchiveRead::new(&src_dir.path); + let mut archive_bytes = Vec::new(); + for i in 0..=128 { + let (chunk, last) = reader.read_chunk().unwrap(); + archive_bytes.extend(&chunk); + if last { + break; + } + assert!(i < 128); // timeout not reached + } + + reader.join_threads().unwrap(); + + // Verify we got some archive bytes + assert!(!archive_bytes.is_empty()); + + // Delete the original files before extracting to verify recreation + fs::remove_dir_all(&src_dir.path).unwrap(); + assert!(!src_dir.path.exists()); + + // Perform extraction using ArchiveWrite by feeding it in small chunks + let mut writer = ArchiveWrite::new(); + for chunk in archive_bytes.chunks(100) { + writer.write_files(chunk).unwrap(); + } + writer.write_others().unwrap(); + + // Verify structure is fully recreated + assert!(src_dir.path.exists()); + assert!(file1_path.exists()); + assert!(empty_file_path.exists()); + assert!(sub_dir.exists()); + assert!(file2_path.exists()); + assert!(sparse_path.exists()); + + // Verify contents + assert_eq!(fs::read(&file1_path).unwrap(), content1); + assert_eq!(fs::read(&empty_file_path).unwrap(), b""); + assert_eq!(fs::read(&file2_path).unwrap(), content2); + + // Verify sparse file content + { + let mut f = File::open(&sparse_path).unwrap(); + let mut start_buf = vec![0; 65536]; + f.read_exact(&mut start_buf).unwrap(); + assert_eq!(start_buf, vec![1u8; 65536]); + + f.seek(SeekFrom::Start(131072)).unwrap(); + let mut end_buf = vec![0; 65536]; + f.read_exact(&mut end_buf).unwrap(); + assert_eq!(end_buf, vec![2u8; 65536]); + + assert_eq!(fs::metadata(&sparse_path).unwrap().len(), 196608); + + // Verify it is actually a sparse file (has 1 hole) + let segs = f.scan_chunks().unwrap(); + assert_eq!(segs.holes().count(), 1); + let hole = segs.holes().next().unwrap(); + assert_eq!(hole.start, 65536); + assert_eq!(hole.end, 131072); + } + + // Verify modified time of file1 is restored (seconds precision) + let meta_restored1 = fs::metadata(&file1_path).unwrap(); + let modified_restored1 = meta_restored1.modified().unwrap(); + assert_eq!( + modified_orig1.duration_since(UNIX_EPOCH).unwrap().as_secs(), + modified_restored1.duration_since(UNIX_EPOCH).unwrap().as_secs() + ); + + // Verify modified time of subdir is restored (seconds precision) + let meta_restored2 = fs::metadata(&sub_dir).unwrap(); + let modified_restored2 = meta_restored2.modified().unwrap(); + assert_eq!( + modified_orig2.duration_since(UNIX_EPOCH).unwrap().as_secs(), + modified_restored2.duration_since(UNIX_EPOCH).unwrap().as_secs() + ); + + // Verify symlink + assert!(symlink_path.exists()); + let symlink_metadata = fs::symlink_metadata(&symlink_path).unwrap(); + assert!(symlink_metadata.file_type().is_symlink()); + let target = fs::read_link(&symlink_path).unwrap(); + assert_eq!(target, Path::new("file1.txt")); + + // Verify hardlink (Unix only) + #[cfg(unix)] + { + assert!(hardlink_path.exists()); + use std::os::unix::fs::MetadataExt; + let meta_f1 = fs::metadata(&file1_path).unwrap(); + let meta_hl = fs::metadata(&hardlink_path).unwrap(); + assert_eq!(meta_f1.ino(), meta_hl.ino()); + } + } + + #[test] + fn test_archive_sparse_files() { + let src_dir = TestDir::new("test_archive_sparse"); + + // hole only + let hole_filepath = src_dir.path.join("hole.bin"); + let mut hole_file = File::create(&hole_filepath).unwrap(); + let hole_filesize = 131072; + #[cfg(windows)] + ArchiveWrite::set_sparse_file_on_windows(&hole_file).unwrap(); + hole_file.seek(SeekFrom::Start(hole_filesize - 1)).unwrap(); + hole_file.write_all(&[0]).unwrap(); + hole_file.drill_hole(0, hole_filesize).unwrap(); + hole_file.sync_all().unwrap(); + + // hole-data + let hole_data_filepath = src_dir.path.join("hole_data.bin"); + let mut hole_data_file = File::create(&hole_data_filepath).unwrap(); + #[cfg(windows)] + ArchiveWrite::set_sparse_file_on_windows(&hole_data_file).unwrap(); + hole_data_file.seek_relative(65536).unwrap(); + hole_data_file.write_all(&[0x0Du8; 256]).unwrap(); + hole_data_file.drill_hole(0, 65536).unwrap(); + hole_data_file.sync_all().unwrap(); + + // data-hole + let data_hole_filepath = src_dir.path.join("data_hole.bin"); + let mut data_hole_file = File::create(&data_hole_filepath).unwrap(); + #[cfg(windows)] + ArchiveWrite::set_sparse_file_on_windows(&data_hole_file).unwrap(); + data_hole_file.write_all(&vec![0xD0u8; 131072]).unwrap(); + data_hole_file.seek_relative(131072 - 1).unwrap(); + data_hole_file.write_all(&[0]).unwrap(); + data_hole_file.drill_hole(131072, 131072 + 131072).unwrap(); + data_hole_file.sync_all().unwrap(); + + // hole-data-hole + let hdh_filepath = src_dir.path.join("hole_data_hole.bin"); + let mut hdh_file = File::create(&hdh_filepath).unwrap(); + #[cfg(windows)] + ArchiveWrite::set_sparse_file_on_windows(&hdh_file).unwrap(); + hdh_file.seek_relative(CHUNK_SIZE as i64).unwrap(); + hdh_file.write_all(&vec![3; CHUNK_SIZE]).unwrap(); + hdh_file.seek_relative(CHUNK_SIZE as i64 - 1).unwrap(); + hdh_file.write_all(&[0]).unwrap(); + hdh_file.drill_hole(0, CHUNK_SIZE as u64).unwrap(); + hdh_file.drill_hole(2 * CHUNK_SIZE as u64, 3 * CHUNK_SIZE as u64).unwrap(); + hdh_file.sync_all().unwrap(); + + + // Perform archiving using ArchiveRead + let mut reader = ArchiveRead::new(&src_dir.path); + let mut archive_bytes = Vec::new(); + for i in 0..=128 { + let (chunk, last_chunk) = reader.read_chunk().unwrap(); + archive_bytes.extend(&chunk); + if last_chunk { break; } + assert!(i < 128); // timeout not reached + } + reader.join_threads().unwrap(); + + // Verify we got some archive bytes + assert!(!archive_bytes.is_empty()); + + // Delete the original files before extracting to verify recreation + fs::remove_dir_all(&src_dir.path).unwrap(); + assert!(!src_dir.path.exists()); + + + // Perform extraction using ArchiveWrite + let mut writer = ArchiveWrite::new(); + for chunk in archive_bytes.chunks(CHUNK_SIZE) { + writer.write_files(chunk).unwrap(); + } + writer.write_others().unwrap(); + + // Verify structure is fully recreated + assert!(src_dir.path.exists()); + assert!(hole_filepath.exists()); + assert!(hole_data_filepath.exists()); + assert!(data_hole_filepath.exists()); + + + // Verify hole-only file + let mut hole_file = File::open(&hole_filepath).unwrap(); + let segs = hole_file.scan_chunks().unwrap(); + assert_eq!(segs.len(), 1); + assert_eq!(segs.holes().count(), 1); + assert_eq!(segs.data().count(), 0); + let hole = segs.first().unwrap(); + assert_eq!(hole.segment_type, SegmentType::Hole); + assert_eq!(hole.range.start, 0); + assert_eq!(hole.range.end, hole_filesize); + + // Verify hole-data file + let mut hole_data_file = File::open(&hole_data_filepath).unwrap(); + let segs = hole_data_file.scan_chunks().unwrap(); + assert_eq!(segs.len(), 2); + assert_eq!(segs.holes().count(), 1); + assert_eq!(segs.data().count(), 1); + let hole = segs.first().unwrap(); // check hole-segment + assert_eq!(hole.segment_type, SegmentType::Hole); + assert_eq!(hole.range.start, 0); + assert_eq!(hole.range.end, 65536); + let data = segs.last().unwrap(); // check data-segment + assert_eq!(data.segment_type, SegmentType::Data); + assert_eq!(data.range.start, 65536); + assert_eq!(data.range.end, 65536 + 256); + hole_data_file.rewind().unwrap(); // check file contents + let mut buf = vec![]; + let file_size = hole_data_file.read_to_end(&mut buf).unwrap(); + assert_eq!(file_size, 65536 + 256); + assert_eq!(buf[..65536], vec![0; 65536]); + assert_eq!(buf[65536..], vec![0x0Du8; 256]); + + // Verify data-hole file + let mut data_hole_file = File::open(&data_hole_filepath).unwrap(); + let segs = data_hole_file.scan_chunks().unwrap(); + assert_eq!(segs.len(), 2); + assert_eq!(segs.holes().count(), 1); + assert_eq!(segs.data().count(), 1); + let data = segs.first().unwrap(); // check data-segment + assert_eq!(data.segment_type, SegmentType::Data); + assert_eq!(data.range.start, 0); + assert_eq!(data.range.end, 131072); + let hole = segs.last().unwrap(); // check hole-segment + assert_eq!(hole.segment_type, SegmentType::Hole); + assert_eq!(hole.range.start, 131072); + assert_eq!(hole.range.end, 131072 + 131072); + data_hole_file.rewind().unwrap(); // check file contents + let mut buf = vec![]; + let file_size = data_hole_file.read_to_end(&mut buf).unwrap(); + assert_eq!(file_size, 131072 + 131072); + assert_eq!(buf[..131072], vec![0xD0u8; 131072]); + assert_eq!(buf[131072..], vec![0; 131072]); + + // Verify hole-data-hole file + let mut hdh_file = File::open(&hdh_filepath).unwrap(); + let segs = hdh_file.scan_chunks().unwrap(); + assert_eq!(segs.len(), 3); + assert_eq!(segs.holes().count(), 2); + assert_eq!(segs.data().count(), 1); + let hole = segs.first().unwrap(); // check hole-segment + assert_eq!(hole.segment_type, SegmentType::Hole); + assert_eq!(hole.range.start, 0); + assert_eq!(hole.range.end, CHUNK_SIZE as u64); + let data = segs.get(1).unwrap(); // check data-segment + assert_eq!(data.segment_type, SegmentType::Data); + assert_eq!(data.range.start, CHUNK_SIZE as u64); + assert_eq!(data.range.end, 2 * CHUNK_SIZE as u64); + let hole = segs.last().unwrap(); // check hole-segment + assert_eq!(hole.segment_type, SegmentType::Hole); + assert_eq!(hole.range.start, 2 * CHUNK_SIZE as u64); + assert_eq!(hole.range.end, 3 * CHUNK_SIZE as u64); + hdh_file.rewind().unwrap(); // check file contents + let mut buf = vec![]; + let file_size = hdh_file.read_to_end(&mut buf).unwrap(); + assert_eq!(file_size, 3 * CHUNK_SIZE); + assert_eq!(buf[..CHUNK_SIZE], vec![0; CHUNK_SIZE]); + assert_eq!(buf[CHUNK_SIZE..2 * CHUNK_SIZE], vec![3; CHUNK_SIZE]); + assert_eq!(buf[2 * CHUNK_SIZE..], vec![0; CHUNK_SIZE]); + } + + + #[test] + fn test_archive_nonexistent_dir() { + let mut reader = ArchiveRead::new(Path::new("test_archive_does_not_exist")); + let (chunk, last) = reader.read_chunk().unwrap(); + assert!(last); + assert!(chunk.is_empty()); + reader.join_threads().unwrap(); + } + + #[test] + fn test_archive_empty_dir() { + let src_dir = TestDir::new("test_archive_empty"); + + let mut reader = ArchiveRead::new(&src_dir.path); + let mut archive_bytes = Vec::new(); + + let (chunk, last) = reader.read_chunk().unwrap(); + archive_bytes.extend(&chunk); + assert!(last); + + reader.join_threads().unwrap(); + + // Delete source + fs::remove_dir_all(&src_dir.path).unwrap(); + + // Extract + let mut writer = ArchiveWrite::new(); + writer.write_files(&archive_bytes).unwrap(); + writer.write_others().unwrap(); + + // Recreated directory should exist and be empty + assert!(src_dir.path.exists()); + let count = fs::read_dir(&src_dir.path).unwrap().count(); + assert_eq!(count, 0); + } +} + diff --git a/src/encryption.rs b/src/encryption.rs index b9aac42..ff82913 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -806,6 +806,45 @@ mod tests { let _ = fs::remove_file("test_cc_split.bin.c03"); } + #[test] + fn test_crypt_archive() { + // Create directory with files that should be archived + let dir_path = PathBuf::from("test_archive_toplevel"); + fs::create_dir_all(&dir_path).unwrap(); + + let file1 = dir_path.join("file1.bin"); + let mut data1 = vec![0; CHUNK_SIZE * 10 + 12345]; + rand::rng().fill_bytes(&mut data1); + fs::write(&file1, &data1).unwrap(); + + let file2 = dir_path.join("file2.bin"); + let mut data2 = vec![0; CHUNK_SIZE]; + rand::rng().fill_bytes(&mut data2); + fs::write(&file2, &data2).unwrap(); + + // Build archive of directory and encrypt it + Encryption::encrypt(&dir_path, None, false, vec![]).unwrap(); + + // Delete the original files before extracting to verify recreation + fs::remove_dir_all(&dir_path).unwrap(); + assert!(!dir_path.exists()); + + // Decrypt and rebuild archived directory + let arch_path = dir_path.with_extension(ENCRYPTED_FILE_EXT); + Decryption::decrypt(&arch_path, None).unwrap(); + + // Verify structure is fully recreated + assert!(dir_path.exists()); + assert!(file1.exists()); + assert!(file2.exists()); + + // Verify contents + assert_eq!(fs::read(&file1).unwrap(), data1); + assert_eq!(fs::read(&file2).unwrap(), data2); + + let _ = fs::remove_dir_all(&dir_path); + } + #[test] #[ignore="only for benchmarking"] fn test_crypt_bench() { From cd024da1c49488f77e2c1402feff3448d962bf6c Mon Sep 17 00:00:00 2001 From: JoergDF Date: Thu, 9 Jul 2026 17:30:33 +0200 Subject: [PATCH 10/16] Add hard link detection for windows. Rework errors in sparse file handling. --- Cargo.lock | 24 +++---- Cargo.toml | 4 +- src/archive.rs | 162 ++++++++++++++++++++++++++++++++++------------ src/encryption.rs | 3 +- 4 files changed, 135 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 925442d..88b8df8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,9 +144,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" dependencies = [ "find-msvc-tools", "shlex", @@ -288,22 +288,22 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "cryptcrypt" -version = "0.3.0" +version = "0.4.0" dependencies = [ "aes-gcm-siv", "argon2", @@ -325,7 +325,7 @@ dependencies = [ "typed-path", "typenum", "walkdir", - "winapi", + "windows-sys 0.61.2", ] [[package]] @@ -972,18 +972,18 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 58dd0f0..f49e1be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cryptcrypt" -version = "0.3.0" +version = "0.4.0" edition = "2024" [dependencies] @@ -24,7 +24,7 @@ sha3 = "0.12.0" typed-path = "0.12.3" typenum = "1.19.0" walkdir = "2.5.0" -winapi = "0.3.9" +windows-sys = { version = "0.61.2", features = ["Win32_System_Ioctl"] } [profile.release] lto = "thin" diff --git a/src/archive.rs b/src/archive.rs index a4b141c..6e71804 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -5,11 +5,10 @@ use std::path::{Path, PathBuf}; use std::thread; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use typed_path::Utf8WindowsPath; -use walkdir::WalkDir; +use walkdir::{DirEntry, WalkDir}; use crossbeam_channel::{Receiver, Select, TryRecvError, bounded}; use num_cpus; use filetime; -#[cfg(unix)] use std::collections::HashMap; use drill_press::{SegmentType, Segment, Segments, SparseFile}; @@ -66,35 +65,63 @@ impl ArchiveRead { let f_in_path = f_in_path.to_path_buf(); let tx_paths = tx_paths.clone(); thread_handles.push(thread::spawn(move || -> std::result::Result<(), String> { + + /// Helper function to get a file's metadata for detecting hardlinks (Unix version) + /// Files with hard links have a number-of-links greater 1. + /// A hard link and the file it is pointing to have the same file id. + /// + /// # Arguments + /// - `entry`: Directory entry which should be a file + /// + /// # Returns + /// - `(num_links, file_id)`: (number of links, file id) #[cfg(unix)] - let mut hard_link_files: HashMap = HashMap::new(); + fn file_meta_for_hardlink(entry: &DirEntry) -> (u64, u128) { + use std::os::unix::fs::MetadataExt; + + let mut num_links = 0; + let mut file_id = 0; + + if let Ok(meta) = entry.metadata() { + num_links = meta.nlink(); + file_id = meta.ino() as u128; + } + + (num_links, file_id) + } + + /// Helper function to get a file's metadata for detecting hardlinks (Windows version) + #[cfg(windows)] + fn file_meta_for_hardlink(entry: &DirEntry) -> (u64, u128) { + match ArchiveRead::file_meta_for_hardlink_on_windows(entry.path()) { + Ok(values) => values, + Err(e) => { + eprintln!("Could not get hard link information of file {} - Reason: {e}", entry.file_name().display()); + (0, 0) + } + } + } + + let mut hard_link_files: HashMap = HashMap::new(); for entry in WalkDir::new(f_in_path) { match entry { Ok(entry) => { // check if entry is a hard link - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - use walkdir::DirEntryExt; - - let mut hard_link_target: Option = None; - if entry.file_type().is_file() - && let Ok(meta) = entry.metadata() && meta.nlink() > 1 { - let file_id = entry.ino(); - if let Some(hl_target) = hard_link_files.get(&file_id) { - // entry is hard link - hard_link_target = Some(hl_target.to_owned()); - } else { - // entry is taken as original file path (i.e. target of hard link) - hard_link_files.insert(file_id, entry.clone().into_path()); - } + let mut hard_link_target: Option = None; + if entry.file_type().is_file() { + let (num_links, file_id) = file_meta_for_hardlink(&entry); + if num_links > 1 { + if let Some(hl_target) = hard_link_files.get(&file_id) { + // entry is hard link + hard_link_target = Some(hl_target.to_owned()); + } else { + // entry is taken as original file path (i.e. target of hard link) + hard_link_files.insert(file_id, entry.clone().into_path()); + } } - let _ = tx_paths.send((entry, hard_link_target)); } - #[cfg(windows)] - let _ = tx_paths.send((entry, None)); - // windows: use number_of_links() and file_index() of std::os::windows::fs::MetadataExt, when supported by stable rust + let _ = tx_paths.send((entry, hard_link_target)); } Err(ref e) => { eprintln!("Skipped entry while walking directory tree - Reason: {e}"); } } @@ -200,6 +227,44 @@ impl ArchiveRead { Self { thread_handles, rx_out_receivers, channel_index: None, channel_finished: vec![false; num_workers], buf_out } } + /// Gets a file's metadata for detecting hardlinks (Windows version) + /// + /// # Arguments + /// - `filepath` - path to file + /// + /// # Returns + /// `Ok((num_links, file_id))` on success (number of links, file id) + /// `Èrr` if the system call fails. + #[cfg(windows)] + fn file_meta_for_hardlink_on_windows(filepath: &Path) -> std::io::Result<(u64, u128)> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION}; + use windows_sys::Win32::Foundation::HANDLE; + + let mut num_links = 0; + let mut file_id = 0; + + if let Ok(file) = File::open(filepath) { + let file_handle = file.as_raw_handle(); + unsafe { + let mut info: BY_HANDLE_FILE_INFORMATION = mem::zeroed(); + let result = GetFileInformationByHandle( + file_handle as HANDLE, + &mut info + ); + if result == 0 { + return Err(std::io::Error::last_os_error()); + } else { + num_links = info.nNumberOfLinks as u64; + file_id = ((info.dwVolumeSerialNumber as u128) << 64) + | ((info.nFileIndexHigh as u128) << 32) | (info.nFileIndexLow as u128); + } + } + } + Ok((num_links, file_id)) + } + /// Appends the file path length and path string to the archive header buffer. /// /// # Arguments @@ -231,7 +296,7 @@ impl ArchiveRead { /// - `Ok((archive_header, filepath_and_size, sparse_segments))` on success. /// - `Err` if metadata retrieval or OS-specific operations fail. #[allow(clippy::type_complexity)] - fn build_archive_header(entry: &walkdir::DirEntry, hard_link_target: &Option) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { + fn build_archive_header(entry: &DirEntry, hard_link_target: &Option) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { // archive header initialized with place holder for header size let mut archive_header = vec![0u8; ARCHIVE_HEADER_LENGTH_SIZE]; @@ -310,7 +375,7 @@ impl ArchiveRead { && let Ok(segs) = f_in.scan_chunks() { sparse_segments = segs; - println!("arch: {} {:?}", entry.path().display(), sparse_segments); + // println!("arch: {} {:?}", entry.path().display(), sparse_segments); // number of holes let holes_count = sparse_segments.holes().count(); @@ -536,14 +601,15 @@ impl ArchiveWrite { #[cfg(windows)] fn set_sparse_file_on_windows(file: &File) -> std::io::Result<()> { use std::os::windows::io::AsRawHandle; - use winapi::um::ioapiset::DeviceIoControl; - use winapi::um::winioctl::FSCTL_SET_SPARSE; + use windows_sys::Win32::System::IO::DeviceIoControl; + use windows_sys::Win32::System::Ioctl::FSCTL_SET_SPARSE; + use windows_sys::Win32::Foundation::HANDLE; let handle = file.as_raw_handle(); let mut bytes_returned = 0; unsafe { let result = DeviceIoControl( - handle as _, + handle as HANDLE, FSCTL_SET_SPARSE, std::ptr::null_mut(), 0, @@ -654,11 +720,13 @@ impl ArchiveWrite { if data_start != self.file_size { self.sparse_segments.push( Segment { segment_type: SegmentType::Data, range: data_start..self.file_size} ); } - println!("unar: {:?} {:?}", entry_path.display(), self.sparse_segments); + // println!("unar: {:?} {:?}", entry_path.display(), self.sparse_segments); // Windows requires to set sparse flag for a sparse file #[cfg(windows)] - Self::set_sparse_file_on_windows(self.f_out.as_ref().unwrap())?; + if Self::set_sparse_file_on_windows(self.f_out.as_ref().unwrap()).is_err() { + eprintln!("Could not set sparse option for file {}", entry_path.display()); + } } } else if file_type == TYPE_SYMLINK_FILE || file_type == TYPE_SYMLINK_DIR { @@ -669,9 +737,6 @@ impl ArchiveWrite { let target_path; (target_path, e) = Self::get_path_from_header(header, e, created_on_os_type)?; - // remove symlink, if it already exists, otherwise it can't be created - let _ = fs::remove_file(&entry_path); - // create symlink #[cfg(unix)] { @@ -787,9 +852,12 @@ impl WriteFiles for ArchiveWrite { } if self.file_size == 0 { - // holes must not be set before the whole file was written + // holes must not be set before the whole file was written; + // on failure, the file should become non-sparse, do not break execution, just continue for hole in self.sparse_segments.holes() { - f_out.drill_hole(hole.start, hole.end)?; + if f_out.drill_hole(hole.start, hole.end).is_err() { + eprintln!("Could not set sparse region for file {}", self.file_path.display()); + } } self.data_segment_size = 0; @@ -851,9 +919,6 @@ impl WriteFiles for ArchiveWrite { // create directory (of hard link), if it doesn't exists Self::create_parent_directory(entry_path)?; - // remove hard link, if it already exists, otherwise it can't be created - let _ = fs::remove_file(entry_path); - fs::hard_link(target_path, entry_path)?; } @@ -1006,20 +1071,21 @@ mod tests { f.drill_hole(65536, 131072).unwrap(); } - // Create symlink (windows and unix) and hardlink (Unix only) + // Create symlink let symlink_path = src_dir.path.join("link.txt"); #[cfg(unix)] - let hardlink_path = src_dir.path.join("hardlink.txt"); - #[cfg(unix)] { std::os::unix::fs::symlink("file1.txt", &symlink_path).unwrap(); - fs::hard_link(&file1_path, &hardlink_path).unwrap(); } #[cfg(windows)] { std::os::windows::fs::symlink_file("file1.txt", &symlink_path).unwrap(); } + // Create hardlink + let hardlink_path = src_dir.path.join("hardlink.txt"); + fs::hard_link(&file1_path, &hardlink_path).unwrap(); + // Get original modified time of file1 let meta_orig1 = fs::metadata(&file1_path).unwrap(); let modified_orig1 = meta_orig1.modified().unwrap(); @@ -1114,7 +1180,7 @@ mod tests { let target = fs::read_link(&symlink_path).unwrap(); assert_eq!(target, Path::new("file1.txt")); - // Verify hardlink (Unix only) + // Verify hardlink #[cfg(unix)] { assert!(hardlink_path.exists()); @@ -1122,6 +1188,16 @@ mod tests { let meta_f1 = fs::metadata(&file1_path).unwrap(); let meta_hl = fs::metadata(&hardlink_path).unwrap(); assert_eq!(meta_f1.ino(), meta_hl.ino()); + assert_eq!(meta_f1.nlink(), meta_hl.nlink()); + } + #[cfg(windows)] + { + assert!(hardlink_path.exists()); + let (num_links_f1, file_id_f1) = ArchiveRead::file_meta_for_hardlink_on_windows(&file1_path).unwrap(); + let (num_links_hl, file_id_hl) = ArchiveRead::file_meta_for_hardlink_on_windows(&hardlink_path).unwrap(); + assert_eq!(file_id_f1, file_id_hl); + assert_eq!(num_links_f1, 2); + assert_eq!(num_links_hl, 2); } } diff --git a/src/encryption.rs b/src/encryption.rs index ff82913..c0d3c7e 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -806,7 +806,7 @@ mod tests { let _ = fs::remove_file("test_cc_split.bin.c03"); } - #[test] + #[test] fn test_crypt_archive() { // Create directory with files that should be archived let dir_path = PathBuf::from("test_archive_toplevel"); @@ -843,6 +843,7 @@ mod tests { assert_eq!(fs::read(&file2).unwrap(), data2); let _ = fs::remove_dir_all(&dir_path); + let _ = fs::remove_file(&arch_path); } #[test] From 17acd417d6789d0be0ffdebe6676164f7de6333a Mon Sep 17 00:00:00 2001 From: JoergDF Date: Thu, 16 Jul 2026 13:02:40 +0200 Subject: [PATCH 11/16] Add output path option to command line. Add exclusion of split-files from archiving. --- src/archive.rs | 138 ++++++++++++++++++++++++--- src/decryption.rs | 42 +++++++- src/encryption.rs | 238 ++++++++++++++++++++++++++++++++++++++-------- src/main.rs | 25 +++-- 4 files changed, 376 insertions(+), 67 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 6e71804..39e1f4c 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; use drill_press::{SegmentType, Segment, Segments, SparseFile}; use crate::common_io::{ReadChunk, WriteFiles}; -use crate::{CHUNK_SIZE, Result}; +use crate::{CHUNK_SIZE, Result, ENCRYPTED_FILE_EXT, SPLIT_ENC_FILE_EXT}; const TYPE_FILE: u8 = 0x00; @@ -51,10 +51,11 @@ impl ArchiveRead { /// /// # Arguments /// - `f_in_path`: The root path of the directory tree to archive. + /// - `exclude_path`: Path to be excluded from archive /// /// # Returns /// - A new `ArchiveRead` instance. - pub fn new(f_in_path: &Path) -> Self { + pub fn new(f_in_path: &Path, exclude_path: &Path) -> Self { let num_workers = num_cpus::get(); let mut thread_handles = Vec::with_capacity(num_workers + 1); let mut rx_out_receivers = Vec::with_capacity(num_workers); @@ -63,12 +64,13 @@ impl ArchiveRead { { let f_in_path = f_in_path.to_path_buf(); + let exclude_path = exclude_path.to_path_buf(); let tx_paths = tx_paths.clone(); thread_handles.push(thread::spawn(move || -> std::result::Result<(), String> { /// Helper function to get a file's metadata for detecting hardlinks (Unix version) /// Files with hard links have a number-of-links greater 1. - /// A hard link and the file it is pointing to have the same file id. + /// A hard link and the file it is pointing to have the same file id (inode). /// /// # Arguments /// - `entry`: Directory entry which should be a file @@ -101,15 +103,19 @@ impl ArchiveRead { } } } - + let mut hard_link_files: HashMap = HashMap::new(); for entry in WalkDir::new(f_in_path) { match entry { Ok(entry) => { - // check if entry is a hard link let mut hard_link_target: Option = None; if entry.file_type().is_file() { + // exclude path from archiving (archive output files should not be used) + if Self::exclude_file(entry.path(), &exclude_path) { + continue; + } + // check if entry is a hard link let (num_links, file_id) = file_meta_for_hardlink(&entry); if num_links > 1 { if let Some(hl_target) = hard_link_files.get(&file_id) { @@ -227,6 +233,53 @@ impl ArchiveRead { Self { thread_handles, rx_out_receivers, channel_index: None, channel_finished: vec![false; num_workers], buf_out } } + /// Checks if a file entry path matches the exclude path (the target archive output file(s)). + /// + /// This prevents the archiver from reading and archiving its own output file(s) + /// (e.g. single `.cce` files or split `.cXX` archive volumes) when they are stored + /// within the directory being archived. + /// + /// # Arguments + /// - `entry_path`: The path of the file entry to check. + /// - `exclude_path`: The target archive output path to exclude. + /// + /// # Returns + /// - `true` if the entry path matches the target archive path and should be excluded. + /// - `false` otherwise. + fn exclude_file(entry_path: &Path, exclude_path: &Path) -> bool { + let Some(exclude_ext) = exclude_path.extension() else { + return false; + }; + if exclude_ext == ENCRYPTED_FILE_EXT { + // single output file .cce + + // exclude_path is an absolute path, entry.path() is a relative path + return exclude_path.ends_with(entry_path); + + } else if exclude_ext == SPLIT_ENC_FILE_EXT { + // split file .c00, .c01, .c02, ... + + // entry.path() is relative, therefore make it absolute for following comparison + let Ok(entry_path) = entry_path.canonicalize() else { + return false; + }; + if entry_path.parent() != exclude_path.parent() { + return false; + } + if entry_path.file_stem() != exclude_path.file_stem() { + return false; + } + if let Some(entry_ext) = entry_path.extension() && let Some(entry_ext) = entry_ext.to_str() { + return entry_ext.starts_with('c') && entry_ext[1..].chars().all(|c| c.is_ascii_digit()); + } + + } else { + panic!("Unknown file extension {}", exclude_ext.display()); + } + + false + } + /// Gets a file's metadata for detecting hardlinks (Windows version) /// /// # Arguments @@ -676,7 +729,7 @@ impl ArchiveWrite { // create type if file_type == TYPE_DIRECTORY { // create directory - // it could be an empty one therefore it would not be created by other entries + // it could be an empty directory therefore it would not be created by other entries fs::create_dir_all(&entry_path)?; // save timestamps for restoring them at the end @@ -1095,7 +1148,7 @@ mod tests { let modified_orig2 = meta_orig2.modified().unwrap(); // Perform archiving using ArchiveRead - let mut reader = ArchiveRead::new(&src_dir.path); + let mut reader = ArchiveRead::new(&src_dir.path, Path::new("")); let mut archive_bytes = Vec::new(); for i in 0..=128 { let (chunk, last) = reader.read_chunk().unwrap(); @@ -1252,7 +1305,7 @@ mod tests { // Perform archiving using ArchiveRead - let mut reader = ArchiveRead::new(&src_dir.path); + let mut reader = ArchiveRead::new(&src_dir.path, Path::new("")); let mut archive_bytes = Vec::new(); for i in 0..=128 { let (chunk, last_chunk) = reader.read_chunk().unwrap(); @@ -1364,10 +1417,9 @@ mod tests { assert_eq!(buf[2 * CHUNK_SIZE..], vec![0; CHUNK_SIZE]); } - #[test] fn test_archive_nonexistent_dir() { - let mut reader = ArchiveRead::new(Path::new("test_archive_does_not_exist")); + let mut reader = ArchiveRead::new(Path::new("test_archive_does_not_exist"), Path::new("")); let (chunk, last) = reader.read_chunk().unwrap(); assert!(last); assert!(chunk.is_empty()); @@ -1378,7 +1430,7 @@ mod tests { fn test_archive_empty_dir() { let src_dir = TestDir::new("test_archive_empty"); - let mut reader = ArchiveRead::new(&src_dir.path); + let mut reader = ArchiveRead::new(&src_dir.path, Path::new("")); let mut archive_bytes = Vec::new(); let (chunk, last) = reader.read_chunk().unwrap(); @@ -1400,5 +1452,69 @@ mod tests { let count = fs::read_dir(&src_dir.path).unwrap().count(); assert_eq!(count, 0); } + + #[test] + fn test_exclude_file() { + // Case 1: No extension on exclude_path -> returns false + assert!(!ArchiveRead::exclude_file(Path::new("file.txt"), Path::new("no_ext"))); + + // Case 2: Unknown extension on exclude_path -> panics! + let result = std::panic::catch_unwind(|| { + ArchiveRead::exclude_file(Path::new("file.txt"), Path::new("file.txt")); + }); + assert!(result.is_err()); + + // Case 3: ENCRYPTED_FILE_EXT ("cce") + let exclude_cce = Path::new("/path/to/my_archive.cce"); + assert!(ArchiveRead::exclude_file(Path::new("my_archive.cce"), exclude_cce)); + assert!(ArchiveRead::exclude_file(Path::new("to/my_archive.cce"), exclude_cce)); + assert!(ArchiveRead::exclude_file(Path::new("path/to/my_archive.cce"), exclude_cce)); + assert!(!ArchiveRead::exclude_file(Path::new("other.cce"), exclude_cce)); + + // Case 4: SPLIT_ENC_FILE_EXT ("c00") + let temp_dir = TestDir::new("test_exclude_file_split"); + let base_dir = temp_dir.path.canonicalize().unwrap(); + + let exclude_c00 = base_dir.join("archive.c00"); + + // Non-existent entry path should fail canonicalize and return false + assert!(!ArchiveRead::exclude_file(Path::new("non_existent_archive.c00"), &exclude_c00)); + + // Create actual files to test successful canonicalization + let entry_c00 = base_dir.join("archive.c00"); + File::create(&entry_c00).unwrap(); + let entry_c01 = base_dir.join("archive.c01"); + File::create(&entry_c01).unwrap(); + let entry_c99 = base_dir.join("archive.c99"); + File::create(&entry_c99).unwrap(); + let entry_c100 = base_dir.join("archive.c100"); + File::create(&entry_c100).unwrap(); + let entry_txt = base_dir.join("archive.txt"); + File::create(&entry_txt).unwrap(); + let entry_other_stem = base_dir.join("other.c00"); + File::create(&entry_other_stem).unwrap(); + + // Create a file in a different directory with same name/extension + let other_dir = base_dir.join("subdir"); + fs::create_dir_all(&other_dir).unwrap(); + let entry_diff_dir = other_dir.join("archive.c00"); + File::create(&entry_diff_dir).unwrap(); + + // Assertions for c00 splits: + // matching splits + assert!(ArchiveRead::exclude_file(&entry_c00, &exclude_c00)); + assert!(ArchiveRead::exclude_file(&entry_c01, &exclude_c00)); + assert!(ArchiveRead::exclude_file(&entry_c99, &exclude_c00)); + assert!(ArchiveRead::exclude_file(&entry_c100, &exclude_c00)); + + // relative matching path (canonicalize converts to absolute) + let relative_c00 = Path::new("test_exclude_file_split").join("archive.c00"); + assert!(ArchiveRead::exclude_file(&relative_c00, &exclude_c00)); + + // non-matching extensions or stems + assert!(!ArchiveRead::exclude_file(&entry_txt, &exclude_c00)); + assert!(!ArchiveRead::exclude_file(&entry_other_stem, &exclude_c00)); + assert!(!ArchiveRead::exclude_file(&entry_diff_dir, &exclude_c00)); + } } diff --git a/src/decryption.rs b/src/decryption.rs index 6bac3af..5301368 100644 --- a/src/decryption.rs +++ b/src/decryption.rs @@ -2,6 +2,8 @@ use std::thread; use std::io::Read; use std::path::PathBuf; use std::collections::HashMap; +use std::fs; +use std::env; use argon2::Argon2; use chacha20poly1305::{XChaCha20Poly1305, XNonce}; use aes_gcm_siv::{aead::{Aead, KeyInit}, Aes256GcmSiv, Nonce}; @@ -311,12 +313,13 @@ impl Decryption { /// /// # Arguments /// - `filepath_in`: Path to encrypted input file (must end with `.cce`) + /// - `dirpath_out`: Option path to an output directory /// - `keyfilepath`: Optional path to an additional key file /// /// # Returns /// - `Ok(())` on successful decryption /// - `Err` if file operations, password handling, or decryption fails - pub fn decrypt(filepath_in: &PathBuf, keyfilepath: Option<&PathBuf>) -> Result<()> { + pub fn decrypt(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, keyfilepath: Option<&PathBuf>) -> Result<()> { if filepath_in.is_dir() { return Err("Cannot decrypt a directory".into()); } @@ -329,7 +332,7 @@ impl Decryption { } else { return Err(format!("Invalid filename, it does not end with .{ENCRYPTED_FILE_EXT} or .{SPLIT_ENC_FILE_EXT}").into()) } - + // set read parameters let mut read_input = Box:: new( ReadInput::new( filepath_in, @@ -354,12 +357,41 @@ impl Decryption { ).into()); } - // get keys + let compress = (file_format & 0x01) != 0; + let archive = (file_format & 0x02) != 0; + + let mut working_dir = &env::current_dir()?; + // output directory path is used + if let Some(dir_out) = dirpath_out { + if archive { + working_dir = dir_out; + } else { + let filename_out = filepath_out.file_name().unwrap(); + filepath_out = dir_out.join(filename_out); + } + } + + if archive { + println!("Archive file will be extracted to directory {}", working_dir.display()); + } else { + println!("Output will be written to file {}", filepath_out.display()); + } + + // get password and keys let key = Self::hash_password(salt_pw, keyfilepath)?; let (key_cha, key_aes) = Self::derive_keys(salt_cha, salt_aes, &key)?; - let compress = (file_format & 0x01) != 0; - let archive = (file_format & 0x02) != 0; + // output directory actions: create new directory, change working directory for an archive + // create a new directory after password entry: + // if password entry failed or user breaks execution on password entry, filesystem stays unchanged + if let Some(dir_out) = dirpath_out { + if !dir_out.exists() { + fs::create_dir_all(dir_out)?; + } + if archive { + env::set_current_dir(dir_out)?; + } + } let write_output: Box = if archive { Box::new( ArchiveWrite::new() ) diff --git a/src/encryption.rs b/src/encryption.rs index c0d3c7e..2bee138 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -1,6 +1,7 @@ use std::io::Read; -use std::path::PathBuf; -use std::thread; +use std::path::{self, PathBuf}; +use std::{fs, thread}; +use std::env; use argon2::Argon2; use chacha20poly1305::{XChaCha20Poly1305}; use rand::{Rng, SeedableRng}; @@ -339,15 +340,78 @@ impl Encryption { thread_handles } - /// Encrypts a file using dual-layer encryption (ChaCha20 + AES-256-GCM-SIV) with optional compression. + /// Resolves and prepares input and output paths for the encryption process. + /// + /// Determines whether the input path represents a directory (which requires + /// archiving). Constructs the final output file path by applying the correct + /// extension depending on whether splitting is enabled, and redirects output + /// to the target directory if specified. If archiving a directory, it returns + /// the target working directory that should be set during archiving to keep + /// paths relative. + /// + /// # Arguments + /// - `filepath_in`: Path to the input file or directory to be encrypted. + /// - `dirpath_out`: Optional path to a directory where the encrypted output should be saved. + /// - `no_split`: Boolean flag indicating if output splitting is disabled. + /// + /// # Returns + /// - `Ok((processed_path_in, filepath_out, build_archive, new_working_dir))` containing: + /// - `processed_path_in`: The relative input path if archiving, or the original input path. + /// - `filepath_out`: The resolved absolute path of the output encrypted file. + /// - `build_archive`: Boolean indicating whether the input is a directory. + /// - `new_working_dir`: Optional path to the directory that the process should change to before archiving. + /// - `Err` if absolute path resolution fails. + fn set_paths(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, no_split: bool) -> Result<(PathBuf, PathBuf, bool, Option)> { + let build_archive = filepath_in.is_dir(); + + let mut filepath_out = filepath_in.clone(); + // if input path is the root or relative (e.g. "/", "."), so without directory name, add a name for the archive + if build_archive && filepath_in.file_name().is_none() { + filepath_out.push("archive"); + } + if no_split { + filepath_out.add_extension(ENCRYPTED_FILE_EXT); + } else { + filepath_out.add_extension(SPLIT_ENC_FILE_EXT); + } + // output directory path is used + // build path, but create directory after password entry + if let Some(dir_out) = dirpath_out { + let filename_out = filepath_out.file_name().unwrap(); + filepath_out = dir_out.join(filename_out); + } + filepath_out = path::absolute(filepath_out)?; + + let mut new_working_dir = None; + let processed_path_in = if build_archive { + // change directory to parent directory + if let Some(dir_path) = filepath_in.file_name() { + if let Some(parent_path) = filepath_in.parent() && parent_path != "" { + new_working_dir = Some(parent_path.to_path_buf()); + } + PathBuf::from(dir_path) + } else { + new_working_dir = Some(filepath_in.to_path_buf()); + PathBuf::from(".") + } + } else { + filepath_in.to_path_buf() + }; + + Ok((processed_path_in, filepath_out, build_archive, new_working_dir)) + } + + /// Encrypts a file or a directory using dual-layer encryption (ChaCha20 + AES-256-GCM-SIV) + /// with optional compression. /// /// Prompts user for password, derives master key using Argon2, derives keys for - /// ChaCha20 and AES-256-GCM-SIV, compresses (on demand) and encrypts the file in + /// ChaCha20 and AES-256-GCM-SIV, compresses (on demand) and encrypts the file/directory in /// chunks across multiple threads. Output file gets `.cce` extension. Or output can /// be split into several files, which get extensions `.c00`, `.c01`, `.c02`, ... /// /// # Arguments - /// - `filepath_in`: Path to input file to encrypt + /// - `filepath_in`: Path to input file or directory to encrypt + /// - `dirpath_out`: Optional path to an output directory /// - `keyfilepath`: Optional path to an additional key file /// - `compress`: Compress input file before encryption /// - `split`: List of output split sizes; if empty, no split is done. @@ -355,24 +419,30 @@ impl Encryption { /// # Returns /// - `Ok(())` on successful encryption /// - `Err` if file operations, password handling, or encryption fails - pub fn encrypt(filepath_in: &PathBuf, keyfilepath: Option<&PathBuf>, compress: bool, split: Vec) -> Result<()> { - let mut filepath_out = filepath_in.clone(); - if split.is_empty() { - filepath_out.add_extension(ENCRYPTED_FILE_EXT); - } else { - filepath_out.add_extension(SPLIT_ENC_FILE_EXT); + pub fn encrypt(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, keyfilepath: Option<&PathBuf>, compress: bool, split: Vec) -> Result<()> { + + let (processed_path_in, filepath_out, build_archive, new_working_dir) = Self::set_paths(filepath_in, dirpath_out, split.is_empty())?; + if build_archive && let Some(work_dir) = new_working_dir { + env::set_current_dir(work_dir)?; } - + + println!("Output will be written to {}", filepath_out.display()); + // ask for password, before there can be error messages of archive let (salt_pw, key) = Self::hash_password(keyfilepath)?; let (salt_cha, key_cha, salt_aes, key_aes) = Self::derive_keys(&key)?; - let build_archive = filepath_in.is_dir(); + // output directory path is used + // create a new directory after password entry: + // if password entry failed or user breaks execution on password entry, filesystem stays unchanged + if let Some(dir_out) = dirpath_out && !dir_out.exists() { + fs::create_dir_all(dir_out)?; + } let read_input: Box = if build_archive { - Box::new( ArchiveRead::new(filepath_in) ) + Box::new( ArchiveRead::new(&processed_path_in, &filepath_out) ) } else { - Box::new( ReadInput::new(filepath_in, CHUNK_SIZE, 0)? ) + Box::new( ReadInput::new(&processed_path_in, CHUNK_SIZE, 0)? ) }; // file header @@ -681,10 +751,11 @@ mod tests { fs::write(&filepath_in, &data).unwrap(); // encrypt, decrypt - Encryption::encrypt(&filepath_in, None, false, vec![]).unwrap(); + Encryption::encrypt(&filepath_in, None, None, false, vec![]).unwrap(); + assert!(filepath_out.exists()); // encrypted file must be different than original data assert_ne!(data, fs::read(&filepath_out).unwrap()); - Decryption::decrypt(&filepath_out, None).unwrap(); + Decryption::decrypt(&filepath_out, None, None).unwrap(); // read and compare decrypted file against backup let decrypt_data = fs::read(&filepath_in).unwrap(); @@ -693,12 +764,12 @@ mod tests { // decrypt with keyfile should fail let filepath_kf = PathBuf::from("test_another_key.bin"); fs::write(&filepath_kf, vec![0; 1024]).unwrap(); - assert!(Decryption::decrypt(&filepath_out, Some(&filepath_kf)).is_err()); + assert!(Decryption::decrypt(&filepath_out, None, Some(&filepath_kf)).is_err()); // with compression fs::write(&filepath_in, &data).unwrap(); - Encryption::encrypt(&filepath_in, None, true, vec![]).unwrap(); - Decryption::decrypt(&filepath_out, None).unwrap(); + Encryption::encrypt(&filepath_in, None, None, true, vec![]).unwrap(); + Decryption::decrypt(&filepath_out, None, None).unwrap(); let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -725,30 +796,30 @@ mod tests { fs::write(&filepath_kf, &data_kf).unwrap(); // use keyfile, encrypt, decrypt - Encryption::encrypt(&filepath_in, Some(&filepath_kf), false, vec![]).unwrap(); - Decryption::decrypt(&filepath_out, Some(&filepath_kf)).unwrap(); + Encryption::encrypt(&filepath_in, None, Some(&filepath_kf), false, vec![]).unwrap(); + Decryption::decrypt(&filepath_out, None, Some(&filepath_kf)).unwrap(); // read and compare decrypted file against original data let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); // decrypt without key file - assert!(Decryption::decrypt(&filepath_out, None).is_err()); + assert!(Decryption::decrypt(&filepath_out, None, None).is_err()); // key file does not exist - assert!(Encryption::encrypt(&filepath_in, Some(&PathBuf::from("test_miss")), false, vec![]).is_err()); - assert!(Decryption::decrypt(&filepath_out, Some(&PathBuf::from("test_miss"))).is_err()); + assert!(Encryption::encrypt(&filepath_in, None, Some(&PathBuf::from("test_miss")), false, vec![]).is_err()); + assert!(Decryption::decrypt(&filepath_out, None, Some(&PathBuf::from("test_miss"))).is_err()); // input file does not exist - assert!(Encryption::encrypt(&PathBuf::from("test_miss"), None, false, vec![]).is_err()); - assert!(Decryption::decrypt(&PathBuf::from("test_miss.cce"), None).is_err()); + assert!(Encryption::encrypt(&PathBuf::from("test_miss"), None, None, false, vec![]).is_err()); + assert!(Decryption::decrypt(&PathBuf::from("test_miss.cce"), None, None).is_err()); assert!(!fs::exists("test_miss").unwrap()); assert!(!fs::exists("test_miss.cce").unwrap()); // with compression fs::write(&filepath_in, &data).unwrap(); - Encryption::encrypt(&filepath_in, Some(&filepath_kf), true, vec![]).unwrap(); - Decryption::decrypt(&filepath_out, Some(&filepath_kf)).unwrap(); + Encryption::encrypt(&filepath_in, None, Some(&filepath_kf), true, vec![]).unwrap(); + Decryption::decrypt(&filepath_out, None, Some(&filepath_kf)).unwrap(); let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -770,7 +841,7 @@ mod tests { fs::write(&filepath_in, &data).unwrap(); // encrypt and split output - Encryption::encrypt(&filepath_in, None, false, vec![1048576, 12]).unwrap(); + Encryption::encrypt(&filepath_in, None, None, false, vec![1048576, 12]).unwrap(); // concatenate spilt output files let mut data_concat = fs::read("test_cc_split.bin.c00").unwrap(); @@ -778,7 +849,7 @@ mod tests { data_concat.extend(fs::read("test_cc_split.bin.c02").unwrap()); fs::write(&filepath_out, &data_concat).unwrap(); - Decryption::decrypt(&filepath_out, None).unwrap(); + Decryption::decrypt(&filepath_out, None, None).unwrap(); // read and compare decrypted file against original data let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -786,14 +857,14 @@ mod tests { // concatenate files with decrypt let _ = fs::remove_file(&filepath_in); let _ = fs::remove_file(&filepath_out); - Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None).unwrap(); + Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None).unwrap(); // read and compare decrypted file against original data let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); // with compression - Encryption::encrypt(&filepath_in, None, true, vec![11, 12, 1024*100]).unwrap(); - Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None).unwrap(); + Encryption::encrypt(&filepath_in, None, None, true, vec![11, 12, 1024*100]).unwrap(); + Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None).unwrap(); let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -810,6 +881,7 @@ mod tests { fn test_crypt_archive() { // Create directory with files that should be archived let dir_path = PathBuf::from("test_archive_toplevel"); + let _ = fs::remove_dir_all(&dir_path); fs::create_dir_all(&dir_path).unwrap(); let file1 = dir_path.join("file1.bin"); @@ -823,7 +895,7 @@ mod tests { fs::write(&file2, &data2).unwrap(); // Build archive of directory and encrypt it - Encryption::encrypt(&dir_path, None, false, vec![]).unwrap(); + Encryption::encrypt(&dir_path, None, None, false, vec![]).unwrap(); // Delete the original files before extracting to verify recreation fs::remove_dir_all(&dir_path).unwrap(); @@ -831,7 +903,7 @@ mod tests { // Decrypt and rebuild archived directory let arch_path = dir_path.with_extension(ENCRYPTED_FILE_EXT); - Decryption::decrypt(&arch_path, None).unwrap(); + Decryption::decrypt(&arch_path, None, None).unwrap(); // Verify structure is fully recreated assert!(dir_path.exists()); @@ -846,6 +918,96 @@ mod tests { let _ = fs::remove_file(&arch_path); } + #[test] + fn test_set_paths() { + // Create temporary test files and directories + let test_dir = PathBuf::from("test_set_paths_dir"); + let _ = fs::remove_dir_all(&test_dir); + fs::create_dir_all(&test_dir).unwrap(); + + let test_file = test_dir.join("test_file.txt"); + fs::write(&test_file, b"test data").unwrap(); + + let sub_dir = test_dir.join("sub_dir"); + fs::create_dir_all(&sub_dir).unwrap(); + + let output_dir = test_dir.join("output_dir"); + fs::create_dir_all(&output_dir).unwrap(); + + // 1. File input, no output dir, no split + { + let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&test_file, None, true).unwrap(); + assert!(!archive); + assert_eq!(work_dir, None); + assert_eq!(processed_in, test_file); + let mut expected = test_file.clone(); + expected.add_extension(ENCRYPTED_FILE_EXT); + assert_eq!(file_out, path::absolute(expected).unwrap()); + } + + // 2. File input, no output dir, with split + { + let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&test_file, None, false).unwrap(); + assert!(!archive); + assert_eq!(work_dir, None); + assert_eq!(processed_in, test_file); + let mut expected = test_file.clone(); + expected.add_extension(SPLIT_ENC_FILE_EXT); + assert_eq!(file_out, path::absolute(expected).unwrap()); + } + + // 3. File input, with output dir, no split + { + let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&test_file, Some(&output_dir), true).unwrap(); + assert!(!archive); + assert_eq!(work_dir, None); + assert_eq!(processed_in, test_file); + let mut expected = test_file.clone(); + expected.add_extension(ENCRYPTED_FILE_EXT); + let expected_out = path::absolute(output_dir.join(expected.file_name().unwrap())).unwrap(); + assert_eq!(file_out, expected_out); + } + + // 4. Directory input (sub_dir), no output dir, no split + { + let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&sub_dir, None, true).unwrap(); + assert!(archive); + assert_eq!(work_dir, Some(test_dir.clone())); + assert_eq!(processed_in, PathBuf::from("sub_dir")); + let mut expected = sub_dir.clone(); + expected.add_extension(ENCRYPTED_FILE_EXT); + assert_eq!(file_out, path::absolute(expected).unwrap()); + } + + // 5. Directory input (sub_dir), with output dir, with split + { + let relative_sub_dir = PathBuf::from("test_set_paths_dir/sub_dir"); + let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&relative_sub_dir, Some(&output_dir), false).unwrap(); + assert!(archive); + assert_eq!(work_dir, Some(test_dir.clone())); + assert_eq!(processed_in, PathBuf::from("sub_dir")); + let mut expected = PathBuf::from("sub_dir"); + expected.add_extension(SPLIT_ENC_FILE_EXT); + let expected_out = path::absolute(output_dir.join(expected.file_name().unwrap())).unwrap(); + assert_eq!(file_out, expected_out); + } + + // 6. Directory input without parent/file_name (e.g. ".") + { + let dot_path = PathBuf::from("."); + let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&dot_path, None, true).unwrap(); + assert!(archive); + assert_eq!(work_dir, Some(PathBuf::from("."))); + assert_eq!(processed_in, PathBuf::from(".")); + let mut expected = PathBuf::from("archive"); + expected.add_extension(ENCRYPTED_FILE_EXT); + assert_eq!(file_out, path::absolute(expected).unwrap()); + } + + // Clean up + let _ = fs::remove_dir_all(&test_dir); + } + #[test] #[ignore="only for benchmarking"] fn test_crypt_bench() { @@ -854,7 +1016,7 @@ mod tests { let mut filepath_out = filepath_in.clone(); filepath_out.add_extension(ENCRYPTED_FILE_EXT); - Encryption::encrypt(&filepath_in, None, false, vec![]).unwrap(); - Decryption::decrypt(&filepath_out, None).unwrap(); + Encryption::encrypt(&filepath_in, None, None, false, vec![]).unwrap(); + Decryption::decrypt(&filepath_out, None, None).unwrap(); } } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 1f9ccff..bd3650b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{self, PathBuf}; use clap::Parser; use std::process::ExitCode; use parse_size::Config; @@ -14,6 +14,10 @@ use cryptcrypt::Result; /// With option -s the encrypted output is split into files with extensions .c00, .c01, .c02, ... /// If a file ending on .c00 is decrypted, the whole split series will be read. struct Args { + /// Output directory, it is created if it does not exist + #[arg(short, long)] + out_dir: Option, + /// Decrypt file (with extension '.cce' or for split series '.c00') #[arg(short, long, default_value_t = false)] decrypt: bool, @@ -32,8 +36,8 @@ struct Args { split: Vec, /// File that should be encrypted or decrypted. - /// If a directory is given, all its files and sub-directories are concatenated and encrypted. - file_or_dir: PathBuf, + /// If a directory is given, its contents is archived and encrypted. + file_or_directory: PathBuf, } /// Main entry point for the cryptcrypt application. @@ -60,19 +64,14 @@ fn main() -> ExitCode { fn run() -> Result<()> { let args = Args::parse(); - let filepath = if args.file_or_dir.is_dir() { - // a directory should be used as is (relative or absolute), therefore do not canonicalize, which results in an absolute path - args.file_or_dir - } else { - args.file_or_dir.canonicalize()? - }; - - let keyfilepath = args.keyfile.map(|path| path.canonicalize()).transpose()?; + let filepath = args.file_or_directory.canonicalize()?; + let keyfilepath = args.keyfile.map(|keyf| keyf.canonicalize()).transpose()?; + let output_dir = args.out_dir.map(path::absolute).transpose()?; if args.decrypt { - Decryption::decrypt(&filepath, keyfilepath.as_ref())?; + Decryption::decrypt(&filepath, output_dir.as_ref(), keyfilepath.as_ref())?; } else { - Encryption::encrypt(&filepath, keyfilepath.as_ref(), args.compress, args.split)?; + Encryption::encrypt(&filepath, output_dir.as_ref(), keyfilepath.as_ref(), args.compress, args.split)?; } Ok(()) From dae12277e1528536243d3ea6ca1a1d9b572e9bd6 Mon Sep 17 00:00:00 2001 From: JoergDF Date: Wed, 22 Jul 2026 17:49:59 +0200 Subject: [PATCH 12/16] For archiving replace changing working directory. Adapt the stored path in archive and change unit tests accordingly. --- src/archive.rs | 73 +++++++++++------- src/decryption.rs | 19 ++--- src/encryption.rs | 184 ++++++++++++++++++++++++++++++---------------- 3 files changed, 173 insertions(+), 103 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 39e1f4c..01fad0c 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -52,10 +52,11 @@ impl ArchiveRead { /// # Arguments /// - `f_in_path`: The root path of the directory tree to archive. /// - `exclude_path`: Path to be excluded from archive + /// - `strip_dir`: Path to be stripped from the start of each path in the archive to make paths relative /// /// # Returns /// - A new `ArchiveRead` instance. - pub fn new(f_in_path: &Path, exclude_path: &Path) -> Self { + pub fn new(f_in_path: &Path, exclude_path: &Path, strip_dir: &Path) -> Self { let num_workers = num_cpus::get(); let mut thread_handles = Vec::with_capacity(num_workers + 1); let mut rx_out_receivers = Vec::with_capacity(num_workers); @@ -142,6 +143,7 @@ impl ArchiveRead { let rx_paths = rx_paths.clone(); let (tx_out, rx_out) = bounded(num_workers); rx_out_receivers.push(rx_out); + let strip_dir = strip_dir.to_path_buf(); thread_handles.push(thread::spawn(move || -> std::result::Result<(), String> { for (entry, hard_link_target) in rx_paths { @@ -149,7 +151,7 @@ impl ArchiveRead { let filepath_and_size; let sparse_segments; - match Self::build_archive_header(&entry, &hard_link_target) { + match Self::build_archive_header(&entry, &hard_link_target, &strip_dir) { Ok(values) => (archive_header, filepath_and_size, sparse_segments) = values, Err(e) => { eprintln!("Skipped entry on building archive header for {} - Reason: {e}", entry.path().display()); @@ -344,12 +346,13 @@ impl ArchiveRead { /// # Arguments /// - `entry`: The directory entry to construct the header for. /// - `hard_link_target`: Optional path pointing to the target if this is a hard link. + /// - `strip_dir`: Path to be stripped from the start of each path in the archive to make paths relative /// /// # Returns /// - `Ok((archive_header, filepath_and_size, sparse_segments))` on success. /// - `Err` if metadata retrieval or OS-specific operations fail. #[allow(clippy::type_complexity)] - fn build_archive_header(entry: &DirEntry, hard_link_target: &Option) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { + fn build_archive_header(entry: &DirEntry, hard_link_target: &Option, strip_dir: &PathBuf) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { // archive header initialized with place holder for header size let mut archive_header = vec![0u8; ARCHIVE_HEADER_LENGTH_SIZE]; @@ -392,14 +395,15 @@ impl ArchiveRead { let os_type = if cfg!(unix) { TYPE_UNIX } else { TYPE_WINDOWS }; archive_header.push(os_type | entry_type); - // path length and path (including filename) - Self::add_path_to_header(entry.path(), &mut archive_header)?; + // path length and path (including filename) (converting to relative path) + let stripped_entry_path = entry.path().strip_prefix(strip_dir)?; + Self::add_path_to_header(stripped_entry_path, &mut archive_header)?; // println!("{}", entry.path().display()); if entry_type == TYPE_HARDLINK { // target path of hard link - let target_path = hard_link_target.as_ref().unwrap(); + let target_path = hard_link_target.as_ref().unwrap().strip_prefix(strip_dir)?; Self::add_path_to_header(target_path, &mut archive_header)?; // header size @@ -575,18 +579,23 @@ pub struct ArchiveWrite { sparse_segments_index: usize, /// Size in bytes of the current sparse data segment. data_segment_size: u64, + /// Optional output directory. + dirpath_out: Option, } impl ArchiveWrite { /// Initializes a new, empty `ArchiveWrite` instance. /// + /// # Arguments + /// - `dirpath_out`: Optional output directory + /// /// # Returns /// - A default `ArchiveWrite` with allocated output buffer. - pub fn new() -> Self { + pub fn new(dirpath_out: Option) -> Self { let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), file_path: PathBuf::new(), dir_times: vec![], pending_hardlinks: vec![], sparse_segments: vec![], - sparse_segments_index: 0, data_segment_size: 0} + sparse_segments_index: 0, data_segment_size: 0, dirpath_out } } /// Ensures that the parent directory of the given path exists. @@ -701,16 +710,22 @@ impl ArchiveWrite { // entry's path let entry_path_string; (entry_path_string, e) = Self::get_path_from_header(header, e, created_on_os_type)?; - let entry_path = PathBuf::from(&entry_path_string); + let mut entry_path = PathBuf::from(&entry_path_string); + // add optional output directory to entry's path + if let Some(dir_out) = &self.dirpath_out { + entry_path = dir_out.join(entry_path); + } // println!("{}", entry_path.display()); if file_type == TYPE_HARDLINK { // hard link's target path - let target_path; - (target_path, _) = Self::get_path_from_header(header, e, created_on_os_type)?; - - self.pending_hardlinks.push((PathBuf::from(target_path), entry_path)); + let (target_path_string, _) = Self::get_path_from_header(header, e, created_on_os_type)?; + let mut target_path = PathBuf::from(target_path_string); + if let Some(dir_out) = &self.dirpath_out { + target_path = dir_out.join(target_path); + } + self.pending_hardlinks.push((target_path, entry_path)); return Ok(()); } @@ -736,7 +751,7 @@ impl ArchiveWrite { self.dir_times.push((entry_path.clone(), time_accessed, time_modified)); } else if file_type == TYPE_FILE { - // create directory (of file), if it doesn't exists + // create directory (of file), if it doesn't exist Self::create_parent_directory(&entry_path)?; // create file @@ -783,7 +798,7 @@ impl ArchiveWrite { } } else if file_type == TYPE_SYMLINK_FILE || file_type == TYPE_SYMLINK_DIR { - // create directory (of symlink), if it doesn't exists + // create directory (of symlink), if it doesn't exist Self::create_parent_directory(&entry_path)?; // symlink's target path @@ -969,12 +984,10 @@ impl WriteFiles for ArchiveWrite { fn write_others(&self) -> Result<()> { // create hard links, if there are any for (target_path, entry_path) in &self.pending_hardlinks { - // create directory (of hard link), if it doesn't exists + // create directory (of hard link), if it doesn't exist Self::create_parent_directory(entry_path)?; - fs::hard_link(target_path, entry_path)?; } - // set timestamps of directories // need to be done after all elements have been created, as creation of an element // updates timestamp of its parent directory to now @@ -1139,16 +1152,17 @@ mod tests { let hardlink_path = src_dir.path.join("hardlink.txt"); fs::hard_link(&file1_path, &hardlink_path).unwrap(); - // Get original modified time of file1 + // Get original modified/accessed time of file1 let meta_orig1 = fs::metadata(&file1_path).unwrap(); let modified_orig1 = meta_orig1.modified().unwrap(); + let accessed_orig1 = meta_orig1.accessed().unwrap(); // Get original modified time of subdirectory let meta_orig2 = fs::metadata(&sub_dir).unwrap(); let modified_orig2 = meta_orig2.modified().unwrap(); // Perform archiving using ArchiveRead - let mut reader = ArchiveRead::new(&src_dir.path, Path::new("")); + let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new("")); let mut archive_bytes = Vec::new(); for i in 0..=128 { let (chunk, last) = reader.read_chunk().unwrap(); @@ -1169,7 +1183,7 @@ mod tests { assert!(!src_dir.path.exists()); // Perform extraction using ArchiveWrite by feeding it in small chunks - let mut writer = ArchiveWrite::new(); + let mut writer = ArchiveWrite::new(None); for chunk in archive_bytes.chunks(100) { writer.write_files(chunk).unwrap(); } @@ -1210,13 +1224,18 @@ mod tests { assert_eq!(hole.end, 131072); } - // Verify modified time of file1 is restored (seconds precision) + // Verify modified/accessed time of file1 is restored (seconds precision) let meta_restored1 = fs::metadata(&file1_path).unwrap(); let modified_restored1 = meta_restored1.modified().unwrap(); assert_eq!( modified_orig1.duration_since(UNIX_EPOCH).unwrap().as_secs(), modified_restored1.duration_since(UNIX_EPOCH).unwrap().as_secs() ); + let accessed_restored1 = meta_restored1.accessed().unwrap(); + assert_eq!( + accessed_orig1.duration_since(UNIX_EPOCH).unwrap().as_secs(), + accessed_restored1.duration_since(UNIX_EPOCH).unwrap().as_secs() + ); // Verify modified time of subdir is restored (seconds precision) let meta_restored2 = fs::metadata(&sub_dir).unwrap(); @@ -1305,7 +1324,7 @@ mod tests { // Perform archiving using ArchiveRead - let mut reader = ArchiveRead::new(&src_dir.path, Path::new("")); + let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new("")); let mut archive_bytes = Vec::new(); for i in 0..=128 { let (chunk, last_chunk) = reader.read_chunk().unwrap(); @@ -1324,7 +1343,7 @@ mod tests { // Perform extraction using ArchiveWrite - let mut writer = ArchiveWrite::new(); + let mut writer = ArchiveWrite::new(None); for chunk in archive_bytes.chunks(CHUNK_SIZE) { writer.write_files(chunk).unwrap(); } @@ -1419,7 +1438,7 @@ mod tests { #[test] fn test_archive_nonexistent_dir() { - let mut reader = ArchiveRead::new(Path::new("test_archive_does_not_exist"), Path::new("")); + let mut reader = ArchiveRead::new(Path::new("test_archive_does_not_exist"), Path::new(""), Path::new("")); let (chunk, last) = reader.read_chunk().unwrap(); assert!(last); assert!(chunk.is_empty()); @@ -1430,7 +1449,7 @@ mod tests { fn test_archive_empty_dir() { let src_dir = TestDir::new("test_archive_empty"); - let mut reader = ArchiveRead::new(&src_dir.path, Path::new("")); + let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new("")); let mut archive_bytes = Vec::new(); let (chunk, last) = reader.read_chunk().unwrap(); @@ -1443,7 +1462,7 @@ mod tests { fs::remove_dir_all(&src_dir.path).unwrap(); // Extract - let mut writer = ArchiveWrite::new(); + let mut writer = ArchiveWrite::new(None); writer.write_files(&archive_bytes).unwrap(); writer.write_others().unwrap(); diff --git a/src/decryption.rs b/src/decryption.rs index 5301368..2e8b36a 100644 --- a/src/decryption.rs +++ b/src/decryption.rs @@ -360,11 +360,11 @@ impl Decryption { let compress = (file_format & 0x01) != 0; let archive = (file_format & 0x02) != 0; - let mut working_dir = &env::current_dir()?; + let mut output_dir = &env::current_dir()?; // output directory path is used if let Some(dir_out) = dirpath_out { if archive { - working_dir = dir_out; + output_dir = dir_out; } else { let filename_out = filepath_out.file_name().unwrap(); filepath_out = dir_out.join(filename_out); @@ -372,7 +372,7 @@ impl Decryption { } if archive { - println!("Archive file will be extracted to directory {}", working_dir.display()); + println!("Archive file will be extracted to directory {}", output_dir.display()); } else { println!("Output will be written to file {}", filepath_out.display()); } @@ -381,20 +381,15 @@ impl Decryption { let key = Self::hash_password(salt_pw, keyfilepath)?; let (key_cha, key_aes) = Self::derive_keys(salt_cha, salt_aes, &key)?; - // output directory actions: create new directory, change working directory for an archive + // output directory action: // create a new directory after password entry: // if password entry failed or user breaks execution on password entry, filesystem stays unchanged - if let Some(dir_out) = dirpath_out { - if !dir_out.exists() { - fs::create_dir_all(dir_out)?; - } - if archive { - env::set_current_dir(dir_out)?; - } + if let Some(dir_out) = dirpath_out && !dir_out.exists() { + fs::create_dir_all(dir_out)?; } let write_output: Box = if archive { - Box::new( ArchiveWrite::new() ) + Box::new( ArchiveWrite::new(dirpath_out.cloned()) ) } else { // set write parameters and create output file Box::new( WriteOutput::new(filepath_out, vec![])? ) diff --git a/src/encryption.rs b/src/encryption.rs index 2bee138..e4fe592 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -1,7 +1,6 @@ use std::io::Read; -use std::path::{self, PathBuf}; +use std::path::{Path, PathBuf}; use std::{fs, thread}; -use std::env; use argon2::Argon2; use chacha20poly1305::{XChaCha20Poly1305}; use rand::{Rng, SeedableRng}; @@ -346,7 +345,7 @@ impl Encryption { /// archiving). Constructs the final output file path by applying the correct /// extension depending on whether splitting is enabled, and redirects output /// to the target directory if specified. If archiving a directory, it returns - /// the target working directory that should be set during archiving to keep + /// the directory that should be stripped during archiving to keep /// paths relative. /// /// # Arguments @@ -355,17 +354,18 @@ impl Encryption { /// - `no_split`: Boolean flag indicating if output splitting is disabled. /// /// # Returns - /// - `Ok((processed_path_in, filepath_out, build_archive, new_working_dir))` containing: - /// - `processed_path_in`: The relative input path if archiving, or the original input path. + /// - `Ok((filepath_out, build_archive, strip_dir_in))` containing: /// - `filepath_out`: The resolved absolute path of the output encrypted file. /// - `build_archive`: Boolean indicating whether the input is a directory. - /// - `new_working_dir`: Optional path to the directory that the process should change to before archiving. + /// - `strip_dir_in`: Optional directory path that should be stripped from the start of the absolute archive path. /// - `Err` if absolute path resolution fails. - fn set_paths(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, no_split: bool) -> Result<(PathBuf, PathBuf, bool, Option)> { + fn set_paths(filepath_in: &Path, dirpath_out: Option<&PathBuf>, no_split: bool) -> Result<(PathBuf, bool, Option)> { + assert!(filepath_in.is_absolute()); + let build_archive = filepath_in.is_dir(); - let mut filepath_out = filepath_in.clone(); - // if input path is the root or relative (e.g. "/", "."), so without directory name, add a name for the archive + let mut filepath_out = filepath_in.to_path_buf(); + // if input path is the root (e.g. "/"), so without directory name, add a name for the archive if build_archive && filepath_in.file_name().is_none() { filepath_out.push("archive"); } @@ -377,28 +377,21 @@ impl Encryption { // output directory path is used // build path, but create directory after password entry if let Some(dir_out) = dirpath_out { + assert!(dir_out.is_absolute()); let filename_out = filepath_out.file_name().unwrap(); filepath_out = dir_out.join(filename_out); } - filepath_out = path::absolute(filepath_out)?; - - let mut new_working_dir = None; - let processed_path_in = if build_archive { - // change directory to parent directory - if let Some(dir_path) = filepath_in.file_name() { - if let Some(parent_path) = filepath_in.parent() && parent_path != "" { - new_working_dir = Some(parent_path.to_path_buf()); - } - PathBuf::from(dir_path) + + let mut strip_dir_in = None; + if build_archive { + if let Some(parent_path) = filepath_in.parent() { + strip_dir_in = Some(parent_path.to_path_buf()); } else { - new_working_dir = Some(filepath_in.to_path_buf()); - PathBuf::from(".") + strip_dir_in = Some(filepath_in.to_path_buf()); } - } else { - filepath_in.to_path_buf() - }; + } - Ok((processed_path_in, filepath_out, build_archive, new_working_dir)) + Ok((filepath_out, build_archive, strip_dir_in)) } /// Encrypts a file or a directory using dual-layer encryption (ChaCha20 + AES-256-GCM-SIV) @@ -421,13 +414,10 @@ impl Encryption { /// - `Err` if file operations, password handling, or encryption fails pub fn encrypt(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, keyfilepath: Option<&PathBuf>, compress: bool, split: Vec) -> Result<()> { - let (processed_path_in, filepath_out, build_archive, new_working_dir) = Self::set_paths(filepath_in, dirpath_out, split.is_empty())?; - if build_archive && let Some(work_dir) = new_working_dir { - env::set_current_dir(work_dir)?; - } + let (filepath_out, build_archive, strip_dir) = Self::set_paths(filepath_in, dirpath_out, split.is_empty())?; println!("Output will be written to {}", filepath_out.display()); - + // ask for password, before there can be error messages of archive let (salt_pw, key) = Self::hash_password(keyfilepath)?; let (salt_cha, key_cha, salt_aes, key_aes) = Self::derive_keys(&key)?; @@ -440,9 +430,9 @@ impl Encryption { } let read_input: Box = if build_archive { - Box::new( ArchiveRead::new(&processed_path_in, &filepath_out) ) + Box::new( ArchiveRead::new(filepath_in, &filepath_out, &strip_dir.unwrap()) ) } else { - Box::new( ReadInput::new(&processed_path_in, CHUNK_SIZE, 0)? ) + Box::new( ReadInput::new(filepath_in, CHUNK_SIZE, 0)? ) }; // file header @@ -468,7 +458,7 @@ impl Encryption { write_output.write_files(&header)?; CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::encrypt_pipe, read_input, write_output)?; - + Ok(()) } } @@ -482,6 +472,7 @@ impl Encryption { mod tests { use super::*; use std::fs; + use std::path; use crate::decryption::Decryption; #[test] @@ -741,7 +732,7 @@ mod tests { #[test] fn test_crypt() { // create file with random data for encryption - let filepath_in = PathBuf::from("test_cc.bin"); + let filepath_in = path::absolute(PathBuf::from("test_cc.bin")).unwrap(); let mut filepath_out = filepath_in.clone(); filepath_out.add_extension(ENCRYPTED_FILE_EXT); @@ -773,16 +764,28 @@ mod tests { let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); + // with output directory + let out_enc_dir = path::absolute(PathBuf::from("test_cc_enc_dir")).unwrap(); + Encryption::encrypt(&filepath_in, Some(&out_enc_dir), None, false, vec![]).unwrap(); + assert!(&out_enc_dir.join(&filepath_out).exists()); + let out_dec_dir = path::absolute(PathBuf::from("test_cc_dec_dir")).unwrap(); + Decryption::decrypt(&out_enc_dir.join(&filepath_out), Some(&out_dec_dir), None).unwrap(); + assert!(&out_dec_dir.join(&filepath_in).exists()); + let decrypt_data = fs::read(out_dec_dir.join(&filepath_in)).unwrap(); + assert_eq!(data, decrypt_data[..]); + // cleanup let _ = fs::remove_file(&filepath_in); let _ = fs::remove_file(&filepath_out); let _ = fs::remove_file(&filepath_kf); + let _ = fs::remove_dir_all(&out_enc_dir); + let _ = fs::remove_dir_all(&out_dec_dir); } #[test] fn test_crypt_with_keyfile() { // create file with random data for encryption - let filepath_in = PathBuf::from("test_cc_kf.bin"); + let filepath_in = path::absolute(PathBuf::from("test_cc_kf.bin")).unwrap(); let mut filepath_out = filepath_in.clone(); filepath_out.add_extension(ENCRYPTED_FILE_EXT); let filepath_kf = PathBuf::from("test_key.bin"); @@ -811,7 +814,7 @@ mod tests { assert!(Decryption::decrypt(&filepath_out, None, Some(&PathBuf::from("test_miss"))).is_err()); // input file does not exist - assert!(Encryption::encrypt(&PathBuf::from("test_miss"), None, None, false, vec![]).is_err()); + assert!(Encryption::encrypt(&path::absolute(PathBuf::from("test_miss")).unwrap(), None, None, false, vec![]).is_err()); assert!(Decryption::decrypt(&PathBuf::from("test_miss.cce"), None, None).is_err()); assert!(!fs::exists("test_miss").unwrap()); assert!(!fs::exists("test_miss.cce").unwrap()); @@ -831,7 +834,7 @@ mod tests { #[test] fn test_crypt_split() { - let filepath_in = PathBuf::from("test_cc_split.bin"); + let filepath_in = path::absolute(PathBuf::from("test_cc_split.bin")).unwrap(); let mut filepath_out = filepath_in.clone(); filepath_out.add_extension(ENCRYPTED_FILE_EXT); @@ -880,7 +883,8 @@ mod tests { #[test] fn test_crypt_archive() { // Create directory with files that should be archived - let dir_path = PathBuf::from("test_archive_toplevel"); + let dir_name = "test_cc_archive"; + let dir_path = path::absolute(PathBuf::from(dir_name)).unwrap(); let _ = fs::remove_dir_all(&dir_path); fs::create_dir_all(&dir_path).unwrap(); @@ -894,6 +898,25 @@ mod tests { rand::rng().fill_bytes(&mut data2); fs::write(&file2, &data2).unwrap(); + // sub-directory for links + let sub_dir = dir_path.join("links"); + fs::create_dir(&sub_dir).unwrap(); + + // create symlink + let symlink1 = sub_dir.join("symlink1.bin"); + #[cfg(unix)] + { + std::os::unix::fs::symlink(&file1, &symlink1).unwrap(); + } + #[cfg(windows)] + { + std::os::windows::fs::symlink_file(&file1, &symlink1).unwrap(); + } + + // create hardlink + let hardlink2 = sub_dir.join("hardlink2.bin"); + fs::hard_link(&file2, &hardlink2).unwrap(); + // Build archive of directory and encrypt it Encryption::encrypt(&dir_path, None, None, false, vec![]).unwrap(); @@ -909,21 +932,56 @@ mod tests { assert!(dir_path.exists()); assert!(file1.exists()); assert!(file2.exists()); + assert!(&symlink1.exists()); + assert!(&hardlink2.exists()); // Verify contents assert_eq!(fs::read(&file1).unwrap(), data1); assert_eq!(fs::read(&file2).unwrap(), data2); + assert_eq!(fs::read(&symlink1).unwrap(), data1); + assert_eq!(fs::read(&hardlink2).unwrap(), data2); + + + // With output directory, with split, check exclusion of output archive files + let out_enc_dir = &dir_path; + Encryption::encrypt(&dir_path, Some(&out_enc_dir), None, false, vec![1000,10000]).unwrap(); + assert!(out_enc_dir.join(dir_name).with_extension(SPLIT_ENC_FILE_EXT).exists()); + + let out_dec_dir = path::absolute(PathBuf::from("test_cc_dec_dir")).unwrap(); + let _ = fs::remove_dir_all(&out_dec_dir); + Decryption::decrypt(&out_enc_dir.join(dir_name).with_extension(SPLIT_ENC_FILE_EXT), Some(&out_dec_dir), None).unwrap(); + + // Verify structure is fully recreated + assert!(out_dec_dir.join(dir_name).exists()); + assert!(out_dec_dir.join(&file1).exists()); + assert!(out_dec_dir.join(&file2).exists()); + assert!(out_dec_dir.join(&symlink1).exists()); + assert!(out_dec_dir.join(&hardlink2).exists()); + + // Verify contents + assert_eq!(fs::read(out_dec_dir.join(&file1)).unwrap(), data1); + assert_eq!(fs::read(out_dec_dir.join(&file2)).unwrap(), data2); + assert_eq!(fs::read(out_dec_dir.join(&symlink1)).unwrap(), data1); + assert_eq!(fs::read(out_dec_dir.join(&hardlink2)).unwrap(), data2); + + // Verify exclude of archive files + assert!(!out_dec_dir.join(dir_name).with_added_extension(SPLIT_ENC_FILE_EXT).exists()); + assert!(!out_dec_dir.join(dir_name).with_added_extension("c01").exists()); + assert!(!out_dec_dir.join(dir_name).with_added_extension("c02").exists()); - let _ = fs::remove_dir_all(&dir_path); let _ = fs::remove_file(&arch_path); + let _ = fs::remove_dir_all(&dir_path); + let _ = fs::remove_dir_all(&out_enc_dir); + let _ = fs::remove_dir_all(&out_dec_dir); } #[test] fn test_set_paths() { - // Create temporary test files and directories - let test_dir = PathBuf::from("test_set_paths_dir"); + // Create test files and directories + let mut test_dir = PathBuf::from("test_set_paths_dir"); let _ = fs::remove_dir_all(&test_dir); fs::create_dir_all(&test_dir).unwrap(); + test_dir = test_dir.canonicalize().unwrap(); let test_file = test_dir.join("test_file.txt"); fs::write(&test_file, b"test data").unwrap(); @@ -936,10 +994,9 @@ mod tests { // 1. File input, no output dir, no split { - let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&test_file, None, true).unwrap(); + let (file_out, archive, strip_dir) = Encryption::set_paths(&test_file, None, true).unwrap(); assert!(!archive); - assert_eq!(work_dir, None); - assert_eq!(processed_in, test_file); + assert_eq!(strip_dir, None); let mut expected = test_file.clone(); expected.add_extension(ENCRYPTED_FILE_EXT); assert_eq!(file_out, path::absolute(expected).unwrap()); @@ -947,10 +1004,9 @@ mod tests { // 2. File input, no output dir, with split { - let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&test_file, None, false).unwrap(); + let (file_out, archive, strip_dir) = Encryption::set_paths(&test_file, None, false).unwrap(); assert!(!archive); - assert_eq!(work_dir, None); - assert_eq!(processed_in, test_file); + assert_eq!(strip_dir, None); let mut expected = test_file.clone(); expected.add_extension(SPLIT_ENC_FILE_EXT); assert_eq!(file_out, path::absolute(expected).unwrap()); @@ -958,10 +1014,9 @@ mod tests { // 3. File input, with output dir, no split { - let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&test_file, Some(&output_dir), true).unwrap(); + let (file_out, archive, strip_dir) = Encryption::set_paths(&test_file, Some(&output_dir), true).unwrap(); assert!(!archive); - assert_eq!(work_dir, None); - assert_eq!(processed_in, test_file); + assert_eq!(strip_dir, None); let mut expected = test_file.clone(); expected.add_extension(ENCRYPTED_FILE_EXT); let expected_out = path::absolute(output_dir.join(expected.file_name().unwrap())).unwrap(); @@ -970,10 +1025,9 @@ mod tests { // 4. Directory input (sub_dir), no output dir, no split { - let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&sub_dir, None, true).unwrap(); + let (file_out, archive, strip_dir) = Encryption::set_paths(&sub_dir, None, true).unwrap(); assert!(archive); - assert_eq!(work_dir, Some(test_dir.clone())); - assert_eq!(processed_in, PathBuf::from("sub_dir")); + assert_eq!(strip_dir, Some(test_dir.clone())); let mut expected = sub_dir.clone(); expected.add_extension(ENCRYPTED_FILE_EXT); assert_eq!(file_out, path::absolute(expected).unwrap()); @@ -981,27 +1035,29 @@ mod tests { // 5. Directory input (sub_dir), with output dir, with split { - let relative_sub_dir = PathBuf::from("test_set_paths_dir/sub_dir"); - let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&relative_sub_dir, Some(&output_dir), false).unwrap(); + let (file_out, archive, strip_dir) = Encryption::set_paths(&sub_dir, Some(&output_dir), false).unwrap(); assert!(archive); - assert_eq!(work_dir, Some(test_dir.clone())); - assert_eq!(processed_in, PathBuf::from("sub_dir")); + assert_eq!(strip_dir, Some(test_dir.clone())); let mut expected = PathBuf::from("sub_dir"); expected.add_extension(SPLIT_ENC_FILE_EXT); let expected_out = path::absolute(output_dir.join(expected.file_name().unwrap())).unwrap(); assert_eq!(file_out, expected_out); } - // 6. Directory input without parent/file_name (e.g. ".") + // 6. Directory input without parent/file_name (e.g. "/") { - let dot_path = PathBuf::from("."); - let (processed_in, file_out, archive, work_dir) = Encryption::set_paths(&dot_path, None, true).unwrap(); + let root_path = + if cfg!(unix) { + PathBuf::from("/") + } else { + PathBuf::from("c:/") + }; + let (file_out, archive, strip_dir) = Encryption::set_paths(&root_path, None, true).unwrap(); assert!(archive); - assert_eq!(work_dir, Some(PathBuf::from("."))); - assert_eq!(processed_in, PathBuf::from(".")); - let mut expected = PathBuf::from("archive"); + assert_eq!(strip_dir, Some(root_path.clone())); + let mut expected = root_path.join("archive"); expected.add_extension(ENCRYPTED_FILE_EXT); - assert_eq!(file_out, path::absolute(expected).unwrap()); + assert_eq!(file_out, expected); } // Clean up From a364278b9df16af71b82470c5ba41c901a5c0a0b Mon Sep 17 00:00:00 2001 From: JoergDF Date: Tue, 28 Jul 2026 11:28:43 +0200 Subject: [PATCH 13/16] Add option verbose and print output, increase file format version, minor changes. - New command line option -v prints archive file names. - Increase file format version to 5. - Minor efficiency improvements. - Extend unit tests of archive. --- src/archive.rs | 190 +++++++++++++++++++++++++++++++++------------- src/decryption.rs | 14 +++- src/encryption.rs | 68 +++++++++-------- src/lib.rs | 2 +- src/main.rs | 10 ++- 5 files changed, 195 insertions(+), 89 deletions(-) diff --git a/src/archive.rs b/src/archive.rs index 01fad0c..359a4df 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -56,7 +56,7 @@ impl ArchiveRead { /// /// # Returns /// - A new `ArchiveRead` instance. - pub fn new(f_in_path: &Path, exclude_path: &Path, strip_dir: &Path) -> Self { + pub fn new(f_in_path: &Path, exclude_path: &Path, strip_dir: &Path, verbose: bool) -> Self { let num_workers = num_cpus::get(); let mut thread_handles = Vec::with_capacity(num_workers + 1); let mut rx_out_receivers = Vec::with_capacity(num_workers); @@ -151,7 +151,8 @@ impl ArchiveRead { let filepath_and_size; let sparse_segments; - match Self::build_archive_header(&entry, &hard_link_target, &strip_dir) { + // archive header + match Self::build_archive_header(&entry, &hard_link_target, &strip_dir, verbose) { Ok(values) => (archive_header, filepath_and_size, sparse_segments) = values, Err(e) => { eprintln!("Skipped entry on building archive header for {} - Reason: {e}", entry.path().display()); @@ -159,6 +160,7 @@ impl ArchiveRead { } } + // read data from file and send it to chunk reader if let Some((filepath, mut file_size)) = filepath_and_size { match File::open(&filepath) { Ok(mut f_in) => { @@ -352,9 +354,10 @@ impl ArchiveRead { /// - `Ok((archive_header, filepath_and_size, sparse_segments))` on success. /// - `Err` if metadata retrieval or OS-specific operations fail. #[allow(clippy::type_complexity)] - fn build_archive_header(entry: &DirEntry, hard_link_target: &Option, strip_dir: &PathBuf) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { + fn build_archive_header(entry: &DirEntry, hard_link_target: &Option, strip_dir: &PathBuf, verbose: bool) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { // archive header initialized with place holder for header size - let mut archive_header = vec![0u8; ARCHIVE_HEADER_LENGTH_SIZE]; + let mut archive_header = Vec::with_capacity(1024); + archive_header.extend([0u8; ARCHIVE_HEADER_LENGTH_SIZE]); /// Computes and sets the final header size at the beginning of the header buffer. /// It is called after all other header fields have been added to the header buffer. @@ -399,7 +402,9 @@ impl ArchiveRead { let stripped_entry_path = entry.path().strip_prefix(strip_dir)?; Self::add_path_to_header(stripped_entry_path, &mut archive_header)?; - // println!("{}", entry.path().display()); + if verbose { + println!("{}", stripped_entry_path.display()); + } if entry_type == TYPE_HARDLINK { // target path of hard link @@ -412,18 +417,21 @@ impl ArchiveRead { return Ok((archive_header, None, vec![])); } + // metadata of entry + let meta_entry = entry.metadata()?; + // last access time - let time_accessed = entry.metadata()?.accessed()?.duration_since(UNIX_EPOCH)?.as_secs(); + let time_accessed = meta_entry.accessed()?.duration_since(UNIX_EPOCH)?.as_secs(); archive_header.extend(time_accessed.to_le_bytes()); // last modification time - let time_modified = entry.metadata()?.modified()?.duration_since(UNIX_EPOCH)?.as_secs(); + let time_modified = meta_entry.modified()?.duration_since(UNIX_EPOCH)?.as_secs(); archive_header.extend(time_modified.to_le_bytes()); let mut file_size = 0; let mut sparse_segments = vec![]; if entry_type == TYPE_FILE { // file size - file_size = entry.metadata()?.len(); + file_size = meta_entry.len(); archive_header.extend(file_size.to_le_bytes()); // sparse file @@ -459,7 +467,7 @@ impl ArchiveRead { let mut perm: u16 = 0; if entry_type == TYPE_FILE || entry_type == TYPE_DIRECTORY { - let permission_mode = entry.metadata()?.permissions().mode(); + let permission_mode = meta_entry.permissions().mode(); // use 12 least significant bits perm = (permission_mode & 0x0FFF) as u16; } @@ -581,6 +589,8 @@ pub struct ArchiveWrite { data_segment_size: u64, /// Optional output directory. dirpath_out: Option, + /// Enable verbose prints. + verbose: bool, } impl ArchiveWrite { @@ -591,11 +601,11 @@ impl ArchiveWrite { /// /// # Returns /// - A default `ArchiveWrite` with allocated output buffer. - pub fn new(dirpath_out: Option) -> Self { + pub fn new(dirpath_out: Option, verbose: bool) -> Self { let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), file_path: PathBuf::new(), dir_times: vec![], pending_hardlinks: vec![], sparse_segments: vec![], - sparse_segments_index: 0, data_segment_size: 0, dirpath_out } + sparse_segments_index: 0, data_segment_size: 0, dirpath_out, verbose } } /// Ensures that the parent directory of the given path exists. @@ -716,7 +726,9 @@ impl ArchiveWrite { entry_path = dir_out.join(entry_path); } - // println!("{}", entry_path.display()); + if self.verbose { + println!("{}", entry_path.display()); + } if file_type == TYPE_HARDLINK { // hard link's target path @@ -788,7 +800,6 @@ impl ArchiveWrite { if data_start != self.file_size { self.sparse_segments.push( Segment { segment_type: SegmentType::Data, range: data_start..self.file_size} ); } - // println!("unar: {:?} {:?}", entry_path.display(), self.sparse_segments); // Windows requires to set sparse flag for a sparse file #[cfg(windows)] @@ -844,7 +855,11 @@ impl ArchiveWrite { s = e; e += size_of::(); let perm = u16::from_le_bytes(header[s..e].try_into()?); - let fd = File::open(&entry_path)?; + let fd = if file_type == TYPE_DIRECTORY { + &File::open(&entry_path)? + } else { + self.f_out.as_ref().unwrap() + }; let mut permissions = fd.metadata()?.permissions(); let mode_masked = permissions.mode() & 0xFFFF_F000; permissions.set_mode(mode_masked | u32::from(perm & 0x0FFF)); @@ -1152,17 +1167,57 @@ mod tests { let hardlink_path = src_dir.path.join("hardlink.txt"); fs::hard_link(&file1_path, &hardlink_path).unwrap(); - // Get original modified/accessed time of file1 + #[cfg(unix)] + { + use std::{fs::Permissions, os::unix::fs::PermissionsExt}; + + // set permissions of file1 + let perm1 = Permissions::from_mode(0o700); + fs::set_permissions(&file1_path, perm1).unwrap(); + + // set permissions of subdirectory + let perm2 = Permissions::from_mode(0o777); + fs::set_permissions(&sub_dir, perm2).unwrap(); + } + + // Set modified/accessed time of file1 + filetime::set_file_times( + &file1_path, + filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(2000)), + filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(1000)) + ).unwrap(); + + // Set modified/accessed time of subdirectory + filetime::set_file_times( + &sub_dir, + filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(2222)), + filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(1111)) + ).unwrap(); + + // Set modified/accessed time of symlink + filetime::set_symlink_file_times( + &symlink_path, + filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(4444)), + filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(3333)) + ).unwrap(); + + // Get modified/accessed time of file1 let meta_orig1 = fs::metadata(&file1_path).unwrap(); let modified_orig1 = meta_orig1.modified().unwrap(); let accessed_orig1 = meta_orig1.accessed().unwrap(); - // Get original modified time of subdirectory + // Get modified/accessed time of subdirectory let meta_orig2 = fs::metadata(&sub_dir).unwrap(); let modified_orig2 = meta_orig2.modified().unwrap(); + let accessed_orig2 = meta_orig2.accessed().unwrap(); + + // Get modified/accessed time of symlink + let meta_orig3 = fs::symlink_metadata(&symlink_path).unwrap(); + let modified_orig3 = meta_orig3.modified().unwrap(); + let accessed_orig3 = meta_orig3.accessed().unwrap(); // Perform archiving using ArchiveRead - let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new("")); + let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new(""), false); let mut archive_bytes = Vec::new(); for i in 0..=128 { let (chunk, last) = reader.read_chunk().unwrap(); @@ -1183,7 +1238,7 @@ mod tests { assert!(!src_dir.path.exists()); // Perform extraction using ArchiveWrite by feeding it in small chunks - let mut writer = ArchiveWrite::new(None); + let mut writer = ArchiveWrite::new(None, false); for chunk in archive_bytes.chunks(100) { writer.write_files(chunk).unwrap(); } @@ -1197,33 +1252,6 @@ mod tests { assert!(file2_path.exists()); assert!(sparse_path.exists()); - // Verify contents - assert_eq!(fs::read(&file1_path).unwrap(), content1); - assert_eq!(fs::read(&empty_file_path).unwrap(), b""); - assert_eq!(fs::read(&file2_path).unwrap(), content2); - - // Verify sparse file content - { - let mut f = File::open(&sparse_path).unwrap(); - let mut start_buf = vec![0; 65536]; - f.read_exact(&mut start_buf).unwrap(); - assert_eq!(start_buf, vec![1u8; 65536]); - - f.seek(SeekFrom::Start(131072)).unwrap(); - let mut end_buf = vec![0; 65536]; - f.read_exact(&mut end_buf).unwrap(); - assert_eq!(end_buf, vec![2u8; 65536]); - - assert_eq!(fs::metadata(&sparse_path).unwrap().len(), 196608); - - // Verify it is actually a sparse file (has 1 hole) - let segs = f.scan_chunks().unwrap(); - assert_eq!(segs.holes().count(), 1); - let hole = segs.holes().next().unwrap(); - assert_eq!(hole.start, 65536); - assert_eq!(hole.end, 131072); - } - // Verify modified/accessed time of file1 is restored (seconds precision) let meta_restored1 = fs::metadata(&file1_path).unwrap(); let modified_restored1 = meta_restored1.modified().unwrap(); @@ -1237,13 +1265,31 @@ mod tests { accessed_restored1.duration_since(UNIX_EPOCH).unwrap().as_secs() ); - // Verify modified time of subdir is restored (seconds precision) + // Verify modified/accessed time of subdir is restored (seconds precision) let meta_restored2 = fs::metadata(&sub_dir).unwrap(); let modified_restored2 = meta_restored2.modified().unwrap(); assert_eq!( modified_orig2.duration_since(UNIX_EPOCH).unwrap().as_secs(), modified_restored2.duration_since(UNIX_EPOCH).unwrap().as_secs() ); + let accessed_restored2 = meta_restored2.accessed().unwrap(); + assert_eq!( + accessed_orig2.duration_since(UNIX_EPOCH).unwrap().as_secs(), + accessed_restored2.duration_since(UNIX_EPOCH).unwrap().as_secs() + ); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + // Verify permissions of file1 + let permission_mode1 = meta_restored1.permissions().mode() & 0x0FFF; + assert_eq!(permission_mode1, 0o700); + + // Verify permissions of subdir + let permission_mode2 = meta_restored2.permissions().mode() & 0x0FFF; + assert_eq!(permission_mode2, 0o777); + } // Verify symlink assert!(symlink_path.exists()); @@ -1251,7 +1297,19 @@ mod tests { assert!(symlink_metadata.file_type().is_symlink()); let target = fs::read_link(&symlink_path).unwrap(); assert_eq!(target, Path::new("file1.txt")); - + + // Verify modified/accessed time of symlink is restored (seconds precision) + let modified_restored3 = symlink_metadata.modified().unwrap(); + assert_eq!( + modified_orig3.duration_since(UNIX_EPOCH).unwrap().as_secs(), + modified_restored3.duration_since(UNIX_EPOCH).unwrap().as_secs() + ); + let accessed_restored3 = symlink_metadata.accessed().unwrap(); + assert_eq!( + accessed_orig3.duration_since(UNIX_EPOCH).unwrap().as_secs(), + accessed_restored3.duration_since(UNIX_EPOCH).unwrap().as_secs() + ); + // Verify hardlink #[cfg(unix)] { @@ -1271,6 +1329,34 @@ mod tests { assert_eq!(num_links_f1, 2); assert_eq!(num_links_hl, 2); } + + // Verify contents + assert_eq!(fs::read(&file1_path).unwrap(), content1); + assert_eq!(fs::read(&empty_file_path).unwrap(), b""); + assert_eq!(fs::read(&file2_path).unwrap(), content2); + + // Verify sparse file content + { + let mut f = File::open(&sparse_path).unwrap(); + let mut start_buf = vec![0; 65536]; + f.read_exact(&mut start_buf).unwrap(); + assert_eq!(start_buf, vec![1u8; 65536]); + + f.seek(SeekFrom::Start(131072)).unwrap(); + let mut end_buf = vec![0; 65536]; + f.read_exact(&mut end_buf).unwrap(); + assert_eq!(end_buf, vec![2u8; 65536]); + + assert_eq!(fs::metadata(&sparse_path).unwrap().len(), 196608); + + // Verify it is actually a sparse file (has 1 hole) + let segs = f.scan_chunks().unwrap(); + assert_eq!(segs.holes().count(), 1); + let hole = segs.holes().next().unwrap(); + assert_eq!(hole.start, 65536); + assert_eq!(hole.end, 131072); + } + } #[test] @@ -1324,7 +1410,7 @@ mod tests { // Perform archiving using ArchiveRead - let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new("")); + let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new(""), false); let mut archive_bytes = Vec::new(); for i in 0..=128 { let (chunk, last_chunk) = reader.read_chunk().unwrap(); @@ -1343,7 +1429,7 @@ mod tests { // Perform extraction using ArchiveWrite - let mut writer = ArchiveWrite::new(None); + let mut writer = ArchiveWrite::new(None, false); for chunk in archive_bytes.chunks(CHUNK_SIZE) { writer.write_files(chunk).unwrap(); } @@ -1438,7 +1524,7 @@ mod tests { #[test] fn test_archive_nonexistent_dir() { - let mut reader = ArchiveRead::new(Path::new("test_archive_does_not_exist"), Path::new(""), Path::new("")); + let mut reader = ArchiveRead::new(Path::new("test_archive_does_not_exist"), Path::new(""), Path::new(""), false); let (chunk, last) = reader.read_chunk().unwrap(); assert!(last); assert!(chunk.is_empty()); @@ -1449,7 +1535,7 @@ mod tests { fn test_archive_empty_dir() { let src_dir = TestDir::new("test_archive_empty"); - let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new("")); + let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new(""), false); let mut archive_bytes = Vec::new(); let (chunk, last) = reader.read_chunk().unwrap(); @@ -1462,7 +1548,7 @@ mod tests { fs::remove_dir_all(&src_dir.path).unwrap(); // Extract - let mut writer = ArchiveWrite::new(None); + let mut writer = ArchiveWrite::new(None, false); writer.write_files(&archive_bytes).unwrap(); writer.write_others().unwrap(); diff --git a/src/decryption.rs b/src/decryption.rs index 2e8b36a..5c88ced 100644 --- a/src/decryption.rs +++ b/src/decryption.rs @@ -319,7 +319,7 @@ impl Decryption { /// # Returns /// - `Ok(())` on successful decryption /// - `Err` if file operations, password handling, or decryption fails - pub fn decrypt(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, keyfilepath: Option<&PathBuf>) -> Result<()> { + pub fn decrypt(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, keyfilepath: Option<&PathBuf>, verbose: bool) -> Result<()> { if filepath_in.is_dir() { return Err("Cannot decrypt a directory".into()); } @@ -359,7 +359,7 @@ impl Decryption { let compress = (file_format & 0x01) != 0; let archive = (file_format & 0x02) != 0; - + let mut output_dir = &env::current_dir()?; // output directory path is used if let Some(dir_out) = dirpath_out { @@ -388,8 +388,16 @@ impl Decryption { fs::create_dir_all(dir_out)?; } + if verbose { + println!("--------------------------"); + println!("File format version: {}", file_format_version); + println!("Compressed: {}", compress); + println!("Archived: {}", archive); + println!("--------------------------"); + } + let write_output: Box = if archive { - Box::new( ArchiveWrite::new(dirpath_out.cloned()) ) + Box::new( ArchiveWrite::new(dirpath_out.cloned(), verbose) ) } else { // set write parameters and create output file Box::new( WriteOutput::new(filepath_out, vec![])? ) diff --git a/src/encryption.rs b/src/encryption.rs index e4fe592..b796025 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -412,7 +412,7 @@ impl Encryption { /// # Returns /// - `Ok(())` on successful encryption /// - `Err` if file operations, password handling, or encryption fails - pub fn encrypt(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, keyfilepath: Option<&PathBuf>, compress: bool, split: Vec) -> Result<()> { + pub fn encrypt(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, keyfilepath: Option<&PathBuf>, compress: bool, split: Vec, verbose: bool) -> Result<()> { let (filepath_out, build_archive, strip_dir) = Self::set_paths(filepath_in, dirpath_out, split.is_empty())?; @@ -430,7 +430,7 @@ impl Encryption { } let read_input: Box = if build_archive { - Box::new( ArchiveRead::new(filepath_in, &filepath_out, &strip_dir.unwrap()) ) + Box::new( ArchiveRead::new(filepath_in, &filepath_out, &strip_dir.unwrap(), verbose) ) } else { Box::new( ReadInput::new(filepath_in, CHUNK_SIZE, 0)? ) }; @@ -451,6 +451,14 @@ impl Encryption { header.extend(salt_cha); header.extend(salt_aes); + if verbose { + println!("------------------------"); + println!("File format version: {}", FILE_FORMAT_VERSION); + println!("Compression: {}", if compress {"on"} else {"off"} ); + println!("Archiving: {}", if build_archive {"on"} else {"off"} ); + println!("------------------------"); + } + // set write parameters and create output file let mut write_output = Box::new( WriteOutput::new(filepath_out, split)? ); @@ -742,11 +750,11 @@ mod tests { fs::write(&filepath_in, &data).unwrap(); // encrypt, decrypt - Encryption::encrypt(&filepath_in, None, None, false, vec![]).unwrap(); + Encryption::encrypt(&filepath_in, None, None, false, vec![], false).unwrap(); assert!(filepath_out.exists()); // encrypted file must be different than original data assert_ne!(data, fs::read(&filepath_out).unwrap()); - Decryption::decrypt(&filepath_out, None, None).unwrap(); + Decryption::decrypt(&filepath_out, None, None, false).unwrap(); // read and compare decrypted file against backup let decrypt_data = fs::read(&filepath_in).unwrap(); @@ -755,21 +763,21 @@ mod tests { // decrypt with keyfile should fail let filepath_kf = PathBuf::from("test_another_key.bin"); fs::write(&filepath_kf, vec![0; 1024]).unwrap(); - assert!(Decryption::decrypt(&filepath_out, None, Some(&filepath_kf)).is_err()); + assert!(Decryption::decrypt(&filepath_out, None, Some(&filepath_kf), false).is_err()); // with compression fs::write(&filepath_in, &data).unwrap(); - Encryption::encrypt(&filepath_in, None, None, true, vec![]).unwrap(); - Decryption::decrypt(&filepath_out, None, None).unwrap(); + Encryption::encrypt(&filepath_in, None, None, true, vec![], false).unwrap(); + Decryption::decrypt(&filepath_out, None, None, false).unwrap(); let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); // with output directory let out_enc_dir = path::absolute(PathBuf::from("test_cc_enc_dir")).unwrap(); - Encryption::encrypt(&filepath_in, Some(&out_enc_dir), None, false, vec![]).unwrap(); + Encryption::encrypt(&filepath_in, Some(&out_enc_dir), None, false, vec![], false).unwrap(); assert!(&out_enc_dir.join(&filepath_out).exists()); let out_dec_dir = path::absolute(PathBuf::from("test_cc_dec_dir")).unwrap(); - Decryption::decrypt(&out_enc_dir.join(&filepath_out), Some(&out_dec_dir), None).unwrap(); + Decryption::decrypt(&out_enc_dir.join(&filepath_out), Some(&out_dec_dir), None, false).unwrap(); assert!(&out_dec_dir.join(&filepath_in).exists()); let decrypt_data = fs::read(out_dec_dir.join(&filepath_in)).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -799,30 +807,30 @@ mod tests { fs::write(&filepath_kf, &data_kf).unwrap(); // use keyfile, encrypt, decrypt - Encryption::encrypt(&filepath_in, None, Some(&filepath_kf), false, vec![]).unwrap(); - Decryption::decrypt(&filepath_out, None, Some(&filepath_kf)).unwrap(); + Encryption::encrypt(&filepath_in, None, Some(&filepath_kf), false, vec![], false).unwrap(); + Decryption::decrypt(&filepath_out, None, Some(&filepath_kf), false).unwrap(); // read and compare decrypted file against original data let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); // decrypt without key file - assert!(Decryption::decrypt(&filepath_out, None, None).is_err()); + assert!(Decryption::decrypt(&filepath_out, None, None, false).is_err()); // key file does not exist - assert!(Encryption::encrypt(&filepath_in, None, Some(&PathBuf::from("test_miss")), false, vec![]).is_err()); - assert!(Decryption::decrypt(&filepath_out, None, Some(&PathBuf::from("test_miss"))).is_err()); + assert!(Encryption::encrypt(&filepath_in, None, Some(&PathBuf::from("test_miss")), false, vec![], false).is_err()); + assert!(Decryption::decrypt(&filepath_out, None, Some(&PathBuf::from("test_miss")), false).is_err()); // input file does not exist - assert!(Encryption::encrypt(&path::absolute(PathBuf::from("test_miss")).unwrap(), None, None, false, vec![]).is_err()); - assert!(Decryption::decrypt(&PathBuf::from("test_miss.cce"), None, None).is_err()); + assert!(Encryption::encrypt(&path::absolute(PathBuf::from("test_miss")).unwrap(), None, None, false, vec![], false).is_err()); + assert!(Decryption::decrypt(&PathBuf::from("test_miss.cce"), None, None, false).is_err()); assert!(!fs::exists("test_miss").unwrap()); assert!(!fs::exists("test_miss.cce").unwrap()); // with compression fs::write(&filepath_in, &data).unwrap(); - Encryption::encrypt(&filepath_in, None, Some(&filepath_kf), true, vec![]).unwrap(); - Decryption::decrypt(&filepath_out, None, Some(&filepath_kf)).unwrap(); + Encryption::encrypt(&filepath_in, None, Some(&filepath_kf), true, vec![], false).unwrap(); + Decryption::decrypt(&filepath_out, None, Some(&filepath_kf), false).unwrap(); let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -844,7 +852,7 @@ mod tests { fs::write(&filepath_in, &data).unwrap(); // encrypt and split output - Encryption::encrypt(&filepath_in, None, None, false, vec![1048576, 12]).unwrap(); + Encryption::encrypt(&filepath_in, None, None, false, vec![1048576, 12], false).unwrap(); // concatenate spilt output files let mut data_concat = fs::read("test_cc_split.bin.c00").unwrap(); @@ -852,7 +860,7 @@ mod tests { data_concat.extend(fs::read("test_cc_split.bin.c02").unwrap()); fs::write(&filepath_out, &data_concat).unwrap(); - Decryption::decrypt(&filepath_out, None, None).unwrap(); + Decryption::decrypt(&filepath_out, None, None, false).unwrap(); // read and compare decrypted file against original data let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -860,14 +868,14 @@ mod tests { // concatenate files with decrypt let _ = fs::remove_file(&filepath_in); let _ = fs::remove_file(&filepath_out); - Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None).unwrap(); + Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None, false).unwrap(); // read and compare decrypted file against original data let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); // with compression - Encryption::encrypt(&filepath_in, None, None, true, vec![11, 12, 1024*100]).unwrap(); - Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None).unwrap(); + Encryption::encrypt(&filepath_in, None, None, true, vec![11, 12, 1024*100], false).unwrap(); + Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None, false).unwrap(); let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -918,7 +926,7 @@ mod tests { fs::hard_link(&file2, &hardlink2).unwrap(); // Build archive of directory and encrypt it - Encryption::encrypt(&dir_path, None, None, false, vec![]).unwrap(); + Encryption::encrypt(&dir_path, None, None, false, vec![], false).unwrap(); // Delete the original files before extracting to verify recreation fs::remove_dir_all(&dir_path).unwrap(); @@ -926,7 +934,7 @@ mod tests { // Decrypt and rebuild archived directory let arch_path = dir_path.with_extension(ENCRYPTED_FILE_EXT); - Decryption::decrypt(&arch_path, None, None).unwrap(); + Decryption::decrypt(&arch_path, None, None, false).unwrap(); // Verify structure is fully recreated assert!(dir_path.exists()); @@ -944,12 +952,12 @@ mod tests { // With output directory, with split, check exclusion of output archive files let out_enc_dir = &dir_path; - Encryption::encrypt(&dir_path, Some(&out_enc_dir), None, false, vec![1000,10000]).unwrap(); + Encryption::encrypt(&dir_path, Some(out_enc_dir), None, false, vec![1000,10000], false).unwrap(); assert!(out_enc_dir.join(dir_name).with_extension(SPLIT_ENC_FILE_EXT).exists()); let out_dec_dir = path::absolute(PathBuf::from("test_cc_dec_dir")).unwrap(); let _ = fs::remove_dir_all(&out_dec_dir); - Decryption::decrypt(&out_enc_dir.join(dir_name).with_extension(SPLIT_ENC_FILE_EXT), Some(&out_dec_dir), None).unwrap(); + Decryption::decrypt(&out_enc_dir.join(dir_name).with_extension(SPLIT_ENC_FILE_EXT), Some(&out_dec_dir), None, false).unwrap(); // Verify structure is fully recreated assert!(out_dec_dir.join(dir_name).exists()); @@ -971,7 +979,7 @@ mod tests { let _ = fs::remove_file(&arch_path); let _ = fs::remove_dir_all(&dir_path); - let _ = fs::remove_dir_all(&out_enc_dir); + let _ = fs::remove_dir_all(out_enc_dir); let _ = fs::remove_dir_all(&out_dec_dir); } @@ -1072,7 +1080,7 @@ mod tests { let mut filepath_out = filepath_in.clone(); filepath_out.add_extension(ENCRYPTED_FILE_EXT); - Encryption::encrypt(&filepath_in, None, None, false, vec![]).unwrap(); - Decryption::decrypt(&filepath_out, None, None).unwrap(); + Encryption::encrypt(&filepath_in, None, None, false, vec![], false).unwrap(); + Decryption::decrypt(&filepath_out, None, None, false).unwrap(); } } \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index eae2fb3..fe94a6a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ pub mod decryption; pub mod archive; -pub const FILE_FORMAT_VERSION: u8 = 4; +pub const FILE_FORMAT_VERSION: u8 = 5; pub const ENCRYPTED_FILE_EXT: &str = "cce"; pub const SPLIT_ENC_FILE_EXT: &str = "c00"; pub const CHUNK_SIZE: usize = 1_048_576; // 1024 * 1024 bytes diff --git a/src/main.rs b/src/main.rs index bd3650b..df72079 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ use cryptcrypt::Result; #[derive(Parser)] #[command(version, about, verbatim_doc_comment, long_about = None)] -/// Program for encryption and decryption of file or directory. +/// Application for encryption and decryption of file or directory. /// If no option is given, input is encrypted. A directory as input causes the build of an encrypted archive. /// With option -s the encrypted output is split into files with extensions .c00, .c01, .c02, ... /// If a file ending on .c00 is decrypted, the whole split series will be read. @@ -35,6 +35,10 @@ struct Args { value_parser = |s: &str| { let cfg = Config::new().with_binary(); cfg.parse_size(s) })] split: Vec, + /// Show entries during archive operation + #[arg(short, long, default_value_t = false)] + verbose: bool, + /// File that should be encrypted or decrypted. /// If a directory is given, its contents is archived and encrypted. file_or_directory: PathBuf, @@ -69,9 +73,9 @@ fn run() -> Result<()> { let output_dir = args.out_dir.map(path::absolute).transpose()?; if args.decrypt { - Decryption::decrypt(&filepath, output_dir.as_ref(), keyfilepath.as_ref())?; + Decryption::decrypt(&filepath, output_dir.as_ref(), keyfilepath.as_ref(), args.verbose)?; } else { - Encryption::encrypt(&filepath, output_dir.as_ref(), keyfilepath.as_ref(), args.compress, args.split)?; + Encryption::encrypt(&filepath, output_dir.as_ref(), keyfilepath.as_ref(), args.compress, args.split, args.verbose)?; } Ok(()) From df88752eab06f4f8f064f5afc9a756b088b856ac Mon Sep 17 00:00:00 2001 From: JoergDF Date: Thu, 30 Jul 2026 17:52:07 +0200 Subject: [PATCH 14/16] Fix of sparse file handling. Update fuzz code to changed interface. - if scan of sparse file fails, add zero for sparse holes in header - support archive in fuzzer --- Cargo.lock | 57 +-- fuzz/Cargo.lock | 559 ++++++++++------------------- fuzz/fuzz_targets/fuzzer_encdec.rs | 55 ++- fuzz/fuzz_targets/fuzzer_split.rs | 10 +- src/archive.rs | 10 +- src/decryption.rs | 12 +- src/encryption.rs | 4 +- src/main.rs | 2 +- 8 files changed, 286 insertions(+), 423 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88b8df8..cb70d75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,9 +144,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.66" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "shlex", @@ -212,9 +212,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -222,9 +222,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -234,14 +234,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -544,9 +544,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "num_cpus" @@ -621,18 +621,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -766,9 +766,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -792,7 +803,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -972,22 +983,22 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index b48c304..edf4ef3 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -18,7 +18,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", "cpufeatures 0.2.17", ] @@ -88,12 +88,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "anyhow" -version = "1.0.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" - [[package]] name = "arbitrary" version = "1.4.2" @@ -121,12 +115,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "bitflags" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" - [[package]] name = "blake2" version = "0.10.6" @@ -147,9 +135,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", ] @@ -165,9 +153,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.62" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -175,6 +163,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.4" @@ -187,18 +181,18 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", "cpufeatures 0.2.17", ] [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -229,9 +223,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -239,9 +233,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -251,14 +245,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -269,9 +263,9 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "colorchoice" @@ -305,22 +299,22 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "cryptcrypt" -version = "0.3.0" +version = "0.4.0" dependencies = [ "aes-gcm-siv", "argon2", @@ -328,6 +322,8 @@ dependencies = [ "chacha20poly1305", "clap", "crossbeam-channel", + "drill-press", + "filetime", "hkdf", "num_cpus", "parse-size", @@ -337,7 +333,10 @@ dependencies = [ "secrecy", "sha2", "sha3", + "typed-path", "typenum", + "walkdir", + "windows-sys 0.61.2", ] [[package]] @@ -365,9 +364,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ "hybrid-array", ] @@ -398,7 +397,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -418,29 +417,61 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer 0.12.0", + "block-buffer 0.12.1", "const-oid", - "crypto-common 0.2.1", + "crypto-common 0.2.2", "ctutils", ] [[package]] -name = "equivalent" -version = "1.0.2" +name = "drill-press" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "f980f20e7c746f567c2c335924468fa647ca38f23d87fdcfbd74340d02783210" +dependencies = [ + "cfg-if 0.1.10", + "errno", + "libc", + "thiserror", + "winapi", +] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "errno" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] [[package]] -name = "foldhash" -version = "0.1.5" +name = "errno-dragonfly" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if 1.0.4", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "generic-array" @@ -458,57 +489,28 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", "wasi", ] [[package]] name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - -[[package]] -name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", - "r-efi 6.0.0", + "r-efi", "rand_core 0.10.1", - "wasip2", - "wasip3", ] [[package]] name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "heck" @@ -542,31 +544,13 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ "typenum", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - -[[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", -] - [[package]] name = "inout" version = "0.1.4" @@ -582,19 +566,13 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] @@ -604,16 +582,10 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.3.0", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "libbz2-rs-sys" version = "0.2.5" @@ -622,32 +594,20 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfuzzer-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12a681b7dd8ce12bff52488013ba614b869148d54dd79836ab85aafdd53f08d" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" dependencies = [ "arbitrary", "cc", ] -[[package]] -name = "log" -version = "0.4.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" - -[[package]] -name = "memchr" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" - [[package]] name = "num_cpus" version = "1.17.0" @@ -704,7 +664,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.2.17", "opaque-debug", "universal-hash", @@ -719,40 +679,24 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "r-efi" version = "6.0.0" @@ -761,12 +705,12 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ - "chacha20 0.10.0", - "getrandom 0.4.2", + "chacha20 0.10.1", + "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -797,9 +741,9 @@ checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rpassword" -version = "7.5.2" +version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac5b223d9738ef56e0b98305410be40fa0941bf6036c56f1506751e43552d64" +checksum = "2da316a15f47e3d053de9cb2c439650bd8fa4aaeb9365f2e5f27f492ff73c196" dependencies = [ "libc", "rtoolbox", @@ -817,60 +761,21 @@ dependencies = [ ] [[package]] -name = "secrecy" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" -dependencies = [ - "zeroize", -] - -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" +name = "same-file" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" dependencies = [ - "proc-macro2", - "quote", - "syn", + "winapi-util", ] [[package]] -name = "serde_json" -version = "1.0.149" +name = "secrecy" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", + "zeroize", ] [[package]] @@ -879,7 +784,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures 0.3.0", "digest 0.11.3", ] @@ -897,9 +802,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "sponge-cursor" @@ -921,20 +826,57 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" @@ -942,12 +884,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "universal-hash" version = "0.5.1" @@ -971,62 +907,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" +name = "walkdir" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] [[package]] -name = "wasip2" -version = "1.0.3+wasi-0.2.9" +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" -dependencies = [ - "wit-bindgen 0.57.1", -] +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +name = "winapi" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ - "wit-bindgen 0.51.0", + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] [[package]] -name = "wasm-encoder" -version = "0.244.0" +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] -name = "wasm-metadata" -version = "0.244.0" +name = "winapi-util" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", + "windows-sys 0.61.2", ] [[package]] -name = "wasmparser" -version = "0.244.0" +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-link" @@ -1116,128 +1041,28 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "zeroize" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" - -[[package]] -name = "zmij" -version = "1.0.21" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" diff --git a/fuzz/fuzz_targets/fuzzer_encdec.rs b/fuzz/fuzz_targets/fuzzer_encdec.rs index c640039..14deb49 100644 --- a/fuzz/fuzz_targets/fuzzer_encdec.rs +++ b/fuzz/fuzz_targets/fuzzer_encdec.rs @@ -2,6 +2,7 @@ use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; +use rand::Rng; use rand::RngExt; use std::path::PathBuf; use std::fs; @@ -15,7 +16,7 @@ use cryptcrypt::SPLIT_ENC_FILE_EXT; #[derive(Arbitrary, Clone, Debug)] struct FuzzInput { - data: Vec, + file_sizes: Vec, keydata: Option>, compress: bool, split: Vec, @@ -29,23 +30,42 @@ const DATA_PATH: &str = "/Volumes/RAMDisk1GB/"; // input data is written to files hence encryption can read it from there, // output data is read from files which are written by decryption fuzz_target!(|arb_in: FuzzInput| { + let base_dir = PathBuf::from(DATA_PATH); + let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_nanos(); - let random: u32 = rand::rng().random(); - // For filenames use a random number and nano seconds, required for multiple jobs - let filepath_in = PathBuf::from(format!("{DATA_PATH}dat{:x}{:x}", random, now)); - let keypath = PathBuf::from(format!("{DATA_PATH}key{:x}{:x}", random, now)); + let random: u32 = rand::rng().random(); + let keypath = base_dir.join( PathBuf::from(format!("key{:x}{:x}", random, now)) ); - // add file extension of encrypted file depending on whether file should be split - let mut filepath_out = filepath_in.clone(); + // write data to a file, hence it can be read by encryption + // For file-/pathnames use a random number and nano seconds, required for multiple jobs + let path_in; + if arb_in.file_sizes.len() == 0 { + return; + } else if arb_in.file_sizes.len() == 1 { + let mut data = vec![0; arb_in.file_sizes[0] as usize]; + rand::rng().fill_bytes(&mut data); + path_in = base_dir.join( PathBuf::from(format!("dat{:x}{:x}", random, now)) ); + fs::write(&path_in, &data).unwrap(); + } else { + path_in = base_dir.join( PathBuf::from(format!("dir{:x}{:x}", random, now)) ); + fs::create_dir(&path_in).unwrap(); + + for (i, fsize) in arb_in.file_sizes.iter().enumerate() { + let mut data = vec![0; *fsize as usize]; + rand::rng().fill_bytes(&mut data); + let filepath_in = path_in.join( PathBuf::from(format!("dat{:x}{:x}{:x}", random, now, i)) ); + fs::write(&filepath_in, &data).unwrap(); + } + } + + // add file extension of encrypted file depending on whether file should be split + let mut filepath_out = path_in.clone(); if arb_in.split.is_empty() { filepath_out.add_extension(ENCRYPTED_FILE_EXT); } else { filepath_out.add_extension(SPLIT_ENC_FILE_EXT); } - // write data to a file, hence it can be read by encryption - fs::write(&filepath_in, &arb_in.data).unwrap(); - // if there are data for a keyfile, write it to a file, hence it can be read by encryption let mut filepath_key: Option = None; if let Some(keydata) = arb_in.keydata { @@ -54,17 +74,16 @@ fuzz_target!(|arb_in: FuzzInput| { } // encrypt data and decrypt its output - Encryption::encrypt(&filepath_in, filepath_key.as_ref(), arb_in.compress, arb_in.split).unwrap(); - Decryption::decrypt(&filepath_out, filepath_key.as_ref()).unwrap(); + Encryption::encrypt(&path_in, None, filepath_key.as_ref(), arb_in.compress, arb_in.split, false).unwrap(); - // read decrypted output - let data_out = fs::read(&filepath_in).unwrap(); - - // input to encryption and output of decryption should be the same - assert_eq!(arb_in.data, data_out); + let path_extract = base_dir.join( PathBuf::from(format!("out{:x}{:x}", random, now)) ); + Decryption::decrypt(&filepath_out, Some(&path_extract), filepath_key.as_ref(), false).unwrap(); - // delete files + + // clean up, delete files for file in glob(format!("{DATA_PATH}*{:x}{:x}*", random, now).as_str()).unwrap() { let _ = fs::remove_file(&file.unwrap()); } + let _ = fs::remove_dir_all(&path_in); + let _ = fs::remove_dir_all(&path_extract); }); diff --git a/fuzz/fuzz_targets/fuzzer_split.rs b/fuzz/fuzz_targets/fuzzer_split.rs index 16e5478..99df669 100644 --- a/fuzz/fuzz_targets/fuzzer_split.rs +++ b/fuzz/fuzz_targets/fuzzer_split.rs @@ -7,7 +7,9 @@ use rand::RngExt; use std::time::SystemTime; use glob::glob; use cryptcrypt::common_io; -use cryptcrypt::SPLIT_ENC_FILE_EXT; +use cryptcrypt::common_io::WriteFiles; +use cryptcrypt::common_io::ReadChunk; +use cryptcrypt::{AES_NONCE_SIZE, AES_TAG_SIZE, CHA_NONCE_SIZE, CHA_TAG_SIZE, SPLIT_ENC_FILE_EXT}; // directory for temporary input/output files (could be a RAM disk) @@ -29,9 +31,10 @@ fuzz_target!(|input: (&[u8], Vec)| { let mut wr = common_io::WriteOutput::new(filepath.clone(), split_u64).unwrap(); wr.write_files(&[0]).unwrap(); // header wr.write_files(&data).unwrap(); - + wr.write_files(&[0; CHA_NONCE_SIZE + CHA_TAG_SIZE + AES_NONCE_SIZE + AES_TAG_SIZE]).unwrap(); + // read split files - let mut rd = common_io::ReadInput::new(filepath, 100, 1).unwrap(); + let mut rd = common_io::ReadInput::new(&filepath, 100, 1).unwrap(); // header let mut hdr = [1u8]; rd.read_files(&mut hdr).unwrap(); @@ -44,6 +47,7 @@ fuzz_target!(|input: (&[u8], Vec)| { (dat, final_chunk) = rd.read_chunk().unwrap(); data_out.extend(dat); } + data_out.truncate(data_out.len() - (CHA_NONCE_SIZE + CHA_TAG_SIZE + AES_NONCE_SIZE + AES_TAG_SIZE)); // input data and read data should be the same assert_eq!(data, data_out); diff --git a/src/archive.rs b/src/archive.rs index 359a4df..a0f9ffc 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -436,11 +436,8 @@ impl ArchiveRead { // sparse file // if the files can be scanned for sparse parts, the holes are saved in the archive header - if let Ok(mut f_in) = File::open(entry.path()) - && let Ok(segs) = f_in.scan_chunks() { - sparse_segments = segs; - - // println!("arch: {} {:?}", entry.path().display(), sparse_segments); + if let Ok(mut f_in) = File::open(entry.path()) && let Ok(segs) = f_in.scan_chunks() { + sparse_segments = segs; // number of holes let holes_count = sparse_segments.holes().count(); @@ -452,6 +449,9 @@ impl ArchiveRead { archive_header.extend(hole.start.to_le_bytes()); archive_header.extend(hole.end.to_le_bytes()); } + } else { + // scan for sparse holes failed, handle file as non-sparse and add holes_count = 0 + archive_header.extend(0u16.to_le_bytes()); } } else if entry_type == TYPE_SYMLINK_FILE || entry_type == TYPE_SYMLINK_DIR { diff --git a/src/decryption.rs b/src/decryption.rs index 5c88ced..505b06d 100644 --- a/src/decryption.rs +++ b/src/decryption.rs @@ -371,12 +371,14 @@ impl Decryption { } } - if archive { - println!("Archive file will be extracted to directory {}", output_dir.display()); - } else { - println!("Output will be written to file {}", filepath_out.display()); + if verbose { + if archive { + println!("Archive file will be extracted to directory {}", output_dir.display()); + } else { + println!("Output will be written to file {}", filepath_out.display()); + } } - + // get password and keys let key = Self::hash_password(salt_pw, keyfilepath)?; let (key_cha, key_aes) = Self::derive_keys(salt_cha, salt_aes, &key)?; diff --git a/src/encryption.rs b/src/encryption.rs index b796025..a1b2b69 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -416,7 +416,9 @@ impl Encryption { let (filepath_out, build_archive, strip_dir) = Self::set_paths(filepath_in, dirpath_out, split.is_empty())?; - println!("Output will be written to {}", filepath_out.display()); + if verbose { + println!("Output will be written to {}", filepath_out.display()); + } // ask for password, before there can be error messages of archive let (salt_pw, key) = Self::hash_password(keyfilepath)?; diff --git a/src/main.rs b/src/main.rs index df72079..9387f73 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,7 +35,7 @@ struct Args { value_parser = |s: &str| { let cfg = Config::new().with_binary(); cfg.parse_size(s) })] split: Vec, - /// Show entries during archive operation + /// Show details about operations #[arg(short, long, default_value_t = false)] verbose: bool, From 958eea213ec8c74b1ba1b4db56fc2e6d823a24db Mon Sep 17 00:00:00 2001 From: JoergDF Date: Sat, 1 Aug 2026 17:35:41 +0200 Subject: [PATCH 15/16] In output file header encrypt all data which is not required in cleartext. --- src/decryption.rs | 102 +++++++++++++++++++++++++++++----------------- src/encryption.rs | 90 ++++++++++++++++++++++++---------------- src/lib.rs | 3 +- 3 files changed, 120 insertions(+), 75 deletions(-) diff --git a/src/decryption.rs b/src/decryption.rs index 505b06d..6b316ce 100644 --- a/src/decryption.rs +++ b/src/decryption.rs @@ -12,7 +12,7 @@ use bzip2::read::BzDecoder; use crossbeam_channel::{bounded, Sender, Receiver}; use crate::{Result, KEY_SIZE, CHA_NONCE_SIZE, AES_NONCE_SIZE, CHUNK_SIZE, COMPRESS_LENGTH_SIZE, ENCRYPTED_FILE_EXT, - SPLIT_ENC_FILE_EXT, CHA_TAG_SIZE, AES_TAG_SIZE, HEADER_SIZE, FILE_FORMAT_VERSION}; + SPLIT_ENC_FILE_EXT, CHA_TAG_SIZE, AES_TAG_SIZE, HEADER_SIZE, SALT_SIZE, FILE_FORMAT_VERSION}; use crate::common::{get_pass_bytes, key_derivation}; use crate::common_io::{CryptIo, ReadInput, WriteFiles, WriteOutput}; use crate::archive::ArchiveWrite; @@ -55,13 +55,13 @@ impl Decryption { /// callers keep key material in secure containers. /// /// # Arguments - /// - `salt_cha` — salt for the ChaCha key derivation (expected length: `SALT_SIZE`) - /// - `salt_aes` — salt for the AES key derivation (expected length: `SALT_SIZE`) - /// - `key` — master secret material to expand (type: `SecretSlice`) + /// - `salt_cha`: salt for the ChaCha key derivation (expected length: `SALT_SIZE`) + /// - `salt_aes`: salt for the AES key derivation (expected length: `SALT_SIZE`) + /// - `key`: master secret material to expand (type: `SecretSlice`) /// /// # Returns /// - `Ok((key_cha, key_aes))` — tuple of derived keys (`SecretSlice`) each `KEY_SIZE` bytes long - /// - `Err` — if HKDF expansion or underlying operations fail + /// - `Err` if HKDF expansion or underlying operations fail fn derive_keys(salt_cha: &[u8], salt_aes: &[u8], key: &SecretSlice) -> Result<(SecretSlice, SecretSlice)> { let key_cha = key_derivation(key, salt_cha, "xchacha20poly1305".as_bytes())?; let key_aes = key_derivation(key, salt_aes, "-aes-256-gcm-siv-".as_bytes())?; @@ -78,10 +78,10 @@ impl Decryption { /// - `nonce[1..]` with the little‑endian bytes of `chunk_count` (applied starting at index 1). /// /// # Arguments - /// - `key` — 32‑byte ChaCha key stored in a `SecretSlice`. + /// - `key`: 32‑byte ChaCha key stored in a `SecretSlice`. /// - `buf`: Data containing nonce + ciphertext (+ authentication tag) - /// - `chunk_count` — zero‑based chunk index; must match the value used during encryption. - /// - `final_chunk` — `true` if this is the last chunk; must match the value used during encryption. + /// - `chunk_count`: zero‑based chunk index; must match the value used during encryption. + /// - `final_chunk`: `true` if this is the last chunk; must match the value used during encryption. /// /// # Returns /// - `Ok(plaintext)` containing decrypted data @@ -305,6 +305,44 @@ impl Decryption { thread_handles } + /// Gets unencrypted data of header, i.e. the 3 salt values + /// + /// # Argument: + /// - `header`: header bytes + /// + /// # Returns: + /// - `(salt_pw, salt_cha, salt_aes)`: salts for password, chacha key, aes key + fn get_unencrypted_header_items(header: &[u8]) -> (&[u8], &[u8], &[u8]) { + let salt_pw = &header[..SALT_SIZE]; + let salt_cha = &header[SALT_SIZE..(2 * SALT_SIZE)]; + let salt_aes = &header[(2 * SALT_SIZE)..(3 * SALT_SIZE)]; + + (salt_pw, salt_cha, salt_aes) + } + + /// Gets encrypted data of header + /// + /// # Argument: + /// - `header`: header bytes + /// - `key_cha`: key for chacha decryption + /// - `key_aes`: key for aes decryption + /// + /// # Returns: + /// - `Ok((file_format_version, compress, archive))` contains on success: version of file format, compression status, whether it's an archive + fn get_encrypted_header_items(header: &[u8], key_cha: &SecretSlice, key_aes: &SecretSlice) -> Result<(u8, bool, bool)> { + let enc_head = &header[(3 * SALT_SIZE)..HEADER_SIZE]; + + let buf_aes = Self::aes_decrypt_buffer(key_aes, enc_head)?; + let buf_cha = Self::cha_decrypt_buffer(key_cha, &buf_aes, 0, false)?; + + let file_format_version = buf_cha[0]; + let file_format = buf_cha[1]; + let compress = (file_format & 0x01) != 0; + let archive = (file_format & 0x02) != 0; + + Ok((file_format_version, compress, archive)) + } + /// Decrypts a file encrypted with dual-layer encryption (AES-256-GCM-SIV + ChaCha20). /// /// Reads and evaluates header from file, prompts user for password, derives master key using Argon2, @@ -332,7 +370,7 @@ impl Decryption { } else { return Err(format!("Invalid filename, it does not end with .{ENCRYPTED_FILE_EXT} or .{SPLIT_ENC_FILE_EXT}").into()) } - + // set read parameters let mut read_input = Box:: new( ReadInput::new( filepath_in, @@ -344,12 +382,16 @@ impl Decryption { let mut header = [0u8; HEADER_SIZE]; read_input.read_files(&mut header)?; - let file_format_version = header[0]; - let file_format = header[1]; - let salt_pw = &header[2..34]; - let salt_cha = &header[34..66]; - let salt_aes = &header[66..98]; - + // get salts from header + let (salt_pw, salt_cha, salt_aes) = Self::get_unencrypted_header_items(&header); + + // get password and keys + let key = Self::hash_password(salt_pw, keyfilepath)?; + let (key_cha, key_aes) = Self::derive_keys(salt_cha, salt_aes, &key)?; + + // get rest of header + let (file_format_version, compress, archive) = Self::get_encrypted_header_items(&header, &key_cha, &key_aes)?; + // check format version if file_format_version != FILE_FORMAT_VERSION { return Err(format!( @@ -357,12 +399,12 @@ impl Decryption { ).into()); } - let compress = (file_format & 0x01) != 0; - let archive = (file_format & 0x02) != 0; - let mut output_dir = &env::current_dir()?; // output directory path is used if let Some(dir_out) = dirpath_out { + if !dir_out.exists() { + fs::create_dir_all(dir_out)?; + } if archive { output_dir = dir_out; } else { @@ -370,25 +412,6 @@ impl Decryption { filepath_out = dir_out.join(filename_out); } } - - if verbose { - if archive { - println!("Archive file will be extracted to directory {}", output_dir.display()); - } else { - println!("Output will be written to file {}", filepath_out.display()); - } - } - - // get password and keys - let key = Self::hash_password(salt_pw, keyfilepath)?; - let (key_cha, key_aes) = Self::derive_keys(salt_cha, salt_aes, &key)?; - - // output directory action: - // create a new directory after password entry: - // if password entry failed or user breaks execution on password entry, filesystem stays unchanged - if let Some(dir_out) = dirpath_out && !dir_out.exists() { - fs::create_dir_all(dir_out)?; - } if verbose { println!("--------------------------"); @@ -396,6 +419,11 @@ impl Decryption { println!("Compressed: {}", compress); println!("Archived: {}", archive); println!("--------------------------"); + if archive { + println!("Archive file will be extracted to directory {}", output_dir.display()); + } else { + println!("Output will be written to file {}", filepath_out.display()); + } } let write_output: Box = if archive { diff --git a/src/encryption.rs b/src/encryption.rs index a1b2b69..6261ed8 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -33,8 +33,8 @@ impl Encryption { /// /// # Returns /// - /// - `Ok([u8; SALT_SIZE])` — newly generated salt on success. - /// - `Err` — if RNG initialization fails. + /// - `Ok([u8; SALT_SIZE])` newly generated salt on success. + /// - `Err` if RNG initialization fails. fn create_salt() -> Result<[u8; SALT_SIZE]> { let mut salt = [0u8; SALT_SIZE]; let mut rng = ChaCha20Rng::try_from_rng(&mut SysRng)?; @@ -71,11 +71,11 @@ impl Encryption { /// HKDF-SHA256 to produce two 32-byte keys. /// /// # Arguments - /// - `key` — master secret material to expand (kept in `SecretSlice`). + /// - `key`: master secret material to expand (kept in `SecretSlice`). /// /// # Returns /// - `Ok(( [u8; SALT_SIZE], SecretSlice, [u8; SALT_SIZE], SecretSlice ))` on success. - /// - `Err` — if HKDF expansion or random salt generation fails. + /// - `Err` if HKDF expansion or random salt generation fails. #[allow(clippy::type_complexity)] fn derive_keys(key: &SecretSlice) -> Result<([u8; SALT_SIZE], SecretSlice, [u8; SALT_SIZE], SecretSlice)> { let salt_cha = Self::create_salt()?; @@ -107,11 +107,11 @@ impl Encryption { /// explicit chunk metadata. /// /// # Arguments - /// - `key` — 32-byte ChaCha key held in a `SecretSlice`. - /// - `buf` — plaintext bytes to encrypt (one chunk). - /// - `chunk_count` — zero-based chunk index (incremented per chunk). Must be + /// - `key`: 32-byte ChaCha key held in a `SecretSlice`. + /// - `buf`: plaintext bytes to encrypt (one chunk). + /// - `chunk_count`: zero-based chunk index (incremented per chunk). Must be /// the same value used when decrypting this chunk. - /// - `final_chunk` — `true` if this is the last chunk of the file, + /// - `final_chunk`: `true` if this is the last chunk of the file, /// `false` otherwise. Also must match the value used at decryption. /// /// # Returns @@ -266,7 +266,7 @@ impl Encryption { /// - `cpu_count`: number of worker threads to spawn. /// /// # Returns - /// - `Vec>>` — handles for all spawned threads. + /// - `Vec>>` handles for all spawned threads. pub fn encrypt_pipe( key_cha: &SecretSlice, key_aes: &SecretSlice, @@ -394,6 +394,42 @@ impl Encryption { Ok((filepath_out, build_archive, strip_dir_in)) } + /// Creates header of output file. + /// + /// Salts are required unencrypted, the other data is encrypted. + /// + /// # Arguments + /// - `salt_pw`: salt of password hash + /// - `salt_cha`: salt of the ChaCha key derivation + /// - `salt_aes`: salt of the AES key derivation + /// - `compress`: compression enabled + /// - `archive`: archiving enabled + /// - `key_cha`: key for ChaCha encryption + /// - `key_aes`: hey for AES encryption + /// + /// # Returns + /// - `Ok(header)` contains header on success + /// - `Err` if encryption fails + fn create_header(salt_pw: &[u8], salt_cha: &[u8], salt_aes: &[u8], compress: bool, archive: bool, key_cha: &SecretSlice, key_aes: &SecretSlice) -> Result> { + let mut header = Vec::with_capacity(HEADER_SIZE); + + // cleartext header part + header.extend(salt_pw); + header.extend(salt_cha); + header.extend(salt_aes); + + // encrypted header part + let buf_in = [ + FILE_FORMAT_VERSION, + u8::from(compress) | (u8::from(archive) << 1) + ]; + let buf_cha = Self::cha_encrypt_buffer(key_cha, &buf_in, 0, false)?; + let buf_aes = Self::aes_encrypt_buffer(key_aes, &buf_cha)?; + header.extend(buf_aes); + + Ok(header) + } + /// Encrypts a file or a directory using dual-layer encryption (ChaCha20 + AES-256-GCM-SIV) /// with optional compression. /// @@ -417,7 +453,12 @@ impl Encryption { let (filepath_out, build_archive, strip_dir) = Self::set_paths(filepath_in, dirpath_out, split.is_empty())?; if verbose { - println!("Output will be written to {}", filepath_out.display()); + println!("------------------------"); + println!("File format version: {}", FILE_FORMAT_VERSION); + println!("Compression: {}", if compress {"on"} else {"off"} ); + println!("Archiving: {}", if build_archive {"on"} else {"off"} ); + println!("------------------------"); + println!("Output will be written to file {}", filepath_out.display()); } // ask for password, before there can be error messages of archive @@ -437,34 +478,11 @@ impl Encryption { Box::new( ReadInput::new(filepath_in, CHUNK_SIZE, 0)? ) }; - // file header - // byte description - // 0 version of file format - // 1 info about file format - // bit 0: compression on(1)/off(0) - // bit 1: archive - // 2..33 32-byte-salt of password hash - // 34..65 32-byte-salt of cha key derivation - // 66..97 32-byte-salt of aes key derivation - let mut header = Vec::with_capacity(HEADER_SIZE); - header.push(FILE_FORMAT_VERSION); - header.push(u8::from(compress) | (u8::from(build_archive) << 1)); - header.extend(salt_pw); - header.extend(salt_cha); - header.extend(salt_aes); - - if verbose { - println!("------------------------"); - println!("File format version: {}", FILE_FORMAT_VERSION); - println!("Compression: {}", if compress {"on"} else {"off"} ); - println!("Archiving: {}", if build_archive {"on"} else {"off"} ); - println!("------------------------"); - } - // set write parameters and create output file let mut write_output = Box::new( WriteOutput::new(filepath_out, split)? ); - - // write header + + // create and write header + let header = Self::create_header(&salt_pw, &salt_cha, &salt_aes, compress, build_archive, &key_cha, &key_aes)?; write_output.write_files(&header)?; CryptIo::io_chunks(&key_cha, &key_aes, compress, Self::encrypt_pipe, read_input, write_output)?; diff --git a/src/lib.rs b/src/lib.rs index fe94a6a..e7c4ad9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,11 +19,10 @@ pub const MAX_KEYFILE_CHUNKS: usize = 64; pub const SALT_SIZE: usize = 32; pub const KEY_SIZE: usize = 32; pub const COMPRESS_LENGTH_SIZE: usize = 3; -pub const HEADER_SIZE: usize = 2 + 3 * SALT_SIZE; pub const AES_NONCE_SIZE: usize = ::NonceSize::USIZE; // 12 bytes pub const CHA_NONCE_SIZE: usize = ::NonceSize::USIZE; // 24 bytes pub const AES_TAG_SIZE: usize = ::TagSize::USIZE; // 16 bytes pub const CHA_TAG_SIZE: usize = ::TagSize::USIZE; // 16 bytes - +pub const HEADER_SIZE: usize = 3 * SALT_SIZE + 2 + CHA_NONCE_SIZE + CHA_TAG_SIZE + AES_NONCE_SIZE + AES_TAG_SIZE; pub type Result = std::result::Result>; From 14097693cdc2bf272bf278362a1ac91128fc05ac Mon Sep 17 00:00:00 2001 From: JoergDF Date: Mon, 10 Aug 2026 15:38:33 +0200 Subject: [PATCH 16/16] Archive list option, bugfixes - Add archive list option. - Add tests for archive listing. - Move code of archive header into new struct. - Fix setting of file times: if file write ended and there was no more data in archive, file times were not set for last file. - Fix unit test test_archive_read_write(): setting of access time might not work, as walkdir used in archiving, might change the access time. Hence remove check of access time. - Add random filler bytes to encrypted header bytes for improved security. --- README.md | 56 ++- src/archive.rs | 969 +++++++++++++++++++++++++--------------------- src/decryption.rs | 18 +- src/encryption.rs | 118 +++--- src/lib.rs | 2 +- src/main.rs | 10 +- 6 files changed, 647 insertions(+), 526 deletions(-) diff --git a/README.md b/README.md index 10bf5cd..21ba3b2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # cryptcrypt -A command-line tool for encrypting and decrypting a file using modern encryption algorithms and password-based key derivation. -Additionally it can use a key file, bzip2 compression and an output file split. +A command-line tool for encrypting and decrypting a file or a directory using modern encryption algorithms and password-based key derivation. +Additionally, it can use a key file, bzip2 compression and an output file split. The contents of a directory are concatenated into an archive. **Security Notes**: This software is experimental. There wasn't any security audit. Do not use in production. USE AT YOUR OWN RISK! @@ -15,27 +15,39 @@ Additionally it can use a key file, bzip2 compression and an output file split. - Handles large files efficiently by parallel processing them in chunks - CLI interface - Rust toolchain -- For MacOS, Linux, Windows - +- Supports MacOS, Linux, Windows + +### Archiving supports +- Cross-platform: e.g. an archive created on Linux can be extracted on Windows and vice versa +- Symbolic links +- Hard links +- Sparse files: the size of sparse holes might change on cross-platform extraction +- Modification and access time stamps +- File permissions: only on Unix-like systems (MacOS, Linux) +- Stores relative paths +- Building an archive (i.e. reading files) is done in parallel for improved performance ## Usage ``` -Program for encryption and decryption of a file. -If no option is given, file is encrypted. +Application for encryption and decryption of file or directory. +If no option is given, input is encrypted. Providing a directory creates an encrypted archive. With option -s the encrypted output is split into files with extensions .c00, .c01, .c02, ... -If a file ending on .c00 is decrypted, the whole split series will be read. +If a file ending in .c00 is decrypted, the whole split series will be read. -Usage: cryptcrypt [OPTIONS] +Usage: cryptcrypt [OPTIONS] Arguments: - File that should be encrypted or decrypted + File that should be encrypted or decrypted. If a directory is given, its contents is archived and encrypted Options: + -o, --out-dir Output directory, it is created if it does not exist -d, --decrypt Decrypt file (with extension '.cce' or for split series '.c00') -k, --keyfile Additional key file to supplement the password -z, --compress Compress data before encryption, automatically detected on decryption -s, --split Split encrypted data into pieces of binary byte sizes (e.g. 2g,3g,1g) [G|g|M|m|K|k] + -l, --list-archive List elements of an archive file, do not create its elements + -v, --verbose Show details about operations -h, --help Print help -V, --version Print version ``` @@ -57,9 +69,15 @@ cargo run --release Prompts you to enter a password (with confirmation). Creates output file `file.bin.cce`. Overwrites file, if it already exists. -- Encrypt a file with additional key file, compress and split output files into sizes of 1 KBytes, 2 MBytes and remaining bytes: +- Archive and encrypt a directory (with all its sub-directories and files): + ``` + cryptcrypt directory + ``` + Creates output file `directory.cce`. Overwrites file, if it already exists. + +- Encrypt a file with additional key file, compress and split output files into sizes of 1 KBytes, 2 MBytes and remaining bytes, write output files into folder `output_enc`: ``` - cryptcrypt -k keyfile.bin -z -s 1k,2m file.bin + cryptcrypt -k keyfile.bin -z -s 1k,2m -o output_enc file.bin ``` - Decrypt a file: @@ -69,28 +87,28 @@ cargo run --release Prompts you to enter a password. Creates output file `file.bin`. Overwrites file, if it already exists. -- Decrypt a split series of files with additional key file (compression usage is coded in the encrypted file): +- Decrypt a split series of files with additional key file (compression usage is coded in the encrypted file), write output into folder `output_dec`: ``` - cryptcrypt -k keyfile.bin -d file.bin.c00 + cryptcrypt -k keyfile.bin -o output_dec -d file.bin.c00 ``` ## Encryption details 1. If a key file is used, hash its first 64 MByte (maximum) with [sha3-512](https://github.com/RustCrypto/hashes/tree/master/sha3). -2. Write file format version and file format (i.e. compression status) to start of the output file. -3. Derive encryption key from password and the optional key file hash using Argon2id with a random salt. Write the salt to the output file. -4. From the master key derive two independent 32‑byte keys: one for XChaCha20-Poly1305 and one for AES-256-GCM-SIV. +2. Derive encryption key from password and the optional key file hash using Argon2id with a random salt. Write the salt to the output file. +3. From the master key derive two independent 32‑byte keys: one for XChaCha20-Poly1305 and one for AES-256-GCM-SIV. Each key is derived with [HKDF-SHA256](https://github.com/RustCrypto/KDFs/tree/master/hkdf) using its own fresh random salt. -Write the ChaCha salt then the AES salt immediately after the password salt (file header order: file format version, file format, password salt, ChaCha salt, AES salt). +Write the ChaCha salt then the AES salt immediately after the password salt. +4. Encrypt and write file format version and file format (i.e. compression status) to the output file. (file header: password salt, ChaCha salt, AES salt, encrypted file format version, encrypted file format). 5. Read a 1 MByte chunk from the input file (the last chunk may be smaller). Keep a zero-based sequence number for each chunk and mark the final chunk with a flag. -6. If compression is switched on, compress chunk with bzip2. +6. If compression is switched on, compress chunk with bzip2. Its variably sized output is copied to fixed sized, 1 MB chunks. The chunk size should not give a hint about the cleartext. 7. First-pass encrypt the chunk with [XChaCha20-Poly1305](https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305) using a fresh random base nonce generated per chunk. For the actual encryption nonce the implementation derives a per-chunk nonce by XOR’ing the base nonce with the chunk’s sequence number and the final-chunk flag; the original base nonce is stored before the ChaCha ciphertext so the per-chunk nonce can be recomputed during decryption. Reordering or truncating the chunk sequence would cause a decryption error. 8. Second-pass encrypt the output of step 7 with [AES-256-GCM-SIV](https://github.com/RustCrypto/AEADs/tree/master/aes-gcm-siv) -using a fresh random nonce; the AES nonce is stored before the AES ciphertext and written to (split) output file. +using a fresh random nonce; the AES nonce is stored before the AES ciphertext and written to the (split) output file. 9. Repeat steps 5–8 until all input is processed. ## License diff --git a/src/archive.rs b/src/archive.rs index a0f9ffc..7acee78 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -25,6 +25,351 @@ const TYPE_UNIX: u8 = 0x00; const TYPE_WINDOWS: u8 = 0x10; const ARCHIVE_HEADER_LENGTH_SIZE: usize = 2; + + +/// Represents a parsed archive entry header. +/// +/// Stores metadata decoded from the archive stream for one entry. +struct ArchiveHeader { + file_type: u8, + created_on_os_type: u8, + entry_path: PathBuf, + link_target_path: PathBuf, + time_accessed: SystemTime, + time_modified: SystemTime, + file_size: u64, + sparse_holes_count: u16, + sparse_segments: Vec, + permissions: u16, +} + +impl ArchiveHeader { + /// Appends the file path length and path string to the archive header buffer. + /// + /// # Arguments + /// - `path`: The file system path to encode. + /// - `archive_header`: The mutable buffer to append the encoded path to. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` if the path length exceeds `u16` capacity. + fn add_path_to_header(path: &Path, archive_header: &mut Vec) -> Result<()> { + // path length and path + let path_string = path.to_string_lossy(); + let path_len: u16 = path_string.len().try_into()?; + archive_header.extend(path_len.to_le_bytes()); + archive_header.extend(path_string.as_bytes()); + Ok(()) + } + + /// Computes and sets the final header size at the beginning of the header buffer. + /// It is called after all other header fields have been added to the header buffer. + /// + /// # Arguments + /// - `archive_header`: The mutable slice representing the archive header. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` if the header length cannot be converted to `u16`. + fn set_header_size(archive_header: &mut [u8]) -> Result<()> { + let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = + u16::try_from(archive_header.len() - ARCHIVE_HEADER_LENGTH_SIZE)?.to_le_bytes(); + archive_header[0] = header_size[0]; + archive_header[1] = header_size[1]; + Ok(()) + } + + /// Builds the archive header bytes for a given file system entry. + /// + /// Creates a metadata block containing file type, path, timestamps, size, + /// sparse segments (holes), and permissions. + /// + /// # Arguments + /// - `entry`: The directory entry to construct the header for. + /// - `hard_link_target`: Optional path pointing to the target if this is a hard link. + /// - `strip_dir`: Path to be stripped from the start of each path in the archive to make paths relative + /// + /// # Returns + /// - `Ok((archive_header, filepath_and_size, sparse_segments))` on success. + /// - `Err` if metadata retrieval or OS-specific operations fail. + #[allow(clippy::type_complexity)] + fn build_header(entry: &DirEntry, hard_link_target: &Option, strip_dir: &PathBuf, verbose: bool) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { + // archive header initialized with place holder for header size + let mut archive_header = Vec::with_capacity(1024); + archive_header.extend([0u8; ARCHIVE_HEADER_LENGTH_SIZE]); + + let entry_type = if hard_link_target.is_some() { + TYPE_HARDLINK + } else if entry.file_type().is_file() { + TYPE_FILE + } else if entry.file_type().is_dir() { + TYPE_DIRECTORY + } else if entry.file_type().is_symlink() { + // whether a symlink is a file or a directory is only relevant for windows (when creating them there) + if entry.path().is_dir() { + TYPE_SYMLINK_DIR + } else { + // if target of symlink does not exist (hence it can't be evaluated + // whether it is a file or a directory), type file is used + TYPE_SYMLINK_FILE + } + } else { + return Err("Unsupported file type".into()); + }; + + let os_type = if cfg!(unix) { TYPE_UNIX } else { TYPE_WINDOWS }; + archive_header.push(os_type | entry_type); + + // path length and path (including filename) (converting to relative path) + let stripped_entry_path = entry.path().strip_prefix(strip_dir)?; + Self::add_path_to_header(stripped_entry_path, &mut archive_header)?; + + if verbose { + println!("{}", stripped_entry_path.display()); + } + + if entry_type == TYPE_HARDLINK { + // target path of hard link + let target_path = hard_link_target.as_ref().unwrap().strip_prefix(strip_dir)?; + Self::add_path_to_header(target_path, &mut archive_header)?; + + // header size + Self::set_header_size(&mut archive_header)?; + + return Ok((archive_header, None, vec![])); + } + + // metadata of entry + let meta_entry = entry.metadata()?; + + // last access time + let time_accessed = meta_entry.accessed()?.duration_since(UNIX_EPOCH)?.as_secs(); + archive_header.extend(time_accessed.to_le_bytes()); + // last modification time + let time_modified = meta_entry.modified()?.duration_since(UNIX_EPOCH)?.as_secs(); + archive_header.extend(time_modified.to_le_bytes()); + + let mut file_size = 0; + let mut sparse_segments = vec![]; + if entry_type == TYPE_FILE { + // file size + file_size = meta_entry.len(); + archive_header.extend(file_size.to_le_bytes()); + + // sparse file + // if the files can be scanned for sparse parts, the holes are saved in the archive header + if let Ok(mut f_in) = File::open(entry.path()) && let Ok(segs) = f_in.scan_chunks() { + sparse_segments = segs; + + // number of holes + let holes_count = sparse_segments.holes().count(); + archive_header.extend(u16::try_from( holes_count )?.to_le_bytes()); + + // start and end index of holes, if any + for hole in sparse_segments.holes() { + // start and end are of type u64 + archive_header.extend(hole.start.to_le_bytes()); + archive_header.extend(hole.end.to_le_bytes()); + } + } else { + // scan for sparse holes failed, handle file as non-sparse and add holes_count = 0 + archive_header.extend(0u16.to_le_bytes()); + } + + } else if entry_type == TYPE_SYMLINK_FILE || entry_type == TYPE_SYMLINK_DIR { + // target path of symlink + let target_path = fs::read_link(entry.path())?; + Self::add_path_to_header(&target_path, &mut archive_header)?; + } + + // permissions + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + let mut perm: u16 = 0; + if entry_type == TYPE_FILE || entry_type == TYPE_DIRECTORY { + let permission_mode = meta_entry.permissions().mode(); + // use 12 least significant bits + perm = (permission_mode & 0x0FFF) as u16; + } + archive_header.extend(perm.to_le_bytes()); + } + #[cfg(windows)] + { + archive_header.extend(0u16.to_le_bytes()); + } + + // header size + Self::set_header_size(&mut archive_header)?; + + let mut filepath_and_size = None; + if entry_type == TYPE_FILE { + filepath_and_size = Some((entry.clone().into_path(), file_size)); + } + + Ok((archive_header, filepath_and_size, sparse_segments)) + } + + /// Creates a new empty archive header with default values. + /// + /// Used to initialize a header before parsing archive bytes into it. + pub fn parse_new() -> Self { + Self { + file_type: 0, + created_on_os_type: 0, + entry_path: PathBuf::new(), + link_target_path: PathBuf::new(), + time_accessed: UNIX_EPOCH, + time_modified: UNIX_EPOCH, + file_size: 0, + sparse_holes_count: 0, + sparse_segments: Vec::new(), + permissions: 0 + } + } + + /// Parses archive header bytes and fills this `ArchiveHeader`. + /// + /// Reads the encoded entry path, optional hardlink or symlink target, + /// timestamps, file size, sparse hole descriptors, and permission data. + /// + /// # Arguments + /// - `header`: The raw archive header bytes to parse. + /// - `dirpath_out`: Optional output directory to prepend to restored paths. + /// + /// # Returns + /// - `Ok(())` on success. + /// - `Err` if the header data is malformed or an OS-specific conversion fails. + fn parse_header(&mut self, header: &[u8], dirpath_out: &Option) -> Result<()> { + self.file_type = header[0] & 0x0F; + self.created_on_os_type = header[0] & 0xF0; + + let mut s; + let mut e = 1; + + // entry's path + let entry_path_string; + (entry_path_string, e) = Self::get_path_from_header(header, e, self.created_on_os_type)?; + self.entry_path = PathBuf::from(&entry_path_string); + // add optional output directory to entry's path + if let Some(dir_out) = dirpath_out { + self.entry_path = dir_out.join(self.entry_path.clone()); + } + + if self.file_type == TYPE_HARDLINK { + // hard link's target path + let (target_path_string, _) = Self::get_path_from_header(header, e, self.created_on_os_type)?; + self.link_target_path = PathBuf::from(target_path_string); + if let Some(dir_out) = dirpath_out { + self.link_target_path = dir_out.join(self.link_target_path.clone()); + } + return Ok(()); + } + + // access time + s = e; e += size_of::(); + let time_accessed_seconds = u64::from_le_bytes( header[s..e].try_into()? ); + self.time_accessed = UNIX_EPOCH + Duration::from_secs(time_accessed_seconds); + + // modification time + s = e; e += size_of::(); + let time_modified_seconds = u64::from_le_bytes( header[s..e].try_into()? ); + self.time_modified = UNIX_EPOCH + Duration::from_secs(time_modified_seconds); + + if self.file_type == TYPE_FILE { + // file size + s = e; e += size_of::(); + self.file_size = u64::from_le_bytes( header[s..e].try_into()? ); + + // holes of a sparse file + s = e; e += size_of::(); + self.sparse_holes_count = u16::from_le_bytes( header[s..e].try_into()? ); + + if self.sparse_holes_count > 0 { + // restore data- and hole-segments of sparse file + let mut data_start = 0; + for _ in 0..self.sparse_holes_count { + s = e; e += size_of::(); + let hole_start = u64::from_le_bytes( header[s..e].try_into()? ); + s = e; e += size_of::(); + let hole_end = u64::from_le_bytes( header[s..e].try_into()? ); + + if hole_start != 0 { + // if first segment is not a hole, add a data segment + self.sparse_segments.push( Segment { segment_type: SegmentType::Data, range: data_start..hole_start} ); + } + self.sparse_segments.push( Segment { segment_type: SegmentType::Hole, range: hole_start..hole_end } ); + data_start = hole_end; + } + // if last segment is not a hole, add a data segment + if data_start != self.file_size { + self.sparse_segments.push( Segment { segment_type: SegmentType::Data, range: data_start..self.file_size} ); + } + } + + } else if self.file_type == TYPE_SYMLINK_FILE || self.file_type == TYPE_SYMLINK_DIR { + // symlink's target path + let target_path_string; + (target_path_string, e) = Self::get_path_from_header(header, e, self.created_on_os_type)?; + self.link_target_path = PathBuf::from(target_path_string); + } + + // permissions + // if this is a unix system and the archive was created on a unix system, set permission mode + #[cfg(unix)] + { + if self.created_on_os_type == TYPE_UNIX && (self.file_type == TYPE_DIRECTORY || self.file_type == TYPE_FILE) { + s = e; e += size_of::(); + self.permissions = u16::from_le_bytes(header[s..e].try_into()?); + } + } + #[cfg(windows)] + { + // keep compiler quiet + s = e; e += size_of::(); + self.permissions = u16::from_le_bytes(header[s..e].try_into()?); + } + + Ok(()) + } + + /// Parses a file path from the archive header slice. + /// + /// Reads the path length, extracts the path bytes, converts Windows path separators + /// to Unix format if running on Unix, and returns the path string along with the new end index. + /// + /// # Arguments + /// - `header`: The archive header bytes. + /// - `current_end_index`: The starting index in the header to read from. + /// - `created_on_os_type`: OS type flag indicating which system the archive was created on. + /// + /// # Returns + /// - `Ok((parsed_path, next_index))` on success. + /// - `Err` on parse or UTF-8 decoding failure. + fn get_path_from_header(header: &[u8], current_end_index: usize, created_on_os_type: u8) -> Result<(String, usize)> { + // new start index of header field + let mut s = current_end_index; + // new end index of header field + let mut e = current_end_index + size_of::(); + // path length + let path_len = u16::from_le_bytes(header[s..e].try_into()?); + // path + s = e; e += usize::from(path_len); + let path_bytes = &header[s..e]; + let path_str = str::from_utf8(path_bytes)?; + // convert Windows path to unix path, if on unix (windows can handle unix path) + let entry_path = if cfg!(unix) && created_on_os_type == TYPE_WINDOWS { + Utf8WindowsPath::new(path_str).with_unix_encoding().to_string() + } else { + path_str.to_string() + }; + + Ok((entry_path, e)) + } +} + + /// Handles the reading and archiving of files/directories. /// /// Walks the file system directory tree, processes files in parallel, @@ -152,7 +497,7 @@ impl ArchiveRead { let sparse_segments; // archive header - match Self::build_archive_header(&entry, &hard_link_target, &strip_dir, verbose) { + match ArchiveHeader::build_header(&entry, &hard_link_target, &strip_dir, verbose) { Ok(values) => (archive_header, filepath_and_size, sparse_segments) = values, Err(e) => { eprintln!("Skipped entry on building archive header for {} - Reason: {e}", entry.path().display()); @@ -229,264 +574,97 @@ impl ArchiveRead { } Ok(()) - })); - } - - let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); - - Self { thread_handles, rx_out_receivers, channel_index: None, channel_finished: vec![false; num_workers], buf_out } - } - - /// Checks if a file entry path matches the exclude path (the target archive output file(s)). - /// - /// This prevents the archiver from reading and archiving its own output file(s) - /// (e.g. single `.cce` files or split `.cXX` archive volumes) when they are stored - /// within the directory being archived. - /// - /// # Arguments - /// - `entry_path`: The path of the file entry to check. - /// - `exclude_path`: The target archive output path to exclude. - /// - /// # Returns - /// - `true` if the entry path matches the target archive path and should be excluded. - /// - `false` otherwise. - fn exclude_file(entry_path: &Path, exclude_path: &Path) -> bool { - let Some(exclude_ext) = exclude_path.extension() else { - return false; - }; - if exclude_ext == ENCRYPTED_FILE_EXT { - // single output file .cce - - // exclude_path is an absolute path, entry.path() is a relative path - return exclude_path.ends_with(entry_path); - - } else if exclude_ext == SPLIT_ENC_FILE_EXT { - // split file .c00, .c01, .c02, ... - - // entry.path() is relative, therefore make it absolute for following comparison - let Ok(entry_path) = entry_path.canonicalize() else { - return false; - }; - if entry_path.parent() != exclude_path.parent() { - return false; - } - if entry_path.file_stem() != exclude_path.file_stem() { - return false; - } - if let Some(entry_ext) = entry_path.extension() && let Some(entry_ext) = entry_ext.to_str() { - return entry_ext.starts_with('c') && entry_ext[1..].chars().all(|c| c.is_ascii_digit()); - } - - } else { - panic!("Unknown file extension {}", exclude_ext.display()); - } - - false - } - - /// Gets a file's metadata for detecting hardlinks (Windows version) - /// - /// # Arguments - /// - `filepath` - path to file - /// - /// # Returns - /// `Ok((num_links, file_id))` on success (number of links, file id) - /// `Èrr` if the system call fails. - #[cfg(windows)] - fn file_meta_for_hardlink_on_windows(filepath: &Path) -> std::io::Result<(u64, u128)> { - use std::os::windows::io::AsRawHandle; - use windows_sys::Win32::Storage::FileSystem::{ - GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION}; - use windows_sys::Win32::Foundation::HANDLE; - - let mut num_links = 0; - let mut file_id = 0; - - if let Ok(file) = File::open(filepath) { - let file_handle = file.as_raw_handle(); - unsafe { - let mut info: BY_HANDLE_FILE_INFORMATION = mem::zeroed(); - let result = GetFileInformationByHandle( - file_handle as HANDLE, - &mut info - ); - if result == 0 { - return Err(std::io::Error::last_os_error()); - } else { - num_links = info.nNumberOfLinks as u64; - file_id = ((info.dwVolumeSerialNumber as u128) << 64) - | ((info.nFileIndexHigh as u128) << 32) | (info.nFileIndexLow as u128); - } - } - } - Ok((num_links, file_id)) - } - - /// Appends the file path length and path string to the archive header buffer. - /// - /// # Arguments - /// - `path`: The file system path to encode. - /// - `archive_header`: The mutable buffer to append the encoded path to. - /// - /// # Returns - /// - `Ok(())` on success. - /// - `Err` if the path length exceeds `u16` capacity. - fn add_path_to_header(path: &Path, archive_header: &mut Vec) -> Result<()> { - // path length and path - let path_string = path.to_string_lossy(); - let path_len: u16 = path_string.len().try_into()?; - archive_header.extend(path_len.to_le_bytes()); - archive_header.extend(path_string.as_bytes()); - Ok(()) - } - - /// Builds the archive header bytes for a given file system entry. - /// - /// Creates a metadata block containing file type, path, timestamps, size, - /// sparse segments (holes), and permissions. - /// - /// # Arguments - /// - `entry`: The directory entry to construct the header for. - /// - `hard_link_target`: Optional path pointing to the target if this is a hard link. - /// - `strip_dir`: Path to be stripped from the start of each path in the archive to make paths relative - /// - /// # Returns - /// - `Ok((archive_header, filepath_and_size, sparse_segments))` on success. - /// - `Err` if metadata retrieval or OS-specific operations fail. - #[allow(clippy::type_complexity)] - fn build_archive_header(entry: &DirEntry, hard_link_target: &Option, strip_dir: &PathBuf, verbose: bool) -> Result<(Vec, Option<(PathBuf, u64)>, Vec)> { - // archive header initialized with place holder for header size - let mut archive_header = Vec::with_capacity(1024); - archive_header.extend([0u8; ARCHIVE_HEADER_LENGTH_SIZE]); - - /// Computes and sets the final header size at the beginning of the header buffer. - /// It is called after all other header fields have been added to the header buffer. - /// - /// # Arguments - /// - `archive_header`: The mutable slice representing the archive header. - /// - /// # Returns - /// - `Ok(())` on success. - /// - `Err` if the header length cannot be converted to `u16`. - fn set_header_size(archive_header: &mut [u8]) -> Result<()> { - let header_size: [u8; ARCHIVE_HEADER_LENGTH_SIZE] = - u16::try_from(archive_header.len() - ARCHIVE_HEADER_LENGTH_SIZE)?.to_le_bytes(); - archive_header[0] = header_size[0]; - archive_header[1] = header_size[1]; - Ok(()) - } - - let entry_type = if hard_link_target.is_some() { - TYPE_HARDLINK - } else if entry.file_type().is_file() { - TYPE_FILE - } else if entry.file_type().is_dir() { - TYPE_DIRECTORY - } else if entry.file_type().is_symlink() { - // whether a symlink is a file or a directory is only relevant for windows (when creating them there) - if entry.path().is_dir() { - TYPE_SYMLINK_DIR - } else { - // if target of symlink does not exist (hence it can't be evaluated - // whether it is a file or a directory), type file is used - TYPE_SYMLINK_FILE - } - } else { - return Err("Unsupported file type".into()); - }; - - let os_type = if cfg!(unix) { TYPE_UNIX } else { TYPE_WINDOWS }; - archive_header.push(os_type | entry_type); - - // path length and path (including filename) (converting to relative path) - let stripped_entry_path = entry.path().strip_prefix(strip_dir)?; - Self::add_path_to_header(stripped_entry_path, &mut archive_header)?; - - if verbose { - println!("{}", stripped_entry_path.display()); - } - - if entry_type == TYPE_HARDLINK { - // target path of hard link - let target_path = hard_link_target.as_ref().unwrap().strip_prefix(strip_dir)?; - Self::add_path_to_header(target_path, &mut archive_header)?; - - // header size - set_header_size(&mut archive_header)?; - - return Ok((archive_header, None, vec![])); - } - - // metadata of entry - let meta_entry = entry.metadata()?; + })); + } - // last access time - let time_accessed = meta_entry.accessed()?.duration_since(UNIX_EPOCH)?.as_secs(); - archive_header.extend(time_accessed.to_le_bytes()); - // last modification time - let time_modified = meta_entry.modified()?.duration_since(UNIX_EPOCH)?.as_secs(); - archive_header.extend(time_modified.to_le_bytes()); + let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); - let mut file_size = 0; - let mut sparse_segments = vec![]; - if entry_type == TYPE_FILE { - // file size - file_size = meta_entry.len(); - archive_header.extend(file_size.to_le_bytes()); + Self { thread_handles, rx_out_receivers, channel_index: None, channel_finished: vec![false; num_workers], buf_out } + } - // sparse file - // if the files can be scanned for sparse parts, the holes are saved in the archive header - if let Ok(mut f_in) = File::open(entry.path()) && let Ok(segs) = f_in.scan_chunks() { - sparse_segments = segs; + /// Checks if a file entry path matches the exclude path (the target archive output file(s)). + /// + /// This prevents the archiver from reading and archiving its own output file(s) + /// (e.g. single `.cce` files or split `.cXX` archive volumes) when they are stored + /// within the directory being archived. + /// + /// # Arguments + /// - `entry_path`: The path of the file entry to check. + /// - `exclude_path`: The target archive output path to exclude. + /// + /// # Returns + /// - `true` if the entry path matches the target archive path and should be excluded. + /// - `false` otherwise. + fn exclude_file(entry_path: &Path, exclude_path: &Path) -> bool { + let Some(exclude_ext) = exclude_path.extension() else { + return false; + }; + if exclude_ext == ENCRYPTED_FILE_EXT { + // single output file .cce - // number of holes - let holes_count = sparse_segments.holes().count(); - archive_header.extend(u16::try_from( holes_count )?.to_le_bytes()); + // exclude_path is an absolute path, entry.path() is a relative path + return exclude_path.ends_with(entry_path); - // start and end index of holes, if any - for hole in sparse_segments.holes() { - // start and end are of type u64 - archive_header.extend(hole.start.to_le_bytes()); - archive_header.extend(hole.end.to_le_bytes()); - } - } else { - // scan for sparse holes failed, handle file as non-sparse and add holes_count = 0 - archive_header.extend(0u16.to_le_bytes()); + } else if exclude_ext == SPLIT_ENC_FILE_EXT { + // split file .c00, .c01, .c02, ... + + // entry.path() is relative, therefore make it absolute for following comparison + let Ok(entry_path) = entry_path.canonicalize() else { + return false; + }; + if entry_path.parent() != exclude_path.parent() { + return false; + } + if entry_path.file_stem() != exclude_path.file_stem() { + return false; + } + if let Some(entry_ext) = entry_path.extension() && let Some(entry_ext) = entry_ext.to_str() { + return entry_ext.starts_with('c') && entry_ext[1..].chars().all(|c| c.is_ascii_digit()); } - } else if entry_type == TYPE_SYMLINK_FILE || entry_type == TYPE_SYMLINK_DIR { - // target path of symlink - let target_path = fs::read_link(entry.path())?; - Self::add_path_to_header(&target_path, &mut archive_header)?; + } else { + panic!("Unknown file extension {}", exclude_ext.display()); } - // permissions - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; + false + } - let mut perm: u16 = 0; - if entry_type == TYPE_FILE || entry_type == TYPE_DIRECTORY { - let permission_mode = meta_entry.permissions().mode(); - // use 12 least significant bits - perm = (permission_mode & 0x0FFF) as u16; - } - archive_header.extend(perm.to_le_bytes()); - } - #[cfg(windows)] - { - archive_header.extend(0u16.to_le_bytes()); - } + /// Gets a file's metadata for detecting hardlinks (Windows version) + /// + /// # Arguments + /// - `filepath` - path to file + /// + /// # Returns + /// `Ok((num_links, file_id))` on success (number of links, file id) + /// `Èrr` if the system call fails. + #[cfg(windows)] + fn file_meta_for_hardlink_on_windows(filepath: &Path) -> std::io::Result<(u64, u128)> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION}; + use windows_sys::Win32::Foundation::HANDLE; - // header size - set_header_size(&mut archive_header)?; + let mut num_links = 0; + let mut file_id = 0; - let mut filepath_and_size = None; - if entry_type == TYPE_FILE { - filepath_and_size = Some((entry.clone().into_path(), file_size)); + if let Ok(file) = File::open(filepath) { + let file_handle = file.as_raw_handle(); + unsafe { + let mut info: BY_HANDLE_FILE_INFORMATION = mem::zeroed(); + let result = GetFileInformationByHandle( + file_handle as HANDLE, + &mut info + ); + if result == 0 { + return Err(std::io::Error::last_os_error()); + } else { + num_links = info.nNumberOfLinks as u64; + file_id = ((info.dwVolumeSerialNumber as u128) << 64) + | ((info.nFileIndexHigh as u128) << 32) | (info.nFileIndexLow as u128); + } + } } - - Ok((archive_header, filepath_and_size, sparse_segments)) + Ok((num_links, file_id)) } } @@ -563,7 +741,6 @@ impl ReadChunk for ArchiveRead { /// /// Decodes the incoming archive stream, creating files, directories, symlinks, /// and hard links, restoring their permissions and timestamps. -#[derive(Default)] pub struct ArchiveWrite { /// Active file handle for the entry currently being written. f_out: Option, @@ -591,6 +768,8 @@ pub struct ArchiveWrite { dirpath_out: Option, /// Enable verbose prints. verbose: bool, + /// Enable list mode: Print elements in archive, but do not create them. + list: bool, } impl ArchiveWrite { @@ -601,11 +780,11 @@ impl ArchiveWrite { /// /// # Returns /// - A default `ArchiveWrite` with allocated output buffer. - pub fn new(dirpath_out: Option, verbose: bool) -> Self { + pub fn new(dirpath_out: Option, verbose: bool, list: bool) -> Self { let buf_out = Vec::with_capacity(CHUNK_SIZE * 2); Self { f_out: None, buf_out, header_length: None, file_size: 0, file_times: FileTimes::new(), file_path: PathBuf::new(), dir_times: vec![], pending_hardlinks: vec![], sparse_segments: vec![], - sparse_segments_index: 0, data_segment_size: 0, dirpath_out, verbose } + sparse_segments_index: 0, data_segment_size: 0, dirpath_out, verbose, list } } /// Ensures that the parent directory of the given path exists. @@ -627,41 +806,6 @@ impl ArchiveWrite { Ok(()) } - /// Parses a file path from the archive header slice. - /// - /// Reads the path length, extracts the path bytes, converts Windows path separators - /// to Unix format if running on Unix, and returns the path string along with the new end index. - /// - /// # Arguments - /// - `header`: The archive header bytes. - /// - `current_end_index`: The starting index in the header to read from. - /// - `created_on_os_type`: OS type flag indicating which system the archive was created on. - /// - /// # Returns - /// - `Ok((parsed_path, next_index))` on success. - /// - `Err` on parse or UTF-8 decoding failure. - fn get_path_from_header(header: &[u8], current_end_index: usize, created_on_os_type: u8) -> Result<(String, usize)> { - // new start index of header field - let mut s = current_end_index; - // new end index of header field - let mut e = current_end_index + size_of::(); - - // path length - let path_len = u16::from_le_bytes(header[s..e].try_into()?); - // path - s = e; e += usize::from(path_len); - let path_bytes = &header[s..e]; - let path_str = str::from_utf8(path_bytes)?; - // convert Windows path to unix path, if on unix (windows can handle unix path) - let entry_path = if cfg!(unix) && created_on_os_type == TYPE_WINDOWS { - Utf8WindowsPath::new(path_str).with_unix_encoding().to_string() - } else { - path_str.to_string() - }; - - Ok((entry_path, e)) - } - /// Configures a file as a sparse file on Windows. /// /// # Arguments @@ -698,151 +842,94 @@ impl ArchiveWrite { Ok(()) } - /// Evaluates a parsed archive header to create the corresponding file system entry. - /// - /// Handles directories, files (including sparse configuration), symlinks, and hard links. - /// Sets file size, times, and system-level permissions depending on OS. - /// - /// # Arguments - /// - `header`: The raw header bytes. - /// - /// # Returns - /// - `Ok(())` on success. - /// - `Err` on creation, I/O, or permission errors. fn eval_header(&mut self, header: &[u8]) -> Result<()> { - // type - let file_type = header[0] & 0x0F; - let created_on_os_type = header[0] & 0xF0; - - let mut s; - let mut e = 1; + let mut hdr = ArchiveHeader::parse_new(); + hdr.parse_header(header, &self.dirpath_out)?; - // entry's path - let entry_path_string; - (entry_path_string, e) = Self::get_path_from_header(header, e, created_on_os_type)?; - let mut entry_path = PathBuf::from(&entry_path_string); - // add optional output directory to entry's path - if let Some(dir_out) = &self.dirpath_out { - entry_path = dir_out.join(entry_path); + if self.verbose || self.list { + println!("{}", hdr.entry_path.display()); } - - if self.verbose { - println!("{}", entry_path.display()); + if self.list { + if hdr.file_type == TYPE_FILE { + self.file_size = hdr.file_size; + self.sparse_segments = hdr.sparse_segments; + } + return Ok(()); } - if file_type == TYPE_HARDLINK { - // hard link's target path - let (target_path_string, _) = Self::get_path_from_header(header, e, created_on_os_type)?; - let mut target_path = PathBuf::from(target_path_string); - if let Some(dir_out) = &self.dirpath_out { - target_path = dir_out.join(target_path); - } - self.pending_hardlinks.push((target_path, entry_path)); + if hdr.file_type == TYPE_HARDLINK { + self.pending_hardlinks.push((hdr.link_target_path, hdr.entry_path)); return Ok(()); } - // access time - s = e; e += size_of::(); - let time_accessed_seconds = u64::from_le_bytes( header[s..e].try_into()? ); - let time_accessed = UNIX_EPOCH + Duration::from_secs(time_accessed_seconds); - // modification time - s = e; e += size_of::(); - let time_modified_seconds = u64::from_le_bytes( header[s..e].try_into()? ); - let time_modified = UNIX_EPOCH + Duration::from_secs(time_modified_seconds); + // access/modification time self.file_times = FileTimes::new() - .set_accessed(time_accessed) - .set_modified(time_modified); + .set_accessed(hdr.time_accessed) + .set_modified(hdr.time_modified); // create type - if file_type == TYPE_DIRECTORY { + if hdr.file_type == TYPE_DIRECTORY { // create directory - // it could be an empty directory therefore it would not be created by other entries - fs::create_dir_all(&entry_path)?; - + // it could be an empty directory therefore it would not be created for other entries + if !hdr.entry_path.exists() { + fs::create_dir_all(&hdr.entry_path)?; + } // save timestamps for restoring them at the end - self.dir_times.push((entry_path.clone(), time_accessed, time_modified)); + self.dir_times.push((hdr.entry_path.clone(), hdr.time_accessed, hdr.time_modified)); - } else if file_type == TYPE_FILE { + } else if hdr.file_type == TYPE_FILE { // create directory (of file), if it doesn't exist - Self::create_parent_directory(&entry_path)?; + Self::create_parent_directory(&hdr.entry_path)?; // create file - self.f_out = Some(File::create(&entry_path)?); + self.f_out = Some(File::create(&hdr.entry_path)?); // for printing errors - self.file_path = entry_path.clone(); + self.file_path = hdr.entry_path.clone(); // file size - s = e; e += size_of::(); - self.file_size = u64::from_le_bytes( header[s..e].try_into()? ); - - // holes of a sparse file - s = e; e += size_of::(); - let holes_count = u16::from_le_bytes( header[s..e].try_into()? ); - - if holes_count > 0 { - // restore data- and hole-segments of sparse file - let mut data_start = 0; - for _ in 0..holes_count { - s = e; e += size_of::(); - let hole_start = u64::from_le_bytes( header[s..e].try_into()? ); - s = e; e += size_of::(); - let hole_end = u64::from_le_bytes( header[s..e].try_into()? ); + self.file_size = hdr.file_size; - if hole_start != 0 { - // if first segment is not a hole, add a data segment - self.sparse_segments.push( Segment { segment_type: SegmentType::Data, range: data_start..hole_start} ); - } - self.sparse_segments.push( Segment { segment_type: SegmentType::Hole, range: hole_start..hole_end } ); - data_start = hole_end; - } - // if last segment is not a hole, add a data segment - if data_start != self.file_size { - self.sparse_segments.push( Segment { segment_type: SegmentType::Data, range: data_start..self.file_size} ); - } + if hdr.sparse_holes_count > 0 { + self.sparse_segments = hdr.sparse_segments; // Windows requires to set sparse flag for a sparse file #[cfg(windows)] if Self::set_sparse_file_on_windows(self.f_out.as_ref().unwrap()).is_err() { - eprintln!("Could not set sparse option for file {}", entry_path.display()); + eprintln!("Could not set sparse option for file {}", hdr.entry_path.display()); } } - } else if file_type == TYPE_SYMLINK_FILE || file_type == TYPE_SYMLINK_DIR { + } else if hdr.file_type == TYPE_SYMLINK_FILE || hdr.file_type == TYPE_SYMLINK_DIR { // create directory (of symlink), if it doesn't exist - Self::create_parent_directory(&entry_path)?; - - // symlink's target path - let target_path; - (target_path, e) = Self::get_path_from_header(header, e, created_on_os_type)?; + Self::create_parent_directory(&hdr.entry_path)?; // create symlink #[cfg(unix)] { - std::os::unix::fs::symlink(&target_path, &entry_path)?; + std::os::unix::fs::symlink(&hdr.link_target_path, &hdr.entry_path)?; } #[cfg(windows)] { - if file_type == TYPE_SYMLINK_FILE { - std::os::windows::fs::symlink_file(&target_path, &entry_path)?; + if hdr.file_type == TYPE_SYMLINK_FILE { + std::os::windows::fs::symlink_file(&hdr.link_target_path, &hdr.entry_path)?; } - if file_type == TYPE_SYMLINK_DIR { - std::os::windows::fs::symlink_dir(&target_path, &entry_path)?; + if hdr.file_type == TYPE_SYMLINK_DIR { + std::os::windows::fs::symlink_dir(&hdr.link_target_path, &hdr.entry_path)?; } } // set timestamps of symlink - // replace with fs::set_times_nofollow() when stable rust version supports it match filetime::set_symlink_file_times( - &entry_path, - filetime::FileTime::from_system_time(time_accessed), - filetime::FileTime::from_system_time(time_modified) + &hdr.entry_path, + filetime::FileTime::from_system_time(hdr.time_accessed), + filetime::FileTime::from_system_time(hdr.time_modified) ) { Ok(()) => {}, - Err(e) => eprintln!("Could not set original timestamps for symlink {}: {e}", entry_path.display()), + Err(e) => eprintln!("Could not set original timestamps for symlink {}: {e}", hdr.entry_path.display()), } } else { - return Err(format!("Archive contains unknown file type: {file_type}").into()); + return Err(format!("Archive contains unknown file type: {}", hdr.file_type).into()); } // permissions @@ -851,27 +938,18 @@ impl ArchiveWrite { { use std::os::unix::fs::PermissionsExt; - if created_on_os_type == TYPE_UNIX && (file_type == TYPE_DIRECTORY || file_type == TYPE_FILE) { - s = e; e += size_of::(); - let perm = u16::from_le_bytes(header[s..e].try_into()?); - - let fd = if file_type == TYPE_DIRECTORY { - &File::open(&entry_path)? + if hdr.created_on_os_type == TYPE_UNIX && (hdr.file_type == TYPE_DIRECTORY || hdr.file_type == TYPE_FILE) { + let fd = if hdr.file_type == TYPE_DIRECTORY { + &File::open(&hdr.entry_path)? } else { self.f_out.as_ref().unwrap() }; let mut permissions = fd.metadata()?.permissions(); let mode_masked = permissions.mode() & 0xFFFF_F000; - permissions.set_mode(mode_masked | u32::from(perm & 0x0FFF)); + permissions.set_mode(mode_masked | u32::from(hdr.permissions & 0x0FFF)); fd.set_permissions(permissions)?; } } - #[cfg(windows)] - { - // keep compiler quiet - s = e; e += size_of::(); - let _perm = u16::from_le_bytes(header[s..e].try_into()?); - } Ok(()) } @@ -893,12 +971,14 @@ impl WriteFiles for ArchiveWrite { self.buf_out.extend(buf_in); loop { - if let Some(mut f_out) = self.f_out.as_ref() { + if self.file_size > 0 { // Local helper closure to write buffered data to file let mut write_data = |f_out_size: u64| -> Result { let data_size = self.buf_out.len().min(f_out_size.try_into()?); let buf_out_slice: Vec = self.buf_out.drain(..data_size).collect(); - f_out.write_all(&buf_out_slice)?; + if let Some(mut f_out) = self.f_out.as_ref() { + f_out.write_all(&buf_out_slice)?; + } Ok(data_size as u64) }; @@ -906,6 +986,15 @@ impl WriteFiles for ArchiveWrite { // write non-sparse file let write_size = write_data(self.file_size)?; self.file_size -= write_size; + + // set file times after all data has been written + if self.file_size == 0 { + if let Some(f_out) = self.f_out.as_ref() && f_out.set_times(self.file_times).is_err() { + eprintln!("Could not set original timestamps for file {}", self.file_path.display()); + } + self.f_out = None; + } + if self.buf_out.is_empty() { break; } @@ -927,8 +1016,10 @@ impl WriteFiles for ArchiveWrite { } } SegmentType::Hole => { - f_out.seek_relative((segment.len() - 1).try_into()?)?; - f_out.write_all(&[0])?; + if let Some(mut f_out) = self.f_out.as_ref() { + f_out.seek_relative((segment.len() - 1).try_into()?)?; + f_out.write_all(&[0])?; + } self.file_size -= segment.len(); self.sparse_segments_index += 1; } @@ -937,15 +1028,22 @@ impl WriteFiles for ArchiveWrite { if self.file_size == 0 { // holes must not be set before the whole file was written; // on failure, the file should become non-sparse, do not break execution, just continue - for hole in self.sparse_segments.holes() { - if f_out.drill_hole(hole.start, hole.end).is_err() { - eprintln!("Could not set sparse region for file {}", self.file_path.display()); + if let Some(f_out) = self.f_out.as_ref() { + for hole in self.sparse_segments.holes() { + if f_out.drill_hole(hole.start, hole.end).is_err() { + eprintln!("Could not set sparse region for file {}", self.file_path.display()); + } } } - self.data_segment_size = 0; self.sparse_segments_index = 0; self.sparse_segments.clear(); + + // set file times after all data has been written + if let Some(f_out) = self.f_out.as_ref() && f_out.set_times(self.file_times).is_err() { + eprintln!("Could not set original timestamps for file {}", self.file_path.display()); + } + self.f_out = None; } // no data left and no hole as next segment @@ -955,15 +1053,6 @@ impl WriteFiles for ArchiveWrite { break; } } - - if self.file_size == 0 { - // set file times after all data has been written - if f_out.set_times(self.file_times).is_err() { - eprintln!("Could not set original timestamps for file {}", self.file_path.display()); - } - self.f_out = None; - } - } else if let Some(header_length) = self.header_length { if self.buf_out.len() >= header_length { // get header @@ -1058,7 +1147,7 @@ mod tests { #[test] fn test_add_path_to_header() { let mut buf = Vec::new(); - ArchiveRead::add_path_to_header(Path::new("hello/world.txt"), &mut buf).unwrap(); + ArchiveHeader::add_path_to_header(Path::new("hello/world.txt"), &mut buf).unwrap(); // Path length should be 15 (2 bytes, little-endian) let expected_len = 15u16.to_le_bytes(); @@ -1090,7 +1179,7 @@ mod tests { header.extend_from_slice(&path_len.to_le_bytes()); header.extend_from_slice(path_str.as_bytes()); - let (decoded, end_idx) = ArchiveWrite::get_path_from_header(&header, 0, TYPE_UNIX).unwrap(); + let (decoded, end_idx) = ArchiveHeader::get_path_from_header(&header, 0, TYPE_UNIX).unwrap(); assert_eq!(decoded, "foo/bar/baz.txt"); assert_eq!(end_idx, header.len()); @@ -1101,7 +1190,7 @@ mod tests { header_win.extend_from_slice(&path_len_win.to_le_bytes()); header_win.extend_from_slice(path_str_win.as_bytes()); - let (decoded_win, end_idx_win) = ArchiveWrite::get_path_from_header(&header_win, 0, TYPE_WINDOWS).unwrap(); + let (decoded_win, end_idx_win) = ArchiveHeader::get_path_from_header(&header_win, 0, TYPE_WINDOWS).unwrap(); if cfg!(unix) { assert_eq!(decoded_win, "foo/bar/baz.txt"); } else { @@ -1181,40 +1270,35 @@ mod tests { } // Set modified/accessed time of file1 - filetime::set_file_times( + filetime::set_file_mtime( &file1_path, - filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(2000)), - filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(1000)) + filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(2000)) ).unwrap(); - - // Set modified/accessed time of subdirectory - filetime::set_file_times( - &sub_dir, - filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(2222)), - filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(1111)) - ).unwrap(); - + // Set modified/accessed time of symlink - filetime::set_symlink_file_times( + filetime::set_file_mtime( &symlink_path, - filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(4444)), - filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(3333)) + filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(4444)) + ).unwrap(); + + // Set modified/accessed time of subdirectory + filetime::set_file_mtime( + &sub_dir, + filetime::FileTime::from_system_time(UNIX_EPOCH + Duration::from_secs(2222)) ).unwrap(); + // Get modified/accessed time of file1 let meta_orig1 = fs::metadata(&file1_path).unwrap(); let modified_orig1 = meta_orig1.modified().unwrap(); - let accessed_orig1 = meta_orig1.accessed().unwrap(); // Get modified/accessed time of subdirectory let meta_orig2 = fs::metadata(&sub_dir).unwrap(); let modified_orig2 = meta_orig2.modified().unwrap(); - let accessed_orig2 = meta_orig2.accessed().unwrap(); // Get modified/accessed time of symlink let meta_orig3 = fs::symlink_metadata(&symlink_path).unwrap(); let modified_orig3 = meta_orig3.modified().unwrap(); - let accessed_orig3 = meta_orig3.accessed().unwrap(); // Perform archiving using ArchiveRead let mut reader = ArchiveRead::new(&src_dir.path, Path::new(""), Path::new(""), false); @@ -1238,7 +1322,7 @@ mod tests { assert!(!src_dir.path.exists()); // Perform extraction using ArchiveWrite by feeding it in small chunks - let mut writer = ArchiveWrite::new(None, false); + let mut writer = ArchiveWrite::new(None, false, false); for chunk in archive_bytes.chunks(100) { writer.write_files(chunk).unwrap(); } @@ -1259,11 +1343,6 @@ mod tests { modified_orig1.duration_since(UNIX_EPOCH).unwrap().as_secs(), modified_restored1.duration_since(UNIX_EPOCH).unwrap().as_secs() ); - let accessed_restored1 = meta_restored1.accessed().unwrap(); - assert_eq!( - accessed_orig1.duration_since(UNIX_EPOCH).unwrap().as_secs(), - accessed_restored1.duration_since(UNIX_EPOCH).unwrap().as_secs() - ); // Verify modified/accessed time of subdir is restored (seconds precision) let meta_restored2 = fs::metadata(&sub_dir).unwrap(); @@ -1272,11 +1351,6 @@ mod tests { modified_orig2.duration_since(UNIX_EPOCH).unwrap().as_secs(), modified_restored2.duration_since(UNIX_EPOCH).unwrap().as_secs() ); - let accessed_restored2 = meta_restored2.accessed().unwrap(); - assert_eq!( - accessed_orig2.duration_since(UNIX_EPOCH).unwrap().as_secs(), - accessed_restored2.duration_since(UNIX_EPOCH).unwrap().as_secs() - ); #[cfg(unix)] { @@ -1304,11 +1378,6 @@ mod tests { modified_orig3.duration_since(UNIX_EPOCH).unwrap().as_secs(), modified_restored3.duration_since(UNIX_EPOCH).unwrap().as_secs() ); - let accessed_restored3 = symlink_metadata.accessed().unwrap(); - assert_eq!( - accessed_orig3.duration_since(UNIX_EPOCH).unwrap().as_secs(), - accessed_restored3.duration_since(UNIX_EPOCH).unwrap().as_secs() - ); // Verify hardlink #[cfg(unix)] @@ -1429,7 +1498,7 @@ mod tests { // Perform extraction using ArchiveWrite - let mut writer = ArchiveWrite::new(None, false); + let mut writer = ArchiveWrite::new(None, false, false); for chunk in archive_bytes.chunks(CHUNK_SIZE) { writer.write_files(chunk).unwrap(); } @@ -1548,7 +1617,7 @@ mod tests { fs::remove_dir_all(&src_dir.path).unwrap(); // Extract - let mut writer = ArchiveWrite::new(None, false); + let mut writer = ArchiveWrite::new(None, false, false); writer.write_files(&archive_bytes).unwrap(); writer.write_others().unwrap(); diff --git a/src/decryption.rs b/src/decryption.rs index 6b316ce..8fd31a5 100644 --- a/src/decryption.rs +++ b/src/decryption.rs @@ -333,10 +333,11 @@ impl Decryption { let enc_head = &header[(3 * SALT_SIZE)..HEADER_SIZE]; let buf_aes = Self::aes_decrypt_buffer(key_aes, enc_head)?; - let buf_cha = Self::cha_decrypt_buffer(key_cha, &buf_aes, 0, false)?; - - let file_format_version = buf_cha[0]; - let file_format = buf_cha[1]; + let buf_cha = Self::cha_decrypt_buffer(key_cha, &buf_aes, u32::MAX, false)?; + + // evaluate data, ignore random bytes + let file_format_version = buf_cha[1]; + let file_format = buf_cha[3]; let compress = (file_format & 0x01) != 0; let archive = (file_format & 0x02) != 0; @@ -357,7 +358,7 @@ impl Decryption { /// # Returns /// - `Ok(())` on successful decryption /// - `Err` if file operations, password handling, or decryption fails - pub fn decrypt(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, keyfilepath: Option<&PathBuf>, verbose: bool) -> Result<()> { + pub fn decrypt(filepath_in: &PathBuf, dirpath_out: Option<&PathBuf>, keyfilepath: Option<&PathBuf>, verbose: bool, list_archive: bool) -> Result<()> { if filepath_in.is_dir() { return Err("Cannot decrypt a directory".into()); } @@ -402,7 +403,8 @@ impl Decryption { let mut output_dir = &env::current_dir()?; // output directory path is used if let Some(dir_out) = dirpath_out { - if !dir_out.exists() { + // create user-specified output directory, if it is not an archive or archive is not just listed (and directory is missing) + if ((archive && !list_archive) || !archive) && !dir_out.exists() { fs::create_dir_all(dir_out)?; } if archive { @@ -426,10 +428,10 @@ impl Decryption { } } + // set write parameters and create output file let write_output: Box = if archive { - Box::new( ArchiveWrite::new(dirpath_out.cloned(), verbose) ) + Box::new( ArchiveWrite::new(dirpath_out.cloned(), verbose, list_archive) ) } else { - // set write parameters and create output file Box::new( WriteOutput::new(filepath_out, vec![])? ) }; diff --git a/src/encryption.rs b/src/encryption.rs index 6261ed8..831b6d5 100644 --- a/src/encryption.rs +++ b/src/encryption.rs @@ -409,7 +409,7 @@ impl Encryption { /// /// # Returns /// - `Ok(header)` contains header on success - /// - `Err` if encryption fails + /// - `Err` if encryption or random data generation fails fn create_header(salt_pw: &[u8], salt_cha: &[u8], salt_aes: &[u8], compress: bool, archive: bool, key_cha: &SecretSlice, key_aes: &SecretSlice) -> Result> { let mut header = Vec::with_capacity(HEADER_SIZE); @@ -419,11 +419,20 @@ impl Encryption { header.extend(salt_aes); // encrypted header part + + // random filler data + let mut random_data = [0u8; 2]; + let mut rng = ChaCha20Rng::try_from_rng(&mut SysRng)?; + rng.fill_bytes(&mut random_data); + + // add some randomness to the constant data let buf_in = [ + random_data[0], FILE_FORMAT_VERSION, - u8::from(compress) | (u8::from(archive) << 1) + random_data[1], + u8::from(compress) | (u8::from(archive) << 1), ]; - let buf_cha = Self::cha_encrypt_buffer(key_cha, &buf_in, 0, false)?; + let buf_cha = Self::cha_encrypt_buffer(key_cha, &buf_in, u32::MAX, false)?; let buf_aes = Self::aes_encrypt_buffer(key_aes, &buf_cha)?; header.extend(buf_aes); @@ -758,7 +767,7 @@ mod tests { } #[test] - fn test_crypt() { + fn test_crypt_general() { // create file with random data for encryption let filepath_in = path::absolute(PathBuf::from("test_cc.bin")).unwrap(); let mut filepath_out = filepath_in.clone(); @@ -774,7 +783,7 @@ mod tests { assert!(filepath_out.exists()); // encrypted file must be different than original data assert_ne!(data, fs::read(&filepath_out).unwrap()); - Decryption::decrypt(&filepath_out, None, None, false).unwrap(); + Decryption::decrypt(&filepath_out, None, None, false, false).unwrap(); // read and compare decrypted file against backup let decrypt_data = fs::read(&filepath_in).unwrap(); @@ -783,24 +792,28 @@ mod tests { // decrypt with keyfile should fail let filepath_kf = PathBuf::from("test_another_key.bin"); fs::write(&filepath_kf, vec![0; 1024]).unwrap(); - assert!(Decryption::decrypt(&filepath_out, None, Some(&filepath_kf), false).is_err()); + assert!(Decryption::decrypt(&filepath_out, None, Some(&filepath_kf), false, false).is_err()); // with compression fs::write(&filepath_in, &data).unwrap(); Encryption::encrypt(&filepath_in, None, None, true, vec![], false).unwrap(); - Decryption::decrypt(&filepath_out, None, None, false).unwrap(); + Decryption::decrypt(&filepath_out, None, None, false, false).unwrap(); let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); - // with output directory + // encrypt with output directory let out_enc_dir = path::absolute(PathBuf::from("test_cc_enc_dir")).unwrap(); Encryption::encrypt(&filepath_in, Some(&out_enc_dir), None, false, vec![], false).unwrap(); assert!(&out_enc_dir.join(&filepath_out).exists()); + // decrypt with output directory let out_dec_dir = path::absolute(PathBuf::from("test_cc_dec_dir")).unwrap(); - Decryption::decrypt(&out_enc_dir.join(&filepath_out), Some(&out_dec_dir), None, false).unwrap(); + Decryption::decrypt(&out_enc_dir.join(&filepath_out), Some(&out_dec_dir), None, false, false).unwrap(); assert!(&out_dec_dir.join(&filepath_in).exists()); let decrypt_data = fs::read(out_dec_dir.join(&filepath_in)).unwrap(); assert_eq!(data, decrypt_data[..]); + // decrypt with output directory and list-archive mode - shouldn't have any effect + Decryption::decrypt(&out_enc_dir.join(&filepath_out), Some(&out_dec_dir), None, false, true).unwrap(); + assert!(&out_dec_dir.join(&filepath_in).exists()); // cleanup let _ = fs::remove_file(&filepath_in); @@ -828,29 +841,29 @@ mod tests { // use keyfile, encrypt, decrypt Encryption::encrypt(&filepath_in, None, Some(&filepath_kf), false, vec![], false).unwrap(); - Decryption::decrypt(&filepath_out, None, Some(&filepath_kf), false).unwrap(); + Decryption::decrypt(&filepath_out, None, Some(&filepath_kf), false, false).unwrap(); // read and compare decrypted file against original data let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); // decrypt without key file - assert!(Decryption::decrypt(&filepath_out, None, None, false).is_err()); + assert!(Decryption::decrypt(&filepath_out, None, None, false, false).is_err()); // key file does not exist assert!(Encryption::encrypt(&filepath_in, None, Some(&PathBuf::from("test_miss")), false, vec![], false).is_err()); - assert!(Decryption::decrypt(&filepath_out, None, Some(&PathBuf::from("test_miss")), false).is_err()); + assert!(Decryption::decrypt(&filepath_out, None, Some(&PathBuf::from("test_miss")), false, false).is_err()); // input file does not exist assert!(Encryption::encrypt(&path::absolute(PathBuf::from("test_miss")).unwrap(), None, None, false, vec![], false).is_err()); - assert!(Decryption::decrypt(&PathBuf::from("test_miss.cce"), None, None, false).is_err()); + assert!(Decryption::decrypt(&PathBuf::from("test_miss.cce"), None, None, false, false).is_err()); assert!(!fs::exists("test_miss").unwrap()); assert!(!fs::exists("test_miss.cce").unwrap()); // with compression fs::write(&filepath_in, &data).unwrap(); Encryption::encrypt(&filepath_in, None, Some(&filepath_kf), true, vec![], false).unwrap(); - Decryption::decrypt(&filepath_out, None, Some(&filepath_kf), false).unwrap(); + Decryption::decrypt(&filepath_out, None, Some(&filepath_kf), false, false).unwrap(); let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -880,7 +893,7 @@ mod tests { data_concat.extend(fs::read("test_cc_split.bin.c02").unwrap()); fs::write(&filepath_out, &data_concat).unwrap(); - Decryption::decrypt(&filepath_out, None, None, false).unwrap(); + Decryption::decrypt(&filepath_out, None, None, false, false).unwrap(); // read and compare decrypted file against original data let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -888,14 +901,14 @@ mod tests { // concatenate files with decrypt let _ = fs::remove_file(&filepath_in); let _ = fs::remove_file(&filepath_out); - Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None, false).unwrap(); + Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None, false, false).unwrap(); // read and compare decrypted file against original data let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); // with compression Encryption::encrypt(&filepath_in, None, None, true, vec![11, 12, 1024*100], false).unwrap(); - Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None, false).unwrap(); + Decryption::decrypt(&PathBuf::from("test_cc_split.bin.c00"), None, None, false, false).unwrap(); let decrypt_data = fs::read(&filepath_in).unwrap(); assert_eq!(data, decrypt_data[..]); @@ -911,8 +924,7 @@ mod tests { #[test] fn test_crypt_archive() { // Create directory with files that should be archived - let dir_name = "test_cc_archive"; - let dir_path = path::absolute(PathBuf::from(dir_name)).unwrap(); + let dir_path = PathBuf::from("test_cc_archive"); let _ = fs::remove_dir_all(&dir_path); fs::create_dir_all(&dir_path).unwrap(); @@ -934,11 +946,11 @@ mod tests { let symlink1 = sub_dir.join("symlink1.bin"); #[cfg(unix)] { - std::os::unix::fs::symlink(&file1, &symlink1).unwrap(); + std::os::unix::fs::symlink(PathBuf::from("..").join("file1.bin"), &symlink1).unwrap(); } #[cfg(windows)] { - std::os::windows::fs::symlink_file(&file1, &symlink1).unwrap(); + std::os::windows::fs::symlink_file(PathBuf::from("..").join("file1.bin"), &symlink1).unwrap(); } // create hardlink @@ -946,15 +958,15 @@ mod tests { fs::hard_link(&file2, &hardlink2).unwrap(); // Build archive of directory and encrypt it - Encryption::encrypt(&dir_path, None, None, false, vec![], false).unwrap(); + Encryption::encrypt(&dir_path.canonicalize().unwrap(), None, None, false, vec![], false).unwrap(); // Delete the original files before extracting to verify recreation fs::remove_dir_all(&dir_path).unwrap(); assert!(!dir_path.exists()); // Decrypt and rebuild archived directory - let arch_path = dir_path.with_extension(ENCRYPTED_FILE_EXT); - Decryption::decrypt(&arch_path, None, None, false).unwrap(); + let archive_path = dir_path.with_extension(ENCRYPTED_FILE_EXT); + Decryption::decrypt(&archive_path, None, None, false, false).unwrap(); // Verify structure is fully recreated assert!(dir_path.exists()); @@ -970,37 +982,53 @@ mod tests { assert_eq!(fs::read(&hardlink2).unwrap(), data2); + // With output directory and list-archive mode - no output should be created + let out_dec_dir1 = path::absolute(PathBuf::from("test_cc_dec_dir1")).unwrap(); + let _ = fs::remove_dir_all(&out_dec_dir1); + Decryption::decrypt(&archive_path, Some(&out_dec_dir1), None, false, true).unwrap(); + assert!(!out_dec_dir1.exists()); + assert!(!out_dec_dir1.join(&file1).exists()); + + // With output directory and without list-archive mode - output should be created + Decryption::decrypt(&archive_path, Some(&out_dec_dir1), None, false, false).unwrap(); + assert!(out_dec_dir1.exists()); + assert!(out_dec_dir1.join(&file1).exists()); + assert_eq!(fs::read(out_dec_dir1.join(&file1)).unwrap(), data1); + // With output directory, with split, check exclusion of output archive files let out_enc_dir = &dir_path; - Encryption::encrypt(&dir_path, Some(out_enc_dir), None, false, vec![1000,10000], false).unwrap(); - assert!(out_enc_dir.join(dir_name).with_extension(SPLIT_ENC_FILE_EXT).exists()); - - let out_dec_dir = path::absolute(PathBuf::from("test_cc_dec_dir")).unwrap(); - let _ = fs::remove_dir_all(&out_dec_dir); - Decryption::decrypt(&out_enc_dir.join(dir_name).with_extension(SPLIT_ENC_FILE_EXT), Some(&out_dec_dir), None, false).unwrap(); + Encryption::encrypt(&dir_path.canonicalize().unwrap(), Some(&out_enc_dir.canonicalize().unwrap()), None, false, vec![1000,10000], false).unwrap(); + + let archive_path2 = out_enc_dir.join(&dir_path).with_extension(SPLIT_ENC_FILE_EXT); + assert!(archive_path2.exists()); + let out_dec_dir2 = path::absolute(PathBuf::from("test_cc_dec_dir2")).unwrap(); + let _ = fs::remove_dir_all(&out_dec_dir2); + Decryption::decrypt(&archive_path2, Some(&out_dec_dir2), None, false, false).unwrap(); // Verify structure is fully recreated - assert!(out_dec_dir.join(dir_name).exists()); - assert!(out_dec_dir.join(&file1).exists()); - assert!(out_dec_dir.join(&file2).exists()); - assert!(out_dec_dir.join(&symlink1).exists()); - assert!(out_dec_dir.join(&hardlink2).exists()); + assert!(out_dec_dir2.join(&dir_path).exists()); + assert!(out_dec_dir2.join(&file1).exists()); + assert!(out_dec_dir2.join(&file2).exists()); + assert!(out_dec_dir2.join(&symlink1).exists()); + assert!(out_dec_dir2.join(&hardlink2).exists()); // Verify contents - assert_eq!(fs::read(out_dec_dir.join(&file1)).unwrap(), data1); - assert_eq!(fs::read(out_dec_dir.join(&file2)).unwrap(), data2); - assert_eq!(fs::read(out_dec_dir.join(&symlink1)).unwrap(), data1); - assert_eq!(fs::read(out_dec_dir.join(&hardlink2)).unwrap(), data2); + assert_eq!(fs::read(out_dec_dir2.join(&file1)).unwrap(), data1); + assert_eq!(fs::read(out_dec_dir2.join(&file2)).unwrap(), data2); + assert_eq!(fs::read(out_dec_dir2.join(&symlink1)).unwrap(), data1); + assert_eq!(fs::read(out_dec_dir2.join(&hardlink2)).unwrap(), data2); // Verify exclude of archive files - assert!(!out_dec_dir.join(dir_name).with_added_extension(SPLIT_ENC_FILE_EXT).exists()); - assert!(!out_dec_dir.join(dir_name).with_added_extension("c01").exists()); - assert!(!out_dec_dir.join(dir_name).with_added_extension("c02").exists()); + assert!(!out_dec_dir2.join(&dir_path).with_added_extension(SPLIT_ENC_FILE_EXT).exists()); + assert!(!out_dec_dir2.join(&dir_path).with_added_extension("c01").exists()); + assert!(!out_dec_dir2.join(&dir_path).with_added_extension("c02").exists()); - let _ = fs::remove_file(&arch_path); + let _ = fs::remove_file(&archive_path); + let _ = fs::remove_file(&archive_path2); let _ = fs::remove_dir_all(&dir_path); let _ = fs::remove_dir_all(out_enc_dir); - let _ = fs::remove_dir_all(&out_dec_dir); + let _ = fs::remove_dir_all(&out_dec_dir1); + let _ = fs::remove_dir_all(&out_dec_dir2); } #[test] @@ -1101,6 +1129,6 @@ mod tests { filepath_out.add_extension(ENCRYPTED_FILE_EXT); Encryption::encrypt(&filepath_in, None, None, false, vec![], false).unwrap(); - Decryption::decrypt(&filepath_out, None, None, false).unwrap(); + Decryption::decrypt(&filepath_out, None, None, false, false).unwrap(); } } \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index e7c4ad9..6d21f8a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,6 +23,6 @@ pub const AES_NONCE_SIZE: usize = ::NonceSize::USI pub const CHA_NONCE_SIZE: usize = ::NonceSize::USIZE; // 24 bytes pub const AES_TAG_SIZE: usize = ::TagSize::USIZE; // 16 bytes pub const CHA_TAG_SIZE: usize = ::TagSize::USIZE; // 16 bytes -pub const HEADER_SIZE: usize = 3 * SALT_SIZE + 2 + CHA_NONCE_SIZE + CHA_TAG_SIZE + AES_NONCE_SIZE + AES_TAG_SIZE; +pub const HEADER_SIZE: usize = 3 * SALT_SIZE + 4 + CHA_NONCE_SIZE + CHA_TAG_SIZE + AES_NONCE_SIZE + AES_TAG_SIZE; pub type Result = std::result::Result>; diff --git a/src/main.rs b/src/main.rs index 9387f73..8dc47cc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,9 +10,9 @@ use cryptcrypt::Result; #[derive(Parser)] #[command(version, about, verbatim_doc_comment, long_about = None)] /// Application for encryption and decryption of file or directory. -/// If no option is given, input is encrypted. A directory as input causes the build of an encrypted archive. +/// If no option is given, input is encrypted. Providing a directory creates an encrypted archive. /// With option -s the encrypted output is split into files with extensions .c00, .c01, .c02, ... -/// If a file ending on .c00 is decrypted, the whole split series will be read. +/// If a file ending in .c00 is decrypted, the whole split series will be read. struct Args { /// Output directory, it is created if it does not exist #[arg(short, long)] @@ -35,6 +35,10 @@ struct Args { value_parser = |s: &str| { let cfg = Config::new().with_binary(); cfg.parse_size(s) })] split: Vec, + #[arg(short, long, default_value_t = false)] + /// List elements of an archive file, do not create its elements + list_archive: bool, + /// Show details about operations #[arg(short, long, default_value_t = false)] verbose: bool, @@ -73,7 +77,7 @@ fn run() -> Result<()> { let output_dir = args.out_dir.map(path::absolute).transpose()?; if args.decrypt { - Decryption::decrypt(&filepath, output_dir.as_ref(), keyfilepath.as_ref(), args.verbose)?; + Decryption::decrypt(&filepath, output_dir.as_ref(), keyfilepath.as_ref(), args.verbose, args.list_archive)?; } else { Encryption::encrypt(&filepath, output_dir.as_ref(), keyfilepath.as_ref(), args.compress, args.split, args.verbose)?; }