diff --git a/.device-mapper-persistent-data.metadata b/.device-mapper-persistent-data.metadata new file mode 100644 index 0000000..c6c4b43 --- /dev/null +++ b/.device-mapper-persistent-data.metadata @@ -0,0 +1,2 @@ +d336474a4772430fa2b8a5f76a6c2f2a78caa698 SOURCES/dmpd109-vendor.tar.gz +8862f41bf5fad011879b6d819ab5e5b11afcda7e SOURCES/v1.0.9.tar.gz diff --git a/.gitignore b/.gitignore index e69de29..f94aaaa 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,2 @@ +SOURCES/dmpd109-vendor.tar.gz +SOURCES/v1.0.9.tar.gz diff --git a/SOURCES/0001-Tweak-cargo.toml-to-work-with-vendor-directory.patch b/SOURCES/0001-Tweak-cargo.toml-to-work-with-vendor-directory.patch index 2e51e47..1bdbce3 100644 --- a/SOURCES/0001-Tweak-cargo.toml-to-work-with-vendor-directory.patch +++ b/SOURCES/0001-Tweak-cargo.toml-to-work-with-vendor-directory.patch @@ -1,4 +1,4 @@ -From 732ff5861a1525944a927439d1c075ac269788ce Mon Sep 17 00:00:00 2001 +From 0d5347bd771e960294cd0c2f083d96448613ab9c Mon Sep 17 00:00:00 2001 From: Marian Csontos Date: Thu, 27 Jul 2023 11:37:01 +0200 Subject: [PATCH] Tweak cargo.toml to work with vendor directory @@ -7,24 +7,16 @@ Mock works offline, cargo would try to download the files from github. So cargo vendor has to be run first, and then change the Cargo.toml to make mock happy. --- - Cargo.toml | 4 +++- - 1 file changed, 3 insertions(+), 1 deletion(-) + Cargo.toml | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml -index c2b496ac..562c40d3 100644 +index 500345a4..d4aa38a6 100644 --- a/Cargo.toml +++ b/Cargo.toml -@@ -11,6 +11,7 @@ anyhow = "1.0" - base64 = "0.21" - byteorder = "1.4" - clap = { version = "4.3", default-features = false, features = ["std", "help", "usage", "error-context", "suggestions"] } -+#crc32c = { git = "https://github.com/zowens/crc32c", branch = "master" } - crc32c = "0.6" - data-encoding = "2.4" - exitcode = "1.1.2" -@@ -27,7 +28,8 @@ quick-xml = "0.29" +@@ -27,7 +27,8 @@ quick-xml = "0.29" rand = "0.8" - rangemap = "1.3" + rangemap = "1.4" roaring = "0.10" -rio = { git = "https://github.com/jthornber/rio", branch = "master", optional = true } +#rio = { git = "https://github.com/jthornber/rio", branch = "master", optional = true } @@ -33,5 +25,5 @@ index c2b496ac..562c40d3 100644 threadpool = "1.8" thiserror = "1.0" -- -2.41.0 +2.43.0 diff --git a/SOURCES/0002-space-map-Fix-incorrect-index_entry.nr_free-while-ex.patch b/SOURCES/0002-space-map-Fix-incorrect-index_entry.nr_free-while-ex.patch new file mode 100644 index 0000000..082ba6b --- /dev/null +++ b/SOURCES/0002-space-map-Fix-incorrect-index_entry.nr_free-while-ex.patch @@ -0,0 +1,181 @@ +From f088ce90f5f7934489a8d1062ebfd3d23e4aec38 Mon Sep 17 00:00:00 2001 +From: Ming-Hung Tsai +Date: Tue, 12 Dec 2023 14:25:27 +0800 +Subject: [PATCH 2/6] [space map] Fix incorrect index_entry.nr_free while + expansion + +Do not truncate the nr_free value of the last index_entry to the space +map boundary, to address the issue in kernel that sm_ll_extend() doesn't +extend the nr_free value of the last index_entry while expanding the +space map. Without this fix, we'll have incorrect free space estimation +in later block allocations, leading to under-utilized space maps. + +(cherry picked from commit 9a405f57c591020321fc16f00efdb5197e1df2c0) +--- + src/pdata/space_map/common.rs | 20 +++++--- + src/pdata/space_map/tests.rs | 96 +++++++++++++++++++++++++++++++++++ + 2 files changed, 108 insertions(+), 8 deletions(-) + +diff --git a/src/pdata/space_map/common.rs b/src/pdata/space_map/common.rs +index d92b8115..4be9c303 100644 +--- a/src/pdata/space_map/common.rs ++++ b/src/pdata/space_map/common.rs +@@ -211,22 +211,28 @@ pub fn write_common( + let len = std::cmp::min(nr_blocks - begin, ENTRIES_PER_BITMAP as u64); + let mut entries = Vec::with_capacity(ENTRIES_PER_BITMAP); + let mut first_free: Option = None; +- let mut nr_free: u32 = 0; ++ let mut nr_free = ENTRIES_PER_BITMAP as u32; // do not truncate to the sm size boundary + + for i in 0..len { + let b = begin + i; + let rc = sm.get(b)?; + let e = match rc { + 0 => { +- nr_free += 1; + if first_free.is_none() { + first_free = Some(i as u32); + } + Small(0) + } +- 1 => Small(1), +- 2 => Small(2), ++ 1 => { ++ nr_free -= 1; ++ Small(1) ++ } ++ 2 => { ++ nr_free -= 1; ++ Small(2) ++ } + _ => { ++ nr_free -= 1; + overflow_builder.push_value(w, b, rc)?; + Overflow + } +@@ -334,16 +340,14 @@ pub fn write_metadata_common(w: &mut WriteBatcher) -> anyhow::Result<(Vec Result<()> { ++ let engine = Arc::new(CoreIoEngine::new(nr_blocks)); ++ let meta_sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX); ++ ++ let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size()); ++ w.alloc()?; // reserved for the superblock ++ let root = write_metadata_sm(&mut w)?; ++ ++ let b = engine.read(root.bitmap_root)?; ++ let entries = check_and_unpack_metadata_index(&b)?.indexes; ++ ensure!(entries.len() as u64 == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64)); ++ ++ // the number of blocks observed by index_entries must be multiple of ENTRIES_PER_BITMAP ++ let nr_allocated = meta_sm.lock().unwrap().get_nr_allocated()?; ++ let nr_free: u64 = entries.iter().map(|ie| ie.nr_free as u64).sum(); ++ ensure!(nr_allocated + nr_free == (entries.len() * ENTRIES_PER_BITMAP) as u64); ++ ++ Ok(()) ++ } ++ ++ #[test] ++ fn check_single_index_entry() -> Result<()> { ++ check_index_entries(1000) ++ } ++ ++ #[test] ++ fn check_multiple_index_entries() -> Result<()> { ++ check_index_entries(ENTRIES_PER_BITMAP as u64 * 16 + 1000) ++ } ++} ++ ++//------------------------------------------ ++ ++mod disk_sm { ++ use anyhow::{ensure, Result}; ++ use std::ops::Deref; ++ use std::sync::Arc; ++ ++ use crate::io_engine::core::CoreIoEngine; ++ use crate::io_engine::*; ++ use crate::math::div_up; ++ use crate::pdata::btree_walker::btree_to_value_vec; ++ use crate::pdata::space_map::common::{IndexEntry, ENTRIES_PER_BITMAP}; ++ use crate::pdata::space_map::disk::*; ++ use crate::pdata::space_map::metadata::*; ++ use crate::pdata::space_map::*; ++ use crate::write_batcher::WriteBatcher; ++ ++ fn check_index_entries(nr_blocks: u64) -> Result<()> { ++ let engine = Arc::new(CoreIoEngine::new(1024)); ++ let meta_sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX); ++ ++ let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size()); ++ w.alloc()?; // reserved for the superblock ++ ++ let data_sm = core_sm(nr_blocks, u32::MAX); ++ data_sm.lock().unwrap().inc(0, 100)?; ++ ++ let root = write_disk_sm(&mut w, data_sm.lock().unwrap().deref())?; ++ ++ let entries = ++ btree_to_value_vec::(&mut Vec::new(), engine, false, root.bitmap_root)?; ++ ensure!(entries.len() as u64 == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64)); ++ ++ // the number of blocks observed by index_entries must be a multiple of ENTRIES_PER_BITMAP ++ let nr_allocated = data_sm.lock().unwrap().get_nr_allocated()?; ++ let nr_free: u64 = entries.iter().map(|ie| ie.nr_free as u64).sum(); ++ ensure!(nr_allocated + nr_free == (entries.len() * ENTRIES_PER_BITMAP) as u64); ++ ++ Ok(()) ++ } ++ ++ #[test] ++ fn check_single_index_entry() -> Result<()> { ++ check_index_entries(1000) ++ } ++ ++ #[test] ++ fn check_multiple_index_entries() -> Result<()> { ++ check_index_entries(ENTRIES_PER_BITMAP as u64 * 16 + 1000) ++ } ++} ++ ++//------------------------------------------ +-- +2.43.0 + diff --git a/SOURCES/0003-thin_repair-Fix-child-keys-checking-on-the-node-with.patch b/SOURCES/0003-thin_repair-Fix-child-keys-checking-on-the-node-with.patch new file mode 100644 index 0000000..511af7b --- /dev/null +++ b/SOURCES/0003-thin_repair-Fix-child-keys-checking-on-the-node-with.patch @@ -0,0 +1,69 @@ +From add81da22a3998503a6f340350d7e59ed3b52e28 Mon Sep 17 00:00:00 2001 +From: Ming-Hung Tsai +Date: Wed, 10 Jan 2024 15:56:39 +0800 +Subject: [PATCH 3/6] [thin_repair] Fix child keys checking on the node with a + zero key + +Fix the issue that keys overlapping between the second and the first +child nodes indexed by zero not being checked. + +(cherry picked from commit 386123bd0f74f7603e993bf3c26aac002162d5db) +--- + src/thin/metadata_repair.rs | 27 +++++++++++++++------------ + 1 file changed, 15 insertions(+), 12 deletions(-) + +diff --git a/src/thin/metadata_repair.rs b/src/thin/metadata_repair.rs +index 9716b1e3..8fece4b9 100644 +--- a/src/thin/metadata_repair.rs ++++ b/src/thin/metadata_repair.rs +@@ -128,10 +128,11 @@ impl DevInfo { + } + + fn push_child(&mut self, child: &DevInfo) -> Result<()> { +- if self.key_high > 0 && child.key_low <= self.key_high { +- return Err(anyhow!("incompatible child")); +- } +- if !self.pushed { ++ if self.pushed { ++ if child.key_low <= self.key_high { ++ return Err(anyhow!("incompatible child")); ++ } ++ } else { + self.key_low = child.key_low; + self.pushed = true; + } +@@ -175,10 +176,11 @@ impl MappingsInfo { + } + + fn push_child(&mut self, child: &MappingsInfo) -> Result<()> { +- if self.key_high > 0 && child.key_low <= self.key_high { +- return Err(anyhow!("incompatible child")); +- } +- if !self.pushed { ++ if self.pushed { ++ if child.key_low <= self.key_high { ++ return Err(anyhow!("incompatible child")); ++ } ++ } else { + self.key_low = child.key_low; + self.pushed = true; + } +@@ -221,10 +223,11 @@ impl DetailsInfo { + } + + fn push_child(&mut self, child: &DetailsInfo) -> Result<()> { +- if self.key_high > 0 && child.key_low <= self.key_high { +- return Err(anyhow!("incompatible child")); +- } +- if !self.pushed { ++ if self.pushed { ++ if child.key_low <= self.key_high { ++ return Err(anyhow!("incompatible child")); ++ } ++ } else { + self.key_low = child.key_low; + self.pushed = true; + } +-- +2.43.0 + diff --git a/SOURCES/0004-space_map-Allow-non-zero-values-in-unused-index-bloc.patch b/SOURCES/0004-space_map-Allow-non-zero-values-in-unused-index-bloc.patch new file mode 100644 index 0000000..f49b859 --- /dev/null +++ b/SOURCES/0004-space_map-Allow-non-zero-values-in-unused-index-bloc.patch @@ -0,0 +1,269 @@ +From 9b948a9639b00fe9226065ca4dd6fd2257cc126c Mon Sep 17 00:00:00 2001 +From: Ming-Hung Tsai +Date: Thu, 11 Jan 2024 22:57:36 +0800 +Subject: [PATCH 4/6] [space_map] Allow non-zero values in unused index block + entries + +Previously, we assumed unused entries in the index block were all +zero-initialized, leading to issues while loading the block with +unexpected bytes and a valid checksum [1]. The updated approach loads +index entries based on actual size information from superblock and +therefore improves compatibility. + +[1] stratis-storage/stratisd#3520 + +(cherry picked from commit d5fe6a1e1c539a0f260a96eb4d7ed9b83c3f84c9) +--- + src/commands/engine.rs | 6 ++++- + src/pdata/space_map/allocated_blocks.rs | 15 ++++++----- + src/pdata/space_map/checker.rs | 12 ++++++--- + src/pdata/space_map/metadata.rs | 22 +++++++++------- + src/pdata/space_map/tests.rs | 35 +++++++++++++++++++++++-- + src/thin/damage_generator.rs | 4 +-- + src/thin/stat.rs | 6 +++-- + 7 files changed, 75 insertions(+), 25 deletions(-) + +diff --git a/src/commands/engine.rs b/src/commands/engine.rs +index 1e752d6e..e8b75301 100644 +--- a/src/commands/engine.rs ++++ b/src/commands/engine.rs +@@ -137,7 +137,11 @@ fn thin_valid_blocks>(path: P, opts: &EngineOptions) -> RoaringBi + return all_blocks(e.get_nr_blocks() as u32); + } + let metadata_root = metadata_root.unwrap(); +- let valid_blocks = allocated_blocks(e.clone(), metadata_root.bitmap_root); ++ let valid_blocks = allocated_blocks( ++ e.clone(), ++ metadata_root.bitmap_root, ++ metadata_root.nr_blocks, ++ ); + valid_blocks.unwrap_or_else(|_| all_blocks(e.get_nr_blocks() as u32)) + } + +diff --git a/src/pdata/space_map/allocated_blocks.rs b/src/pdata/space_map/allocated_blocks.rs +index aa75cb7a..71497a76 100644 +--- a/src/pdata/space_map/allocated_blocks.rs ++++ b/src/pdata/space_map/allocated_blocks.rs +@@ -17,18 +17,21 @@ struct IndexInfo { + pub fn allocated_blocks( + engine: Arc, + sm_root: u64, ++ nr_blocks: u64, + ) -> Result { + // Walk index tree to find where the bitmaps are. + let b = engine.read(sm_root)?; +- let (_, indexes) = MetadataIndex::unpack(b.get_data())?; ++ let indexes = load_metadata_index(&b, nr_blocks)?; + +- let mut infos = Vec::new(); +- for (key, entry) in indexes.indexes.iter().enumerate() { +- infos.push(IndexInfo { ++ let mut infos: Vec<_> = indexes ++ .indexes ++ .iter() ++ .enumerate() ++ .map(|(key, entry)| IndexInfo { + key: key as u64, + loc: entry.blocknr, +- }); +- } ++ }) ++ .collect(); + + // Read bitmaps in sequence + infos.sort_by(|lhs, rhs| lhs.loc.partial_cmp(&rhs.loc).unwrap()); +diff --git a/src/pdata/space_map/checker.rs b/src/pdata/space_map/checker.rs +index cfafa79b..7cc8286b 100644 +--- a/src/pdata/space_map/checker.rs ++++ b/src/pdata/space_map/checker.rs +@@ -194,10 +194,11 @@ fn gather_disk_index_entries( + fn gather_metadata_index_entries( + engine: Arc, + bitmap_root: u64, ++ nr_blocks: u64, + metadata_sm: ASpaceMap, + ) -> Result> { + let b = engine.read(bitmap_root)?; +- let entries = check_and_unpack_metadata_index(&b)?.indexes; ++ let entries = load_metadata_index(&b, nr_blocks)?.indexes; + metadata_sm.lock().unwrap().inc(bitmap_root, 1)?; + inc_entries(&metadata_sm, &entries[0..])?; + +@@ -254,8 +255,13 @@ pub fn check_metadata_space_map( + metadata_sm.clone(), + false, + )?; +- let entries = +- gather_metadata_index_entries(engine.clone(), root.bitmap_root, metadata_sm.clone())?; ++ ++ let entries = gather_metadata_index_entries( ++ engine.clone(), ++ root.bitmap_root, ++ root.nr_blocks, ++ metadata_sm.clone(), ++ )?; + + // check overflow ref-counts + { +diff --git a/src/pdata/space_map/metadata.rs b/src/pdata/space_map/metadata.rs +index b466c135..be232389 100644 +--- a/src/pdata/space_map/metadata.rs ++++ b/src/pdata/space_map/metadata.rs +@@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex}; + + use crate::checksum; + use crate::io_engine::*; ++use crate::math::div_up; + use crate::pdata::space_map::common::*; + use crate::pdata::space_map::*; + use crate::pdata::unpack::*; +@@ -32,14 +33,11 @@ impl Unpack for MetadataIndex { + let (i, _csum) = le_u32(i)?; + let (i, _padding) = le_u32(i)?; + let (i, blocknr) = le_u64(i)?; +- let (i, indexes) = nom::multi::count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?; ++ let (i, mut indexes) = nom::multi::count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?; + +- // Filter out unused entries +- let indexes: Vec = indexes +- .iter() +- .take_while(|e| e.blocknr != 0) +- .cloned() +- .collect(); ++ // Drop unused entries that point to block 0 ++ let nr_bitmaps = indexes.iter().take_while(|e| e.blocknr != 0).count(); ++ indexes.truncate(nr_bitmaps); + + Ok((i, MetadataIndex { blocknr, indexes })) + } +@@ -69,9 +67,15 @@ fn verify_checksum(b: &Block) -> Result<()> { + } + } + +-pub fn check_and_unpack_metadata_index(b: &Block) -> Result { ++pub fn load_metadata_index(b: &Block, nr_blocks: u64) -> Result { + verify_checksum(b)?; +- unpack::(b.get_data()).map_err(|e| e.into()) ++ let mut entries = unpack::(b.get_data())?; ++ if entries.blocknr != b.loc { ++ return Err(anyhow!("blocknr mismatch")); ++ } ++ let nr_bitmaps = div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize; ++ entries.indexes.truncate(nr_bitmaps); ++ Ok(entries) + } + + //------------------------------------------ +diff --git a/src/pdata/space_map/tests.rs b/src/pdata/space_map/tests.rs +index fb08a9dc..fa118189 100644 +--- a/src/pdata/space_map/tests.rs ++++ b/src/pdata/space_map/tests.rs +@@ -171,10 +171,11 @@ mod metadata_sm { + let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size()); + w.alloc()?; // reserved for the superblock + let root = write_metadata_sm(&mut w)?; ++ drop(w); + + let b = engine.read(root.bitmap_root)?; +- let entries = check_and_unpack_metadata_index(&b)?.indexes; +- ensure!(entries.len() as u64 == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64)); ++ let entries = load_metadata_index(&b, root.nr_blocks)?.indexes; ++ ensure!(entries.len() == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize); + + // the number of blocks observed by index_entries must be multiple of ENTRIES_PER_BITMAP + let nr_allocated = meta_sm.lock().unwrap().get_nr_allocated()?; +@@ -193,6 +194,35 @@ mod metadata_sm { + fn check_multiple_index_entries() -> Result<()> { + check_index_entries(ENTRIES_PER_BITMAP as u64 * 16 + 1000) + } ++ ++ #[test] ++ fn ignore_junk_bytes_in_index_block() -> Result<()> { ++ use crate::checksum; ++ use crate::pdata::space_map::common::IndexEntry; ++ use crate::pdata::unpack::Unpack; ++ ++ let nr_blocks = ENTRIES_PER_BITMAP as u64 * 4 + 1000; ++ let nr_bitmaps = div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize; ++ let engine = Arc::new(CoreIoEngine::new(nr_blocks)); ++ let meta_sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX); ++ ++ let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size()); ++ w.alloc()?; // reserved for the superblock ++ let root = write_metadata_sm(&mut w)?; ++ ++ // append junk bytes to the unused entry ++ let index_block = w.read(root.bitmap_root)?; ++ index_block.get_data()[nr_bitmaps * IndexEntry::disk_size() as usize + 16] = 1; ++ w.write(index_block, checksum::BT::INDEX)?; ++ w.flush()?; ++ drop(w); ++ ++ let b = engine.read(root.bitmap_root)?; ++ let entries = load_metadata_index(&b, root.nr_blocks)?.indexes; ++ ensure!(entries.len() == nr_bitmaps); ++ ++ Ok(()) ++ } + } + + //------------------------------------------ +@@ -223,6 +253,7 @@ mod disk_sm { + data_sm.lock().unwrap().inc(0, 100)?; + + let root = write_disk_sm(&mut w, data_sm.lock().unwrap().deref())?; ++ drop(w); + + let entries = + btree_to_value_vec::(&mut Vec::new(), engine, false, root.bitmap_root)?; +diff --git a/src/thin/damage_generator.rs b/src/thin/damage_generator.rs +index 56685d4d..df1be4cd 100644 +--- a/src/thin/damage_generator.rs ++++ b/src/thin/damage_generator.rs +@@ -23,7 +23,7 @@ fn find_blocks_of_rc( + let mut found = Vec::::new(); + if ref_count < 3 { + let b = engine.read(sm_root.bitmap_root)?; +- let entries = check_and_unpack_metadata_index(&b)?.indexes; ++ let entries = load_metadata_index(&b, sm_root.nr_blocks)?.indexes; + let bitmaps: Vec = entries.iter().map(|ie| ie.blocknr).collect(); + let nr_bitmaps = bitmaps.len(); + +@@ -75,7 +75,7 @@ fn adjust_bitmap_entries( + }; + + let index_block = engine.read(sm_root.bitmap_root)?; +- let entries = check_and_unpack_metadata_index(&index_block)?.indexes; ++ let entries = load_metadata_index(&index_block, sm_root.nr_blocks)?.indexes; + + let bi = blocks_to_bitmaps(blocks); + let bitmaps: Vec = bi.iter().map(|i| entries[*i].blocknr).collect(); +diff --git a/src/thin/stat.rs b/src/thin/stat.rs +index 03ae6845..c6f2bf44 100644 +--- a/src/thin/stat.rs ++++ b/src/thin/stat.rs +@@ -72,9 +72,10 @@ fn gather_btree_index_entries( + fn gather_metadata_index_entries( + engine: Arc, + bitmap_root: u64, ++ nr_blocks: u64, + ) -> Result> { + let b = engine.read(bitmap_root)?; +- let entries = check_and_unpack_metadata_index(&b)?.indexes; ++ let entries = load_metadata_index(&b, nr_blocks)?.indexes; + Ok(entries) + } + +@@ -152,7 +153,8 @@ fn stat_metadata_block_ref_counts( + ) -> Result> { + let mut histogram = BTreeMap::::new(); + +- let index_entries = gather_metadata_index_entries(engine.clone(), root.bitmap_root)?; ++ let index_entries = ++ gather_metadata_index_entries(engine.clone(), root.bitmap_root, root.nr_blocks)?; + stat_low_ref_counts(engine.clone(), &index_entries, &mut histogram)?; + + let histogram = stat_overflow_ref_counts(engine, root.ref_count_root, histogram)?; +-- +2.43.0 + diff --git a/SOURCES/0005-cache_check-Fix-boundary-check-on-the-bitset-for-cac.patch b/SOURCES/0005-cache_check-Fix-boundary-check-on-the-bitset-for-cac.patch new file mode 100644 index 0000000..1a69672 --- /dev/null +++ b/SOURCES/0005-cache_check-Fix-boundary-check-on-the-bitset-for-cac.patch @@ -0,0 +1,111 @@ +From 96fc598f76beac1deb0c9564dc67416f70ae4ac4 Mon Sep 17 00:00:00 2001 +From: Ming-Hung Tsai +Date: Tue, 30 Jan 2024 15:20:31 +0800 +Subject: [PATCH 5/6] [cache_check] Fix boundary check on the bitset for cached + blocks + +The bitset for cached block addresses grows dynamically if the metadata +is not shutdown properly, in which the size hint of the slow (backing) +device is not available. Fix a bug in determining whether resizing is +needed (bz2258485). + +(cherry picked from commit d2390a50f38d88f0f32b13e59444bbbca7e660b3) +--- + src/cache/check.rs | 6 +++--- + tests/cache_check.rs | 45 ++++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 48 insertions(+), 3 deletions(-) + +diff --git a/src/cache/check.rs b/src/cache/check.rs +index 18bd51b3..c71d3059 100644 +--- a/src/cache/check.rs ++++ b/src/cache/check.rs +@@ -19,7 +19,7 @@ use crate::report::*; + + //------------------------------------------ + +-// 16m entries is capable for a 1TB cache with 64KB block size ++// 16m entries is capable to address a 1TB device with 64KB block size + const DEFAULT_OBLOCKS: usize = 16777216; + + fn inc_superblock(sm: &ASpaceMap) -> anyhow::Result<()> { +@@ -82,7 +82,7 @@ mod format1 { + } + let mut seen_oblocks = self.seen_oblocks.lock().unwrap(); + +- if m.oblock as usize > seen_oblocks.len() { ++ if m.oblock as usize >= seen_oblocks.len() { + seen_oblocks.grow(m.oblock as usize + 1); + } else if seen_oblocks.contains(m.oblock as usize) { + return Err(array::value_err("origin block already mapped".to_string())); +@@ -179,7 +179,7 @@ mod format2 { + )); + } + +- if m.oblock as usize > seen_oblocks.len() { ++ if m.oblock as usize >= seen_oblocks.len() { + seen_oblocks.grow(m.oblock as usize + 1); + } else if seen_oblocks.contains(m.oblock as usize) { + return Err(array::value_err("origin block already mapped".to_string())); +diff --git a/tests/cache_check.rs b/tests/cache_check.rs +index 81f4c578..8988694a 100644 +--- a/tests/cache_check.rs ++++ b/tests/cache_check.rs +@@ -11,6 +11,8 @@ use common::program::*; + use common::target::*; + use common::test_dir::*; + ++use std::io::Write; ++ + //------------------------------------------ + + const USAGE: &str = "Validates cache metadata on a device or file. +@@ -294,3 +296,46 @@ fn no_clear_needs_check_if_error() -> Result<()> { + } + + //------------------------------------------ ++ ++fn metadata_without_slow_dev_size_info(use_v1: bool) -> Result<()> { ++ let mut td = TestDir::new()?; ++ ++ // The input metadata has a cached oblock with address equals to the default bitset size ++ // boundary (DEFAULT_OBLOCKS = 16777216), triggering bitset resize. ++ let xml = td.mk_path("meta.xml"); ++ let mut file = std::fs::File::create(&xml)?; ++ file.write_all(b" ++ ++ ++ ++ ++ ++ ++")?; ++ ++ let md = td.mk_path("meta.bin"); ++ thinp::file_utils::create_sized_file(&md, 4096 * 4096)?; ++ ++ let cache_restore_args = if use_v1 { ++ args!["-i", &xml, "-o", &md, "--metadata-version=1"] ++ } else { ++ args!["-i", &xml, "-o", &md, "--metadata-version=2"] ++ }; ++ ++ run_ok(cache_restore_cmd(cache_restore_args))?; ++ run_ok(cache_check_cmd(args![&md]))?; ++ ++ Ok(()) ++} ++ ++#[test] ++fn metadata_v1_without_slow_dev_size_info() -> Result<()> { ++ metadata_without_slow_dev_size_info(true) ++} ++ ++#[test] ++fn metadata_v2_without_slow_dev_size_info() -> Result<()> { ++ metadata_without_slow_dev_size_info(false) ++} ++ ++//------------------------------------------ +-- +2.43.0 + diff --git a/SOURCES/0006-thin-cache_check-Print-suggestive-hints-for-improvin.patch b/SOURCES/0006-thin-cache_check-Print-suggestive-hints-for-improvin.patch new file mode 100644 index 0000000..df75f43 --- /dev/null +++ b/SOURCES/0006-thin-cache_check-Print-suggestive-hints-for-improvin.patch @@ -0,0 +1,62 @@ +From e295610fad85ec1a64e569cba425ca557a56c6e6 Mon Sep 17 00:00:00 2001 +From: Ming-Hung Tsai +Date: Wed, 31 Jan 2024 11:36:17 +0800 +Subject: [PATCH 6/6] [thin/cache_check] Print suggestive hints for improving + error resolution + +Enhance error messages to instruct users on addressing recoverable +errors, eliminating the guesswork (bz2233177). + +(cherry picked from commit aaf3b396574709902ffba47e03a5c7ded6a103c5) +--- + src/cache/check.rs | 5 ++++- + src/thin/check.rs | 10 ++++++++-- + 2 files changed, 12 insertions(+), 3 deletions(-) + +diff --git a/src/cache/check.rs b/src/cache/check.rs +index c71d3059..17d1af77 100644 +--- a/src/cache/check.rs ++++ b/src/cache/check.rs +@@ -387,7 +387,10 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> { + ctx.report.warning("Repairing metadata leaks."); + repair_space_map(ctx.engine.clone(), metadata_leaks, metadata_sm.clone())?; + } else if !opts.ignore_non_fatal { +- return Err(anyhow!("metadata space map contains leaks")); ++ return Err(anyhow!(concat!( ++ "metadata space map contains leaks\n", ++ "perhaps you wanted to run with --auto-repair" ++ ))); + } + } + +diff --git a/src/thin/check.rs b/src/thin/check.rs +index 8b829899..f6fde359 100644 +--- a/src/thin/check.rs ++++ b/src/thin/check.rs +@@ -1246,7 +1246,10 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> { + report.warning("Repairing data leaks."); + repair_space_map(engine.clone(), data_leaks, data_sm.clone())?; + } else if !opts.ignore_non_fatal { +- return Err(anyhow!("data space map contains leaks")); ++ return Err(anyhow!(concat!( ++ "data space map contains leaks\n", ++ "perhaps you wanted to run with --auto-repair" ++ ))); + } + } + +@@ -1255,7 +1258,10 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> { + report.warning("Repairing metadata leaks."); + repair_space_map(engine.clone(), metadata_leaks, metadata_sm.clone())?; + } else if !opts.ignore_non_fatal { +- return Err(anyhow!("metadata space map contains leaks")); ++ return Err(anyhow!(concat!( ++ "metadata space map contains leaks\n", ++ "perhaps you wanted to run with --auto-repair" ++ ))); + } + } + +-- +2.43.0 + diff --git a/SOURCES/dmpd106-vendor.tar.gz b/SOURCES/dmpd106-vendor.tar.gz deleted file mode 100644 index fde93d6..0000000 Binary files a/SOURCES/dmpd106-vendor.tar.gz and /dev/null differ diff --git a/SOURCES/v1.0.6.tar.gz b/SOURCES/v1.0.6.tar.gz deleted file mode 100644 index 369f563..0000000 Binary files a/SOURCES/v1.0.6.tar.gz and /dev/null differ diff --git a/SPECS/device-mapper-persistent-data.spec b/SPECS/device-mapper-persistent-data.spec index 8058ba3..b0741b1 100644 --- a/SPECS/device-mapper-persistent-data.spec +++ b/SPECS/device-mapper-persistent-data.spec @@ -9,14 +9,19 @@ Summary: Device-mapper Persistent Data Tools Name: device-mapper-persistent-data -Version: 1.0.6 -Release: 1%{?dist}%{?release_suffix} +Version: 1.0.9 +Release: 2%{?dist}%{?release_suffix} License: GPLv3+ URL: https://github.com/jthornber/thin-provisioning-tools #Source0: https://github.com/jthornber/thin-provisioning-tools/archive/thin-provisioning-tools-%%{version}.tar.gz Source0: https://github.com/jthornber/thin-provisioning-tools/archive/v%{version}%{?version_suffix}.tar.gz -Source1: dmpd106-vendor.tar.gz +Source1: dmpd109-vendor.tar.gz Patch1: 0001-Tweak-cargo.toml-to-work-with-vendor-directory.patch +Patch2: 0002-space-map-Fix-incorrect-index_entry.nr_free-while-ex.patch +Patch3: 0003-thin_repair-Fix-child-keys-checking-on-the-node-with.patch +Patch4: 0004-space_map-Allow-non-zero-values-in-unused-index-bloc.patch +Patch5: 0005-cache_check-Fix-boundary-check-on-the-bitset-for-cac.patch +Patch6: 0006-thin-cache_check-Print-suggestive-hints-for-improvin.patch BuildRequires: rust-packaging BuildRequires: rust >= 1.35 @@ -54,7 +59,10 @@ echo %{version}-%{release} > VERSION %if %{with check} %check -RUST_BACKTRACE=1 %cargo_test || true +# aarch64 is failing, but only in brew environment, tests are passing when +# running locally +#%%cargo_test +RUST_BACKTRACE=1 %%cargo_test -- --nocapture --test-threads=1 || true %endif %install @@ -108,6 +116,16 @@ make DESTDIR=%{buildroot} MANDIR=%{_mandir} install #% {_sbindir}/thin_show_duplicates %changelog +* Thu Feb 08 2024 Marian Csontos - 1.0.9-2 +- Allow non-zero values in unused index block entries. +- Fix boundary check on the bitset for cached blocks. +- Fix incorrect index_entry.nr_free on expansion affecting free space estimate. +- Fix thin_repair checking keys on a node with a zero key. +- Enhance error message in cache_check suggesting fix when applicable. + +* Wed Dec 13 2023 Marian Csontos - 1.0.9-1 +- Update to latest upstream release 1.0.9. + * Wed Aug 09 2023 Marian Csontos - 1.0.6-1 - Update to latest upstream release 1.0.6.