rpms/device-mapper-persistent-data
7
0
Fork 0
Code Issues Pull Requests Releases Activity

import CS device-mapper-persistent-data-1.0.9-2.el9

Browse Source
This commit is contained in:
eabdullin 2024-03-28 10:01:43 +00:00
parent 586710bc2d
commit 8900263a56
11 changed files with 725 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.device-mapper-persistent-data.metadata Normal file
View File

@ -0,0 +1,2 @@
d336474a4772430fa2b8a5f76a6c2f2a78caa698 SOURCES/dmpd109-vendor.tar.gz
8862f41bf5fad011879b6d819ab5e5b11afcda7e SOURCES/v1.0.9.tar.gz

2
.gitignore vendored
View File

@ -0,0 +1,2 @@
SOURCES/dmpd109-vendor.tar.gz
SOURCES/v1.0.9.tar.gz

22
SOURCES/0001-Tweak-cargo.toml-to-work-with-vendor-directory.patch
View File

@ -1,4 +1,4 @@
From 732ff5861a1525944a927439d1c075ac269788ce Mon Sep 17 00:00:00 2001 From 0d5347bd771e960294cd0c2f083d96448613ab9c Mon Sep 17 00:00:00 2001
From: Marian Csontos <mcsontos@redhat.com> From: Marian Csontos <mcsontos@redhat.com>
Date: Thu, 27 Jul 2023 11:37:01 +0200 Date: Thu, 27 Jul 2023 11:37:01 +0200
Subject: [PATCH] Tweak cargo.toml to work with vendor directory Subject: [PATCH] Tweak cargo.toml to work with vendor directory
@ -7,24 +7,16 @@ Mock works offline, cargo would try to download the files from github.
So cargo vendor has to be run first, and then change the Cargo.toml to So cargo vendor has to be run first, and then change the Cargo.toml to
make mock happy. make mock happy.
--- ---
Cargo.toml | 4 +++- Cargo.toml | 3 ++-
1 file changed, 3 insertions(+), 1 deletion(-) 1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/Cargo.toml b/Cargo.toml diff --git a/Cargo.toml b/Cargo.toml
index c2b496ac..562c40d3 100644 index 500345a4..d4aa38a6 100644
--- a/Cargo.toml --- a/Cargo.toml
+++ b/Cargo.toml +++ b/Cargo.toml
@@ -11,6 +11,7 @@ anyhow = "1.0" @@ -27,7 +27,8 @@ quick-xml = "0.29"
base64 = "0.21"
byteorder = "1.4"
clap = { version = "4.3", default-features = false, features = ["std", "help", "usage", "error-context", "suggestions"] }
+#crc32c = { git = "https://github.com/zowens/crc32c", branch = "master" }
crc32c = "0.6"
data-encoding = "2.4"
exitcode = "1.1.2"
@@ -27,7 +28,8 @@ quick-xml = "0.29"
rand = "0.8" rand = "0.8"
rangemap = "1.3" rangemap = "1.4"
roaring = "0.10" roaring = "0.10"
-rio = { git = "https://github.com/jthornber/rio", branch = "master", optional = true } -rio = { git = "https://github.com/jthornber/rio", branch = "master", optional = true }
+#rio = { git = "https://github.com/jthornber/rio", branch = "master", optional = true } +#rio = { git = "https://github.com/jthornber/rio", branch = "master", optional = true }
@ -33,5 +25,5 @@ index c2b496ac..562c40d3 100644
threadpool = "1.8" threadpool = "1.8"
thiserror = "1.0" thiserror = "1.0"
-- --
2.41.0 2.43.0

181
SOURCES/0002-space-map-Fix-incorrect-index_entry.nr_free-while-ex.patch Normal file
View File

@ -0,0 +1,181 @@
From f088ce90f5f7934489a8d1062ebfd3d23e4aec38 Mon Sep 17 00:00:00 2001
From: Ming-Hung Tsai <mtsai@redhat.com>
Date: Tue, 12 Dec 2023 14:25:27 +0800
Subject: [PATCH 2/6] [space map] Fix incorrect index_entry.nr_free while
expansion
Do not truncate the nr_free value of the last index_entry to the space
map boundary, to address the issue in kernel that sm_ll_extend() doesn't
extend the nr_free value of the last index_entry while expanding the
space map. Without this fix, we'll have incorrect free space estimation
in later block allocations, leading to under-utilized space maps.
(cherry picked from commit 9a405f57c591020321fc16f00efdb5197e1df2c0)
---
src/pdata/space_map/common.rs | 20 +++++---
src/pdata/space_map/tests.rs | 96 +++++++++++++++++++++++++++++++++++
2 files changed, 108 insertions(+), 8 deletions(-)
diff --git a/src/pdata/space_map/common.rs b/src/pdata/space_map/common.rs
index d92b8115..4be9c303 100644
--- a/src/pdata/space_map/common.rs
+++ b/src/pdata/space_map/common.rs
@@ -211,22 +211,28 @@ pub fn write_common(
let len = std::cmp::min(nr_blocks - begin, ENTRIES_PER_BITMAP as u64);
let mut entries = Vec::with_capacity(ENTRIES_PER_BITMAP);
let mut first_free: Option<u32> = None;
- let mut nr_free: u32 = 0;
+ let mut nr_free = ENTRIES_PER_BITMAP as u32; // do not truncate to the sm size boundary
for i in 0..len {
let b = begin + i;
let rc = sm.get(b)?;
let e = match rc {
0 => {
- nr_free += 1;
if first_free.is_none() {
first_free = Some(i as u32);
}
Small(0)
}
- 1 => Small(1),
- 2 => Small(2),
+ 1 => {
+ nr_free -= 1;
+ Small(1)
+ }
+ 2 => {
+ nr_free -= 1;
+ Small(2)
+ }
_ => {
+ nr_free -= 1;
overflow_builder.push_value(w, b, rc)?;
Overflow
}
@@ -334,16 +340,14 @@ pub fn write_metadata_common(w: &mut WriteBatcher) -> anyhow::Result<(Vec<IndexE
let nr_blocks = w.sm.lock().unwrap().get_nr_blocks()?;
let nr_bitmaps = div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize;
let nr_used_bitmaps = index_entries.len();
- for bm in nr_used_bitmaps..nr_bitmaps {
- let begin = bm as u64 * ENTRIES_PER_BITMAP as u64;
- let len = std::cmp::min(nr_blocks - begin, ENTRIES_PER_BITMAP as u64);
+ for _bm in nr_used_bitmaps..nr_bitmaps {
let entries = vec![BitmapEntry::Small(0); ENTRIES_PER_BITMAP];
let blocknr = write_bitmap(w, entries)?;
// Insert into the index list
let ie = IndexEntry {
blocknr,
- nr_free: len as u32,
+ nr_free: ENTRIES_PER_BITMAP as u32, // do not truncate to the sm size boundary
none_free_before: 0,
};
index_entries.push(ie);
diff --git a/src/pdata/space_map/tests.rs b/src/pdata/space_map/tests.rs
index 26d7834d..fb08a9dc 100644
--- a/src/pdata/space_map/tests.rs
+++ b/src/pdata/space_map/tests.rs
@@ -152,3 +152,99 @@ mod core_sm_u8 {
}
//------------------------------------------
+
+mod metadata_sm {
+ use anyhow::{ensure, Result};
+ use std::sync::Arc;
+
+ use crate::io_engine::core::CoreIoEngine;
+ use crate::io_engine::*;
+ use crate::math::div_up;
+ use crate::pdata::space_map::common::ENTRIES_PER_BITMAP;
+ use crate::pdata::space_map::metadata::*;
+ use crate::write_batcher::WriteBatcher;
+
+ fn check_index_entries(nr_blocks: u64) -> Result<()> {
+ let engine = Arc::new(CoreIoEngine::new(nr_blocks));
+ let meta_sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX);
+
+ let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size());
+ w.alloc()?; // reserved for the superblock
+ let root = write_metadata_sm(&mut w)?;
+
+ let b = engine.read(root.bitmap_root)?;
+ let entries = check_and_unpack_metadata_index(&b)?.indexes;
+ ensure!(entries.len() as u64 == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64));
+
+ // the number of blocks observed by index_entries must be multiple of ENTRIES_PER_BITMAP
+ let nr_allocated = meta_sm.lock().unwrap().get_nr_allocated()?;
+ let nr_free: u64 = entries.iter().map(|ie| ie.nr_free as u64).sum();
+ ensure!(nr_allocated + nr_free == (entries.len() * ENTRIES_PER_BITMAP) as u64);
+
+ Ok(())
+ }
+
+ #[test]
+ fn check_single_index_entry() -> Result<()> {
+ check_index_entries(1000)
+ }
+
+ #[test]
+ fn check_multiple_index_entries() -> Result<()> {
+ check_index_entries(ENTRIES_PER_BITMAP as u64 * 16 + 1000)
+ }
+}
+
+//------------------------------------------
+
+mod disk_sm {
+ use anyhow::{ensure, Result};
+ use std::ops::Deref;
+ use std::sync::Arc;
+
+ use crate::io_engine::core::CoreIoEngine;
+ use crate::io_engine::*;
+ use crate::math::div_up;
+ use crate::pdata::btree_walker::btree_to_value_vec;
+ use crate::pdata::space_map::common::{IndexEntry, ENTRIES_PER_BITMAP};
+ use crate::pdata::space_map::disk::*;
+ use crate::pdata::space_map::metadata::*;
+ use crate::pdata::space_map::*;
+ use crate::write_batcher::WriteBatcher;
+
+ fn check_index_entries(nr_blocks: u64) -> Result<()> {
+ let engine = Arc::new(CoreIoEngine::new(1024));
+ let meta_sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX);
+
+ let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size());
+ w.alloc()?; // reserved for the superblock
+
+ let data_sm = core_sm(nr_blocks, u32::MAX);
+ data_sm.lock().unwrap().inc(0, 100)?;
+
+ let root = write_disk_sm(&mut w, data_sm.lock().unwrap().deref())?;
+
+ let entries =
+ btree_to_value_vec::<IndexEntry>(&mut Vec::new(), engine, false, root.bitmap_root)?;
+ ensure!(entries.len() as u64 == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64));
+
+ // the number of blocks observed by index_entries must be a multiple of ENTRIES_PER_BITMAP
+ let nr_allocated = data_sm.lock().unwrap().get_nr_allocated()?;
+ let nr_free: u64 = entries.iter().map(|ie| ie.nr_free as u64).sum();
+ ensure!(nr_allocated + nr_free == (entries.len() * ENTRIES_PER_BITMAP) as u64);
+
+ Ok(())
+ }
+
+ #[test]
+ fn check_single_index_entry() -> Result<()> {
+ check_index_entries(1000)
+ }
+
+ #[test]
+ fn check_multiple_index_entries() -> Result<()> {
+ check_index_entries(ENTRIES_PER_BITMAP as u64 * 16 + 1000)
+ }
+}
+
+//------------------------------------------
--
2.43.0

69
SOURCES/0003-thin_repair-Fix-child-keys-checking-on-the-node-with.patch Normal file
View File

@ -0,0 +1,69 @@
From add81da22a3998503a6f340350d7e59ed3b52e28 Mon Sep 17 00:00:00 2001
From: Ming-Hung Tsai <mtsai@redhat.com>
Date: Wed, 10 Jan 2024 15:56:39 +0800
Subject: [PATCH 3/6] [thin_repair] Fix child keys checking on the node with a
zero key
Fix the issue that keys overlapping between the second and the first
child nodes indexed by zero not being checked.
(cherry picked from commit 386123bd0f74f7603e993bf3c26aac002162d5db)
---
src/thin/metadata_repair.rs | 27 +++++++++++++++------------
1 file changed, 15 insertions(+), 12 deletions(-)
diff --git a/src/thin/metadata_repair.rs b/src/thin/metadata_repair.rs
index 9716b1e3..8fece4b9 100644
--- a/src/thin/metadata_repair.rs
+++ b/src/thin/metadata_repair.rs
@@ -128,10 +128,11 @@ impl DevInfo {
}
fn push_child(&mut self, child: &DevInfo) -> Result<()> {
- if self.key_high > 0 && child.key_low <= self.key_high {
- return Err(anyhow!("incompatible child"));
- }
- if !self.pushed {
+ if self.pushed {
+ if child.key_low <= self.key_high {
+ return Err(anyhow!("incompatible child"));
+ }
+ } else {
self.key_low = child.key_low;
self.pushed = true;
}
@@ -175,10 +176,11 @@ impl MappingsInfo {
}
fn push_child(&mut self, child: &MappingsInfo) -> Result<()> {
- if self.key_high > 0 && child.key_low <= self.key_high {
- return Err(anyhow!("incompatible child"));
- }
- if !self.pushed {
+ if self.pushed {
+ if child.key_low <= self.key_high {
+ return Err(anyhow!("incompatible child"));
+ }
+ } else {
self.key_low = child.key_low;
self.pushed = true;
}
@@ -221,10 +223,11 @@ impl DetailsInfo {
}
fn push_child(&mut self, child: &DetailsInfo) -> Result<()> {
- if self.key_high > 0 && child.key_low <= self.key_high {
- return Err(anyhow!("incompatible child"));
- }
- if !self.pushed {
+ if self.pushed {
+ if child.key_low <= self.key_high {
+ return Err(anyhow!("incompatible child"));
+ }
+ } else {
self.key_low = child.key_low;
self.pushed = true;
}
--
2.43.0

269
SOURCES/0004-space_map-Allow-non-zero-values-in-unused-index-bloc.patch Normal file
View File

@ -0,0 +1,269 @@
From 9b948a9639b00fe9226065ca4dd6fd2257cc126c Mon Sep 17 00:00:00 2001
From: Ming-Hung Tsai <mtsai@redhat.com>
Date: Thu, 11 Jan 2024 22:57:36 +0800
Subject: [PATCH 4/6] [space_map] Allow non-zero values in unused index block
entries
Previously, we assumed unused entries in the index block were all
zero-initialized, leading to issues while loading the block with
unexpected bytes and a valid checksum [1]. The updated approach loads
index entries based on actual size information from superblock and
therefore improves compatibility.
[1] stratis-storage/stratisd#3520
(cherry picked from commit d5fe6a1e1c539a0f260a96eb4d7ed9b83c3f84c9)
---
src/commands/engine.rs | 6 ++++-
src/pdata/space_map/allocated_blocks.rs | 15 ++++++-----
src/pdata/space_map/checker.rs | 12 ++++++---
src/pdata/space_map/metadata.rs | 22 +++++++++-------
src/pdata/space_map/tests.rs | 35 +++++++++++++++++++++++--
src/thin/damage_generator.rs | 4 +--
src/thin/stat.rs | 6 +++--
7 files changed, 75 insertions(+), 25 deletions(-)
diff --git a/src/commands/engine.rs b/src/commands/engine.rs
index 1e752d6e..e8b75301 100644
--- a/src/commands/engine.rs
+++ b/src/commands/engine.rs
@@ -137,7 +137,11 @@ fn thin_valid_blocks<P: AsRef<Path>>(path: P, opts: &EngineOptions) -> RoaringBi
return all_blocks(e.get_nr_blocks() as u32);
}
let metadata_root = metadata_root.unwrap();
- let valid_blocks = allocated_blocks(e.clone(), metadata_root.bitmap_root);
+ let valid_blocks = allocated_blocks(
+ e.clone(),
+ metadata_root.bitmap_root,
+ metadata_root.nr_blocks,
+ );
valid_blocks.unwrap_or_else(|_| all_blocks(e.get_nr_blocks() as u32))
}
diff --git a/src/pdata/space_map/allocated_blocks.rs b/src/pdata/space_map/allocated_blocks.rs
index aa75cb7a..71497a76 100644
--- a/src/pdata/space_map/allocated_blocks.rs
+++ b/src/pdata/space_map/allocated_blocks.rs
@@ -17,18 +17,21 @@ struct IndexInfo {
pub fn allocated_blocks(
engine: Arc<dyn IoEngine + Send + Sync>,
sm_root: u64,
+ nr_blocks: u64,
) -> Result<RoaringBitmap> {
// Walk index tree to find where the bitmaps are.
let b = engine.read(sm_root)?;
- let (_, indexes) = MetadataIndex::unpack(b.get_data())?;
+ let indexes = load_metadata_index(&b, nr_blocks)?;
- let mut infos = Vec::new();
- for (key, entry) in indexes.indexes.iter().enumerate() {
- infos.push(IndexInfo {
+ let mut infos: Vec<_> = indexes
+ .indexes
+ .iter()
+ .enumerate()
+ .map(|(key, entry)| IndexInfo {
key: key as u64,
loc: entry.blocknr,
- });
- }
+ })
+ .collect();
// Read bitmaps in sequence
infos.sort_by(|lhs, rhs| lhs.loc.partial_cmp(&rhs.loc).unwrap());
diff --git a/src/pdata/space_map/checker.rs b/src/pdata/space_map/checker.rs
index cfafa79b..7cc8286b 100644
--- a/src/pdata/space_map/checker.rs
+++ b/src/pdata/space_map/checker.rs
@@ -194,10 +194,11 @@ fn gather_disk_index_entries(
fn gather_metadata_index_entries(
engine: Arc<dyn IoEngine + Send + Sync>,
bitmap_root: u64,
+ nr_blocks: u64,
metadata_sm: ASpaceMap,
) -> Result<Vec<IndexEntry>> {
let b = engine.read(bitmap_root)?;
- let entries = check_and_unpack_metadata_index(&b)?.indexes;
+ let entries = load_metadata_index(&b, nr_blocks)?.indexes;
metadata_sm.lock().unwrap().inc(bitmap_root, 1)?;
inc_entries(&metadata_sm, &entries[0..])?;
@@ -254,8 +255,13 @@ pub fn check_metadata_space_map(
metadata_sm.clone(),
false,
)?;
- let entries =
- gather_metadata_index_entries(engine.clone(), root.bitmap_root, metadata_sm.clone())?;
+
+ let entries = gather_metadata_index_entries(
+ engine.clone(),
+ root.bitmap_root,
+ root.nr_blocks,
+ metadata_sm.clone(),
+ )?;
// check overflow ref-counts
{
diff --git a/src/pdata/space_map/metadata.rs b/src/pdata/space_map/metadata.rs
index b466c135..be232389 100644
--- a/src/pdata/space_map/metadata.rs
+++ b/src/pdata/space_map/metadata.rs
@@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex};
use crate::checksum;
use crate::io_engine::*;
+use crate::math::div_up;
use crate::pdata::space_map::common::*;
use crate::pdata::space_map::*;
use crate::pdata::unpack::*;
@@ -32,14 +33,11 @@ impl Unpack for MetadataIndex {
let (i, _csum) = le_u32(i)?;
let (i, _padding) = le_u32(i)?;
let (i, blocknr) = le_u64(i)?;
- let (i, indexes) = nom::multi::count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?;
+ let (i, mut indexes) = nom::multi::count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?;
- // Filter out unused entries
- let indexes: Vec<IndexEntry> = indexes
- .iter()
- .take_while(|e| e.blocknr != 0)
- .cloned()
- .collect();
+ // Drop unused entries that point to block 0
+ let nr_bitmaps = indexes.iter().take_while(|e| e.blocknr != 0).count();
+ indexes.truncate(nr_bitmaps);
Ok((i, MetadataIndex { blocknr, indexes }))
}
@@ -69,9 +67,15 @@ fn verify_checksum(b: &Block) -> Result<()> {
}
}
-pub fn check_and_unpack_metadata_index(b: &Block) -> Result<MetadataIndex> {
+pub fn load_metadata_index(b: &Block, nr_blocks: u64) -> Result<MetadataIndex> {
verify_checksum(b)?;
- unpack::<MetadataIndex>(b.get_data()).map_err(|e| e.into())
+ let mut entries = unpack::<MetadataIndex>(b.get_data())?;
+ if entries.blocknr != b.loc {
+ return Err(anyhow!("blocknr mismatch"));
+ }
+ let nr_bitmaps = div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize;
+ entries.indexes.truncate(nr_bitmaps);
+ Ok(entries)
}
//------------------------------------------
diff --git a/src/pdata/space_map/tests.rs b/src/pdata/space_map/tests.rs
index fb08a9dc..fa118189 100644
--- a/src/pdata/space_map/tests.rs
+++ b/src/pdata/space_map/tests.rs
@@ -171,10 +171,11 @@ mod metadata_sm {
let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size());
w.alloc()?; // reserved for the superblock
let root = write_metadata_sm(&mut w)?;
+ drop(w);
let b = engine.read(root.bitmap_root)?;
- let entries = check_and_unpack_metadata_index(&b)?.indexes;
- ensure!(entries.len() as u64 == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64));
+ let entries = load_metadata_index(&b, root.nr_blocks)?.indexes;
+ ensure!(entries.len() == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize);
// the number of blocks observed by index_entries must be multiple of ENTRIES_PER_BITMAP
let nr_allocated = meta_sm.lock().unwrap().get_nr_allocated()?;
@@ -193,6 +194,35 @@ mod metadata_sm {
fn check_multiple_index_entries() -> Result<()> {
check_index_entries(ENTRIES_PER_BITMAP as u64 * 16 + 1000)
}
+
+ #[test]
+ fn ignore_junk_bytes_in_index_block() -> Result<()> {
+ use crate::checksum;
+ use crate::pdata::space_map::common::IndexEntry;
+ use crate::pdata::unpack::Unpack;
+
+ let nr_blocks = ENTRIES_PER_BITMAP as u64 * 4 + 1000;
+ let nr_bitmaps = div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize;
+ let engine = Arc::new(CoreIoEngine::new(nr_blocks));
+ let meta_sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX);
+
+ let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size());
+ w.alloc()?; // reserved for the superblock
+ let root = write_metadata_sm(&mut w)?;
+
+ // append junk bytes to the unused entry
+ let index_block = w.read(root.bitmap_root)?;
+ index_block.get_data()[nr_bitmaps * IndexEntry::disk_size() as usize + 16] = 1;
+ w.write(index_block, checksum::BT::INDEX)?;
+ w.flush()?;
+ drop(w);
+
+ let b = engine.read(root.bitmap_root)?;
+ let entries = load_metadata_index(&b, root.nr_blocks)?.indexes;
+ ensure!(entries.len() == nr_bitmaps);
+
+ Ok(())
+ }
}
//------------------------------------------
@@ -223,6 +253,7 @@ mod disk_sm {
data_sm.lock().unwrap().inc(0, 100)?;
let root = write_disk_sm(&mut w, data_sm.lock().unwrap().deref())?;
+ drop(w);
let entries =
btree_to_value_vec::<IndexEntry>(&mut Vec::new(), engine, false, root.bitmap_root)?;
diff --git a/src/thin/damage_generator.rs b/src/thin/damage_generator.rs
index 56685d4d..df1be4cd 100644
--- a/src/thin/damage_generator.rs
+++ b/src/thin/damage_generator.rs
@@ -23,7 +23,7 @@ fn find_blocks_of_rc(
let mut found = Vec::<u64>::new();
if ref_count < 3 {
let b = engine.read(sm_root.bitmap_root)?;
- let entries = check_and_unpack_metadata_index(&b)?.indexes;
+ let entries = load_metadata_index(&b, sm_root.nr_blocks)?.indexes;
let bitmaps: Vec<u64> = entries.iter().map(|ie| ie.blocknr).collect();
let nr_bitmaps = bitmaps.len();
@@ -75,7 +75,7 @@ fn adjust_bitmap_entries(
};
let index_block = engine.read(sm_root.bitmap_root)?;
- let entries = check_and_unpack_metadata_index(&index_block)?.indexes;
+ let entries = load_metadata_index(&index_block, sm_root.nr_blocks)?.indexes;
let bi = blocks_to_bitmaps(blocks);
let bitmaps: Vec<u64> = bi.iter().map(|i| entries[*i].blocknr).collect();
diff --git a/src/thin/stat.rs b/src/thin/stat.rs
index 03ae6845..c6f2bf44 100644
--- a/src/thin/stat.rs
+++ b/src/thin/stat.rs
@@ -72,9 +72,10 @@ fn gather_btree_index_entries(
fn gather_metadata_index_entries(
engine: Arc<dyn IoEngine + Send + Sync>,
bitmap_root: u64,
+ nr_blocks: u64,
) -> Result<Vec<IndexEntry>> {
let b = engine.read(bitmap_root)?;
- let entries = check_and_unpack_metadata_index(&b)?.indexes;
+ let entries = load_metadata_index(&b, nr_blocks)?.indexes;
Ok(entries)
}
@@ -152,7 +153,8 @@ fn stat_metadata_block_ref_counts(
) -> Result<BTreeMap<u32, u64>> {
let mut histogram = BTreeMap::<u32, u64>::new();
- let index_entries = gather_metadata_index_entries(engine.clone(), root.bitmap_root)?;
+ let index_entries =
+ gather_metadata_index_entries(engine.clone(), root.bitmap_root, root.nr_blocks)?;
stat_low_ref_counts(engine.clone(), &index_entries, &mut histogram)?;
let histogram = stat_overflow_ref_counts(engine, root.ref_count_root, histogram)?;
--
2.43.0

111
SOURCES/0005-cache_check-Fix-boundary-check-on-the-bitset-for-cac.patch Normal file
View File

@ -0,0 +1,111 @@
From 96fc598f76beac1deb0c9564dc67416f70ae4ac4 Mon Sep 17 00:00:00 2001
From: Ming-Hung Tsai <mtsai@redhat.com>
Date: Tue, 30 Jan 2024 15:20:31 +0800
Subject: [PATCH 5/6] [cache_check] Fix boundary check on the bitset for cached
blocks
The bitset for cached block addresses grows dynamically if the metadata
is not shutdown properly, in which the size hint of the slow (backing)
device is not available. Fix a bug in determining whether resizing is
needed (bz2258485).
(cherry picked from commit d2390a50f38d88f0f32b13e59444bbbca7e660b3)
---
src/cache/check.rs | 6 +++---
tests/cache_check.rs | 45 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 48 insertions(+), 3 deletions(-)
diff --git a/src/cache/check.rs b/src/cache/check.rs
index 18bd51b3..c71d3059 100644
--- a/src/cache/check.rs
+++ b/src/cache/check.rs
@@ -19,7 +19,7 @@ use crate::report::*;
//------------------------------------------
-// 16m entries is capable for a 1TB cache with 64KB block size
+// 16m entries is capable to address a 1TB device with 64KB block size
const DEFAULT_OBLOCKS: usize = 16777216;
fn inc_superblock(sm: &ASpaceMap) -> anyhow::Result<()> {
@@ -82,7 +82,7 @@ mod format1 {
}
let mut seen_oblocks = self.seen_oblocks.lock().unwrap();
- if m.oblock as usize > seen_oblocks.len() {
+ if m.oblock as usize >= seen_oblocks.len() {
seen_oblocks.grow(m.oblock as usize + 1);
} else if seen_oblocks.contains(m.oblock as usize) {
return Err(array::value_err("origin block already mapped".to_string()));
@@ -179,7 +179,7 @@ mod format2 {
));
}
- if m.oblock as usize > seen_oblocks.len() {
+ if m.oblock as usize >= seen_oblocks.len() {
seen_oblocks.grow(m.oblock as usize + 1);
} else if seen_oblocks.contains(m.oblock as usize) {
return Err(array::value_err("origin block already mapped".to_string()));
diff --git a/tests/cache_check.rs b/tests/cache_check.rs
index 81f4c578..8988694a 100644
--- a/tests/cache_check.rs
+++ b/tests/cache_check.rs
@@ -11,6 +11,8 @@ use common::program::*;
use common::target::*;
use common::test_dir::*;
+use std::io::Write;
+
//------------------------------------------
const USAGE: &str = "Validates cache metadata on a device or file.
@@ -294,3 +296,46 @@ fn no_clear_needs_check_if_error() -> Result<()> {
}
//------------------------------------------
+
+fn metadata_without_slow_dev_size_info(use_v1: bool) -> Result<()> {
+ let mut td = TestDir::new()?;
+
+ // The input metadata has a cached oblock with address equals to the default bitset size
+ // boundary (DEFAULT_OBLOCKS = 16777216), triggering bitset resize.
+ let xml = td.mk_path("meta.xml");
+ let mut file = std::fs::File::create(&xml)?;
+ file.write_all(b"<superblock uuid=\"\" block_size=\"128\" nr_cache_blocks=\"1024\" policy=\"smq\" hint_width=\"4\">
+ <mappings>
+ <mapping cache_block=\"0\" origin_block=\"16777216\" dirty=\"false\"/>
+ </mappings>
+ <hints>
+ <hint cache_block=\"0\" data=\"AAAAAA==\"/>
+ </hints>
+</superblock>")?;
+
+ let md = td.mk_path("meta.bin");
+ thinp::file_utils::create_sized_file(&md, 4096 * 4096)?;
+
+ let cache_restore_args = if use_v1 {
+ args!["-i", &xml, "-o", &md, "--metadata-version=1"]
+ } else {
+ args!["-i", &xml, "-o", &md, "--metadata-version=2"]
+ };
+
+ run_ok(cache_restore_cmd(cache_restore_args))?;
+ run_ok(cache_check_cmd(args![&md]))?;
+
+ Ok(())
+}
+
+#[test]
+fn metadata_v1_without_slow_dev_size_info() -> Result<()> {
+ metadata_without_slow_dev_size_info(true)
+}
+
+#[test]
+fn metadata_v2_without_slow_dev_size_info() -> Result<()> {
+ metadata_without_slow_dev_size_info(false)
+}
+
+//------------------------------------------
--
2.43.0

62
SOURCES/0006-thin-cache_check-Print-suggestive-hints-for-improvin.patch Normal file
View File

@ -0,0 +1,62 @@
From e295610fad85ec1a64e569cba425ca557a56c6e6 Mon Sep 17 00:00:00 2001
From: Ming-Hung Tsai <mtsai@redhat.com>
Date: Wed, 31 Jan 2024 11:36:17 +0800
Subject: [PATCH 6/6] [thin/cache_check] Print suggestive hints for improving
error resolution
Enhance error messages to instruct users on addressing recoverable
errors, eliminating the guesswork (bz2233177).
(cherry picked from commit aaf3b396574709902ffba47e03a5c7ded6a103c5)
---
src/cache/check.rs | 5 ++++-
src/thin/check.rs | 10 ++++++++--
2 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/src/cache/check.rs b/src/cache/check.rs
index c71d3059..17d1af77 100644
--- a/src/cache/check.rs
+++ b/src/cache/check.rs
@@ -387,7 +387,10 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> {
ctx.report.warning("Repairing metadata leaks.");
repair_space_map(ctx.engine.clone(), metadata_leaks, metadata_sm.clone())?;
} else if !opts.ignore_non_fatal {
- return Err(anyhow!("metadata space map contains leaks"));
+ return Err(anyhow!(concat!(
+ "metadata space map contains leaks\n",
+ "perhaps you wanted to run with --auto-repair"
+ )));
}
}
diff --git a/src/thin/check.rs b/src/thin/check.rs
index 8b829899..f6fde359 100644
--- a/src/thin/check.rs
+++ b/src/thin/check.rs
@@ -1246,7 +1246,10 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> {
report.warning("Repairing data leaks.");
repair_space_map(engine.clone(), data_leaks, data_sm.clone())?;
} else if !opts.ignore_non_fatal {
- return Err(anyhow!("data space map contains leaks"));
+ return Err(anyhow!(concat!(
+ "data space map contains leaks\n",
+ "perhaps you wanted to run with --auto-repair"
+ )));
}
}
@@ -1255,7 +1258,10 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> {
report.warning("Repairing metadata leaks.");
repair_space_map(engine.clone(), metadata_leaks, metadata_sm.clone())?;
} else if !opts.ignore_non_fatal {
- return Err(anyhow!("metadata space map contains leaks"));
+ return Err(anyhow!(concat!(
+ "metadata space map contains leaks\n",
+ "perhaps you wanted to run with --auto-repair"
+ )));
}
}
--
2.43.0

BIN
SOURCES/dmpd106-vendor.tar.gz
View File

Binary file not shown.

BIN
SOURCES/v1.0.6.tar.gz
View File

Binary file not shown.

26
SPECS/device-mapper-persistent-data.spec
View File

@ -9,14 +9,19 @@
Summary: Device-mapper Persistent Data Tools Summary: Device-mapper Persistent Data Tools
Name: device-mapper-persistent-data Name: device-mapper-persistent-data
Version: 1.0.6 Version: 1.0.9
Release: 1%{?dist}%{?release_suffix} Release: 2%{?dist}%{?release_suffix}
License: GPLv3+ License: GPLv3+
URL: https://github.com/jthornber/thin-provisioning-tools URL: https://github.com/jthornber/thin-provisioning-tools
#Source0: https://github.com/jthornber/thin-provisioning-tools/archive/thin-provisioning-tools-%%{version}.tar.gz #Source0: https://github.com/jthornber/thin-provisioning-tools/archive/thin-provisioning-tools-%%{version}.tar.gz
Source0: https://github.com/jthornber/thin-provisioning-tools/archive/v%{version}%{?version_suffix}.tar.gz Source0: https://github.com/jthornber/thin-provisioning-tools/archive/v%{version}%{?version_suffix}.tar.gz
Source1: dmpd106-vendor.tar.gz Source1: dmpd109-vendor.tar.gz
Patch1: 0001-Tweak-cargo.toml-to-work-with-vendor-directory.patch Patch1: 0001-Tweak-cargo.toml-to-work-with-vendor-directory.patch
Patch2: 0002-space-map-Fix-incorrect-index_entry.nr_free-while-ex.patch
Patch3: 0003-thin_repair-Fix-child-keys-checking-on-the-node-with.patch
Patch4: 0004-space_map-Allow-non-zero-values-in-unused-index-bloc.patch
Patch5: 0005-cache_check-Fix-boundary-check-on-the-bitset-for-cac.patch
Patch6: 0006-thin-cache_check-Print-suggestive-hints-for-improvin.patch
BuildRequires: rust-packaging BuildRequires: rust-packaging
BuildRequires: rust >= 1.35 BuildRequires: rust >= 1.35
@ -54,7 +59,10 @@ echo %{version}-%{release} > VERSION
%if %{with check} %if %{with check}
%check %check
RUST_BACKTRACE=1 %cargo_test || true # aarch64 is failing, but only in brew environment, tests are passing when
# running locally
#%%cargo_test
RUST_BACKTRACE=1 %%cargo_test -- --nocapture --test-threads=1 || true
%endif %endif
%install %install
@ -108,6 +116,16 @@ make DESTDIR=%{buildroot} MANDIR=%{_mandir} install
#% {_sbindir}/thin_show_duplicates #% {_sbindir}/thin_show_duplicates
%changelog %changelog
* Thu Feb 08 2024 Marian Csontos <mcsontos@redhat.com> - 1.0.9-2
- Allow non-zero values in unused index block entries.
- Fix boundary check on the bitset for cached blocks.
- Fix incorrect index_entry.nr_free on expansion affecting free space estimate.
- Fix thin_repair checking keys on a node with a zero key.
- Enhance error message in cache_check suggesting fix when applicable.
* Wed Dec 13 2023 Marian Csontos <mcsontos@redhat.com> - 1.0.9-1
- Update to latest upstream release 1.0.9.
* Wed Aug 09 2023 Marian Csontos <mcsontos@redhat.com> - 1.0.6-1 * Wed Aug 09 2023 Marian Csontos <mcsontos@redhat.com> - 1.0.6-1
- Update to latest upstream release 1.0.6. - Update to latest upstream release 1.0.6.