import CS device-mapper-persistent-data-1.0.9-2.el9
This commit is contained in:
parent
586710bc2d
commit
8900263a56
2
.device-mapper-persistent-data.metadata
Normal file
2
.device-mapper-persistent-data.metadata
Normal file
@ -0,0 +1,2 @@
|
||||
d336474a4772430fa2b8a5f76a6c2f2a78caa698 SOURCES/dmpd109-vendor.tar.gz
|
||||
8862f41bf5fad011879b6d819ab5e5b11afcda7e SOURCES/v1.0.9.tar.gz
|
2
.gitignore
vendored
2
.gitignore
vendored
@ -0,0 +1,2 @@
|
||||
SOURCES/dmpd109-vendor.tar.gz
|
||||
SOURCES/v1.0.9.tar.gz
|
@ -1,4 +1,4 @@
|
||||
From 732ff5861a1525944a927439d1c075ac269788ce Mon Sep 17 00:00:00 2001
|
||||
From 0d5347bd771e960294cd0c2f083d96448613ab9c Mon Sep 17 00:00:00 2001
|
||||
From: Marian Csontos <mcsontos@redhat.com>
|
||||
Date: Thu, 27 Jul 2023 11:37:01 +0200
|
||||
Subject: [PATCH] Tweak cargo.toml to work with vendor directory
|
||||
@ -7,24 +7,16 @@ Mock works offline, cargo would try to download the files from github.
|
||||
So cargo vendor has to be run first, and then change the Cargo.toml to
|
||||
make mock happy.
|
||||
---
|
||||
Cargo.toml | 4 +++-
|
||||
1 file changed, 3 insertions(+), 1 deletion(-)
|
||||
Cargo.toml | 3 ++-
|
||||
1 file changed, 2 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/Cargo.toml b/Cargo.toml
|
||||
index c2b496ac..562c40d3 100644
|
||||
index 500345a4..d4aa38a6 100644
|
||||
--- a/Cargo.toml
|
||||
+++ b/Cargo.toml
|
||||
@@ -11,6 +11,7 @@ anyhow = "1.0"
|
||||
base64 = "0.21"
|
||||
byteorder = "1.4"
|
||||
clap = { version = "4.3", default-features = false, features = ["std", "help", "usage", "error-context", "suggestions"] }
|
||||
+#crc32c = { git = "https://github.com/zowens/crc32c", branch = "master" }
|
||||
crc32c = "0.6"
|
||||
data-encoding = "2.4"
|
||||
exitcode = "1.1.2"
|
||||
@@ -27,7 +28,8 @@ quick-xml = "0.29"
|
||||
@@ -27,7 +27,8 @@ quick-xml = "0.29"
|
||||
rand = "0.8"
|
||||
rangemap = "1.3"
|
||||
rangemap = "1.4"
|
||||
roaring = "0.10"
|
||||
-rio = { git = "https://github.com/jthornber/rio", branch = "master", optional = true }
|
||||
+#rio = { git = "https://github.com/jthornber/rio", branch = "master", optional = true }
|
||||
@ -33,5 +25,5 @@ index c2b496ac..562c40d3 100644
|
||||
threadpool = "1.8"
|
||||
thiserror = "1.0"
|
||||
--
|
||||
2.41.0
|
||||
2.43.0
|
||||
|
||||
|
@ -0,0 +1,181 @@
|
||||
From f088ce90f5f7934489a8d1062ebfd3d23e4aec38 Mon Sep 17 00:00:00 2001
|
||||
From: Ming-Hung Tsai <mtsai@redhat.com>
|
||||
Date: Tue, 12 Dec 2023 14:25:27 +0800
|
||||
Subject: [PATCH 2/6] [space map] Fix incorrect index_entry.nr_free while
|
||||
expansion
|
||||
|
||||
Do not truncate the nr_free value of the last index_entry to the space
|
||||
map boundary, to address the issue in kernel that sm_ll_extend() doesn't
|
||||
extend the nr_free value of the last index_entry while expanding the
|
||||
space map. Without this fix, we'll have incorrect free space estimation
|
||||
in later block allocations, leading to under-utilized space maps.
|
||||
|
||||
(cherry picked from commit 9a405f57c591020321fc16f00efdb5197e1df2c0)
|
||||
---
|
||||
src/pdata/space_map/common.rs | 20 +++++---
|
||||
src/pdata/space_map/tests.rs | 96 +++++++++++++++++++++++++++++++++++
|
||||
2 files changed, 108 insertions(+), 8 deletions(-)
|
||||
|
||||
diff --git a/src/pdata/space_map/common.rs b/src/pdata/space_map/common.rs
|
||||
index d92b8115..4be9c303 100644
|
||||
--- a/src/pdata/space_map/common.rs
|
||||
+++ b/src/pdata/space_map/common.rs
|
||||
@@ -211,22 +211,28 @@ pub fn write_common(
|
||||
let len = std::cmp::min(nr_blocks - begin, ENTRIES_PER_BITMAP as u64);
|
||||
let mut entries = Vec::with_capacity(ENTRIES_PER_BITMAP);
|
||||
let mut first_free: Option<u32> = None;
|
||||
- let mut nr_free: u32 = 0;
|
||||
+ let mut nr_free = ENTRIES_PER_BITMAP as u32; // do not truncate to the sm size boundary
|
||||
|
||||
for i in 0..len {
|
||||
let b = begin + i;
|
||||
let rc = sm.get(b)?;
|
||||
let e = match rc {
|
||||
0 => {
|
||||
- nr_free += 1;
|
||||
if first_free.is_none() {
|
||||
first_free = Some(i as u32);
|
||||
}
|
||||
Small(0)
|
||||
}
|
||||
- 1 => Small(1),
|
||||
- 2 => Small(2),
|
||||
+ 1 => {
|
||||
+ nr_free -= 1;
|
||||
+ Small(1)
|
||||
+ }
|
||||
+ 2 => {
|
||||
+ nr_free -= 1;
|
||||
+ Small(2)
|
||||
+ }
|
||||
_ => {
|
||||
+ nr_free -= 1;
|
||||
overflow_builder.push_value(w, b, rc)?;
|
||||
Overflow
|
||||
}
|
||||
@@ -334,16 +340,14 @@ pub fn write_metadata_common(w: &mut WriteBatcher) -> anyhow::Result<(Vec<IndexE
|
||||
let nr_blocks = w.sm.lock().unwrap().get_nr_blocks()?;
|
||||
let nr_bitmaps = div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize;
|
||||
let nr_used_bitmaps = index_entries.len();
|
||||
- for bm in nr_used_bitmaps..nr_bitmaps {
|
||||
- let begin = bm as u64 * ENTRIES_PER_BITMAP as u64;
|
||||
- let len = std::cmp::min(nr_blocks - begin, ENTRIES_PER_BITMAP as u64);
|
||||
+ for _bm in nr_used_bitmaps..nr_bitmaps {
|
||||
let entries = vec![BitmapEntry::Small(0); ENTRIES_PER_BITMAP];
|
||||
let blocknr = write_bitmap(w, entries)?;
|
||||
|
||||
// Insert into the index list
|
||||
let ie = IndexEntry {
|
||||
blocknr,
|
||||
- nr_free: len as u32,
|
||||
+ nr_free: ENTRIES_PER_BITMAP as u32, // do not truncate to the sm size boundary
|
||||
none_free_before: 0,
|
||||
};
|
||||
index_entries.push(ie);
|
||||
diff --git a/src/pdata/space_map/tests.rs b/src/pdata/space_map/tests.rs
|
||||
index 26d7834d..fb08a9dc 100644
|
||||
--- a/src/pdata/space_map/tests.rs
|
||||
+++ b/src/pdata/space_map/tests.rs
|
||||
@@ -152,3 +152,99 @@ mod core_sm_u8 {
|
||||
}
|
||||
|
||||
//------------------------------------------
|
||||
+
|
||||
+mod metadata_sm {
|
||||
+ use anyhow::{ensure, Result};
|
||||
+ use std::sync::Arc;
|
||||
+
|
||||
+ use crate::io_engine::core::CoreIoEngine;
|
||||
+ use crate::io_engine::*;
|
||||
+ use crate::math::div_up;
|
||||
+ use crate::pdata::space_map::common::ENTRIES_PER_BITMAP;
|
||||
+ use crate::pdata::space_map::metadata::*;
|
||||
+ use crate::write_batcher::WriteBatcher;
|
||||
+
|
||||
+ fn check_index_entries(nr_blocks: u64) -> Result<()> {
|
||||
+ let engine = Arc::new(CoreIoEngine::new(nr_blocks));
|
||||
+ let meta_sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX);
|
||||
+
|
||||
+ let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size());
|
||||
+ w.alloc()?; // reserved for the superblock
|
||||
+ let root = write_metadata_sm(&mut w)?;
|
||||
+
|
||||
+ let b = engine.read(root.bitmap_root)?;
|
||||
+ let entries = check_and_unpack_metadata_index(&b)?.indexes;
|
||||
+ ensure!(entries.len() as u64 == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64));
|
||||
+
|
||||
+ // the number of blocks observed by index_entries must be multiple of ENTRIES_PER_BITMAP
|
||||
+ let nr_allocated = meta_sm.lock().unwrap().get_nr_allocated()?;
|
||||
+ let nr_free: u64 = entries.iter().map(|ie| ie.nr_free as u64).sum();
|
||||
+ ensure!(nr_allocated + nr_free == (entries.len() * ENTRIES_PER_BITMAP) as u64);
|
||||
+
|
||||
+ Ok(())
|
||||
+ }
|
||||
+
|
||||
+ #[test]
|
||||
+ fn check_single_index_entry() -> Result<()> {
|
||||
+ check_index_entries(1000)
|
||||
+ }
|
||||
+
|
||||
+ #[test]
|
||||
+ fn check_multiple_index_entries() -> Result<()> {
|
||||
+ check_index_entries(ENTRIES_PER_BITMAP as u64 * 16 + 1000)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+//------------------------------------------
|
||||
+
|
||||
+mod disk_sm {
|
||||
+ use anyhow::{ensure, Result};
|
||||
+ use std::ops::Deref;
|
||||
+ use std::sync::Arc;
|
||||
+
|
||||
+ use crate::io_engine::core::CoreIoEngine;
|
||||
+ use crate::io_engine::*;
|
||||
+ use crate::math::div_up;
|
||||
+ use crate::pdata::btree_walker::btree_to_value_vec;
|
||||
+ use crate::pdata::space_map::common::{IndexEntry, ENTRIES_PER_BITMAP};
|
||||
+ use crate::pdata::space_map::disk::*;
|
||||
+ use crate::pdata::space_map::metadata::*;
|
||||
+ use crate::pdata::space_map::*;
|
||||
+ use crate::write_batcher::WriteBatcher;
|
||||
+
|
||||
+ fn check_index_entries(nr_blocks: u64) -> Result<()> {
|
||||
+ let engine = Arc::new(CoreIoEngine::new(1024));
|
||||
+ let meta_sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX);
|
||||
+
|
||||
+ let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size());
|
||||
+ w.alloc()?; // reserved for the superblock
|
||||
+
|
||||
+ let data_sm = core_sm(nr_blocks, u32::MAX);
|
||||
+ data_sm.lock().unwrap().inc(0, 100)?;
|
||||
+
|
||||
+ let root = write_disk_sm(&mut w, data_sm.lock().unwrap().deref())?;
|
||||
+
|
||||
+ let entries =
|
||||
+ btree_to_value_vec::<IndexEntry>(&mut Vec::new(), engine, false, root.bitmap_root)?;
|
||||
+ ensure!(entries.len() as u64 == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64));
|
||||
+
|
||||
+ // the number of blocks observed by index_entries must be a multiple of ENTRIES_PER_BITMAP
|
||||
+ let nr_allocated = data_sm.lock().unwrap().get_nr_allocated()?;
|
||||
+ let nr_free: u64 = entries.iter().map(|ie| ie.nr_free as u64).sum();
|
||||
+ ensure!(nr_allocated + nr_free == (entries.len() * ENTRIES_PER_BITMAP) as u64);
|
||||
+
|
||||
+ Ok(())
|
||||
+ }
|
||||
+
|
||||
+ #[test]
|
||||
+ fn check_single_index_entry() -> Result<()> {
|
||||
+ check_index_entries(1000)
|
||||
+ }
|
||||
+
|
||||
+ #[test]
|
||||
+ fn check_multiple_index_entries() -> Result<()> {
|
||||
+ check_index_entries(ENTRIES_PER_BITMAP as u64 * 16 + 1000)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+//------------------------------------------
|
||||
--
|
||||
2.43.0
|
||||
|
@ -0,0 +1,69 @@
|
||||
From add81da22a3998503a6f340350d7e59ed3b52e28 Mon Sep 17 00:00:00 2001
|
||||
From: Ming-Hung Tsai <mtsai@redhat.com>
|
||||
Date: Wed, 10 Jan 2024 15:56:39 +0800
|
||||
Subject: [PATCH 3/6] [thin_repair] Fix child keys checking on the node with a
|
||||
zero key
|
||||
|
||||
Fix the issue that keys overlapping between the second and the first
|
||||
child nodes indexed by zero not being checked.
|
||||
|
||||
(cherry picked from commit 386123bd0f74f7603e993bf3c26aac002162d5db)
|
||||
---
|
||||
src/thin/metadata_repair.rs | 27 +++++++++++++++------------
|
||||
1 file changed, 15 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/src/thin/metadata_repair.rs b/src/thin/metadata_repair.rs
|
||||
index 9716b1e3..8fece4b9 100644
|
||||
--- a/src/thin/metadata_repair.rs
|
||||
+++ b/src/thin/metadata_repair.rs
|
||||
@@ -128,10 +128,11 @@ impl DevInfo {
|
||||
}
|
||||
|
||||
fn push_child(&mut self, child: &DevInfo) -> Result<()> {
|
||||
- if self.key_high > 0 && child.key_low <= self.key_high {
|
||||
- return Err(anyhow!("incompatible child"));
|
||||
- }
|
||||
- if !self.pushed {
|
||||
+ if self.pushed {
|
||||
+ if child.key_low <= self.key_high {
|
||||
+ return Err(anyhow!("incompatible child"));
|
||||
+ }
|
||||
+ } else {
|
||||
self.key_low = child.key_low;
|
||||
self.pushed = true;
|
||||
}
|
||||
@@ -175,10 +176,11 @@ impl MappingsInfo {
|
||||
}
|
||||
|
||||
fn push_child(&mut self, child: &MappingsInfo) -> Result<()> {
|
||||
- if self.key_high > 0 && child.key_low <= self.key_high {
|
||||
- return Err(anyhow!("incompatible child"));
|
||||
- }
|
||||
- if !self.pushed {
|
||||
+ if self.pushed {
|
||||
+ if child.key_low <= self.key_high {
|
||||
+ return Err(anyhow!("incompatible child"));
|
||||
+ }
|
||||
+ } else {
|
||||
self.key_low = child.key_low;
|
||||
self.pushed = true;
|
||||
}
|
||||
@@ -221,10 +223,11 @@ impl DetailsInfo {
|
||||
}
|
||||
|
||||
fn push_child(&mut self, child: &DetailsInfo) -> Result<()> {
|
||||
- if self.key_high > 0 && child.key_low <= self.key_high {
|
||||
- return Err(anyhow!("incompatible child"));
|
||||
- }
|
||||
- if !self.pushed {
|
||||
+ if self.pushed {
|
||||
+ if child.key_low <= self.key_high {
|
||||
+ return Err(anyhow!("incompatible child"));
|
||||
+ }
|
||||
+ } else {
|
||||
self.key_low = child.key_low;
|
||||
self.pushed = true;
|
||||
}
|
||||
--
|
||||
2.43.0
|
||||
|
@ -0,0 +1,269 @@
|
||||
From 9b948a9639b00fe9226065ca4dd6fd2257cc126c Mon Sep 17 00:00:00 2001
|
||||
From: Ming-Hung Tsai <mtsai@redhat.com>
|
||||
Date: Thu, 11 Jan 2024 22:57:36 +0800
|
||||
Subject: [PATCH 4/6] [space_map] Allow non-zero values in unused index block
|
||||
entries
|
||||
|
||||
Previously, we assumed unused entries in the index block were all
|
||||
zero-initialized, leading to issues while loading the block with
|
||||
unexpected bytes and a valid checksum [1]. The updated approach loads
|
||||
index entries based on actual size information from superblock and
|
||||
therefore improves compatibility.
|
||||
|
||||
[1] stratis-storage/stratisd#3520
|
||||
|
||||
(cherry picked from commit d5fe6a1e1c539a0f260a96eb4d7ed9b83c3f84c9)
|
||||
---
|
||||
src/commands/engine.rs | 6 ++++-
|
||||
src/pdata/space_map/allocated_blocks.rs | 15 ++++++-----
|
||||
src/pdata/space_map/checker.rs | 12 ++++++---
|
||||
src/pdata/space_map/metadata.rs | 22 +++++++++-------
|
||||
src/pdata/space_map/tests.rs | 35 +++++++++++++++++++++++--
|
||||
src/thin/damage_generator.rs | 4 +--
|
||||
src/thin/stat.rs | 6 +++--
|
||||
7 files changed, 75 insertions(+), 25 deletions(-)
|
||||
|
||||
diff --git a/src/commands/engine.rs b/src/commands/engine.rs
|
||||
index 1e752d6e..e8b75301 100644
|
||||
--- a/src/commands/engine.rs
|
||||
+++ b/src/commands/engine.rs
|
||||
@@ -137,7 +137,11 @@ fn thin_valid_blocks<P: AsRef<Path>>(path: P, opts: &EngineOptions) -> RoaringBi
|
||||
return all_blocks(e.get_nr_blocks() as u32);
|
||||
}
|
||||
let metadata_root = metadata_root.unwrap();
|
||||
- let valid_blocks = allocated_blocks(e.clone(), metadata_root.bitmap_root);
|
||||
+ let valid_blocks = allocated_blocks(
|
||||
+ e.clone(),
|
||||
+ metadata_root.bitmap_root,
|
||||
+ metadata_root.nr_blocks,
|
||||
+ );
|
||||
valid_blocks.unwrap_or_else(|_| all_blocks(e.get_nr_blocks() as u32))
|
||||
}
|
||||
|
||||
diff --git a/src/pdata/space_map/allocated_blocks.rs b/src/pdata/space_map/allocated_blocks.rs
|
||||
index aa75cb7a..71497a76 100644
|
||||
--- a/src/pdata/space_map/allocated_blocks.rs
|
||||
+++ b/src/pdata/space_map/allocated_blocks.rs
|
||||
@@ -17,18 +17,21 @@ struct IndexInfo {
|
||||
pub fn allocated_blocks(
|
||||
engine: Arc<dyn IoEngine + Send + Sync>,
|
||||
sm_root: u64,
|
||||
+ nr_blocks: u64,
|
||||
) -> Result<RoaringBitmap> {
|
||||
// Walk index tree to find where the bitmaps are.
|
||||
let b = engine.read(sm_root)?;
|
||||
- let (_, indexes) = MetadataIndex::unpack(b.get_data())?;
|
||||
+ let indexes = load_metadata_index(&b, nr_blocks)?;
|
||||
|
||||
- let mut infos = Vec::new();
|
||||
- for (key, entry) in indexes.indexes.iter().enumerate() {
|
||||
- infos.push(IndexInfo {
|
||||
+ let mut infos: Vec<_> = indexes
|
||||
+ .indexes
|
||||
+ .iter()
|
||||
+ .enumerate()
|
||||
+ .map(|(key, entry)| IndexInfo {
|
||||
key: key as u64,
|
||||
loc: entry.blocknr,
|
||||
- });
|
||||
- }
|
||||
+ })
|
||||
+ .collect();
|
||||
|
||||
// Read bitmaps in sequence
|
||||
infos.sort_by(|lhs, rhs| lhs.loc.partial_cmp(&rhs.loc).unwrap());
|
||||
diff --git a/src/pdata/space_map/checker.rs b/src/pdata/space_map/checker.rs
|
||||
index cfafa79b..7cc8286b 100644
|
||||
--- a/src/pdata/space_map/checker.rs
|
||||
+++ b/src/pdata/space_map/checker.rs
|
||||
@@ -194,10 +194,11 @@ fn gather_disk_index_entries(
|
||||
fn gather_metadata_index_entries(
|
||||
engine: Arc<dyn IoEngine + Send + Sync>,
|
||||
bitmap_root: u64,
|
||||
+ nr_blocks: u64,
|
||||
metadata_sm: ASpaceMap,
|
||||
) -> Result<Vec<IndexEntry>> {
|
||||
let b = engine.read(bitmap_root)?;
|
||||
- let entries = check_and_unpack_metadata_index(&b)?.indexes;
|
||||
+ let entries = load_metadata_index(&b, nr_blocks)?.indexes;
|
||||
metadata_sm.lock().unwrap().inc(bitmap_root, 1)?;
|
||||
inc_entries(&metadata_sm, &entries[0..])?;
|
||||
|
||||
@@ -254,8 +255,13 @@ pub fn check_metadata_space_map(
|
||||
metadata_sm.clone(),
|
||||
false,
|
||||
)?;
|
||||
- let entries =
|
||||
- gather_metadata_index_entries(engine.clone(), root.bitmap_root, metadata_sm.clone())?;
|
||||
+
|
||||
+ let entries = gather_metadata_index_entries(
|
||||
+ engine.clone(),
|
||||
+ root.bitmap_root,
|
||||
+ root.nr_blocks,
|
||||
+ metadata_sm.clone(),
|
||||
+ )?;
|
||||
|
||||
// check overflow ref-counts
|
||||
{
|
||||
diff --git a/src/pdata/space_map/metadata.rs b/src/pdata/space_map/metadata.rs
|
||||
index b466c135..be232389 100644
|
||||
--- a/src/pdata/space_map/metadata.rs
|
||||
+++ b/src/pdata/space_map/metadata.rs
|
||||
@@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::checksum;
|
||||
use crate::io_engine::*;
|
||||
+use crate::math::div_up;
|
||||
use crate::pdata::space_map::common::*;
|
||||
use crate::pdata::space_map::*;
|
||||
use crate::pdata::unpack::*;
|
||||
@@ -32,14 +33,11 @@ impl Unpack for MetadataIndex {
|
||||
let (i, _csum) = le_u32(i)?;
|
||||
let (i, _padding) = le_u32(i)?;
|
||||
let (i, blocknr) = le_u64(i)?;
|
||||
- let (i, indexes) = nom::multi::count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?;
|
||||
+ let (i, mut indexes) = nom::multi::count(IndexEntry::unpack, MAX_METADATA_BITMAPS)(i)?;
|
||||
|
||||
- // Filter out unused entries
|
||||
- let indexes: Vec<IndexEntry> = indexes
|
||||
- .iter()
|
||||
- .take_while(|e| e.blocknr != 0)
|
||||
- .cloned()
|
||||
- .collect();
|
||||
+ // Drop unused entries that point to block 0
|
||||
+ let nr_bitmaps = indexes.iter().take_while(|e| e.blocknr != 0).count();
|
||||
+ indexes.truncate(nr_bitmaps);
|
||||
|
||||
Ok((i, MetadataIndex { blocknr, indexes }))
|
||||
}
|
||||
@@ -69,9 +67,15 @@ fn verify_checksum(b: &Block) -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
-pub fn check_and_unpack_metadata_index(b: &Block) -> Result<MetadataIndex> {
|
||||
+pub fn load_metadata_index(b: &Block, nr_blocks: u64) -> Result<MetadataIndex> {
|
||||
verify_checksum(b)?;
|
||||
- unpack::<MetadataIndex>(b.get_data()).map_err(|e| e.into())
|
||||
+ let mut entries = unpack::<MetadataIndex>(b.get_data())?;
|
||||
+ if entries.blocknr != b.loc {
|
||||
+ return Err(anyhow!("blocknr mismatch"));
|
||||
+ }
|
||||
+ let nr_bitmaps = div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize;
|
||||
+ entries.indexes.truncate(nr_bitmaps);
|
||||
+ Ok(entries)
|
||||
}
|
||||
|
||||
//------------------------------------------
|
||||
diff --git a/src/pdata/space_map/tests.rs b/src/pdata/space_map/tests.rs
|
||||
index fb08a9dc..fa118189 100644
|
||||
--- a/src/pdata/space_map/tests.rs
|
||||
+++ b/src/pdata/space_map/tests.rs
|
||||
@@ -171,10 +171,11 @@ mod metadata_sm {
|
||||
let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size());
|
||||
w.alloc()?; // reserved for the superblock
|
||||
let root = write_metadata_sm(&mut w)?;
|
||||
+ drop(w);
|
||||
|
||||
let b = engine.read(root.bitmap_root)?;
|
||||
- let entries = check_and_unpack_metadata_index(&b)?.indexes;
|
||||
- ensure!(entries.len() as u64 == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64));
|
||||
+ let entries = load_metadata_index(&b, root.nr_blocks)?.indexes;
|
||||
+ ensure!(entries.len() == div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize);
|
||||
|
||||
// the number of blocks observed by index_entries must be multiple of ENTRIES_PER_BITMAP
|
||||
let nr_allocated = meta_sm.lock().unwrap().get_nr_allocated()?;
|
||||
@@ -193,6 +194,35 @@ mod metadata_sm {
|
||||
fn check_multiple_index_entries() -> Result<()> {
|
||||
check_index_entries(ENTRIES_PER_BITMAP as u64 * 16 + 1000)
|
||||
}
|
||||
+
|
||||
+ #[test]
|
||||
+ fn ignore_junk_bytes_in_index_block() -> Result<()> {
|
||||
+ use crate::checksum;
|
||||
+ use crate::pdata::space_map::common::IndexEntry;
|
||||
+ use crate::pdata::unpack::Unpack;
|
||||
+
|
||||
+ let nr_blocks = ENTRIES_PER_BITMAP as u64 * 4 + 1000;
|
||||
+ let nr_bitmaps = div_up(nr_blocks, ENTRIES_PER_BITMAP as u64) as usize;
|
||||
+ let engine = Arc::new(CoreIoEngine::new(nr_blocks));
|
||||
+ let meta_sm = core_metadata_sm(engine.get_nr_blocks(), u32::MAX);
|
||||
+
|
||||
+ let mut w = WriteBatcher::new(engine.clone(), meta_sm.clone(), engine.get_batch_size());
|
||||
+ w.alloc()?; // reserved for the superblock
|
||||
+ let root = write_metadata_sm(&mut w)?;
|
||||
+
|
||||
+ // append junk bytes to the unused entry
|
||||
+ let index_block = w.read(root.bitmap_root)?;
|
||||
+ index_block.get_data()[nr_bitmaps * IndexEntry::disk_size() as usize + 16] = 1;
|
||||
+ w.write(index_block, checksum::BT::INDEX)?;
|
||||
+ w.flush()?;
|
||||
+ drop(w);
|
||||
+
|
||||
+ let b = engine.read(root.bitmap_root)?;
|
||||
+ let entries = load_metadata_index(&b, root.nr_blocks)?.indexes;
|
||||
+ ensure!(entries.len() == nr_bitmaps);
|
||||
+
|
||||
+ Ok(())
|
||||
+ }
|
||||
}
|
||||
|
||||
//------------------------------------------
|
||||
@@ -223,6 +253,7 @@ mod disk_sm {
|
||||
data_sm.lock().unwrap().inc(0, 100)?;
|
||||
|
||||
let root = write_disk_sm(&mut w, data_sm.lock().unwrap().deref())?;
|
||||
+ drop(w);
|
||||
|
||||
let entries =
|
||||
btree_to_value_vec::<IndexEntry>(&mut Vec::new(), engine, false, root.bitmap_root)?;
|
||||
diff --git a/src/thin/damage_generator.rs b/src/thin/damage_generator.rs
|
||||
index 56685d4d..df1be4cd 100644
|
||||
--- a/src/thin/damage_generator.rs
|
||||
+++ b/src/thin/damage_generator.rs
|
||||
@@ -23,7 +23,7 @@ fn find_blocks_of_rc(
|
||||
let mut found = Vec::<u64>::new();
|
||||
if ref_count < 3 {
|
||||
let b = engine.read(sm_root.bitmap_root)?;
|
||||
- let entries = check_and_unpack_metadata_index(&b)?.indexes;
|
||||
+ let entries = load_metadata_index(&b, sm_root.nr_blocks)?.indexes;
|
||||
let bitmaps: Vec<u64> = entries.iter().map(|ie| ie.blocknr).collect();
|
||||
let nr_bitmaps = bitmaps.len();
|
||||
|
||||
@@ -75,7 +75,7 @@ fn adjust_bitmap_entries(
|
||||
};
|
||||
|
||||
let index_block = engine.read(sm_root.bitmap_root)?;
|
||||
- let entries = check_and_unpack_metadata_index(&index_block)?.indexes;
|
||||
+ let entries = load_metadata_index(&index_block, sm_root.nr_blocks)?.indexes;
|
||||
|
||||
let bi = blocks_to_bitmaps(blocks);
|
||||
let bitmaps: Vec<u64> = bi.iter().map(|i| entries[*i].blocknr).collect();
|
||||
diff --git a/src/thin/stat.rs b/src/thin/stat.rs
|
||||
index 03ae6845..c6f2bf44 100644
|
||||
--- a/src/thin/stat.rs
|
||||
+++ b/src/thin/stat.rs
|
||||
@@ -72,9 +72,10 @@ fn gather_btree_index_entries(
|
||||
fn gather_metadata_index_entries(
|
||||
engine: Arc<dyn IoEngine + Send + Sync>,
|
||||
bitmap_root: u64,
|
||||
+ nr_blocks: u64,
|
||||
) -> Result<Vec<IndexEntry>> {
|
||||
let b = engine.read(bitmap_root)?;
|
||||
- let entries = check_and_unpack_metadata_index(&b)?.indexes;
|
||||
+ let entries = load_metadata_index(&b, nr_blocks)?.indexes;
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
@@ -152,7 +153,8 @@ fn stat_metadata_block_ref_counts(
|
||||
) -> Result<BTreeMap<u32, u64>> {
|
||||
let mut histogram = BTreeMap::<u32, u64>::new();
|
||||
|
||||
- let index_entries = gather_metadata_index_entries(engine.clone(), root.bitmap_root)?;
|
||||
+ let index_entries =
|
||||
+ gather_metadata_index_entries(engine.clone(), root.bitmap_root, root.nr_blocks)?;
|
||||
stat_low_ref_counts(engine.clone(), &index_entries, &mut histogram)?;
|
||||
|
||||
let histogram = stat_overflow_ref_counts(engine, root.ref_count_root, histogram)?;
|
||||
--
|
||||
2.43.0
|
||||
|
@ -0,0 +1,111 @@
|
||||
From 96fc598f76beac1deb0c9564dc67416f70ae4ac4 Mon Sep 17 00:00:00 2001
|
||||
From: Ming-Hung Tsai <mtsai@redhat.com>
|
||||
Date: Tue, 30 Jan 2024 15:20:31 +0800
|
||||
Subject: [PATCH 5/6] [cache_check] Fix boundary check on the bitset for cached
|
||||
blocks
|
||||
|
||||
The bitset for cached block addresses grows dynamically if the metadata
|
||||
is not shutdown properly, in which the size hint of the slow (backing)
|
||||
device is not available. Fix a bug in determining whether resizing is
|
||||
needed (bz2258485).
|
||||
|
||||
(cherry picked from commit d2390a50f38d88f0f32b13e59444bbbca7e660b3)
|
||||
---
|
||||
src/cache/check.rs | 6 +++---
|
||||
tests/cache_check.rs | 45 ++++++++++++++++++++++++++++++++++++++++++++
|
||||
2 files changed, 48 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/src/cache/check.rs b/src/cache/check.rs
|
||||
index 18bd51b3..c71d3059 100644
|
||||
--- a/src/cache/check.rs
|
||||
+++ b/src/cache/check.rs
|
||||
@@ -19,7 +19,7 @@ use crate::report::*;
|
||||
|
||||
//------------------------------------------
|
||||
|
||||
-// 16m entries is capable for a 1TB cache with 64KB block size
|
||||
+// 16m entries is capable to address a 1TB device with 64KB block size
|
||||
const DEFAULT_OBLOCKS: usize = 16777216;
|
||||
|
||||
fn inc_superblock(sm: &ASpaceMap) -> anyhow::Result<()> {
|
||||
@@ -82,7 +82,7 @@ mod format1 {
|
||||
}
|
||||
let mut seen_oblocks = self.seen_oblocks.lock().unwrap();
|
||||
|
||||
- if m.oblock as usize > seen_oblocks.len() {
|
||||
+ if m.oblock as usize >= seen_oblocks.len() {
|
||||
seen_oblocks.grow(m.oblock as usize + 1);
|
||||
} else if seen_oblocks.contains(m.oblock as usize) {
|
||||
return Err(array::value_err("origin block already mapped".to_string()));
|
||||
@@ -179,7 +179,7 @@ mod format2 {
|
||||
));
|
||||
}
|
||||
|
||||
- if m.oblock as usize > seen_oblocks.len() {
|
||||
+ if m.oblock as usize >= seen_oblocks.len() {
|
||||
seen_oblocks.grow(m.oblock as usize + 1);
|
||||
} else if seen_oblocks.contains(m.oblock as usize) {
|
||||
return Err(array::value_err("origin block already mapped".to_string()));
|
||||
diff --git a/tests/cache_check.rs b/tests/cache_check.rs
|
||||
index 81f4c578..8988694a 100644
|
||||
--- a/tests/cache_check.rs
|
||||
+++ b/tests/cache_check.rs
|
||||
@@ -11,6 +11,8 @@ use common::program::*;
|
||||
use common::target::*;
|
||||
use common::test_dir::*;
|
||||
|
||||
+use std::io::Write;
|
||||
+
|
||||
//------------------------------------------
|
||||
|
||||
const USAGE: &str = "Validates cache metadata on a device or file.
|
||||
@@ -294,3 +296,46 @@ fn no_clear_needs_check_if_error() -> Result<()> {
|
||||
}
|
||||
|
||||
//------------------------------------------
|
||||
+
|
||||
+fn metadata_without_slow_dev_size_info(use_v1: bool) -> Result<()> {
|
||||
+ let mut td = TestDir::new()?;
|
||||
+
|
||||
+ // The input metadata has a cached oblock with address equals to the default bitset size
|
||||
+ // boundary (DEFAULT_OBLOCKS = 16777216), triggering bitset resize.
|
||||
+ let xml = td.mk_path("meta.xml");
|
||||
+ let mut file = std::fs::File::create(&xml)?;
|
||||
+ file.write_all(b"<superblock uuid=\"\" block_size=\"128\" nr_cache_blocks=\"1024\" policy=\"smq\" hint_width=\"4\">
|
||||
+ <mappings>
|
||||
+ <mapping cache_block=\"0\" origin_block=\"16777216\" dirty=\"false\"/>
|
||||
+ </mappings>
|
||||
+ <hints>
|
||||
+ <hint cache_block=\"0\" data=\"AAAAAA==\"/>
|
||||
+ </hints>
|
||||
+</superblock>")?;
|
||||
+
|
||||
+ let md = td.mk_path("meta.bin");
|
||||
+ thinp::file_utils::create_sized_file(&md, 4096 * 4096)?;
|
||||
+
|
||||
+ let cache_restore_args = if use_v1 {
|
||||
+ args!["-i", &xml, "-o", &md, "--metadata-version=1"]
|
||||
+ } else {
|
||||
+ args!["-i", &xml, "-o", &md, "--metadata-version=2"]
|
||||
+ };
|
||||
+
|
||||
+ run_ok(cache_restore_cmd(cache_restore_args))?;
|
||||
+ run_ok(cache_check_cmd(args![&md]))?;
|
||||
+
|
||||
+ Ok(())
|
||||
+}
|
||||
+
|
||||
+#[test]
|
||||
+fn metadata_v1_without_slow_dev_size_info() -> Result<()> {
|
||||
+ metadata_without_slow_dev_size_info(true)
|
||||
+}
|
||||
+
|
||||
+#[test]
|
||||
+fn metadata_v2_without_slow_dev_size_info() -> Result<()> {
|
||||
+ metadata_without_slow_dev_size_info(false)
|
||||
+}
|
||||
+
|
||||
+//------------------------------------------
|
||||
--
|
||||
2.43.0
|
||||
|
@ -0,0 +1,62 @@
|
||||
From e295610fad85ec1a64e569cba425ca557a56c6e6 Mon Sep 17 00:00:00 2001
|
||||
From: Ming-Hung Tsai <mtsai@redhat.com>
|
||||
Date: Wed, 31 Jan 2024 11:36:17 +0800
|
||||
Subject: [PATCH 6/6] [thin/cache_check] Print suggestive hints for improving
|
||||
error resolution
|
||||
|
||||
Enhance error messages to instruct users on addressing recoverable
|
||||
errors, eliminating the guesswork (bz2233177).
|
||||
|
||||
(cherry picked from commit aaf3b396574709902ffba47e03a5c7ded6a103c5)
|
||||
---
|
||||
src/cache/check.rs | 5 ++++-
|
||||
src/thin/check.rs | 10 ++++++++--
|
||||
2 files changed, 12 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/src/cache/check.rs b/src/cache/check.rs
|
||||
index c71d3059..17d1af77 100644
|
||||
--- a/src/cache/check.rs
|
||||
+++ b/src/cache/check.rs
|
||||
@@ -387,7 +387,10 @@ pub fn check(opts: CacheCheckOptions) -> anyhow::Result<()> {
|
||||
ctx.report.warning("Repairing metadata leaks.");
|
||||
repair_space_map(ctx.engine.clone(), metadata_leaks, metadata_sm.clone())?;
|
||||
} else if !opts.ignore_non_fatal {
|
||||
- return Err(anyhow!("metadata space map contains leaks"));
|
||||
+ return Err(anyhow!(concat!(
|
||||
+ "metadata space map contains leaks\n",
|
||||
+ "perhaps you wanted to run with --auto-repair"
|
||||
+ )));
|
||||
}
|
||||
}
|
||||
|
||||
diff --git a/src/thin/check.rs b/src/thin/check.rs
|
||||
index 8b829899..f6fde359 100644
|
||||
--- a/src/thin/check.rs
|
||||
+++ b/src/thin/check.rs
|
||||
@@ -1246,7 +1246,10 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> {
|
||||
report.warning("Repairing data leaks.");
|
||||
repair_space_map(engine.clone(), data_leaks, data_sm.clone())?;
|
||||
} else if !opts.ignore_non_fatal {
|
||||
- return Err(anyhow!("data space map contains leaks"));
|
||||
+ return Err(anyhow!(concat!(
|
||||
+ "data space map contains leaks\n",
|
||||
+ "perhaps you wanted to run with --auto-repair"
|
||||
+ )));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1255,7 +1258,10 @@ pub fn check(opts: ThinCheckOptions) -> Result<()> {
|
||||
report.warning("Repairing metadata leaks.");
|
||||
repair_space_map(engine.clone(), metadata_leaks, metadata_sm.clone())?;
|
||||
} else if !opts.ignore_non_fatal {
|
||||
- return Err(anyhow!("metadata space map contains leaks"));
|
||||
+ return Err(anyhow!(concat!(
|
||||
+ "metadata space map contains leaks\n",
|
||||
+ "perhaps you wanted to run with --auto-repair"
|
||||
+ )));
|
||||
}
|
||||
}
|
||||
|
||||
--
|
||||
2.43.0
|
||||
|
Binary file not shown.
Binary file not shown.
@ -9,14 +9,19 @@
|
||||
|
||||
Summary: Device-mapper Persistent Data Tools
|
||||
Name: device-mapper-persistent-data
|
||||
Version: 1.0.6
|
||||
Release: 1%{?dist}%{?release_suffix}
|
||||
Version: 1.0.9
|
||||
Release: 2%{?dist}%{?release_suffix}
|
||||
License: GPLv3+
|
||||
URL: https://github.com/jthornber/thin-provisioning-tools
|
||||
#Source0: https://github.com/jthornber/thin-provisioning-tools/archive/thin-provisioning-tools-%%{version}.tar.gz
|
||||
Source0: https://github.com/jthornber/thin-provisioning-tools/archive/v%{version}%{?version_suffix}.tar.gz
|
||||
Source1: dmpd106-vendor.tar.gz
|
||||
Source1: dmpd109-vendor.tar.gz
|
||||
Patch1: 0001-Tweak-cargo.toml-to-work-with-vendor-directory.patch
|
||||
Patch2: 0002-space-map-Fix-incorrect-index_entry.nr_free-while-ex.patch
|
||||
Patch3: 0003-thin_repair-Fix-child-keys-checking-on-the-node-with.patch
|
||||
Patch4: 0004-space_map-Allow-non-zero-values-in-unused-index-bloc.patch
|
||||
Patch5: 0005-cache_check-Fix-boundary-check-on-the-bitset-for-cac.patch
|
||||
Patch6: 0006-thin-cache_check-Print-suggestive-hints-for-improvin.patch
|
||||
|
||||
BuildRequires: rust-packaging
|
||||
BuildRequires: rust >= 1.35
|
||||
@ -54,7 +59,10 @@ echo %{version}-%{release} > VERSION
|
||||
|
||||
%if %{with check}
|
||||
%check
|
||||
RUST_BACKTRACE=1 %cargo_test || true
|
||||
# aarch64 is failing, but only in brew environment, tests are passing when
|
||||
# running locally
|
||||
#%%cargo_test
|
||||
RUST_BACKTRACE=1 %%cargo_test -- --nocapture --test-threads=1 || true
|
||||
%endif
|
||||
|
||||
%install
|
||||
@ -108,6 +116,16 @@ make DESTDIR=%{buildroot} MANDIR=%{_mandir} install
|
||||
#% {_sbindir}/thin_show_duplicates
|
||||
|
||||
%changelog
|
||||
* Thu Feb 08 2024 Marian Csontos <mcsontos@redhat.com> - 1.0.9-2
|
||||
- Allow non-zero values in unused index block entries.
|
||||
- Fix boundary check on the bitset for cached blocks.
|
||||
- Fix incorrect index_entry.nr_free on expansion affecting free space estimate.
|
||||
- Fix thin_repair checking keys on a node with a zero key.
|
||||
- Enhance error message in cache_check suggesting fix when applicable.
|
||||
|
||||
* Wed Dec 13 2023 Marian Csontos <mcsontos@redhat.com> - 1.0.9-1
|
||||
- Update to latest upstream release 1.0.9.
|
||||
|
||||
* Wed Aug 09 2023 Marian Csontos <mcsontos@redhat.com> - 1.0.6-1
|
||||
- Update to latest upstream release 1.0.6.
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user