Rebase to latest stable release.

- use ceph 16.2.4 - update the patches - add the libcephsqlite packages - add gating.yaml Resolves: rhbz#1961080
2021-04-18 11:53:52 +02:00 · 2021-04-18 11:53:52 +02:00 · 84e3167971
parent 6919109e2a
commit 84e3167971
14 changed files with 235 additions and 135 deletions
--- a/0004-src-CMakeLists.txt.patch
+++ b/0004-src-CMakeLists.txt.patch
@ -1,12 +0,0 @@
--- ceph-16.0.0/src/CMakeLists.txt.orig	2021-01-25 13:45:15.316053258 -0500
-+++ ceph-16.0.0/src/CMakeLists.txt	2021-01-25 13:43:34.418305591 -0500
-@@ -29,7 +29,8 @@
-   -D_THREAD_SAFE
-   -D__STDC_FORMAT_MACROS
-   -D_FILE_OFFSET_BITS=64
-  -DBOOST_ASIO_DISABLE_THREAD_KEYWORD_EXTENSION)
-+  -DBOOST_ASIO_DISABLE_THREAD_KEYWORD_EXTENSION
-+  -DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT)
- if(LINUX)
-   add_definitions("-D_GNU_SOURCE")
- endif()
--- a/0004-src-common-ceph-time.h.patch
+++ b/0004-src-common-ceph-time.h.patch
@ -1,10 +0,0 @@
--- ceph-15.2.8/src/common/ceph_time.h.orig	2020-12-24 06:23:32.605762289 -0500
-+++ ceph-15.2.8/src/common/ceph_time.h	2020-12-24 13:50:14.304877611 -0500
-@@ -18,6 +18,7 @@
- #include <chrono>
- #include <iostream>
- #include <string>
-+#include <optional>
- #include <sys/time.h>
- 
- #include "include/ceph_assert.h"
--- a/0005-src-CMakeLists.txt.patch
+++ b/0005-src-CMakeLists.txt.patch
@ -1,22 +0,0 @@
--- ceph-15.2.8/src/CMakeLists.txt.orig	2021-01-25 13:34:31.122206146 -0500
-+++ ceph-15.2.8/src/CMakeLists.txt	2021-01-25 13:37:12.863612216 -0500
-@@ -23,10 +23,16 @@
- set(datadir ${CEPH_INSTALL_DATADIR})
- set(prefix ${CMAKE_INSTALL_PREFIX})
- 
-add_definitions("-DHAVE_CONFIG_H -D__CEPH__ -D_REENTRANT -D_THREAD_SAFE -D__STDC_FORMAT_MACROS")
-add_definitions("-D_FILE_OFFSET_BITS=64")
-+add_definitions(-DHAVE_CONFIG_H
-+  -D__CEPH__
-+  -D_REENTRANT
-+  -D_THREAD_SAFE
-+  -D__STDC_FORMAT_MACROS
-+  -D_FILE_OFFSET_BITS=64
-+  -DBOOST_ASIO_DISABLE_THREAD_KEYWORD_EXTENSION
-+  -DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT)
- if(LINUX)
-  add_definitions("-D_GNU_SOURCE")
-+  add_definitions(-D_GNU_SOURCE)
- endif()
- 
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
--- a/0005-src-librbd-migration.patch
+++ b/0005-src-librbd-migration.patch
@ -1,78 +0,0 @@
--- ceph-16.1.0/src/librbd/AsioEngine.h.orig	2021-01-27 09:52:12.314415539 -0500
-+++ ceph-16.1.0/src/librbd/AsioEngine.h	2021-01-27 09:55:35.289955105 -0500
-@@ -39,7 +39,8 @@
-   inline operator boost::asio::io_context&() {
-     return m_io_context;
-   }
-  inline boost::asio::io_context::executor_type get_executor() {
-+  using executor_type = boost::asio::io_context::executor_type;
-+  inline executor_type get_executor() {
-     return m_io_context.get_executor();
-   }
- 
--- ceph-16.1.0/src/librbd/migration/FileStream.cc.orig	2021-01-27 09:58:21.984505879 -0500
-+++ ceph-16.1.0/src/librbd/migration/FileStream.cc	2021-01-27 09:58:48.945241982 -0500
-@@ -121,7 +121,8 @@
- template <typename I>
- FileStream<I>::FileStream(I* image_ctx, const json_spirit::mObject& json_object)
-   : m_cct(image_ctx->cct), m_asio_engine(image_ctx->asio_engine),
-    m_json_object(json_object), m_strand(*m_asio_engine) {
-+    m_json_object(json_object),
-+    m_strand(boost::asio::make_strand(*m_asio_engine)) {
- }
- 
- template <typename I>
--- ceph-16.1.0/src/librbd/migration/FileStream.h.orig	2021-01-27 09:59:03.542640537 -0500
-+++ ceph-16.1.0/src/librbd/migration/FileStream.h	2021-01-27 09:59:42.054692043 -0500
-@@ -6,7 +6,8 @@
- 
- #include "include/int_types.h"
- #include "librbd/migration/StreamInterface.h"
-#include <boost/asio/io_context_strand.hpp>
-+#include <boost/asio/io_context.hpp>
-+#include <boost/asio/strand.hpp>
- #include <boost/asio/posix/basic_stream_descriptor.hpp>
- #include <json_spirit/json_spirit.h>
- #include <memory>
-@@ -48,7 +49,7 @@
-   std::shared_ptr<AsioEngine> m_asio_engine;
-   json_spirit::mObject m_json_object;
- 
-  boost::asio::io_context::strand m_strand;
-+  boost::asio::strand<boost::asio::io_context::executor_type> m_strand;
- #ifdef BOOST_ASIO_HAS_POSIX_STREAM_DESCRIPTOR
-   std::optional<boost::asio::posix::stream_descriptor> m_stream_descriptor;
- 
--- ceph-16.1.0/src/librbd/migration/HttpClient.cc.orig	2021-01-27 10:00:04.167295789 -0500
-+++ ceph-16.1.0/src/librbd/migration/HttpClient.cc	2021-01-27 10:00:34.303118596 -0500
-@@ -764,7 +764,8 @@
- template <typename I>
- HttpClient<I>::HttpClient(I* image_ctx, const std::string& url)
-   : m_cct(image_ctx->cct), m_image_ctx(image_ctx),
-    m_asio_engine(image_ctx->asio_engine), m_url(url), m_strand(*m_asio_engine),
-+    m_asio_engine(image_ctx->asio_engine), m_url(url),
-+    m_strand(boost::asio::make_strand(*m_asio_engine)),
-     m_ssl_context(boost::asio::ssl::context::sslv23_client) {
-     m_ssl_context.set_default_verify_paths();
- }
--- ceph-16.1.0/src/librbd/migration/HttpClient.h.orig	2021-01-27 10:00:49.626536973 -0500
-+++ ceph-16.1.0/src/librbd/migration/HttpClient.h	2021-01-27 10:01:38.728877629 -0500
-@@ -9,7 +9,8 @@
- #include "librbd/io/Types.h"
- #include "librbd/migration/HttpProcessorInterface.h"
- #include "librbd/migration/Types.h"
-#include <boost/asio/io_context_strand.hpp>
-+#include <boost/asio/io_context.hpp>
-+#include <boost/asio/strand.hpp>
- #include <boost/asio/ip/tcp.hpp>
- #include <boost/asio/ssl/context.hpp>
- #include <boost/beast/version.hpp>
-@@ -171,7 +172,7 @@
- 
-   HttpProcessorInterface* m_http_processor = nullptr;
- 
-  boost::asio::io_context::strand m_strand;
-+  boost::asio::strand<boost::asio::io_context::executor_type> m_strand;
- 
-   boost::asio::ssl::context m_ssl_context;
-   std::unique_ptr<HttpSessionInterface> m_http_session;
--- a/0006-src-blk-CMakeLists.txt.patch
+++ b/0006-src-blk-CMakeLists.txt.patch
--- a/0007-src-test-neorados-CMakeLists.txt.patch
+++ b/0007-src-test-neorados-CMakeLists.txt.patch
@ -0,0 +1,11 @@
+--- ceph-16.1.0-43-g6b74fb5c/src/test/neorados/CMakeLists.txt.orig	2021-02-01 08:25:18.006965821 -0500
+++ ceph-16.1.0-43-g6b74fb5c/src/test/neorados/CMakeLists.txt	2021-02-01 08:25:34.244407147 -0500
+@@ -19,7 +19,7 @@
+ target_link_libraries(ceph_test_neorados_op_speed
+   libneorados fmt::fmt ${unittest_libs})
+ 
+-add_library(neoradostest-support common_tests.cc)
+add_library(neoradostest-support STATIC common_tests.cc)
+ target_link_libraries(neoradostest-support
+   libneorados fmt::fmt)
+ 
--- a/0008-cmake-modules-Finduring.cmake.patch
+++ b/0008-cmake-modules-Finduring.cmake.patch
@ -0,0 +1,11 @@
+--- ceph-16.1.0-43-g6b74fb5c/cmake/modules/Finduring.cmake.orig	2021-02-01 08:45:39.316108287 -0500
+++ ceph-16.1.0-43-g6b74fb5c/cmake/modules/Finduring.cmake	2021-02-01 08:45:59.813665378 -0500
+@@ -5,7 +5,7 @@
+ # uring_FOUND - True if uring found.
+ 
+ find_path(URING_INCLUDE_DIR liburing.h)
+-find_library(URING_LIBRARIES liburing.a liburing)
+find_library(URING_LIBRARIES liburing.so liburing)
+ 
+ include(FindPackageHandleStandardArgs)
+ find_package_handle_standard_args(uring DEFAULT_MSG URING_LIBRARIES URING_INCLUDE_DIR)
--- a/0009-librgw-notifications-initialize-kafka-and-amqp.patch
+++ b/0009-librgw-notifications-initialize-kafka-and-amqp.patch
@ -0,0 +1,63 @@
+From 178f6bdac97b57300bbe0956633cf686a7e3ccee Mon Sep 17 00:00:00 2001
+From: Yuval Lifshitz <ylifshit@redhat.com>
+Date: Fri, 12 Mar 2021 08:56:45 +0200
+Subject: [PATCH] librgw/notifications: initialize kafka and amqp
+
+Fixes: https://tracker.ceph.com/issues/49738
+
+Signed-off-by: Yuval Lifshitz <ylifshit@redhat.com>
+---
+ src/rgw/librgw.cc | 23 +++++++++++++++++++++++
+ 1 file changed, 23 insertions(+)
+
+diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc
+index 012cc54c3b..a5351dbe7c 100644
+--- a/src/rgw/librgw.cc
+++ b/src/rgw/librgw.cc
+@@ -53,6 +53,12 @@
+ #include "rgw_http_client.h"
+ #include "rgw_http_client_curl.h"
+ #include "rgw_perf_counters.h"
+#ifdef WITH_RADOSGW_AMQP_ENDPOINT
+#include "rgw_amqp.h"
+#endif
+#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
+#include "rgw_kafka.h"
+#endif
+ 
+ #include "services/svc_zone.h"
+ 
+@@ -617,6 +623,17 @@ namespace rgw {
+       /* ignore error */
+     }
+ 
+#ifdef WITH_RADOSGW_AMQP_ENDPOINT
+    if (!rgw::amqp::init(cct.get())) {
+      derr << "ERROR: failed to initialize AMQP manager" << dendl;
+    }
+#endif
+#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
+    if (!rgw::kafka::init(cct.get())) {
+      derr << "ERROR: failed to initialize Kafka manager" << dendl;
+    }
+#endif
+
+     return 0;
+   } /* RGWLib::init() */
+ 
+@@ -645,6 +662,12 @@ namespace rgw {
+     rgw_shutdown_resolver();
+     rgw_http_client_cleanup();
+     rgw::curl::cleanup_curl();
+#ifdef WITH_RADOSGW_AMQP_ENDPOINT
+    rgw::amqp::shutdown();
+#endif
+#ifdef WITH_RADOSGW_KAFKA_ENDPOINT
+    rgw::kafka::shutdown();
+#endif
+ 
+     rgw_perf_stop(g_ceph_context);
+ 
+-- 
+2.26.2
+
--- a/0010-os-bluestore-strip-trailing-slash-for-directory-list.patch
+++ b/0010-os-bluestore-strip-trailing-slash-for-directory-list.patch
@ -0,0 +1,57 @@
+From 3aa31813980d22719277a04797df48310acdff66 Mon Sep 17 00:00:00 2001
+From: Jonas Jelten <jj@sft.lol>
+Date: Mon, 15 Mar 2021 23:21:07 +0100
+Subject: [PATCH] os/bluestore: strip trailing slash for directory listings
+
+Calls to BlueRocksEnv::GetChildren may contain a trailing / in the
+queried directory, which is stripped away with this patch.
+
+If it's not stripped, the directory entry is not found in BlueFS:
+```
+10 bluefs readdir db/
+20 bluefs readdir dir db/ not found
+ 3 rocksdb: [db/db_impl/db_impl_open.cc:1785] Persisting Option File error: OK
+```
+
+Fixes: https://tracker.ceph.com/issues/49815
+Signed-off-by: Jonas Jelten <jj@sft.lol>
+---
+ src/os/bluestore/BlueFS.cc | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc
+index ea39626aef..62b9d27f58 100644
+--- a/src/os/bluestore/BlueFS.cc
+++ b/src/os/bluestore/BlueFS.cc
+@@ -3493,9 +3493,14 @@
+ 
+ int BlueFS::readdir(const string& dirname, vector<string> *ls)
+ {
+  std::string dname = dirname;
+  // dirname may contain a trailing /
+  if (!dname.empty() && dname.back() == '/') {
+    dname.pop_back();
+  }
+   std::lock_guard l(lock);
+-  dout(10) << __func__ << " " << dirname << dendl;
+-  if (dirname.empty()) {
+  dout(10) << __func__ << " " << dname << dendl;
+  if (dname.empty()) {
+     // list dirs
+     ls->reserve(dir_map.size() + 2);
+     for (auto& q : dir_map) {
+@@ -3503,9 +3508,9 @@
+     }
+   } else {
+     // list files in dir
+-    map<string,DirRef>::iterator p = dir_map.find(dirname);
+    map<string,DirRef>::iterator p = dir_map.find(dname);
+     if (p == dir_map.end()) {
+-      dout(20) << __func__ << " dir " << dirname << " not found" << dendl;
+      dout(20) << __func__ << " dir " << dname << " not found" << dendl;
+       return -ENOENT;
+     }
+     DirRef dir = p->second;
+-- 
+2.26.2
+
--- a/0011-src-test-rgw-amqp_mock.cc.patch
+++ b/0011-src-test-rgw-amqp_mock.cc.patch
@ -0,0 +1,23 @@
+--- a/src/test/rgw/amqp_mock.cc
+++ b/src/test/rgw/amqp_mock.cc
+@@ -291,7 +291,11 @@ amqp_confirm_select_ok_t* amqp_confirm_select(amqp_connection_state_t state, amq
+   return state->confirm;
+ }
+ 
+-int amqp_simple_wait_frame_noblock(amqp_connection_state_t state, amqp_frame_t *decoded_frame, struct timeval* tv) {
+extern "C" {
+
+int amqp_simple_wait_frame_noblock(amqp_connection_state_t state,
+                                   amqp_frame_t *decoded_frame,
+                                   const struct timeval* tv) {
+   if (state->socket && state->socket->open_called &&
+       state->login_called && state->channel1 && state->channel2 && state->exchange &&
+       state->queue && state->consume && state->confirm && !FAIL_NEXT_READ) {
+@@ -345,6 +349,7 @@ int amqp_simple_wait_frame_noblock(amqp_connection_state_t state, amqp_frame_t *
+   }
+   return AMQP_STATUS_CONNECTION_CLOSED;
+ }
+} // extern "C"
+ 
+ amqp_basic_consume_ok_t* amqp_basic_consume(
+     amqp_connection_state_t state, amqp_channel_t channel, amqp_bytes_t queue,
--- a/0012-src-compressor-snappy-SnappyCompressor.h.patch
+++ b/0012-src-compressor-snappy-SnappyCompressor.h.patch
@ -0,0 +1,11 @@
+--- ceph-16.2.4/src/compressor/snappy/SnappyCompressor.h.orig	2021-05-26 08:30:26.205447315 -0400
+++ ceph-16.2.4/src/compressor/snappy/SnappyCompressor.h	2021-05-26 08:31:22.570443033 -0400
+@@ -97,7 +97,7 @@
+     if (qat_enabled)
+       return qat_accel.decompress(p, compressed_len, dst, compressor_message);
+ #endif
+-    snappy::uint32 res_len = 0;
+    std::uint32_t res_len = 0;
+     BufferlistSource source_1(p, compressed_len);
+     if (!snappy::GetUncompressedLength(&source_1, &res_len)) {
+       return -1;
--- a/ceph.spec
+++ b/ceph.spec
@ -63,8 +63,8 @@
 # main package definition
 #################################################################################
 Name:		ceph
-Version:	16.1.0
-Release:	0.8.snapshot%{?dist}
+Version:	16.2.4
+Release:	1%{?dist}
 %if 0%{?fedora} || 0%{?rhel}
 Epoch:		2
 %endif
@ -80,15 +80,17 @@ License:	(LGPLv2.1 or LGPLv3) and CC-BY-SA-3.0 and GPLv2 and Boost-1.0 and BSD a
 Group:		System/Filesystems
 %endif
 URL:		http://ceph.com/
-#Source0:	%%{?_remote_tarball_prefix}ceph-%%{version}.tar.gz
-# https://2.chacra.ceph.com/r/ceph/pacific/6b74fb5c...
-Source0:        ceph-16.1.0-308-gabe639eb.tar.bz2
+Source0:        ceph-16.2.4.tar.gz
 Patch0001:	0001-src-common-crc32c_intel_fast.patch
 Patch0002:	0002-src-common-CMakeLists.txt.patch
 Patch0003:	0003-src-common-bitstr.h.patch
-Patch0004:	0004-src-CMakeLists.txt.patch
-Patch0005:	0005-src-blk-CMakeLists.txt.patch
-Source1:	cmake-modules-BuildBoost.cmake.noautopatch
+Patch0006:	0006-src-blk-CMakeLists.txt.patch
+Patch0007:	0007-src-test-neorados-CMakeLists.txt.patch
+Patch0008:	0008-cmake-modules-Finduring.cmake.patch
+Patch0009:	0009-librgw-notifications-initialize-kafka-and-amqp.patch
+Patch0010:	0010-os-bluestore-strip-trailing-slash-for-directory-list.patch
+Patch0011:	0011-src-test-rgw-amqp_mock.cc.patch
+Patch0012:	0012-src-compressor-snappy-SnappyCompressor.h.patch
 # ceph 14.0.1 does not support 32-bit architectures, bugs #1727788, #1727787
 ExcludeArch:	i686 armv7hl
 %if 0%{?suse_version}
@ -147,6 +149,7 @@ BuildRequires:	python%{python3_pkgversion}
 BuildRequires:	python%{python3_pkgversion}-devel
 BuildRequires:	python%{python3_pkgversion}-setuptools
 BuildRequires:	snappy-devel
+BuildRequires:	sqlite-devel
 BuildRequires:	sudo
 BuildRequires:	pkgconfig(udev)
 BuildRequires:	util-linux
@ -390,6 +393,32 @@ Obsoletes:	python-rados < %{_epoch_prefix}%{version}-%{release}
 This package contains Python 3 libraries for interacting with Ceph RADOS
 object store.

+%package -n libcephsqlite
+Summary:	SQLite3 VFS for Ceph
+%if 0%{?suse_version}
+Group:		System/Libraries
+%endif
+Requires:	librados2 = %{_epoch_prefix}%{version}-%{release}
+%description -n libcephsqlite
+A SQLite3 VFS for storing and manipulating databases stored on Ceph's RADOS
+distributed object store.
+
+%package -n libcephsqlite-devel
+Summary:	SQLite3 VFS for Ceph headers
+%if 0%{?suse_version}
+Group:		Development/Libraries/C and C++
+%endif
+Requires:	sqlite-devel
+Requires:	libcephsqlite = %{_epoch_prefix}%{version}-%{release}
+Requires:	librados-devel = %{_epoch_prefix}%{version}-%{release}
+Requires:	libradospp-devel = %{_epoch_prefix}%{version}-%{release}
+Obsoletes:	ceph-devel < %{_epoch_prefix}%{version}-%{release}
+Provides:	libcephsqlite-devel = %{_epoch_prefix}%{version}-%{release}
+Obsoletes:	libcephsqlite-devel < %{_epoch_prefix}%{version}-%{release}
+%description -n libcephsqlite-devel
+A SQLite3 VFS for storing and manipulating databases stored on Ceph's RADOS
+distributed object store.
+
 %if 0%{with libradosstriper}
 %package -n libradosstriper1
 Summary:	RADOS striping interface
@ -568,10 +597,7 @@ class plugins.
 # common
 #################################################################################
 %prep
-%autosetup -p1 -n ceph-16.1.0-308-gabe639eb
-%ifarch x86_64
-patch -p1 < %{SOURCE1}
-%endif
+%autosetup -p1 -n ceph-16.2.4

 %build
 # LTO can be enabled as soon as the following GCC bug is fixed:
@ -999,6 +1025,16 @@ fi
 %{python3_sitearch}/rados.cpython*.so
 %{python3_sitearch}/rados-*.egg-info

+%files -n libcephsqlite
+%{_libdir}/libcephsqlite.so
+
+%post -n libcephsqlite -p /sbin/ldconfig
+
+%postun -n libcephsqlite -p /sbin/ldconfig
+
+%files -n libcephsqlite-devel
+%{_includedir}/libcephsqlite.h
+
 %if 0%{with libradosstriper}
 %files -n libradosstriper1
 %{_libdir}/libradosstriper.so.*
@ -1105,6 +1141,10 @@ fi
 %{_includedir}/rados/objclass.h

 %changelog
+* Fri May 28 2021 Boris Ranto <branto@redhat.com> - 2:16.2.4-1
+- Rebase to 16.2.4
+- Add libcephsqlite packages
+
 * Thu Apr 15 2021 Mohan Boddu <mboddu@redhat.com> - 2:16.1.0-0.8.snapshot
 - Rebuilt for RHEL 9 BETA on Apr 15th 2021. Related: rhbz#1947937

--- a/gating.yaml
+++ b/gating.yaml
@ -0,0 +1,6 @@
+--- !Policy
+product_versions:
+  - rhel-9
+decision_context: osci_compose_gate
+rules:
+  - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional}
--- a/2
+++ b/2
@ -1 +1 @@
-SHA512 (ceph-16.1.0-308-gabe639eb.tar.bz2) = 41433257c59a6a9ef852e3810f3ed244d143ba7874a0f896fb5e94b7ed48f282c59b06b406df9cfb4b9b5a48bfdf7ce23acfa1cc82ee8b61c171bfefc75ead98
+SHA512 (ceph-16.2.4.tar.gz) = 8904e7fcabd50276e4e86735fd85dba96f417c194c219025146f994707def7abe372528b0f2e862e274b2f405d7b791bac86b6cbb8d57cecd6f44023a2ffb6f6