Compare commits

...

No commits in common. "c8s" and "imports/c9-beta/jq-1.6-14.el9" have entirely different histories.

23 changed files with 20419 additions and 25535 deletions

View File

@ -1 +0,0 @@
1

1
.gitignore vendored
View File

@ -1,2 +1 @@
SOURCES/jq-1.6.tar.gz
/jq-1.6.tar.gz

1
.jq.metadata Normal file
View File

@ -0,0 +1 @@
02959bca30672e0dfe678e7b36464c8fb08ec389 SOURCES/jq-1.6.tar.gz

View File

@ -1,244 +0,0 @@
commit fead9669cb67badb22789d3ed1888779ed85c679
Author: Tomas Halman <thalman@redhat.com>
Date: Thu Mar 9 11:34:44 2023 +0100
Test that jq works in threads
This patch implements test that searches a key in simple
json in pthread.
diff -up jq-1.6/src/jq_test.c.orig jq-1.6/src/jq_test.c
--- jq-1.6/src/jq_test.c.orig 2023-03-09 14:02:16.980062057 +0100
+++ jq-1.6/src/jq_test.c 2023-03-09 14:03:38.347017032 +0100
@@ -2,12 +2,17 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
+#ifdef HAVE_PTHREAD
+#include <pthread.h>
+#endif
#include "jv.h"
#include "jq.h"
static void jv_test();
static void run_jq_tests(jv, int, FILE *, int, int);
-
+#ifdef HAVE_PTHREAD
+static void run_jq_pthread_tests();
+#endif
int jq_testsuite(jv libdirs, int verbose, int argc, char* argv[]) {
FILE *testdata = stdin;
@@ -32,6 +37,9 @@ int jq_testsuite(jv libdirs, int verbose
}
}
run_jq_tests(libdirs, verbose, testdata, skip, take);
+#ifdef HAVE_PTHREAD
+ run_jq_pthread_tests();
+#endif
return 0;
}
@@ -105,7 +113,7 @@ static void run_jq_tests(jv lib_dirs, in
if (buf[0] == '\n' || (buf[0] == '\r' && buf[1] == '\n'))
break;
}
-
+
must_fail = 0;
check_msg = 0;
@@ -229,7 +237,7 @@ static void run_jq_tests(jv lib_dirs, in
total_skipped = tests_to_skip - skip;
}
- printf("%d of %d tests passed (%d malformed, %d skipped)\n",
+ printf("%d of %d tests passed (%d malformed, %d skipped)\n",
passed, tests, invalid, total_skipped);
if (skip > 0) {
@@ -241,6 +249,88 @@ static void run_jq_tests(jv lib_dirs, in
}
+/// pthread regression test
+#ifdef HAVE_PTHREAD
+#define NUMBER_OF_THREADS 3
+struct test_pthread_data {
+ int result;
+};
+
+static int test_pthread_jq_parse(jq_state *jq, struct jv_parser *parser)
+{
+ int rv = 0;
+ jv value;
+
+ value = jv_parser_next(parser);
+ while (jv_is_valid(value)) {
+ jq_start(jq, value, 0);
+ jv result = jq_next(jq);
+
+ while (jv_is_valid(result)) {
+ jv_free(result);
+ result = jq_next(jq);
+ }
+ jv_free(result);
+ value = jv_parser_next(parser);
+ }
+ jv_free(value);
+ return rv;
+}
+
+static void *test_pthread_run(void *ptr) {
+ int rv;
+ jq_state *jq;
+ const char *prg = ".data";
+ const char *buf = "{ \"data\": 1 }";
+ struct test_pthread_data *data = ptr;
+
+ jq = jq_init();
+ if (jq_compile(jq, prg) == 0) {
+ jq_teardown(&jq);
+ return NULL;
+ }
+
+ struct jv_parser *parser = jv_parser_new(0);
+ jv_parser_set_buf(parser, buf, strlen(buf), 0);
+ rv = test_pthread_jq_parse(jq, parser);
+
+ data->result = rv;
+
+ jv_parser_free(parser);
+ jq_teardown(&jq);
+ return NULL;
+}
+
+static void run_jq_pthread_tests() {
+ pthread_t threads[NUMBER_OF_THREADS];
+ struct test_pthread_data data[NUMBER_OF_THREADS];
+ int createerror;
+ int a;
+
+ memset(&threads, 0, sizeof(threads));
+ memset(&data, 0, sizeof(data));
+
+ // Create all threads
+ for (a = 0; a < NUMBER_OF_THREADS; ++a) {
+ createerror = pthread_create(&threads[a], NULL, test_pthread_run, &data[a]);
+ assert(createerror == 0);
+ }
+
+ // wait for all threads
+ for(a = 0; a < NUMBER_OF_THREADS; ++a) {
+ if (threads[a] != 0) {
+ pthread_join(threads[a], NULL);
+ }
+ }
+
+ // check results
+ for(a = 0; a < NUMBER_OF_THREADS; ++a) {
+ assert(data[a].result == 0);
+ }
+}
+#endif // HAVE_PTHREAD
+
+
static void jv_test() {
/// JSON parser regression tests
{
commit f05c5bb521f404592b137e02a1b8abd50da87ca3
Author: Tomas Halman <thalman@redhat.com>
Date: Wed Mar 1 20:07:28 2023 +0100
Fix segmentation fault when using jq in threads
In previous code nomem_handler in jv_alloc.c is called only once
and therefore the structure is not initialized for second and
following threads.
This leads to segmentation fault in multi-threading environment.
This patch moves initialization of nomem_handler out of the
pthread_once call.
diff -up jq-1.6/src/jv_alloc.c.orig jq-1.6/src/jv_alloc.c
--- jq-1.6/src/jv_alloc.c.orig 2018-11-02 02:49:29.000000000 +0100
+++ jq-1.6/src/jv_alloc.c 2023-03-09 14:13:47.810177037 +0100
@@ -45,9 +45,11 @@ static void memory_exhausted() {
#ifdef HAVE_PTHREAD_KEY_CREATE
#include <pthread.h>
-pthread_key_t nomem_handler_key;
-pthread_once_t mem_once = PTHREAD_ONCE_INIT;
+static pthread_key_t nomem_handler_key;
+static pthread_once_t mem_once = PTHREAD_ONCE_INIT;
+/* tsd_fini is called on application exit */
+/* it clears the nomem_handler allocated in the main thread */
static void tsd_fini(void) {
struct nomem_handler *nomem_handler;
nomem_handler = pthread_getspecific(nomem_handler_key);
@@ -57,30 +59,45 @@ static void tsd_fini(void) {
}
}
+/* The tsd_fini_thread is a destructor set by calling */
+/* pthread_key_create(&nomem_handler_key, tsd_fini_thread) */
+/* It is called when thread ends */
+static void tsd_fini_thread(void *nomem_handler) {
+ free(nomem_handler);
+}
+
static void tsd_init(void) {
- if (pthread_key_create(&nomem_handler_key, NULL) != 0) {
- fprintf(stderr, "error: cannot create thread specific key");
+ if (pthread_key_create(&nomem_handler_key, tsd_fini_thread) != 0) {
+ fprintf(stderr, "jq: error: cannot create thread specific key");
abort();
}
if (atexit(tsd_fini) != 0) {
- fprintf(stderr, "error: cannot set an exit handler");
+ fprintf(stderr, "jq: error: cannot set an exit handler");
abort();
}
- struct nomem_handler *nomem_handler = calloc(1, sizeof(struct nomem_handler));
- if (pthread_setspecific(nomem_handler_key, nomem_handler) != 0) {
- fprintf(stderr, "error: cannot set thread specific data");
- abort();
+}
+
+static void tsd_init_nomem_handler(void)
+{
+ if (pthread_getspecific(nomem_handler_key) == NULL) {
+ struct nomem_handler *nomem_handler = calloc(1, sizeof(struct nomem_handler));
+ if (pthread_setspecific(nomem_handler_key, nomem_handler) != 0) {
+ fprintf(stderr, "jq: error: cannot set thread specific data");
+ abort();
+ }
}
}
void jv_nomem_handler(jv_nomem_handler_f handler, void *data) {
pthread_once(&mem_once, tsd_init); // cannot fail
+ tsd_init_nomem_handler();
+
struct nomem_handler *nomem_handler;
nomem_handler = pthread_getspecific(nomem_handler_key);
if (nomem_handler == NULL) {
handler(data);
- fprintf(stderr, "error: cannot allocate memory\n");
+ fprintf(stderr, "jq: error: cannot allocate memory\n");
abort();
}
nomem_handler->handler = handler;
@@ -91,6 +108,8 @@ static void memory_exhausted() {
struct nomem_handler *nomem_handler;
pthread_once(&mem_once, tsd_init);
+ tsd_init_nomem_handler();
+
nomem_handler = pthread_getspecific(nomem_handler_key);
if (nomem_handler)
nomem_handler->handler(nomem_handler->data); // Maybe handler() will longjmp() to safety

File diff suppressed because it is too large Load Diff

View File

@ -1,52 +0,0 @@
diff -up jq-1.6/src/jq_test.c.orig jq-1.6/src/jq_test.c
--- jq-1.6/src/jq_test.c.orig 2024-05-03 11:47:47.403617188 +0200
+++ jq-1.6/src/jq_test.c 2024-05-03 11:48:46.569675199 +0200
@@ -16,6 +16,7 @@ static void run_jq_pthread_tests();
int jq_testsuite(jv libdirs, int verbose, int argc, char* argv[]) {
FILE *testdata = stdin;
+ const char *testdata_filename = NULL;
int skip = -1;
int take = -1;
jv_test();
@@ -28,18 +29,24 @@ int jq_testsuite(jv libdirs, int verbose
take = atoi(argv[i+1]);
i++;
} else {
- testdata = fopen(argv[i], "r");
- if (!testdata) {
- perror("fopen");
- exit(1);
- }
+ testdata_filename = argv[i];
}
}
}
+ if (testdata_filename) {
+ testdata = fopen(testdata_filename, "r");
+ if (!testdata) {
+ perror("fopen");
+ exit(1);
+ }
+ }
run_jq_tests(libdirs, verbose, testdata, skip, take);
#ifdef HAVE_PTHREAD
run_jq_pthread_tests();
#endif
+ if (testdata_filename) {
+ fclose(testdata);
+ }
return 0;
}
diff -up jq-1.6/src/locfile.c.orig jq-1.6/src/locfile.c
--- jq-1.6/src/locfile.c.orig 2018-11-02 02:49:29.000000000 +0100
+++ jq-1.6/src/locfile.c 2024-05-03 11:15:46.562476303 +0200
@@ -72,6 +72,7 @@ void locfile_locate(struct locfile* l, l
}
jv m1 = jv_string_vfmt(fmt, fmtargs);
+ va_end(fmtargs);
if (!jv_is_valid(m1)) {
jq_report_error(l->jq, m1);
return;

View File

@ -1,46 +1,46 @@
Name: jq
Version: 1.6
Release: 9%{?dist}
Release: 14%{?dist}
Summary: Command-line JSON processor
License: MIT and ASL 2.0 and CC-BY and GPLv3
URL: http://stedolan.github.io/jq/
Source0: https://github.com/stedolan/jq/releases/download/%{name}-%{version}/%{name}-%{version}.tar.gz
Patch0: 0000-jq-decimal-literal-number.patch
# Backport of PR#1752 for RHBZ#2008979
Patch0: jq-decimal-literal-number.patch
Patch1: 0001-iterration-problem-for-non-decimal-string.patch
Patch2: 0002-add-mantest.patch
Patch3: 0003-fix-pthread-segfault.patch
Patch4: 0004-make-jq-fast.patch
Patch5: 0005-sast.patch
BuildRequires: gcc
BuildRequires: flex
BuildRequires: bison
BuildRequires: oniguruma-devel
BuildRequires: autoconf
BuildRequires: automake
BuildRequires: libtool
BuildRequires: chrpath
%ifnarch s390x
%ifarch %{valgrind_arches}
BuildRequires: valgrind
%endif
BuildRequires: make
BuildRequires: autoconf
BuildRequires: automake
BuildRequires: libtool
%description
lightweight and flexible command-line JSON processor
jq is like sed for JSON data you can use it to slice
and filter and map and transform structured data with
the same ease that sed, awk, grep and friends let you
play with text.
jq is like sed for JSON data you can use it to slice
and filter and map and transform structured data with
the same ease that sed, awk, grep and friends let you
play with text.
It is written in portable C, and it has zero runtime
dependencies.
It is written in portable C, and it has zero runtime
dependencies.
jq can mangle the data format that you have into the
one that you want with very little effort, and the
program to do so is often shorter and simpler than
you'd expect.
jq can mangle the data format that you have into the
one that you want with very little effort, and the
program to do so is often shorter and simpler than
you'd expect.
%package devel
Summary: Development files for %{name}
@ -54,7 +54,7 @@ Development files for %{name}
%autosetup -n %{name}-%{version} -p1
%build
autoreconf -fi
autoreconf -if
%configure --disable-static
make %{?_smp_mflags}
# Docs already shipped in jq's tarball.
@ -100,45 +100,52 @@ make check
%changelog
* Fri May 3 2024 Tomas Halman <thalman@redhat.com> - 1.6-9
- Fix SAST findings in jq 1.6
- Resolves: RHEL-37827
* Fri Oct 13 2023 Tomas Halman <thalman@redhat.com> - 1.6-8
- Make jq 1.6 fast
- Resolves: RHEL-5052
* Tue Mar 14 2023 Tomas Halman <thalman@redhat.com> - 1.6-7
- Fix jq segfault when used in threads
- Resolves: rhbz#2092160
* Fri Nov 4 2022 Tomas Halman <thalman@redhat.com> - 1.6-6
- Add mantest to the gating
- Related: rhbz#2049601
- Related: rhbz#2049594
* Tue Oct 11 2022 Tomas Halman <thalman@redhat.com> - 1.6-5
- Remove rpath from jq binary
- Related: rhbz#2049601
* Fri Oct 21 2022 Tomas Halman <thalman@redhat.com> - 1.6-13
- jq try/catch stops iteration over items
Resolves: rhbz#2049594
* Tue Oct 11 2022 Tomas Halman <thalman@redhat.com> - 1.6-4
- Fix iterration problem for non decimal string
- Resolves: rhbz#2049601
* Mon Nov 15 2021 Tomas Halman <thalman@redhat.com>
- Strip rpath from jq binary
Related: rhbz#2008983
* Mon Oct 4 2021 Tomas Halman <thalman@redhat.com>
- Fix big integers issue
- Resolves: bug#2008717
* Wed Sep 29 2021 Davide Cavalca <dcavalca@centosproject.org> - 1.6-10
- Backport PR#1752 to fix an integer logic issue
Resolves: rhbz#2008983
* Mon Oct 4 2021 Tomas Halman <thalman@redhat.com>
- Releasing v1.6
- Resolves: bug#1852514
* Mon Aug 09 2021 Mohan Boddu <mboddu@redhat.com> - 1.6-9
- Rebuilt for IMA sigs, glibc 2.34, aarch64 flags
Related: rhbz#1991688
* Wed Aug 11 2021 Tomas Halman <thalman@redhat.com>
- Publishing devel package
- Resolves: bug#1908928
* Fri Apr 16 2021 Mohan Boddu <mboddu@redhat.com> - 1.6-8
- Rebuilt for RHEL 9 BETA on Apr 15th 2021. Related: rhbz#1947937
* Sat Aug 11 2018 Troy Dawson <tdawson@redhat.com>
- Fix typo: s390 -> s390x
- Related: bug#1614611
* Tue Jan 26 2021 Fedora Release Engineering <releng@fedoraproject.org> - 1.6-7
- Rebuilt for https://fedoraproject.org/wiki/Fedora_34_Mass_Rebuild
* Sat Dec 05 2020 Richard W.M. Jones <rjones@redhat.com> - 1.6-6
- Use correct valgrind_arches macro to check for valgrind.
* Tue Jul 28 2020 Fedora Release Engineering <releng@fedoraproject.org> - 1.6-5
- Rebuilt for https://fedoraproject.org/wiki/Fedora_33_Mass_Rebuild
* Wed Jan 29 2020 Fedora Release Engineering <releng@fedoraproject.org> - 1.6-4
- Rebuilt for https://fedoraproject.org/wiki/Fedora_32_Mass_Rebuild
* Thu Jul 25 2019 Fedora Release Engineering <releng@fedoraproject.org> - 1.6-3
- Rebuilt for https://fedoraproject.org/wiki/Fedora_31_Mass_Rebuild
* Fri Feb 01 2019 Fedora Release Engineering <releng@fedoraproject.org> - 1.6-2
- Rebuilt for https://fedoraproject.org/wiki/Fedora_30_Mass_Rebuild
* Thu Nov 08 2018 David Fetter <david@fetter.org> - 1.6-1
- Upstream 1.6.0
* Fri Jul 13 2018 Fedora Release Engineering <releng@fedoraproject.org> - 1.5-13
- Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild
* Sun Apr 01 2018 Mamoru TASAKA <mtasaka@fedoraproject.org> - 1.5-12
- Rebuild against oniguruma 6.8.1

View File

@ -1,6 +0,0 @@
--- !Policy
product_versions:
- rhel-8
decision_context: osci_compose_gate
rules:
- !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional}

View File

@ -1,23 +0,0 @@
summary: Run gating tests
discover:
how: fmf
dist-git-source: true
dist-git-type: centos
prepare:
- how: install
package:
- dnf-plugins-core
- https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
- how: shell
script: /usr/bin/crb enable
- how: install
package:
- jq
- jq-devel
- gcc
- wget
- how: shell
script: dnf config-manager --disable epel*
execute:
how: tmt

View File

@ -1 +0,0 @@
SHA512 (jq-1.6.tar.gz) = f5ae8be558ca2ff15324c378d623106b74bd0823be50835e23548584aa1eb24eb81f8f054693d5d3fe44f157d0735c5f0f40b9f21899ba068f2a11d1345ace19

View File

@ -1,20 +0,0 @@
#!/bin/bash
export LC_ALL=C
stress_jq() {
`which time` -p -f '%e\n' bash -c 'for i in `seq 1 1000` ; do echo '"'"'{"foo":"bar"}'"'"' | jq '"'"'has("bar")'"'"' > /dev/null ; done' 2>&1 | cut -d. -f1
}
FAIL=0
TIME=`stress_jq`
echo -n "Test jqspeed ... "
if [ $TIME -gt 8 ] ; then
echo "failed"
FAIL=1
else
echo "ok"
fi
echo "Runtime: ${TIME}s "
exit $FAIL

View File

@ -1,6 +0,0 @@
summary: Run jq performance/speed test
require:
- jq
- which
- time
test: ./jqspeed.sh

View File

View File

@ -1,5 +0,0 @@
---
standard-inventory-qcow2:
qemu:
m: 2G

View File

@ -1,19 +0,0 @@
#!/bin/sh
FAIL=0
# Test for segfault in multithreaded environment
gcc -o test_segfault test_segfault_with_multithreaded_env.c -lpthread -ljq && \
./test_segfault
RET=$?
echo -n "Test jqsegfault ... "
if [ $RET != 0 ]; then
echo "failed"
FAIL=1
else
echo "ok"
fi
exit $FAIL

View File

@ -1,8 +0,0 @@
summary: Run segfault threads test
require:
- wget
- dnf-utils
- gcc
- jq
- jq-devel
test: ./jqsegfault.sh

View File

@ -1,90 +0,0 @@
#include <stdlib.h>
#include <jq.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <pthread.h>
int my_jq_parse(jq_state *jq, struct jv_parser *parser)
{
int rv = 0;
jv value;
value = jv_parser_next(parser);
while (jv_is_valid(value)) {
jq_start(jq, value, 0);
jv result;
while (jv_is_valid(result = jq_next(jq))) {
jv dumped = jv_dump_string(result, 0);
const char *str = jv_string_value(dumped);
printf("dumped: %s\n", str);
}
jv_free(result);
value = jv_parser_next(parser);
}
if (jv_invalid_has_msg(jv_copy(value))) {
jv msg = jv_invalid_get_msg(value);
printf("invalid: %s\n", jv_string_value(msg));
jv_free(msg);
rv = 1;
} else {
jv_free(value);
}
return rv;
}
void *run(void *not_used) {
int rv;
jq_state *jq;
const char *prg = ".data";
jq = jq_init();
printf("jq_init jq: %p prg: %s\n", jq, prg);
if (jq_compile(jq, prg) == 0) {
jq_teardown(&jq);
return NULL;
}
printf("compiled\n");
struct jv_parser *parser = jv_parser_new(0);
const char *buf = "{ \"data\": 1 }";
jv_parser_set_buf(parser, buf, strlen(buf), 0);
rv = my_jq_parse(jq, parser);
if (rv != 0) {
printf("my_jq_parse failed!\n");
}
jv_parser_free(parser);
jq_teardown(&jq);
return NULL;
}
#define THREADS 2
/* calling run() twice works fine */
/* calling them in threads causes core-dump */
int main (int argc, char *argv[])
{
pthread_t threads[THREADS];
int createerror;
int a;
memset(&threads, 0, sizeof(threads));
for (a = 0; a < THREADS; ++a) {
// sleep(1); // < if you want to run threads sequentionally
createerror = pthread_create(&threads[a], NULL, run, NULL);
if (createerror) {
printf("create thread error %d\n", a);
}
}
for(a = 0; a < THREADS; ++a) {
if (threads[a] != 0) {
pthread_join(threads[a], NULL);
}
}
return 0;
}

View File

@ -1,27 +0,0 @@
---
- hosts: localhost
roles:
- role: standard-test-source
tags:
- always
- role: standard-test-basic
tags:
- classic
required_packages:
- jq
- gcc
- wget
- jq-devel
- valgrind
- rubygem-rake
tests:
- upstream:
dir: upstream
run: ./jqtests.sh
- segfault:
dir: segfault
run: ./jqsegfault.sh
- performance:
dir: performance
run: ./jqspeed.sh

View File

@ -1,48 +0,0 @@
#!/bin/sh
die () {
echo "$1" >&2
exit 1
}
# If source not found, download it with dnf
if [ ! -d ./source ]; then
# Extract source from srpm
dnf download --source jq && \
rpm2cpio jq*.rpm|cpio -id && \
mkdir source && \
tar -zxf jq-*.tar.gz -C source --strip-components=1
if [ $? != 0 ]; then
echo "Failed to download upstream tests"
exit 1
fi
fi
pushd ./source || die "missing source directory"
rm -f jq tests/*.log 2>/dev/null
ln -s /usr/bin/jq || die "failed to link jq binary"
FAIL=0
# run the tests
# List of tests is taken from Makefile
TESTS="tests/optionaltest tests/mantest tests/jqtest tests/onigtest tests/shtest tests/utf8test tests/base64test"
for t in $TESTS; do
echo -n "Test $t ... "
./${t} >"${t}.log" 2>&1
RET=$?
if [ $RET = 0 ]; then
echo "ok"
else
echo "failed"
echo "-------------------- ${t}.log start -----------------------------"
cat "${t}.log"
echo "-------------------- ${t}.log end -----------------------------"
FAIL=1
fi
done
popd # exit SOURCE_DIR
exit $FAIL

View File

@ -1,11 +0,0 @@
summary: Run jq gating tests
require:
- wget
- dnf-utils
- gcc
- jq
- jq-devel
- valgrind
- diffutils
test: ./jqtests.sh
duration: 30m