Fix various JMAP tests.

Pull a couple of small patches from upstream which fix several
JMAP-related tests.

Rework the test exclusion list; there are now only a few which need to
be excluded.

Also drop a patch to cassandane to handle the quota->cyr_quota rename;
we can achieve the same thing now without patching.

cassandane.ini

@@ -18,6 +18,7 @@ maxworkers = 1
 [cyrus default]
 prefix = /usr
 destdir = BUILDROOT
+quota = cyr_quota
 
 # Replication testing disabled
 # [cyrus replica]

cyrus-imapd.spec

@@ -9,7 +9,7 @@
 
 Name: cyrus-imapd
 Version: 3.0.2
-Release: 4%{?dist}
+Release: 5%{?dist}
 
 %define ssl_pem_file /etc/pki/%name/%name.pem
 
@@ -38,6 +38,11 @@ Patch2: patch-cyrus-pkgconfig-file
 # Fedora-specific patch for the default configuration file
 Patch3: patch-cyrus-default-configs
 
+# Fix bugs in jmap turned up by Cassandane
+# https://github.com/cyrusimap/cyrus-imapd/issues/2042
+Patch4: patch-cyrus-jmapcalendar
+Patch5: patch-cyrus-jmapcalendar2
+
 Source10: cyrus-imapd.logrotate
 Source11: cyrus-imapd.pam-config
 Source12: cyrus-imapd.sysconfig
@@ -78,10 +83,6 @@ Source91: patch-cassandane-no-syslog
 # Upstream ticket https://github.com/cyrusimap/cyrus-imapd/issues/1995
 Source92: patch-cassandane-fix-annotator
 
-# Change hardcoded names from 'quota' to 'cyr_quota'
-# Upstream ticket https://github.com/cyrusimap/cassandane/issues/26
-Source93: patch-cassandane-rename-quota
-
 # Dependency notes:
 # * clamav-devel and mariadb-devel removed because they both depend on
 #   compat-openssl.
@@ -222,7 +223,6 @@ tar xf %SOURCE81
 
 patch -p1 < %SOURCE91
 patch -p1 < %SOURCE92
-patch -p1 < %SOURCE93
 cp %SOURCE82 cassandane.ini
 # RF rpm-buildroot-usage
 sed -i \
@@ -433,33 +433,18 @@ exit 0
 %endif
 
 %ifarch s390x
-# Unfortunately s390x is simply too slow to reliably run the test suite right
+# Uncomment the exit if s390x is being too slow to get a useful build out.
-# now.  The suite brings up and tears down a complete cyrus environment for
+#exit 0
-# each test, and the builders are simply unable to handle this reliably.  Tests
-# will simply fail randomly, and playing whack-a-mole with every potential
-# failure when a build takes seven hours is just too much.  s390x-specific test
-# information is still present below in case situation improves.
-exit 0
 %endif
 
 # Run the Cassandane test suite.  This will exhaustively test the various
 # server components, but running it in a mock chroot is rather an exercise.
-# Sadly Cassandane simply can't run on a 32-bit host at the moment.  It needs
-# Perl to be compiled with support for 64-bit types, and due to an unfortunate
-# typo in the Perl specfile, that's not the case currently.
-# https://bugzilla.redhat.com/show_bug.cgi?id=1268828 There may be other
-# Cassandane issues, so occasional testing on 32-bit architectures is needed
-# until this is all resolved.  Tracked upstream as
-# https://github.com/cyrusimap/cassandane/issues/21
 pushd cassandane
 
 mkdir -p imaptest/src
 ln -s /usr/bin/imaptest imaptest/src
 ln -s /usr/share/imaptest/tests imaptest/src
 
-# Build the syslog interposer; not currently working
-#gcc -fPIC -shared -Wl,-soname,-libredirectsyslog.so.0 -ldl -o libredirectsyslog.so.0.0 redirect-syslog.c
-
 export LD_LIBRARY_PATH=%buildroot/%_libdir
 export CYRUS_USER=$USER
 
@@ -471,55 +456,29 @@ export CYRUS_USER=$USER
 # and are more for testing system performance than for testing Cyrus itself.
 exclude=('!Cassandane::Test::Core' '!Metronome')
 
-# The below non-arch-specific excluded tests are expected to fail in 3.0.2 and
+# The following tests fail on all architectures.
-# should be removed and re-checked at each release.
 tests=(
-    Admin.imap_admins
+    # Fails because our Xapian is too old for proper CJK support.  1.5 will be
+    # OK, but it is not yet released.  The alternative is to bundle.
     SearchFuzzy.cjk_words
-    JMAPCalendars.setcalendarevents_alerts
+
-    JMAPCalendars.setcalendarevents_locations
+    # https://github.com/cyrusimap/cyrus-imapd/issues/2047
-    JMAPCalendars.setcalendarevents_recurrenceoverrides
+    Admin.imap_admins
+
+    # https://github.com/cyrusimap/cyrus-imapd/issues/2048
+   JMAPCalendars.setcalendarevents_alerts
 )
 for i in ${tests[@]}; do exclude+=("!$i"); done
 
-%ifarch i686 armv7hl
-# Just this one test is now failing on 32-bit machines.  I'm not sure why.
-# https://github.com/cyrusimap/cyrus-imapd/issues/2042
-tests=(
-    JMAPCalendars.setcalendarevents_recurrence
-)
-for i in ${tests[@]}; do exclude+=("!$i"); done
-%endif
-
-%ifarch ppc64 s390x
-# Some failures only on big-endian machines for some reason
-# Reported upstream at https://github.com/cyrusimap/cyrus-imapd/issues/2040
-tests=(
-    JMAPCalendars.getcalendareventlist
-    JMAPCalendars.getcalendareventupdates
-    JMAPCalendars.setcalendarevents_recurrence
-    JMAPContacts.getcontactgroupupdates
-    JMAPContacts.getcontactupdates
-    JMAPMail.getmailboxupdates
-    JMAPMail.getmessagelist_window
-    JMAPMail.getmessageupdates
-    JMAPMail.getthreadupdates
-)
-for i in ${tests[@]}; do exclude+=("!$i"); done
-%endif
-
-%ifarch s390x
-# This one test fails occasionally on s390x because the hosts are just too slow
-# to complete it.  It's testing something valid (that the fork rate limiting
-# settings work properly) but s399x can'f fork quickly enough to exceeed the
-# limits it's testing.
-exclude+=("!Master.maxforkrate")
-%endif
-
 %if 0%{?fedora} <= 26
-# For F26 and older, 32-bit Perl doesn't support the quad types in unpack.  The
+# Some F26-specific test exclusions
-# following tests use those, and so they fail on those releases.
 tests=(
+    # These fail because F26 libical has not been compiled with
+    # ICAL_ALLOW_EMPTY_PROPERTIES.
+    JMAPCalendars.setcalendarevents_locations
+    JMAPCalendars.setcalendarevents_recurrenceoverrides
+
+    # These all fail because F26 perl doesn't support quad types in unpack.
     JMAPContacts.setcontacts_importance_later
     Metadata.expunge_messages
     Metadata.msg_replication_new_mas_partial_wwd
@@ -649,6 +608,11 @@ getent passwd cyrus >/dev/null || /usr/sbin/useradd -c "Cyrus IMAP Server" -d /v
 
 
 %changelog
+* Fri Jun 30 2017 Jason L Tibbitts III <tibbs@math.uh.edu> - 3.0.2-5
+- Add two patches from upstream which fix JMAPCalendars issues on 32-bit and
+  big-endian architectures.
+- Clean up test invocation and exclusion list.  More tests pass now.
+
 * Wed Jun 28 2017 Jason L Tibbitts III <tibbs@math.uh.edu> - 3.0.2-4
 - Explicitly set specialusealways: 1 in the default config.
 

patch-cassandane-rename-quota

@@ -1,120 +0,0 @@
-diff --git a/Cassandane/Cyrus/Quota.pm b/Cassandane/Cyrus/Quota.pm
-index b231eda..893e141 100644
---- a/Cassandane/Cyrus/Quota.pm
-+++ b/Cassandane/Cyrus/Quota.pm
-@@ -1000,7 +1000,7 @@ sub test_quota_f_unixhs
-     my @data = $self->{instance}->run_command({
-         cyrus => 1,
-         redirects => { stdout => $self->{instance}{basedir} . '/quota.out' },
--    }, 'quota', '-f');
-+    }, 'cyr_quota', '-f');
- 
-     open(FH, "<", $self->{instance}{basedir} . '/quota.out');
-     local $/ = undef;
-@@ -1095,7 +1095,7 @@ sub test_quota_f
-     );
- 
-     xlog "find and add the quota";
--    $self->{instance}->run_command({ cyrus => 1 }, 'quota', '-f');
-+    $self->{instance}->run_command({ cyrus => 1 }, 'cyr_quota', '-f');
- 
-     xlog "check usages";
-     $self->_check_usages(
-@@ -1112,7 +1112,7 @@ sub test_quota_f
-     );
- 
-     xlog "re-run the quota utility";
--    $self->{instance}->run_command({ cyrus => 1 }, 'quota', '-f');
-+    $self->{instance}->run_command({ cyrus => 1 }, 'cyr_quota', '-f');
- 
-     xlog "check usages";
-     $self->_check_usages(
-@@ -1165,7 +1165,7 @@ sub test_quota_f_vs_update
-     $self->{instance}->quota_Z_go("$basefolder.a");
-     $self->{instance}->quota_Z_go("$basefolder.b");
-     my (@bits) = $self->{instance}->run_command({ cyrus => 1, background => 1 },
--	'quota', '-Z', '-f', $basefolder);
-+	'cyr_quota', '-Z', '-f', $basefolder);
- 
-     # waiting for quota -f to ensure that
-     # a) the -Z mechanism is working and
-@@ -1248,14 +1248,14 @@ sub test_quota_f_nested_qr
-     $self->_check_usages(quotaroot => "$inbox.nnn", storage => 0);
- 
-     xlog "run quota -f to find and add the quota";
--    $self->{instance}->run_command({ cyrus => 1 }, 'quota', '-f');
-+    $self->{instance}->run_command({ cyrus => 1 }, 'cyr_quota', '-f');
- 
-     xlog "check that STORAGE quota is restored for both roots";
-     $self->_check_usages(quotaroot => $inbox, storage => int($ex1/1024));
-     $self->_check_usages(quotaroot => "$inbox.nnn", storage => int($ex2/1024));
- 
-     xlog "run quota -f again";
--    $self->{instance}->run_command({ cyrus => 1 }, 'quota', '-f');
-+    $self->{instance}->run_command({ cyrus => 1 }, 'cyr_quota', '-f');
- 
-     xlog "check that STORAGE quota is still correct for both roots";
-     $self->_check_usages(quotaroot => $inbox, storage => int($ex1/1024));
-@@ -1330,7 +1330,7 @@ sub test_quota_f_prefix
- 			 storage => int($exp_baseplus/1024));
- 
-     xlog "Run quota -f";
--    $self->{instance}->run_command({ cyrus => 1 }, 'quota', '-f');
-+    $self->{instance}->run_command({ cyrus => 1 }, 'cyr_quota', '-f');
- 
-     xlog "Check that the quotas were unchanged by quota -f";
-     $self->_check_usages(quotaroot => 'user.base',
-@@ -1353,7 +1353,7 @@ sub test_quota_f_prefix
- 			 storage => int($bogus_baseplus/1024));
- 
-     xlog "Run quota -f with no prefix";
--    $self->{instance}->run_command({ cyrus => 1 }, 'quota', '-f');
-+    $self->{instance}->run_command({ cyrus => 1 }, 'cyr_quota', '-f');
- 
-     xlog "Check that the quotas were all fixed";
-     $self->_check_usages(quotaroot => 'user.base',
-@@ -1374,7 +1374,7 @@ sub test_quota_f_prefix
- 			 storage => int($bogus_baseplus/1024));
- 
-     xlog "Run quota -f on user.base only";
--    $self->{instance}->run_command({ cyrus => 1 }, 'quota', '-f', 'user.base');
-+    $self->{instance}->run_command({ cyrus => 1 }, 'cyr_quota', '-f', 'user.base');
- 
-     xlog "Check that only the user.base and user.baseplus quotas were fixed";
-     $self->_check_usages(quotaroot => 'user.base',
-@@ -1395,7 +1395,7 @@ sub test_quota_f_prefix
- 			 storage => int($bogus_baseplus/1024));
- 
-     xlog "Run quota -f on user.baseplus only";
--    $self->{instance}->run_command({ cyrus => 1 }, 'quota', '-f', 'user.baseplus');
-+    $self->{instance}->run_command({ cyrus => 1 }, 'cyr_quota', '-f', 'user.baseplus');
- 
-     xlog "Check that only the user.baseplus quotas were fixed";
-     $self->_check_usages(quotaroot => 'user.base',
-@@ -2384,7 +2384,7 @@ sub test_reconstruct
- 				   'reconstruct', 'user.cassandane');
-     xlog "Running quota -f";
-     $self->{instance}->run_command({ cyrus => 1 },
--				   'quota', '-f', "user.cassandane");
-+				   'cyr_quota', '-f', "user.cassandane");
- 
-     $talk = $store->get_client();
- 
-@@ -2520,7 +2520,7 @@ sub test_reconstruct_orphans
- 				   'reconstruct', 'user.cassandane');
-     xlog "Running quota -f";
-     $self->{instance}->run_command({ cyrus => 1 },
--				   'quota', '-f', "user.cassandane");
-+				   'cyr_quota', '-f', "user.cassandane");
- 
-     $talk = $store->get_client();
- 
-@@ -2566,7 +2566,7 @@ sub test_bug3735
-     $self->{instance}->run_command({
- 	cyrus => 1,
- 	redirects => { stdout => $filename },
--    }, 'quota', "user.a");
-+    }, 'cyr_quota', "user.a");
- 
-     open RESULTS, '<', $filename
- 	or die "Cannot open $filename for reading: $!";