diff --git a/.gitignore b/.gitignore index 0392dfc..a8e3a92 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /git-lfs-2.7.0.tar.gz /git-lfs-2.7.1.tar.gz /git-lfs-2.7.2.tar.gz +/git-lfs-v2.8.0.tar.gz diff --git a/3800.patch b/3800.patch new file mode 100644 index 0000000..e6f0283 --- /dev/null +++ b/3800.patch @@ -0,0 +1,124 @@ +From d1ee735a4aacb80d9c3c4c34fc4317c6eef6718a Mon Sep 17 00:00:00 2001 +From: "brian m. carlson" +Date: Wed, 28 Aug 2019 21:02:26 +0000 +Subject: [PATCH] Avoid deadlock when transfer queue fails + +In 1412d6e4 ("Don't fail if we lack objects the server has", +2019-04-30), we changed the code to abort later if a missing object +occurs. In doing so, we had to consider the case where the transfer +queue aborts early for some reason and ensure that the sync.WaitGroup +does not unnecessarily block due to outstanding objects never getting +processed. + +However, the approach we used, which was to explicitly add the number of +items we skipped processing, was error prone and didn't cover all cases. +Notably, a DNS failure could randomly cause a hang during a push. Solve +this by creating a class for a wait group which is abortable and simply +abort it if we encounter an error, preventing any deadlocks caused by +miscounting the number of items. +--- + tq/transfer_queue.go | 55 ++++++++++++++++++++++++++++++++++---------- + 1 file changed, 43 insertions(+), 12 deletions(-) + +diff --git a/tq/transfer_queue.go b/tq/transfer_queue.go +index 89296a646..7d39fe581 100644 +--- a/tq/transfer_queue.go ++++ b/tq/transfer_queue.go +@@ -123,6 +123,43 @@ func (b batch) Len() int { return len(b) } + func (b batch) Less(i, j int) bool { return b[i].Size < b[j].Size } + func (b batch) Swap(i, j int) { b[i], b[j] = b[j], b[i] } + ++type abortableWaitGroup struct { ++ wq sync.WaitGroup ++ counter int ++ mu sync.Mutex ++} ++ ++func newAbortableWaitQueue() *abortableWaitGroup { ++ return &abortableWaitGroup{} ++} ++ ++func (q *abortableWaitGroup) Add(delta int) { ++ q.mu.Lock() ++ defer q.mu.Unlock() ++ ++ q.counter += delta ++ q.wq.Add(delta) ++} ++ ++func (q *abortableWaitGroup) Done() { ++ q.mu.Lock() ++ defer q.mu.Unlock() ++ ++ q.counter -= 1 ++ q.wq.Done() ++} ++ ++func (q *abortableWaitGroup) Abort() { ++ q.mu.Lock() ++ defer q.mu.Unlock() ++ ++ q.wq.Add(-q.counter) ++} ++ ++func (q *abortableWaitGroup) Wait() { ++ q.wq.Wait() ++} ++ + // TransferQueue organises the wider process of uploading and downloading, + // including calling the API, passing the actual transfer request to transfer + // adapters, and dealing with progress, errors and retries. +@@ -150,7 +187,7 @@ type TransferQueue struct { + // wait is used to keep track of pending transfers. It is incremented + // once per unique OID on Add(), and is decremented when that transfer + // is marked as completed or failed, but not retried. +- wait sync.WaitGroup ++ wait *abortableWaitGroup + manifest *Manifest + rc *retryCounter + +@@ -250,6 +287,7 @@ func NewTransferQueue(dir Direction, manifest *Manifest, remote string, options + trMutex: &sync.Mutex{}, + manifest: manifest, + rc: newRetryCounter(), ++ wait: newAbortableWaitQueue(), + } + + for _, opt := range options { +@@ -401,8 +439,11 @@ func (q *TransferQueue) collectBatches() { + collected, closing = q.collectPendingUntil(done) + + // If we've encountered a serious error here, abort immediately; +- // don't process further batches. ++ // don't process further batches. Abort the wait queue so that ++ // we don't deadlock waiting for objects to complete when they ++ // never will. + if err != nil { ++ q.wait.Abort() + break + } + +@@ -497,11 +538,6 @@ func (q *TransferQueue) enqueueAndCollectRetriesFor(batch batch) (batch, error) + } + } + +- if err != nil && bRes != nil { +- // Avoid a hang if we return early. +- q.wait.Add(-len(bRes.Objects)) +- } +- + return next, err + } + } +@@ -521,11 +557,6 @@ func (q *TransferQueue) enqueueAndCollectRetriesFor(batch batch) (batch, error) + // missing in that case, since we don't need to upload + // it. + if o.Missing && len(o.Actions) != 0 { +- // Indicate that we've handled these objects, in +- // this case by ignoring them and aborting +- // early. Failing to do this means we deadlock +- // on this WaitGroup. +- q.wait.Add(-len(bRes.Objects)) + return nil, errors.Errorf("Unable to find source for object %v (try running git lfs fetch --all)", o.Oid) + } + } diff --git a/git-lfs.spec b/git-lfs.spec index d24a462..dbe5e2d 100644 --- a/git-lfs.spec +++ b/git-lfs.spec @@ -2,7 +2,7 @@ # https://github.com/git-lfs/git-lfs %global goipath github.com/git-lfs/git-lfs -Version: 2.7.2 +Version: 2.8.0 %gometa @@ -19,11 +19,14 @@ Summary: Git extension for versioning large files License: MIT URL: https://git-lfs.github.io/ -Source0: %{gosource} +Source0: https://github.com/%{name}/%{name}/releases/download/v%{version}/%{name}-v%{version}.tar.gz +# https://github.com/git-lfs/git-lfs/issues/3798 +Patch0001: https://github.com/git-lfs/git-lfs/pull/3800.patch -BuildRequires: golang(github.com/git-lfs/gitobj) >= 1.1.0 -BuildRequires: golang(github.com/git-lfs/gitobj/errors) >= 1.1.0 +BuildRequires: golang(github.com/git-lfs/gitobj) >= 1.3.1 +BuildRequires: golang(github.com/git-lfs/gitobj/errors) >= 1.3.1 BuildRequires: golang(github.com/git-lfs/go-netrc/netrc) >= 0-0.1.20180827gite0e9ca4 +BuildRequires: golang(github.com/git-lfs/go-ntlm/ntlm) BuildRequires: golang(github.com/git-lfs/wildmatch) >= 1.0.2 BuildRequires: golang(github.com/kr/pty) BuildRequires: golang(github.com/mattn/go-isatty) >= 0.0.4 @@ -31,7 +34,6 @@ BuildRequires: golang(github.com/olekukonko/ts) BuildRequires: golang(github.com/pkg/errors) BuildRequires: golang(github.com/rubyist/tracerx) BuildRequires: golang(github.com/spf13/cobra) >= 0.0.3 -BuildRequires: golang(github.com/ThomsonReutersEikon/go-ntlm/ntlm) BuildRequires: golang(golang.org/x/sync/semaphore) # Generate man pages @@ -62,6 +64,8 @@ storing the file contents on a remote server. %prep %goprep +%patch0001 -p1 + # Modify Makefile so that it expects binaries where we build them. sed -i -e 's!\.\./bin/!/%{gobuilddir}/bin/!g' t/Makefile @@ -121,6 +125,9 @@ PATH=%{buildroot}%{_bindir}:%{gobuilddir}/bin:$PATH \ %changelog +* Fri Aug 30 2019 Elliott Sales de Andrade - 2.8.0-1 +- Update to latest version + * Thu Jul 25 2019 Fedora Release Engineering - 2.7.2-3 - Rebuilt for https://fedoraproject.org/wiki/Fedora_31_Mass_Rebuild diff --git a/sources b/sources index 104fa5d..56e1022 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (git-lfs-2.7.2.tar.gz) = cd71815eb418b7acaf077de4873ff49d332f71151c1212ca4fe3a2d0e079bad873894bb416488f272777153cc3422deecf17a849cd67f150b44eb094a09ae8be +SHA512 (git-lfs-v2.8.0.tar.gz) = e30da595dc2302fef692211da2efe93db9803914603229b46f15199ac9b87fe3d604a0436223d9e47f34b70ea2206ebf69b572003ead8d9f81f036c986281e2c