Merge branch 'f30'
This commit is contained in:
Elliott Sales de Andrade
commit
8b02cd9c47
1
.gitignore
vendored
1
.gitignore
vendored
@ -10,3 +10,4 @@
|
||||
/git-lfs-2.7.0.tar.gz
|
||||
/git-lfs-2.7.1.tar.gz
|
||||
/git-lfs-2.7.2.tar.gz
|
||||
/git-lfs-v2.8.0.tar.gz
|
||||
|
||||
124
3800.patch
Normal file
124
3800.patch
Normal file
@ -0,0 +1,124 @@
|
||||
From d1ee735a4aacb80d9c3c4c34fc4317c6eef6718a Mon Sep 17 00:00:00 2001
|
||||
From: "brian m. carlson" <bk2204@github.com>
|
||||
Date: Wed, 28 Aug 2019 21:02:26 +0000
|
||||
Subject: [PATCH] Avoid deadlock when transfer queue fails
|
||||
|
||||
In 1412d6e4 ("Don't fail if we lack objects the server has",
|
||||
2019-04-30), we changed the code to abort later if a missing object
|
||||
occurs. In doing so, we had to consider the case where the transfer
|
||||
queue aborts early for some reason and ensure that the sync.WaitGroup
|
||||
does not unnecessarily block due to outstanding objects never getting
|
||||
processed.
|
||||
|
||||
However, the approach we used, which was to explicitly add the number of
|
||||
items we skipped processing, was error prone and didn't cover all cases.
|
||||
Notably, a DNS failure could randomly cause a hang during a push. Solve
|
||||
this by creating a class for a wait group which is abortable and simply
|
||||
abort it if we encounter an error, preventing any deadlocks caused by
|
||||
miscounting the number of items.
|
||||
---
|
||||
tq/transfer_queue.go | 55 ++++++++++++++++++++++++++++++++++----------
|
||||
1 file changed, 43 insertions(+), 12 deletions(-)
|
||||
|
||||
diff --git a/tq/transfer_queue.go b/tq/transfer_queue.go
|
||||
index 89296a646..7d39fe581 100644
|
||||
--- a/tq/transfer_queue.go
|
||||
+++ b/tq/transfer_queue.go
|
||||
@@ -123,6 +123,43 @@ func (b batch) Len() int { return len(b) }
|
||||
func (b batch) Less(i, j int) bool { return b[i].Size < b[j].Size }
|
||||
func (b batch) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
|
||||
|
||||
+type abortableWaitGroup struct {
|
||||
+ wq sync.WaitGroup
|
||||
+ counter int
|
||||
+ mu sync.Mutex
|
||||
+}
|
||||
+
|
||||
+func newAbortableWaitQueue() *abortableWaitGroup {
|
||||
+ return &abortableWaitGroup{}
|
||||
+}
|
||||
+
|
||||
+func (q *abortableWaitGroup) Add(delta int) {
|
||||
+ q.mu.Lock()
|
||||
+ defer q.mu.Unlock()
|
||||
+
|
||||
+ q.counter += delta
|
||||
+ q.wq.Add(delta)
|
||||
+}
|
||||
+
|
||||
+func (q *abortableWaitGroup) Done() {
|
||||
+ q.mu.Lock()
|
||||
+ defer q.mu.Unlock()
|
||||
+
|
||||
+ q.counter -= 1
|
||||
+ q.wq.Done()
|
||||
+}
|
||||
+
|
||||
+func (q *abortableWaitGroup) Abort() {
|
||||
+ q.mu.Lock()
|
||||
+ defer q.mu.Unlock()
|
||||
+
|
||||
+ q.wq.Add(-q.counter)
|
||||
+}
|
||||
+
|
||||
+func (q *abortableWaitGroup) Wait() {
|
||||
+ q.wq.Wait()
|
||||
+}
|
||||
+
|
||||
// TransferQueue organises the wider process of uploading and downloading,
|
||||
// including calling the API, passing the actual transfer request to transfer
|
||||
// adapters, and dealing with progress, errors and retries.
|
||||
@@ -150,7 +187,7 @@ type TransferQueue struct {
|
||||
// wait is used to keep track of pending transfers. It is incremented
|
||||
// once per unique OID on Add(), and is decremented when that transfer
|
||||
// is marked as completed or failed, but not retried.
|
||||
- wait sync.WaitGroup
|
||||
+ wait *abortableWaitGroup
|
||||
manifest *Manifest
|
||||
rc *retryCounter
|
||||
|
||||
@@ -250,6 +287,7 @@ func NewTransferQueue(dir Direction, manifest *Manifest, remote string, options
|
||||
trMutex: &sync.Mutex{},
|
||||
manifest: manifest,
|
||||
rc: newRetryCounter(),
|
||||
+ wait: newAbortableWaitQueue(),
|
||||
}
|
||||
|
||||
for _, opt := range options {
|
||||
@@ -401,8 +439,11 @@ func (q *TransferQueue) collectBatches() {
|
||||
collected, closing = q.collectPendingUntil(done)
|
||||
|
||||
// If we've encountered a serious error here, abort immediately;
|
||||
- // don't process further batches.
|
||||
+ // don't process further batches. Abort the wait queue so that
|
||||
+ // we don't deadlock waiting for objects to complete when they
|
||||
+ // never will.
|
||||
if err != nil {
|
||||
+ q.wait.Abort()
|
||||
break
|
||||
}
|
||||
|
||||
@@ -497,11 +538,6 @@ func (q *TransferQueue) enqueueAndCollectRetriesFor(batch batch) (batch, error)
|
||||
}
|
||||
}
|
||||
|
||||
- if err != nil && bRes != nil {
|
||||
- // Avoid a hang if we return early.
|
||||
- q.wait.Add(-len(bRes.Objects))
|
||||
- }
|
||||
-
|
||||
return next, err
|
||||
}
|
||||
}
|
||||
@@ -521,11 +557,6 @@ func (q *TransferQueue) enqueueAndCollectRetriesFor(batch batch) (batch, error)
|
||||
// missing in that case, since we don't need to upload
|
||||
// it.
|
||||
if o.Missing && len(o.Actions) != 0 {
|
||||
- // Indicate that we've handled these objects, in
|
||||
- // this case by ignoring them and aborting
|
||||
- // early. Failing to do this means we deadlock
|
||||
- // on this WaitGroup.
|
||||
- q.wait.Add(-len(bRes.Objects))
|
||||
return nil, errors.Errorf("Unable to find source for object %v (try running git lfs fetch --all)", o.Oid)
|
||||
}
|
||||
}
|
||||
17
git-lfs.spec
17
git-lfs.spec
@ -2,7 +2,7 @@
|
||||
|
||||
# https://github.com/git-lfs/git-lfs
|
||||
%global goipath github.com/git-lfs/git-lfs
|
||||
Version: 2.7.2
|
||||
Version: 2.8.0
|
||||
|
||||
%gometa
|
||||
|
||||
@ -19,11 +19,14 @@ Summary: Git extension for versioning large files
|
||||
|
||||
License: MIT
|
||||
URL: https://git-lfs.github.io/
|
||||
Source0: %{gosource}
|
||||
Source0: https://github.com/%{name}/%{name}/releases/download/v%{version}/%{name}-v%{version}.tar.gz
|
||||
# https://github.com/git-lfs/git-lfs/issues/3798
|
||||
Patch0001: https://github.com/git-lfs/git-lfs/pull/3800.patch
|
||||
|
||||
BuildRequires: golang(github.com/git-lfs/gitobj) >= 1.1.0
|
||||
BuildRequires: golang(github.com/git-lfs/gitobj/errors) >= 1.1.0
|
||||
BuildRequires: golang(github.com/git-lfs/gitobj) >= 1.3.1
|
||||
BuildRequires: golang(github.com/git-lfs/gitobj/errors) >= 1.3.1
|
||||
BuildRequires: golang(github.com/git-lfs/go-netrc/netrc) >= 0-0.1.20180827gite0e9ca4
|
||||
BuildRequires: golang(github.com/git-lfs/go-ntlm/ntlm)
|
||||
BuildRequires: golang(github.com/git-lfs/wildmatch) >= 1.0.2
|
||||
BuildRequires: golang(github.com/kr/pty)
|
||||
BuildRequires: golang(github.com/mattn/go-isatty) >= 0.0.4
|
||||
@ -31,7 +34,6 @@ BuildRequires: golang(github.com/olekukonko/ts)
|
||||
BuildRequires: golang(github.com/pkg/errors)
|
||||
BuildRequires: golang(github.com/rubyist/tracerx)
|
||||
BuildRequires: golang(github.com/spf13/cobra) >= 0.0.3
|
||||
BuildRequires: golang(github.com/ThomsonReutersEikon/go-ntlm/ntlm)
|
||||
BuildRequires: golang(golang.org/x/sync/semaphore)
|
||||
|
||||
# Generate man pages
|
||||
@ -62,6 +64,8 @@ storing the file contents on a remote server.
|
||||
%prep
|
||||
%goprep
|
||||
|
||||
%patch0001 -p1
|
||||
|
||||
# Modify Makefile so that it expects binaries where we build them.
|
||||
sed -i -e 's!\.\./bin/!/%{gobuilddir}/bin/!g' t/Makefile
|
||||
|
||||
@ -121,6 +125,9 @@ PATH=%{buildroot}%{_bindir}:%{gobuilddir}/bin:$PATH \
|
||||
|
||||
|
||||
%changelog
|
||||
* Fri Aug 30 2019 Elliott Sales de Andrade <quantum.analyst@gmail.com> - 2.8.0-1
|
||||
- Update to latest version
|
||||
|
||||
* Thu Jul 25 2019 Fedora Release Engineering <releng@fedoraproject.org> - 2.7.2-3
|
||||
- Rebuilt for https://fedoraproject.org/wiki/Fedora_31_Mass_Rebuild
|
||||
|
||||
|
||||
2
sources
2
sources
@ -1 +1 @@
|
||||
SHA512 (git-lfs-2.7.2.tar.gz) = cd71815eb418b7acaf077de4873ff49d332f71151c1212ca4fe3a2d0e079bad873894bb416488f272777153cc3422deecf17a849cd67f150b44eb094a09ae8be
|
||||
SHA512 (git-lfs-v2.8.0.tar.gz) = e30da595dc2302fef692211da2efe93db9803914603229b46f15199ac9b87fe3d604a0436223d9e47f34b70ea2206ebf69b572003ead8d9f81f036c986281e2c
|
||||
|
||||
Loading…
Reference in New Issue
Block a user