From abe1a4f531168df225e9bbe8a064e5732ac6415a Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Wed, 28 Aug 2019 02:41:19 -0400 Subject: [PATCH 1/2] Switch to releases URL, which has verifiable SHA256 hashes. --- git-lfs.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-lfs.spec b/git-lfs.spec index 38a2937..2fedf2f 100644 --- a/git-lfs.spec +++ b/git-lfs.spec @@ -9,7 +9,7 @@ Summary: Git extension for versioning large files License: MIT URL: https://git-lfs.github.io/ -Source0: %{gosource} +Source0: https://github.com/%{name}/%{name}/releases/download/v%{version}/%{name}-v%{version}.tar.gz BuildRequires: golang(github.com/ThomsonReutersEikon/go-ntlm/ntlm) BuildRequires: golang(github.com/git-lfs/gitobj) >= 1.1.0 From a7b729ae6b3c1a8b024a3bd6f0e3c99c339b9c77 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Fri, 30 Aug 2019 05:49:55 -0400 Subject: [PATCH 2/2] Update to latest version. --- .gitignore | 1 + 3800.patch | 124 +++++++++++++++++++++++++++++++++++++++++++++++++++ git-lfs.spec | 11 +++-- sources | 2 +- 4 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 3800.patch diff --git a/.gitignore b/.gitignore index 0392dfc..a8e3a92 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /git-lfs-2.7.0.tar.gz /git-lfs-2.7.1.tar.gz /git-lfs-2.7.2.tar.gz +/git-lfs-v2.8.0.tar.gz diff --git a/3800.patch b/3800.patch new file mode 100644 index 0000000..e6f0283 --- /dev/null +++ b/3800.patch @@ -0,0 +1,124 @@ +From d1ee735a4aacb80d9c3c4c34fc4317c6eef6718a Mon Sep 17 00:00:00 2001 +From: "brian m. carlson" +Date: Wed, 28 Aug 2019 21:02:26 +0000 +Subject: [PATCH] Avoid deadlock when transfer queue fails + +In 1412d6e4 ("Don't fail if we lack objects the server has", +2019-04-30), we changed the code to abort later if a missing object +occurs. In doing so, we had to consider the case where the transfer +queue aborts early for some reason and ensure that the sync.WaitGroup +does not unnecessarily block due to outstanding objects never getting +processed. + +However, the approach we used, which was to explicitly add the number of +items we skipped processing, was error prone and didn't cover all cases. +Notably, a DNS failure could randomly cause a hang during a push. Solve +this by creating a class for a wait group which is abortable and simply +abort it if we encounter an error, preventing any deadlocks caused by +miscounting the number of items. +--- + tq/transfer_queue.go | 55 ++++++++++++++++++++++++++++++++++---------- + 1 file changed, 43 insertions(+), 12 deletions(-) + +diff --git a/tq/transfer_queue.go b/tq/transfer_queue.go +index 89296a646..7d39fe581 100644 +--- a/tq/transfer_queue.go ++++ b/tq/transfer_queue.go +@@ -123,6 +123,43 @@ func (b batch) Len() int { return len(b) } + func (b batch) Less(i, j int) bool { return b[i].Size < b[j].Size } + func (b batch) Swap(i, j int) { b[i], b[j] = b[j], b[i] } + ++type abortableWaitGroup struct { ++ wq sync.WaitGroup ++ counter int ++ mu sync.Mutex ++} ++ ++func newAbortableWaitQueue() *abortableWaitGroup { ++ return &abortableWaitGroup{} ++} ++ ++func (q *abortableWaitGroup) Add(delta int) { ++ q.mu.Lock() ++ defer q.mu.Unlock() ++ ++ q.counter += delta ++ q.wq.Add(delta) ++} ++ ++func (q *abortableWaitGroup) Done() { ++ q.mu.Lock() ++ defer q.mu.Unlock() ++ ++ q.counter -= 1 ++ q.wq.Done() ++} ++ ++func (q *abortableWaitGroup) Abort() { ++ q.mu.Lock() ++ defer q.mu.Unlock() ++ ++ q.wq.Add(-q.counter) ++} ++ ++func (q *abortableWaitGroup) Wait() { ++ q.wq.Wait() ++} ++ + // TransferQueue organises the wider process of uploading and downloading, + // including calling the API, passing the actual transfer request to transfer + // adapters, and dealing with progress, errors and retries. +@@ -150,7 +187,7 @@ type TransferQueue struct { + // wait is used to keep track of pending transfers. It is incremented + // once per unique OID on Add(), and is decremented when that transfer + // is marked as completed or failed, but not retried. +- wait sync.WaitGroup ++ wait *abortableWaitGroup + manifest *Manifest + rc *retryCounter + +@@ -250,6 +287,7 @@ func NewTransferQueue(dir Direction, manifest *Manifest, remote string, options + trMutex: &sync.Mutex{}, + manifest: manifest, + rc: newRetryCounter(), ++ wait: newAbortableWaitQueue(), + } + + for _, opt := range options { +@@ -401,8 +439,11 @@ func (q *TransferQueue) collectBatches() { + collected, closing = q.collectPendingUntil(done) + + // If we've encountered a serious error here, abort immediately; +- // don't process further batches. ++ // don't process further batches. Abort the wait queue so that ++ // we don't deadlock waiting for objects to complete when they ++ // never will. + if err != nil { ++ q.wait.Abort() + break + } + +@@ -497,11 +538,6 @@ func (q *TransferQueue) enqueueAndCollectRetriesFor(batch batch) (batch, error) + } + } + +- if err != nil && bRes != nil { +- // Avoid a hang if we return early. +- q.wait.Add(-len(bRes.Objects)) +- } +- + return next, err + } + } +@@ -521,11 +557,6 @@ func (q *TransferQueue) enqueueAndCollectRetriesFor(batch batch) (batch, error) + // missing in that case, since we don't need to upload + // it. + if o.Missing && len(o.Actions) != 0 { +- // Indicate that we've handled these objects, in +- // this case by ignoring them and aborting +- // early. Failing to do this means we deadlock +- // on this WaitGroup. +- q.wait.Add(-len(bRes.Objects)) + return nil, errors.Errorf("Unable to find source for object %v (try running git lfs fetch --all)", o.Oid) + } + } diff --git a/git-lfs.spec b/git-lfs.spec index 2fedf2f..d9f6387 100644 --- a/git-lfs.spec +++ b/git-lfs.spec @@ -1,5 +1,5 @@ %global goipath github.com/git-lfs/git-lfs -Version: 2.7.2 +Version: 2.8.0 %gometa @@ -10,10 +10,12 @@ Summary: Git extension for versioning large files License: MIT URL: https://git-lfs.github.io/ Source0: https://github.com/%{name}/%{name}/releases/download/v%{version}/%{name}-v%{version}.tar.gz +# https://github.com/git-lfs/git-lfs/issues/3798 +Patch0001: https://github.com/git-lfs/git-lfs/pull/3800.patch -BuildRequires: golang(github.com/ThomsonReutersEikon/go-ntlm/ntlm) -BuildRequires: golang(github.com/git-lfs/gitobj) >= 1.1.0 +BuildRequires: golang(github.com/git-lfs/gitobj) >= 1.3.1 BuildRequires: golang(github.com/git-lfs/go-netrc/netrc) >= 0-0.1.20180827gite0e9ca4 +BuildRequires: golang(github.com/git-lfs/go-ntlm/ntlm) BuildRequires: golang(github.com/git-lfs/wildmatch) >= 1.0.2 BuildRequires: golang(github.com/kr/pty) BuildRequires: golang(github.com/mattn/go-isatty) >= 0.0.4 @@ -117,6 +119,9 @@ PATH=%{buildroot}%{_bindir}:$PWD/_bin:$PATH \ %changelog +* Fri Aug 30 2019 Elliott Sales de Andrade - 2.8.0-1 +- Update to latest version + * Wed Apr 24 2019 Elliott Sales de Andrade - 2.7.2-1 - Update to latest version diff --git a/sources b/sources index 104fa5d..56e1022 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (git-lfs-2.7.2.tar.gz) = cd71815eb418b7acaf077de4873ff49d332f71151c1212ca4fe3a2d0e079bad873894bb416488f272777153cc3422deecf17a849cd67f150b44eb094a09ae8be +SHA512 (git-lfs-v2.8.0.tar.gz) = e30da595dc2302fef692211da2efe93db9803914603229b46f15199ac9b87fe3d604a0436223d9e47f34b70ea2206ebf69b572003ead8d9f81f036c986281e2c