From 95f873ff983a1ae57415b3c16a881e74432cf8b8 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 9 Feb 2021 14:04:32 +0100 Subject: [PATCH 1/4] runtests.pl: support the nonewline attribute for the data part Added to FILEFORMAT Closes #8239 Upstream-commit: 736847611a40c01e7c290407e22e2f0f5f8efd6a Signed-off-by: Kamil Dudka --- tests/FILEFORMAT.md | 5 ++++- tests/runtests.pl | 7 +++++++ tests/server/getpart.c | 11 ++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/FILEFORMAT.md b/tests/FILEFORMAT.md index b75a02a..0b98787 100644 --- a/tests/FILEFORMAT.md +++ b/tests/FILEFORMAT.md @@ -185,7 +185,7 @@ which are treated together as a single identifier. ## `` -### `` +### `` data to be sent to the client on its request and later verified that it arrived safely. Set `nocheck="yes"` to prevent the test script from verifying @@ -211,6 +211,9 @@ much sense for other sections than "data"). `hex=yes` means that the data is a sequence of hex pairs. It will get decoded and used as "raw" data. +`nonewline=yes` means that the last byte (the trailing newline character) +should be cut off from the data before sending or comparing it. + For FTP file listings, the `` section will be used *only* if you make sure that there has been a CWD done first to a directory named `test-[num]` where [num] is the test case number. Otherwise the ftp server can't know from diff --git a/tests/runtests.pl b/tests/runtests.pl index 40315aa..2e1500d 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -3837,6 +3837,13 @@ sub singletest { else { # check against the data section @reply = getpart("reply", "data"); + if(@reply) { + my %hash = getpartattr("reply", "data"); + if($hash{'nonewline'}) { + # cut off the final newline from the final line of the data + chomp($reply[$#reply]); + } + } # get the mode attribute my $filemode=$replyattr{'mode'}; if($filemode && ($filemode eq "text") && $has_textaware) { diff --git a/tests/server/getpart.c b/tests/server/getpart.c index 32b55bc..f8fe3f6 100644 --- a/tests/server/getpart.c +++ b/tests/server/getpart.c @@ -5,7 +5,7 @@ * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2021, Daniel Stenberg, , et al. + * Copyright (C) 1998 - 2022, Daniel Stenberg, , et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms @@ -295,6 +295,7 @@ int getpart(char **outbuf, size_t *outlen, size_t outalloc = 256; int in_wanted_part = 0; int base64 = 0; + int nonewline = 0; int error; enum { @@ -360,6 +361,8 @@ int getpart(char **outbuf, size_t *outlen, if(error) return error; } + if(nonewline) + (*outlen)--; break; } } @@ -377,6 +380,8 @@ int getpart(char **outbuf, size_t *outlen, if(error) return error; } + if(nonewline) + (*outlen)--; break; } } @@ -451,6 +456,10 @@ int getpart(char **outbuf, size_t *outlen, /* bit rough test, but "mostly" functional, */ /* treat wanted part data as base64 encoded */ base64 = 1; + if(strstr(patt, "nonewline=")) { + show(("* setting nonewline\n")); + nonewline = 1; + } } continue; } -- 2.39.1 From 6e244e1bcb04012e11c537253e76e6f968d8bb72 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Thu, 1 Dec 2022 09:21:04 +0100 Subject: [PATCH 2/4] runtests: do CRLF replacements per section only The `crlf="yes"` attribute and "hyper mode" are now only applied on a subset of dedicated sections: data, datacheck, stdout and protocol. Updated test 2500 accordingly. Also made test1 use crlf="yes" for , mostly because it is often used as a template test case. Going forward, using this attribute we should be able to write test cases using linefeeds only and avoid mixed line ending encodings. Follow-up to ca15b7512e8d11 Fixes #10009 Closes #10010 Upstream-commit: 2f34a7347f315513bfda9ef14770d287fb246bcd Signed-off-by: Kamil Dudka --- tests/FILEFORMAT.md | 22 +++++++++++++++------ tests/data/test1 | 14 ++++++------- tests/runtests.pl | 48 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/tests/FILEFORMAT.md b/tests/FILEFORMAT.md index c1fbc57..dcb5695 100644 --- a/tests/FILEFORMAT.md +++ b/tests/FILEFORMAT.md @@ -185,7 +185,7 @@ which are treated together as a single identifier. ## `` -### `` +### `` data to be sent to the client on its request and later verified that it arrived safely. Set `nocheck="yes"` to prevent the test script from verifying @@ -214,12 +214,16 @@ and used as "raw" data. `nonewline=yes` means that the last byte (the trailing newline character) should be cut off from the data before sending or comparing it. +`crlf=yes` forces *header* newlines to become CRLF even if not written so in +the source file. Note that this makes runtests.pl parse and "guess" what is a +header and what is not in order to apply the CRLF line endings appropriately. + For FTP file listings, the `` section will be used *only* if you make sure that there has been a CWD done first to a directory named `test-[num]` where [num] is the test case number. Otherwise the ftp server can't know from which test file to load the list content. -### `` +### `` Send back this contents instead of the one. The num is set by: @@ -243,7 +247,7 @@ The connect section is used instead of the 'data' for all CONNECT requests. The remainder of the rules for the data section then apply but with a connect prefix. -### `` +### `` if the data is sent but this is what should be checked afterwards. If `nonewline=yes` is set, runtests will cut off the trailing newline from the data before comparing with the one actually received by the client. @@ -251,7 +255,7 @@ data before comparing with the one actually received by the client. Use the `mode="text"` attribute if the output is in text mode on platforms that have a text/binary difference. -### `` +### `` The contents of numbered datacheck sections are appended to the non-numbered one. @@ -528,13 +532,16 @@ changing protocol data such as port numbers or user-agent strings. One perl op per line that operates on the protocol dump. This is pretty advanced. Example: `s/^EPRT .*/EPRT stripped/`. -### `` +### `` the protocol dump curl should transmit, if 'nonewline' is set, we will cut off the trailing newline of this given data before comparing with the one actually sent by the client The `` and `` rules are applied before comparisons are made. +`crlf=yes` forces the newlines to become CRLF even if not written so in the +test. + ### `` The protocol dump curl should transmit to a HTTP proxy (when the http-proxy @@ -551,7 +558,7 @@ have a text/binary difference. If 'nonewline' is set, we will cut off the trailing newline of this given data before comparing with the one actually received by the client -### `` +### `` This verifies that this data was passed to stdout. Use the mode="text" attribute if the output is in text mode on platforms that @@ -560,6 +567,9 @@ have a text/binary difference. If 'nonewline' is set, we will cut off the trailing newline of this given data before comparing with the one actually received by the client +`crlf=yes` forces the newlines to become CRLF even if not written so in the +test. + ### `` The file's contents must be identical to this after the test is complete. Use the mode="text" attribute if the output is in text mode on platforms that have diff --git a/tests/data/test1 b/tests/data/test1 index f39a08b..700bed8 100644 --- a/tests/data/test1 +++ b/tests/data/test1 @@ -9,7 +9,7 @@ HTTP GET # # Server-side - + HTTP/1.1 200 OK Date: Tue, 09 Nov 2010 14:49:00 GMT Server: test-server/fake @@ -42,12 +42,12 @@ http://%HOSTIP:%HTTPPORT/%TESTNUMBER # # Verify data after the test has been "shot" - -GET /%TESTNUMBER HTTP/1.1 -Host: %HOSTIP:%HTTPPORT -User-Agent: curl/%VERSION -Accept: */* - + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + diff --git a/tests/runtests.pl b/tests/runtests.pl index 72a9989..b12a42d 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -3410,7 +3410,13 @@ sub subBase64 { my $prevupdate; sub subNewlines { - my ($thing) = @_; + my ($force, $thing) = @_; + + if($force) { + # enforce CRLF newline + $$thing =~ s/\x0d*\x0a/\x0d\x0a/; + return; + } # When curl is built with Hyper, it gets all response headers delivered as # name/value pairs and curl "invents" the newlines when it saves the @@ -3424,7 +3430,7 @@ sub subNewlines { # skip curl error messages ($$thing !~ /^curl: \(\d+\) /))) { # enforce CRLF newline - $$thing =~ s/\x0a/\x0d\x0a/; + $$thing =~ s/\x0d*\x0a/\x0d\x0a/; $prevupdate = 1; } else { @@ -3496,6 +3502,7 @@ sub prepro { my (@entiretest) = @_; my $show = 1; my @out; + my $data_crlf; for my $s (@entiretest) { my $f = $s; if($s =~ /^ *%if (.*)/) { @@ -3519,9 +3526,19 @@ sub prepro { next; } if($show) { + # The processor does CRLF replacements in the sections if + # necessary since those parts might be read by separate servers. + if($s =~ /^ */) { + if($1 =~ /crlf="yes"/ || $has_hyper) { + $data_crlf = 1; + } + } + elsif(($s =~ /^ *<\/data/) && $data_crlf) { + $data_crlf = 0; + } subVariables(\$s, $testnum, "%"); subBase64(\$s); - subNewlines(\$s) if($has_hyper); + subNewlines(0, \$s) if($data_crlf); push @out, $s; } } @@ -3830,6 +3847,11 @@ sub singletest { # of the datacheck chomp($replycheckpart[$#replycheckpart]); } + if($replycheckpartattr{'crlf'} || + ($has_hyper && ($keywords{"HTTP"} + || $keywords{"HTTPS"}))) { + map subNewlines(0, \$_), @replycheckpart; + } push(@reply, @replycheckpart); } } @@ -3851,6 +3873,11 @@ sub singletest { map s/\r\n/\n/g, @reply; map s/\n/\r\n/g, @reply; } + if($replyattr{'crlf'} || + ($has_hyper && ($keywords{"HTTP"} + || $keywords{"HTTPS"}))) { + map subNewlines(0, \$_), @reply; + } } # this is the valid protocol blurb curl should generate @@ -4287,6 +4314,12 @@ sub singletest { chomp($validstdout[$#validstdout]); } + if($hash{'crlf'} || + ($has_hyper && ($keywords{"HTTP"} + || $keywords{"HTTPS"}))) { + map subNewlines(0, \$_), @validstdout; + } + $res = compare($testnum, $testname, "stdout", \@actual, \@validstdout); if($res) { return $errorreturncode; @@ -4381,6 +4414,10 @@ sub singletest { } } + if($hash{'crlf'}) { + map subNewlines(1, \$_), @protstrip; + } + if((!$out[0] || ($out[0] eq "")) && $protstrip[0]) { logmsg "\n $testnum: protocol FAILED!\n". " There was no content at all in the file $SERVERIN.\n". @@ -4512,6 +4549,11 @@ sub singletest { map s/\r\n/\n/g, @outfile; map s/\n/\r\n/g, @outfile; } + if($hash{'crlf'} || + ($has_hyper && ($keywords{"HTTP"} + || $keywords{"HTTPS"}))) { + map subNewlines(0, \$_), @outfile; + } my $strip; for $strip (@stripfile) { -- 2.39.1 From 228ed11bf33c63d9208a3fb38fe5a0d19c0764bd Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Tue, 27 Dec 2022 11:50:23 +0100 Subject: [PATCH 3/4] runtests: support crlf="yes" for verify/proxy Upstream-commit: dc0725244a3163f1e2d5f51165db3a1a430f3ba0 Signed-off-by: Kamil Dudka --- tests/FILEFORMAT.md | 4 ++-- tests/runtests.pl | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/FILEFORMAT.md b/tests/FILEFORMAT.md index dcb5695..6646793 100644 --- a/tests/FILEFORMAT.md +++ b/tests/FILEFORMAT.md @@ -532,7 +532,7 @@ changing protocol data such as port numbers or user-agent strings. One perl op per line that operates on the protocol dump. This is pretty advanced. Example: `s/^EPRT .*/EPRT stripped/`. -### `` +### `` the protocol dump curl should transmit, if 'nonewline' is set, we will cut off the trailing newline of this given data before comparing with the one actually @@ -542,7 +542,7 @@ comparisons are made. `crlf=yes` forces the newlines to become CRLF even if not written so in the test. -### `` +### `` The protocol dump curl should transmit to a HTTP proxy (when the http-proxy server is used), if 'nonewline' is set, we will cut off the trailing newline diff --git a/tests/runtests.pl b/tests/runtests.pl index b12a42d..5cdc83d 100755 --- a/tests/runtests.pl +++ b/tests/runtests.pl @@ -4510,6 +4510,11 @@ sub singletest { } } + if($hash{'crlf'} || + ($has_hyper && ($keywords{"HTTP"} || $keywords{"HTTPS"}))) { + map subNewlines(0, \$_), @protstrip; + } + $res = compare($testnum, $testname, "proxy", \@out, \@protstrip); if($res) { return $errorreturncode; -- 2.39.1 From bc5fc958b017895728962c9d44c469418cbec1a0 Mon Sep 17 00:00:00 2001 From: Patrick Monnerat Date: Mon, 13 Feb 2023 08:33:09 +0100 Subject: [PATCH 4/4] content_encoding: do not reset stage counter for each header Test 418 verifies Closes #10492 Upstream-commit: 119fb187192a9ea13dc90d9d20c215fc82799ab9 Signed-off-by: Kamil Dudka --- lib/content_encoding.c | 7 +- lib/urldata.h | 1 + tests/data/Makefile.inc | 1 + tests/data/test387 | 2 +- tests/data/test418 | 152 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 5 deletions(-) create mode 100644 tests/data/test418 diff --git a/lib/content_encoding.c b/lib/content_encoding.c index bfc13e2..94344d6 100644 --- a/lib/content_encoding.c +++ b/lib/content_encoding.c @@ -1033,7 +1033,6 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, const char *enclist, int maybechunked) { struct SingleRequest *k = &data->req; - int counter = 0; do { const char *name; @@ -1068,9 +1067,9 @@ CURLcode Curl_build_unencoding_stack(struct Curl_easy *data, if(!encoding) encoding = &error_encoding; /* Defer error at stack use. */ - if(++counter >= MAX_ENCODE_STACK) { - failf(data, "Reject response due to %u content encodings", - counter); + if(k->writer_stack_depth++ >= MAX_ENCODE_STACK) { + failf(data, "Reject response due to more than %u content encodings", + MAX_ENCODE_STACK); return CURLE_BAD_CONTENT_ENCODING; } /* Stack the unencoding stage. */ diff --git a/lib/urldata.h b/lib/urldata.h index 5b4b34f..8c8c20b 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -700,6 +700,7 @@ struct SingleRequest { #ifndef CURL_DISABLE_DOH struct dohdata *doh; /* DoH specific data for this request */ #endif + unsigned char writer_stack_depth; /* Unencoding stack depth. */ BIT(header); /* incoming data has HTTP header */ BIT(content_range); /* set TRUE if Content-Range: was found */ BIT(upload_done); /* set to TRUE when doing chunked transfer-encoding diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index fb51cd6..86b6f85 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -67,6 +67,7 @@ test393 test394 test395 test396 test397 \ \ test400 test401 test402 test403 test404 test405 test406 test407 test408 \ test409 test410 \ + test418 \ \ test430 test431 test432 test433 test434 \ \ diff --git a/tests/data/test387 b/tests/data/test387 index 015ec25..644fc7f 100644 --- a/tests/data/test387 +++ b/tests/data/test387 @@ -47,7 +47,7 @@ Accept: */* 61 -curl: (61) Reject response due to 5 content encodings +curl: (61) Reject response due to more than 5 content encodings diff --git a/tests/data/test418 b/tests/data/test418 new file mode 100644 index 0000000..50e974e --- /dev/null +++ b/tests/data/test418 @@ -0,0 +1,152 @@ + + + +HTTP +gzip + + + +# +# Server-side + + +HTTP/1.1 200 OK +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip +Transfer-Encoding: gzip + +-foo- + + + +# +# Client-side + + +http + + +Response with multiple Transfer-Encoding headers + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER -sS + + + +# +# Verify data after the test has been "shot" + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + + + +# CURLE_BAD_CONTENT_ENCODING is 61 + +61 + + +curl: (61) Reject response due to more than 5 content encodings + + + -- 2.39.1