From 459ed339aebe4a3845f51f98cba301da1151683e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 17 Nov 2012 15:02:18 +0100 Subject: [PATCH 01/33] Add a script to generate an runtests.pl exclude file for Privoxy curl tests that are known not to work with Privoxy are skipped. --- tests/cts/gen-skip-reasons.pl | 189 ++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100755 tests/cts/gen-skip-reasons.pl diff --git a/tests/cts/gen-skip-reasons.pl b/tests/cts/gen-skip-reasons.pl new file mode 100755 index 00000000..e3569d4e --- /dev/null +++ b/tests/cts/gen-skip-reasons.pl @@ -0,0 +1,189 @@ +#!/usr/bin/perl + +################################################################################ +# +# gen-skip-reasons.pl +# +# Generates an exclude file that can be passed to runtests.pl to skip certain +# tests that aren't expected to work when run through Privoxy. +# +# Copyright (c) 2012-2021 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +################################################################################ + +use warnings; +use strict; + +sub main() { + my %related_tests; + + for my $testnum (1..3500) { + my $why; + if ($testnum eq 16 or + $testnum eq 45 or + $testnum eq 63) { + $why = "Not supposed to work with Privoxy. Expected forwarding failure. Workaround probably possible."; + } elsif ($testnum eq 17) { + $why = "Not supposed to work with Privoxy. Invalid METHODs get rejected."; + } elsif ($testnum eq 19 or + $testnum eq 20) { + $why = "Not supposed to work with Privoxy. Tests behaviour with non-existing server and doesn't deal with error messages coming from a proxy."; + } elsif ($testnum eq 30 or + $testnum eq 37 or + $testnum eq 66 or + $testnum eq 1079) { + $why = "Expected to fail with Privoxy. In case of missing server headers Privoxy generates an error message the test doesn't expect."; + } elsif ($testnum eq 31 or $testnum eq 1160) { + $why = "Expected to fail. Privoxy normalizes white-space in a cookie."; + } elsif ($testnum eq 46) { + $why = "Invalid URL and use of --resolv."; + } elsif ($testnum eq 59) { + $why = "Invalid URL gets rejected by Privoxy which the test can't handle."; + } elsif ($testnum eq 129) { + $why = "Invalid HTTP version. Privoxy downgrades it to 1.1."; + } elsif ($testnum eq 187) { + $why = "Expected to fail. Test doesn't deal with Privoxy's error message in case of invalid URLs."; + } elsif ($testnum eq 207) { + $why = "Expected to fail. Test doesn't handle Privoxy's error message. Privoxy doesn't behave correctly, though."; + } elsif ($testnum eq 260) { + $why = "Known to fail. Looks like a curl bug. The URL passed to Privoxy is invalid but the test expect a valid one when not using a proxy"; + } elsif ($testnum eq 262) { + $why = "Not supposed to work with Privoxy. Privoxy doesn't support nul bytes in headers and neither does the spec."; + } elsif ($testnum eq 266 or $testnum eq 1116 or $testnum eq 1540) { + $why = "Known to fail. Uses chunk trailers which Privoxy currently doesn't support."; + } elsif ($testnum eq 309) { + $why = "Known to fail. Uses https and test does not expect the 'Connection established' response from Privoxy"; + } elsif ($testnum eq 339 or $testnum eq 347 or $testnum eq 1591) { + $why = "Chunked transfer with trailers which Privoxy does not understand. Needs investigating."; + } elsif ($testnum eq 1052) { + $why = "Expected to fail. Connection header expected in one response but not in the other. Not yet covered by runtests.pl's proxy mode."; + } elsif ($testnum eq 1118) { + $why = "Expected to fail. Looks like a curl bug although Privoxy's behaviour seems subobtimal as well."; + } elsif ($testnum eq 1310) { + $why = "Known to fail. NTLM-related. Cause not properly diagnosed yet. Privoxy's behaviour seems reasonable."; + } elsif ($testnum eq 155) { + $why = "Known to fail. Not yet analyzed."; + } elsif ($testnum eq 158 or $testnum eq 246 or $testnum eq 565 or $testnum eq 579) { + $why = "Known to fail. Not properly analyzed. Looks like Privoxy's continue hack is insufficient."; + } elsif ($testnum eq 412 or $testnum eq 413) { + $why = "Known to fail as curl is tunneling the request even though it's vanilla HTTP."; + } elsif ($testnum eq 435) { + $why = "Expected to fail. Uses %{remote_port} and expects the port of the server and not the one from Privoxy."; + } elsif ($testnum eq 507) { + $why = "Expected to fail. DNS failures cause a Privoxy error message the test doesn't handle."; + } elsif ($testnum eq 501) { + $why = "Not relevant for a proxy."; + } elsif ($testnum eq 530 or + $testnum eq 584) { + $why = "Known to fail. Test server expects pipelined requests and doesn't respond otherwise."; + } elsif ($testnum eq 556) { + $why = "Expected to fail. Uses HTTP/1.2 which Privoxy rejects as invalid."; + } elsif ($testnum eq 581) { + $why = "Expected to fail. Privoxy removes second Content-Type header."; + } elsif ($testnum eq 587 or $testnum eq 644) { + $why = "Expected to fail. POST request doesn't make it to the server. Needs investigating."; + } elsif ($testnum eq 655) { + $why = "Expected to fail. Uses tool. Failure reason not yet analyzed"; + } elsif ($testnum eq 970) { + $why = "Expected to fail. Privoxy adds a Proxy-Connection header which results in a modified num_headers value in the JSON output"; + } elsif ($testnum eq 1074) { + $why = "Expected to fail. Privoxy doesn't downgrade the forwarded request and doesn't have ". + "to as long as the client is treated like a HTTP/1.0 client. Needs double-checking."; + } elsif ($testnum eq 1144) { + $why = "Expected to fail. Server response is invalid and results in 502 message from Privoxy"; + } elsif ($testnum eq 1147) { + $why = "Expected to fail. Privoxy merges a two-line cookie into a one line cookie."; + } elsif ($testnum eq 1151) { + $why = "Expected to fail. Large cookies that don't make it to the cookie file. Needs investigating."; + } elsif ($testnum eq 1188) { + $why = "Expected to fail. Relies on a connection failure which results in a Privoxy error message the test does not expect"; + } elsif ($testnum eq 1223) { + $why = "Expected to fail. Tests remote address which doesn't work with proxies."; + } elsif ($testnum eq 1433) { + $why = "Expected to fail. Privoxy will enforce a valid HTTP version number"; + } elsif ($testnum eq 1506 or $testnum eq 1510) { + $why = "Expected to fail when using a proxy. Hardcoded addresses in expected output."; + } elsif ($testnum eq 1156) { + $why = "Expected to fail as it relies on Range requests making it to the server."; + } elsif ($testnum eq 1164 or $testnum eq 1172 or $testnum eq 1174) { + $why = "Expected to fail as Privoxy does not support HTTP/0.9."; + } elsif ($testnum eq 1292) { + $why = "Expected to fail as Privoxy replaces the empty Host header."; + } elsif ($testnum eq 1533) { + $why = "Sends an invalid method. Needs investigating."; + } elsif ($testnum eq 1556) { + $why = "Known to fail. Body value changes from 100008 to 100009. Needs investigating."; + } elsif ($testnum eq 1915) { + $why = "Known to fail. Uses tool that doesn't expect a proxy."; + } elsif ($testnum eq 1933) { + $why = "Known to fail. Modified signature in Authorization header. Needs investigating"; + } elsif ($testnum eq 2032 or $testnum eq 2033) { + $why = "Known to fail due to a limitation of the test which doesn't properly deal with interleaved output from two parallel connections"; + } elsif ($testnum eq 2049 or $testnum eq 2052 or $testnum eq 2053 or $testnum eq 2054) { + $why = "Uses --connect-to. Need investigating."; + } elsif ($testnum eq 2082 or $testnum eq 2084 or $testnum eq 2085) { + $why = "Known to fail. Uses %HTTPPORT and does not expect Privoxy's port but the remote one."; + } elsif ($testnum eq 96) { + $why = "Test 96 is incomplete"; + } elsif ($testnum eq 1901 or $testnum eq 1902 or $testnum eq 1903) { + $why = "Known to fail due to different response orders."; + } elsif ($testnum eq 2100) { + $why = "Known to fail. Use DNS-over-HTTP."; + } elsif ($testnum eq 3014 or $testnum eq 3015) { + $why = "Known to fail. Curl adds a Proxy-Connection header while test expect a certain number of headers."; + } + + next unless defined $why; + + if (exists $related_tests{$why}) { + $related_tests{$why} = $related_tests{$why} . ", $testnum"; + } else { + $related_tests{$why} = "$testnum"; + } + } + + foreach my $why (keys %related_tests) { + printf("test:%s: %s\n", $related_tests{$why}, $why); + } + + foreach my $protocol ('FTP', 'POP3', 'IMAP', 'SMTP', 'GOPHER', 'TELNET', 'FILE', 'RTSP') { + # Curl's behaviour when combining --proxy, -H and telnet:// seems strange and may be a bug." + printf("keyword:%s: Protocol %s is not supported by Privoxy.\n", $protocol, $protocol); + } + + foreach my $protocol ('SOCKS4', 'SOCKS5') { + # Curl's behaviour when combining --proxy, -H and telnet:// seems strange and may be a bug." + printf("keyword:%s: Protocol %s is supported by Privoxy but the tests need additional magic before they can be used.\n", $protocol, $protocol); + } + + foreach my $misc ('proxy', '--resolve', '--libcurl', 'CURLOPT_RESOLVE') { + printf("keyword:%s: Tests with keyword '%s' currently don't work with Privoxy as they need additional runtests.pl changes.\n", $misc, $misc); + } + printf("keyword:%s: Tests with keyword '%s' aren't expected to work with Privoxy running in a jail without IPv6 connectivity.\n", 'IPv6', 'IPv6'); + + printf("keyword:%s: Tests with keyword '%s' don't work with Privoxy as they use the OPTIONS method which is currently not properly supported (TODO #186).\n", '--request-target', '--request-target'); + + printf("keyword:%s: Tests with keyword '%s' obviously should be skipped. Check the full keyword for details.\n", 'skip', 'skip'); + + foreach my $keyword ('FAILURE', 'unsupported', 'curl-config') { + printf("keyword:%s: Tests with keyword '%s' do not reach the proxy. Or do they?\n", $keyword, $keyword); + } + + foreach my $tool ('lib517', 'lib543', 'lib543', '--manual', '--help', 'symbols-in-versions', 'memory-includes', 'unittest') { + printf("tool:%s: Tests with tool '%s' are not relevant for proxies.\n", $tool, $tool); + } + +} + +main(); -- 2.35.2 From bbe31fe0fe91a9f91bb18048c262c80fb456c866 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 12 Jul 2014 14:15:23 +0200 Subject: [PATCH 02/33] Add a generated test manifest file .. to skip curl tests that aren't relevant for Privoxy or are known not to work for other reasons. --- tests/cts/curl-test-manifest-for-privoxy | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/cts/curl-test-manifest-for-privoxy diff --git a/tests/cts/curl-test-manifest-for-privoxy b/tests/cts/curl-test-manifest-for-privoxy new file mode 100644 index 00000000..f74d1dfb --- /dev/null +++ b/tests/cts/curl-test-manifest-for-privoxy @@ -0,0 +1,80 @@ +keyword:--libcurl: Tests with keyword '--libcurl' currently don't work with Privoxy as they need additional runtests.pl changes. +keyword:--request-target: Tests with keyword '--request-target' don't work with Privoxy as they use the OPTIONS method which is currently not properly supported (TODO #186). +keyword:--resolve: Tests with keyword '--resolve' currently don't work with Privoxy as they need additional runtests.pl changes. +keyword:curl-config: Tests with keyword 'curl-config' do not reach the proxy. Or do they? +keyword:CURLOPT_RESOLVE: Tests with keyword 'CURLOPT_RESOLVE' currently don't work with Privoxy as they need additional runtests.pl changes. +keyword:FAILURE: Tests with keyword 'FAILURE' do not reach the proxy. Or do they? +keyword:FILE: Protocol FILE is not supported by Privoxy. +keyword:FTP: Protocol FTP is not supported by Privoxy. +keyword:GOPHER: Protocol GOPHER is not supported by Privoxy. +keyword:IMAP: Protocol IMAP is not supported by Privoxy. +keyword:IPv6: Tests with keyword 'IPv6' aren't expected to work with Privoxy running in a jail without IPv6 connectivity. +keyword:POP3: Protocol POP3 is not supported by Privoxy. +keyword:proxy: Tests with keyword 'proxy' currently don't work with Privoxy as they need additional runtests.pl changes. +keyword:RTSP: Protocol RTSP is not supported by Privoxy. +keyword:skip: Tests with keyword 'skip' obviously should be skipped. Check the full keyword for details. +keyword:SMTP: Protocol SMTP is not supported by Privoxy. +keyword:SOCKS4: Protocol SOCKS4 is supported by Privoxy but the tests need additional magic before they can be used. +keyword:SOCKS5: Protocol SOCKS5 is supported by Privoxy but the tests need additional magic before they can be used. +keyword:TELNET: Protocol TELNET is not supported by Privoxy. +keyword:unsupported: Tests with keyword 'unsupported' do not reach the proxy. Or do they? +test:1052: Expected to fail. Connection header expected in one response but not in the other. Not yet covered by runtests.pl's proxy mode. +test:1074: Expected to fail. Privoxy doesn't downgrade the forwarded request and doesn't have to as long as the client is treated like a HTTP/1.0 client. Needs double-checking. +test:1118: Expected to fail. Looks like a curl bug although Privoxy's behaviour seems subobtimal as well. +test:1144: Expected to fail. Server response is invalid and results in 502 message from Privoxy +test:1147: Expected to fail. Privoxy merges a two-line cookie into a one line cookie. +test:1151: Expected to fail. Large cookies that don't make it to the cookie file. Needs investigating. +test:1156: Expected to fail as it relies on Range requests making it to the server. +test:1164, 1172, 1174: Expected to fail as Privoxy does not support HTTP/0.9. +test:1188: Expected to fail. Relies on a connection failure which results in a Privoxy error message the test does not expect +test:1223: Expected to fail. Tests remote address which doesn't work with proxies. +test:129: Invalid HTTP version. Privoxy downgrades it to 1.1. +test:1292: Expected to fail as Privoxy replaces the empty Host header. +test:1310: Known to fail. NTLM-related. Cause not properly diagnosed yet. Privoxy's behaviour seems reasonable. +test:1433: Expected to fail. Privoxy will enforce a valid HTTP version number +test:1506, 1510: Expected to fail when using a proxy. Hardcoded addresses in expected output. +test:1533: Sends an invalid method. Needs investigating. +test:155: Known to fail. Not yet analyzed. +test:1556: Known to fail. Body value changes from 100008 to 100009. Needs investigating. +test:158, 246, 565, 579: Known to fail. Not properly analyzed. Looks like Privoxy's continue hack is insufficient. +test:16, 45, 63: Not supposed to work with Privoxy. Expected forwarding failure. Workaround probably possible. +test:17: Not supposed to work with Privoxy. Invalid METHODs get rejected. +test:187: Expected to fail. Test doesn't deal with Privoxy's error message in case of invalid URLs. +test:19, 20: Not supposed to work with Privoxy. Tests behaviour with non-existing server and doesn't deal with error messages coming from a proxy. +test:1901, 1902, 1903: Known to fail due to different response orders. +test:1915: Known to fail. Uses tool that doesn't expect a proxy. +test:1933: Known to fail. Modified signature in Authorization header. Needs investigating +test:2032, 2033: Known to fail due to a limitation of the test which doesn't properly deal with interleaved output from two parallel connections +test:2049, 2052, 2053, 2054: Uses --connect-to. Need investigating. +test:207: Expected to fail. Test doesn't handle Privoxy's error message. Privoxy doesn't behave correctly, though. +test:2082, 2084, 2085: Known to fail. Uses %HTTPPORT and does not expect Privoxy's port but the remote one. +test:2100: Known to fail. Use DNS-over-HTTP. +test:260: Known to fail. Looks like a curl bug. The URL passed to Privoxy is invalid but the test expect a valid one when not using a proxy +test:262: Not supposed to work with Privoxy. Privoxy doesn't support nul bytes in headers and neither does the spec. +test:266, 1116, 1540: Known to fail. Uses chunk trailers which Privoxy currently doesn't support. +test:30, 37, 66, 1079: Expected to fail with Privoxy. In case of missing server headers Privoxy generates an error message the test doesn't expect. +test:3014, 3015: Known to fail. Curl adds a Proxy-Connection header while test expect a certain number of headers. +test:309: Known to fail. Uses https and test does not expect the 'Connection established' response from Privoxy +test:31, 1160: Expected to fail. Privoxy normalizes white-space in a cookie. +test:339, 347, 1591: Chunked transfer with trailers which Privoxy does not understand. Needs investigating. +test:412, 413: Known to fail as curl is tunneling the request even though it's vanilla HTTP. +test:435: Expected to fail. Uses %{remote_port} and expects the port of the server and not the one from Privoxy. +test:46: Invalid URL and use of --resolv. +test:501: Not relevant for a proxy. +test:507: Expected to fail. DNS failures cause a Privoxy error message the test doesn't handle. +test:530, 584: Known to fail. Test server expects pipelined requests and doesn't respond otherwise. +test:556: Expected to fail. Uses HTTP/1.2 which Privoxy rejects as invalid. +test:581: Expected to fail. Privoxy removes second Content-Type header. +test:587, 644: Expected to fail. POST request doesn't make it to the server. Needs investigating. +test:59: Invalid URL gets rejected by Privoxy which the test can't handle. +test:655: Expected to fail. Uses tool. Failure reason not yet analyzed +test:96: Test 96 is incomplete +test:970: Expected to fail. Privoxy adds a Proxy-Connection header which results in a modified num_headers value in the JSON output +tool:--help: Tests with tool '--help' are not relevant for proxies. +tool:--manual: Tests with tool '--manual' are not relevant for proxies. +tool:lib517: Tests with tool 'lib517' are not relevant for proxies. +tool:lib543: Tests with tool 'lib543' are not relevant for proxies. +tool:lib543: Tests with tool 'lib543' are not relevant for proxies. +tool:memory-includes: Tests with tool 'memory-includes' are not relevant for proxies. +tool:symbols-in-versions: Tests with tool 'symbols-in-versions' are not relevant for proxies. +tool:unittest: Tests with tool 'unittest' are not relevant for proxies. -- 2.35.2 From 7043b8905a9d559e44534f37a9e8754e3be812b1 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 15 Jun 2014 18:28:51 +0200 Subject: [PATCH 03/33] Add privoxy-runtests.pm Overwrites some of the functions used by curl's runtests.pl to let it properly deal with Privoxy being used as proxy. --- tests/cts/privoxy-runtests.pm | 223 ++++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 tests/cts/privoxy-runtests.pm diff --git a/tests/cts/privoxy-runtests.pm b/tests/cts/privoxy-runtests.pm new file mode 100644 index 00000000..10261fab --- /dev/null +++ b/tests/cts/privoxy-runtests.pm @@ -0,0 +1,223 @@ +################################################################################ +# privoxy-runtests.pm +# +# Code that has to be loaded by curl's runtests.pl with the -L option +# to deal with modifications required when using the tests with Privoxy. +# +# Copyright (c) 2014-2021 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +################################################################################ + +use strict; +use warnings; +no warnings "redefine"; + +my $verbose = 0; +my $use_external_proxy = 0; + +BEGIN { + # Keep a couple of functions from getpart.pm accessible so + # our redefinitions don't have to reimplement them. + our $real_showdiff = \&showdiff; + our $real_getpart = \&getpart; + our $real_getpartattr = \&getpartattr; + our $real_compareparts = \&compareparts; + our $real_startnew = \&startnew; +} + +sub print_skipped_header($) { + my $skipped_header = shift; + $skipped_header =~ s@\r?\n$@@; + print "Skipping '$skipped_header'\n"; +} + +# Process headers to ignore differences that are to be expected +# when Privoxy is being used. +# +# - Filter out "Proxy-Connection:" headers when checking for +# test success. +# - Filter out a header that is specified with a "X-Ignore-Header" header. +# - Deal with tests that don't expect CRLF header endings as +# long as the test uses it consistently. +# - Reduce spaces in server headers with a too-simplistic heuristic +# that happens to work for the existing tests. +sub process_headers($$) { + my ($head1_ref, $head2_ref) = @_; + my @head1; + my @head2; + my $crlf_expected = 0; + my $connection_header_expected = 0; + my $proxy_connection_header_expected = 0; + my $parsing_server_headers = 0; + my $ignore_header; + my $ignored_header; + + foreach (@$head2_ref) { + if (/^HTTP/) { + # If it starts like a response line, assume we are + # looking at server headers. + $parsing_server_headers = 1; + } + if (/^\r?\n$/) { + $parsing_server_headers = 0; + } + if (/\r\n$/) { + $crlf_expected = 1; # XXX: assume the expectancy is consistant. + } + + if (/^Connection:/) { + $connection_header_expected = 1; + } + if (/^Proxy-Connection:/) { + $proxy_connection_header_expected = 1; + } + if (/^X-Ignore-Header: (.*)/) { + $ignore_header = $1; + print "Ignoring header '$ignore_header'\n" if $verbose; + } + if (defined $ignore_header and /^$ignore_header: .*/) { + $ignored_header = $_; + } + + if ($parsing_server_headers and not /"/) { + # Normalize spaces in server headers similar to the way Privoxy + # does. This is required for curl tests 29, 40, 42 and 54. + s@ +@ @g; + } + } + + if ($verbose) { + print "Expecting " . ($crlf_expected ? "" : "no ") . "crlf\n"; + print "Expecting " . ($connection_header_expected ? "a" : "no") . " Connection: header\n"; + print "Expecting " . ($proxy_connection_header_expected ? "a" : "no") . " Proxy-Connection: header\n"; + } + + foreach (@$head1_ref) { + + s@\r\n$@\n@ unless ($crlf_expected); + + if ((m/^Connection:/ and not $connection_header_expected) or + (m/^Proxy-Connection:/ and not $proxy_connection_header_expected)) { + print_skipped_header($_) if ($verbose); + next; + } + if (defined $ignore_header) { + if (m/^$ignore_header:/) { + push @head1, "X-Ignore-Header: $ignore_header\n"; + $_ = $ignored_header; + } + } + push @head1, $_; + } + $head1_ref = \@head1; + + return ($head1_ref, $head2_ref); +} + +# Behaves like the real compareparts(), but if a proxy is being used, +# headers are run through process_headers() before checking them for +# differences. +sub compareparts { + my ($head1_ref, $head2_ref) = @_; + our $real_compareparts; + + if ($use_external_proxy) { + ($head1_ref, $head2_ref) = process_headers($head1_ref, $head2_ref); + } + + return &$real_compareparts($head1_ref, $head2_ref); +} + +# Behaves like the real getpart() but if a proxy is being used +# and a proxy-reply section exists, it is returned instead of +# a common reply section. +sub getpart { + my ($section, $part) = @_; + our $real_getpart; + + if ($use_external_proxy and + $section eq 'reply' and + partexists("proxy-reply", $part)) { + $section = "proxy-reply"; + } + + return &$real_getpart($section, $part); +} + +# Behaves like the real getpartattr() but if a proxy is being used +# and a proxy-reply section exists, it is being used instead of +# a common reply section. +sub getpartattr { + my ($section, $part)=@_; + our $real_getpartattr; + + if ($use_external_proxy and + $section eq 'reply' and + partexists("proxy-reply", $part)) { + $section = "proxy-reply"; + } + + return &$real_getpartattr($section, $part); +} + +# Behaves like the real logmsg but suppresses warnings +# about unknown tests +sub logmsg { + for (@_) { + next if /^Warning: test\d+ not present in/; + print "$_"; + } +} + +# Behaves like the real showdiff() but diffs twice, +# the second time after processing the headers. +sub showdiff { + my ($logdir, $head1_ref, $head2_ref) = @_; + our $real_showdiff; + + print "Unprocessed headers:\n"; + print &$real_showdiff($logdir, $head1_ref, $head2_ref); + + print "Processed headers:\n"; + ($head1_ref, $head2_ref) = process_headers($head1_ref, $head2_ref); + return &$real_showdiff($logdir, $head1_ref, $head2_ref); +} + +# Behaves like the real startnew() but sets a static port if +# the started server is httpserver.pl. +sub startnew { + my ($cmd, $pidfile, $timeout, $fake) = @_; + our $real_startnew; + + if ($cmd =~ /httpserver\.pl/) { + $cmd =~ s@--port 0@--port 20000@; + } elsif ($cmd =~ m@server/socksd@) { + $cmd =~ s@--port 0@--port 20001@; + } + + return &$real_startnew($cmd, $pidfile, $timeout, $fake); +} + +sub main() { + + # Look but don't touch, @ARGV is still needed elsewhere + foreach my $arg (@ARGV) { + $use_external_proxy = 1 if ($arg eq "-P"); + $verbose = 1 if ($arg eq "-v"); + } + + return 1; +} + +main(); -- 2.35.2 From 88ac3b4c068cfa3262bab0deda399f92ef3d344e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 Jan 2021 16:49:08 +0100 Subject: [PATCH 04/33] Add runtests-wrapper.sh A wrapper around curl's runtests.pl. --- tests/cts/runtests-wrapper.sh | 128 ++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100755 tests/cts/runtests-wrapper.sh diff --git a/tests/cts/runtests-wrapper.sh b/tests/cts/runtests-wrapper.sh new file mode 100755 index 00000000..e56a408f --- /dev/null +++ b/tests/cts/runtests-wrapper.sh @@ -0,0 +1,128 @@ +#!/bin/sh +################################################################################ +# +# runtests-wrapper.sh +# +# Wrapper around curl's runtests.pl that sets a couple of options +# so Privoxy is being used. +# +# Copyright (c) 2013-2021 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +################################################################################ + +curl_setup_is_sane() { + local curl_source_directory="${1}" + local curl_binary="${curl_source_directory}/src/curl" + local runtests_pl="${curl_source_directory}/tests/runtests.pl" + + if [ ! -d "${curl_source_directory}" ]; then + echo "Missing curl source directory at ${curl_source_directory}" + return 1 + fi + if [ ! -f "${curl_binary}" ]; then + echo "Missing curl binary at ${curl_binary}. Did you compile curl?" + return 1 + fi + if [ ! -f "${runtests_pl}" ]; then + echo "Did not find runtests.pl at ${runtests_pl}" + return 1 + fi + + return 0 +} + +runtests_wrapper() { + local extra_args \ + a_flag proxy_args exclude_file_args testdir_args \ + privoxy_lib privoxy_ip privoxy_source_directory curl_source_directory \ + keyword directory_name test_dir + + directory_name="$(dirname "$0")" + test_dir="$(realpath "${directory_name}")" + privoxy_source_directory="$(realpath "${test_dir}"/../..)" + privoxy_lib="${privoxy_source_directory}/tests/cts/privoxy-runtests.pm" + curl_source_directory="$(realpath "${privoxy_source_directory}"/../curl)" + + curl_setup_is_sane "${curl_source_directory}" || exit 1 + + # Defaults that can be changed through arguments + privoxy_ip=127.0.0.1 + privoxy_port=9119 + a_flag="-a" + proxy_args="-P http://${privoxy_ip}:${privoxy_port}/ -o HOSTIP=${privoxy_ip}" + exclude_file_args="-E ${privoxy_source_directory}/tests/cts/curl-test-manifest-for-privoxy" + testdir_args="-o TESTDIR=${privoxy_source_directory}/tests/cts/data" + keyword=HTTP + + while [ -n "$1" ]; + do + case "$1" in + "-A") + a_flag="" + shift + ;; + "-E") + exclude_file_args="" + shift + ;; + "-k") + shift + keyword="$1" + shift + ;; + "-i") + shift + privoxy_ip="$1" + shift + proxy_args="-P http://${privoxy_ip}:${privoxy_port}/ -o HOSTIP=${privoxy_ip}" + ;; + "-T") + echo "Not setting TESTDIR" + testdir_args="" + shift + ;; + "-t") + shift + echo "Overwriting default TESTDIR with $1" + testdir_args="-o TESTDIR=$1" + shift + ;; + "-p") + shift + privoxy_port="$1" + shift + proxy_args="-P http://${privoxy_ip}:${privoxy_port}/ -o HOSTIP=${privoxy_ip}" + ;; + "-P") + # "Obviously" -P means not setting -P + echo "Not setting '$proxy_args'" + proxy_args="" + shift + ;; + *) + break;; + esac + done + + extra_args="$*" + + cd "${curl_source_directory}/tests" || exit 1 + ./runtests.pl -L "${privoxy_lib}" $proxy_args $exclude_file_args $testdir_args $a_flag -n $keyword !skip $extra_args +} + +main() { + runtests_wrapper "$@" +} + +main "$@" -- 2.35.2 From abc31c5ed2d13ed763d0f0cdc5e03ae7c562828b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 27 Feb 2021 10:54:18 +0100 Subject: [PATCH 05/33] Add a README for the tests based on the curl test suite (cts) Contains improvements made after feedback from Roland. --- tests/cts/README | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/cts/README diff --git a/tests/cts/README b/tests/cts/README new file mode 100644 index 00000000..a741c6d8 --- /dev/null +++ b/tests/cts/README @@ -0,0 +1,63 @@ +This directory contains tests that leverage curl's test suite (cts). + +For details about curl see: https://curl.se/ + +To use the tests, a local curl fork is required as some of +the required patches haven't been accepted upstream yet. + +Some temporary hints are below: + +From the parent directory of the directory with the +Privoxy sources do: + + git clone https://github.com/curl/curl.git + + cd curl + + # Build curl as described in GIT-INFO. Note that the configure + # call needs to explicitly choose a TLS library. For example + # by adding "--with-openssl". + + # Install stunnel + + # Test that curl's tests are working without Privoxy + make test + + # Get and apply required patches on top of a curl revision known to work + curl -O https://www.fabiankeil.de/sourcecode/curl-test-suite-improvements-2022-05-22.diff + git checkout -b privoxy-tests 8ea851b29d4beb + git am --keep-cr curl-test-suite-improvements-2022-05-22.diff + + # Rebuild tests + cd tests + make + +You can then cd back into privoxy/tests/cts and execute the +tests by running: + + run-privoxy-tests.sh + +The script takes care of starting Privoxy with a configuration expected +by the tests. Note that unlike privoxy-regression-test.pl it currently +does not automatically check that Privoxy has been built with all the +features required for the tests to work and simply tries to execute +all tests. + +The tests are divided into multiple test scenarios. + +Each test scenario has its own subdirectory with a Privoxy +configuration file called privoxy.conf and a data directory +containing the test files. + +The tests are written using the "curl test suite file format" +described at: +https://github.com/curl/curl/blob/master/tests/FILEFORMAT.md + +One extension made is that a section is supported +which will be used when checking the test results instead of using +the section that would be used otherwise. + +This extension is supported by loading a custom perl library +(privoxy-runtests.pm) when running runtests.pl. Loading the library +is done with runtests-wrapper.sh which also sets a couple of other +required options and is normally called through run-privoxy-tests.sh. -- 2.35.2 From c8dc0dc076eff264d792cc8ab4a1b2bbe50efc7e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 22 Jan 2021 09:25:39 +0100 Subject: [PATCH 06/33] Add logs directory for the tests based on the curl test suite --- tests/cts/logs/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/cts/logs/.gitignore diff --git a/tests/cts/logs/.gitignore b/tests/cts/logs/.gitignore new file mode 100644 index 00000000..bc74a1e3 --- /dev/null +++ b/tests/cts/logs/.gitignore @@ -0,0 +1,2 @@ +# Ignore the logs in this directory +*.log -- 2.35.2 From 804daed5506557f0eee6630491be6e89fb99e0b4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 Jan 2021 06:50:06 +0100 Subject: [PATCH 07/33] Add run-privoxy-tests.sh ... which takes care of starting Privoxy before running tests. --- tests/cts/run-privoxy-tests.sh | 158 +++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 tests/cts/run-privoxy-tests.sh diff --git a/tests/cts/run-privoxy-tests.sh b/tests/cts/run-privoxy-tests.sh new file mode 100755 index 00000000..c519aff2 --- /dev/null +++ b/tests/cts/run-privoxy-tests.sh @@ -0,0 +1,158 @@ +#!/bin/sh +################################################################################ +# +# run-privoxy-tests.sh +# +# Runs the Privoxy tests based on curl's runtests.pl framework. +# +# Copyright (c) 2021 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +################################################################################ + +UPSTREAM_TEST_SCENARIO=upstream-tests + +# Delaying the test run by a whole second is annoying on fast systems +# and may not be long enough on slow systems. There must be a more +# elegant method to verify that Privoxy is running or failed to start. +SECONDS_TO_WAIT_FOR_PRIVOXY_TO_START=1 + +start_privoxy() { + local test_dir test_scenario + local privoxy_config privoxy_binary pid_file + test_dir="${1}" + test_scenario="${2}" + + privoxy_config_dir="${test_dir}/${test_scenario}" + privoxy_binary="$(realpath "${test_dir}/../../privoxy")" + pid_file="${test_dir}/${test_scenario}/../privoxy.pid" + log_file="${test_dir}/logs/${test_scenario}.log" + + ( + cd "${privoxy_config_dir}" || exit 1 + "${privoxy_binary}" --no-daemon \ + --pidfile "${pid_file}" \ + privoxy.conf > "${log_file}" 2>&1 || exit 1 & + ) + + sleep "${SECONDS_TO_WAIT_FOR_PRIVOXY_TO_START}" + + if [ ! -f "${pid_file}" ]; then + echo "Privoxy failed to start or did not start in time" + if [ -f "${log_file}" ]; then + tail -n 1 "${log_file}" + fi + exit 1 + fi +} + +stop_privoxy() { + local test_dir test_scenario + local pid_file + test_dir="${1}" + test_scenario="${2}" + pid_file="${test_dir}/${test_scenario}/../privoxy.pid" + if [ -f "${pid_file}" ]; then + kill "$(cat "${pid_file}")" + fi +} + +run_privoxy_tests() { + local start_privoxy="$1" + local test_scenario="$2" + local directory_name="$(dirname "$0")" + local test_dir="$(realpath "${directory_name}")" + local ret + + echo "Test scenario: ${test_scenario}" + $start_privoxy && start_privoxy "${test_dir}" "${test_scenario}" + + "${test_dir}/runtests-wrapper.sh" -A -E -t "${test_dir}/${test_scenario}/data" HTTP HTTPS + ret=$? + + $start_privoxy && stop_privoxy "${test_dir}" "${test_scenario}" + + return $ret +} + +run_upstream_tests() { + local start_privoxy="$1" + local directory_name="$(dirname "$0")" + local test_dir="$(realpath "${directory_name}")" + local ret + + echo "Test scenario: ${UPSTREAM_TEST_SCENARIO}" + $start_privoxy && start_privoxy "${test_dir}" "${UPSTREAM_TEST_SCENARIO}" + + "${test_dir}/runtests-wrapper.sh" -A -T HTTP + ret=$? + + $start_privoxy && stop_privoxy "${test_dir}" "${UPSTREAM_TEST_SCENARIO}" + + return $ret +} + +get_test_scenarios() { + local directory_name="$(dirname "$0")" + local test_dir="$(realpath "${directory_name}")" + local test_scenario + local privoxy_config + + for privoxy_config in "${test_dir}"/*/privoxy.conf; do + test_scenario="${privoxy_config%%/privoxy.conf}" + test_scenario="${test_scenario##$test_dir/}" + echo "${test_scenario}" + done +} + +main() { + local test_scenario="" + local test_scenarios="" + local start_privoxy=true + + while [ -n "$1" ]; + do + case "$1" in + "-r") + echo "Not starting privoxy." + start_privoxy=false + shift + ;; + "-t") + shift + test_scenarios="$1" + shift + ;; + *) + echo "Invalid parameter: $1" + exit 1 + ;; + esac + done + + if [ -z "${test_scenarios}" ]; then + test_scenarios="$(get_test_scenarios)" + fi + + for test_scenario in ${test_scenarios}; do + if [ "${test_scenario}" = "${UPSTREAM_TEST_SCENARIO}" ]; then + run_upstream_tests ${start_privoxy} || exit 1 + else + run_privoxy_tests ${start_privoxy} "${test_scenario}" || exit 1 + fi + done + + exit 0 +} + +main "$@" -- 2.35.2 From 900bcd4fd8b0bb992fe6a3c6a29c41d68746860d Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 Jan 2021 16:49:29 +0100 Subject: [PATCH 08/33] Add client-header-order tests --- tests/cts/client-header-order/data/test332 | 52 +++++++++++++++++++++ tests/cts/client-header-order/data/test333 | 52 +++++++++++++++++++++ tests/cts/client-header-order/data/test334 | 42 +++++++++++++++++ tests/cts/client-header-order/data/test335 | 54 ++++++++++++++++++++++ tests/cts/client-header-order/privoxy.conf | 20 ++++++++ 5 files changed, 220 insertions(+) create mode 100644 tests/cts/client-header-order/data/test332 create mode 100644 tests/cts/client-header-order/data/test333 create mode 100644 tests/cts/client-header-order/data/test334 create mode 100644 tests/cts/client-header-order/data/test335 create mode 100644 tests/cts/client-header-order/privoxy.conf diff --git a/tests/cts/client-header-order/data/test332 b/tests/cts/client-header-order/data/test332 new file mode 100644 index 00000000..0cf8407d --- /dev/null +++ b/tests/cts/client-header-order/data/test332 @@ -0,0 +1,52 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + + +http + + +HTTP client header order + + +-H "Origin: https://www.privoxy.org/" -H "Cache-Control: max-age=315576000" -H "If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT" -H "Upgrade-Insecure-Requests: 1" -H "Pragma: no-cache" -H "DNT: 1" -H "Cookie: yolo=123" -H "Referer: https://www.example.org/" -H "Proxy-Connection: keep-alive" -H "Accept-Encoding: gzip" -H "Accept: */*" -H "Host: %HOSTIP:%HTTPPORT" -H "Accept-Language: en-CA" --user-agent "Client with unsorted headers" http://%HOSTIP:%HTTPPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: Client with unsorted headers +Accept: */* +Accept-Language: en-CA +Accept-Encoding: gzip +Referer: https://www.example.org/ +Cookie: yolo=123 +DNT: 1 +Connection: close +Pragma: no-cache +Upgrade-Insecure-Requests: 1 +If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT +Cache-Control: max-age=315576000 +Origin: https://www.privoxy.org/ + + + + diff --git a/tests/cts/client-header-order/data/test333 b/tests/cts/client-header-order/data/test333 new file mode 100644 index 00000000..a67eb4ff --- /dev/null +++ b/tests/cts/client-header-order/data/test333 @@ -0,0 +1,52 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + + +http + + +HTTP client header order 2 + + +-H "Origin: https://www.privoxy.org/" -H "Cache-Control: max-age=315576000" -H "If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT" -H "Upgrade-Insecure-Requests: 1" -H "DNT: 1" -H "Pragma: no-cache" -H "Cookie: yolo=123" -H "Referer: https://www.example.org/" -H "Accept-Encoding: gzip" -H "Proxy-Connection: keep-alive" -H "Accept: */*" -H "Accept-Language: en-CA" -H "Host: %HOSTIP:%HTTPPORT" --user-agent "Client with unsorted headers" http://%HOSTIP:%HTTPPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: Client with unsorted headers +Accept: */* +Accept-Language: en-CA +Accept-Encoding: gzip +Referer: https://www.example.org/ +Cookie: yolo=123 +DNT: 1 +Connection: close +Pragma: no-cache +Upgrade-Insecure-Requests: 1 +If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT +Cache-Control: max-age=315576000 +Origin: https://www.privoxy.org/ + + + + diff --git a/tests/cts/client-header-order/data/test334 b/tests/cts/client-header-order/data/test334 new file mode 100644 index 00000000..2c7ed0f7 --- /dev/null +++ b/tests/cts/client-header-order/data/test334 @@ -0,0 +1,42 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + + +http + + +HTTP client header order 3. Only a few headers to sort. + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Connection: close + + + + diff --git a/tests/cts/client-header-order/data/test335 b/tests/cts/client-header-order/data/test335 new file mode 100644 index 00000000..7c89c6b1 --- /dev/null +++ b/tests/cts/client-header-order/data/test335 @@ -0,0 +1,54 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + + +http + + +HTTP client header order 4. Left over headers. + + +-H "Bum: Tschaka" -H "Blafasel: Die da" -H "Origin: https://www.privoxy.org/" -H "Cache-Control: max-age=315576000" -H "If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT" -H "Upgrade-Insecure-Requests: 1" -H "DNT: 1" -H "Pragma: no-cache" -H "Cookie: yolo=123" -H "Referer: https://www.example.org/" -H "Accept-Encoding: gzip" -H "Proxy-Connection: keep-alive" -H "Accept: */*" -H "Accept-Language: en-CA" -H "Host: %HOSTIP:%HTTPPORT" --user-agent "Client with unsorted headers" http://%HOSTIP:%HTTPPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: Client with unsorted headers +Accept: */* +Accept-Language: en-CA +Accept-Encoding: gzip +Referer: https://www.example.org/ +Cookie: yolo=123 +DNT: 1 +Connection: close +Pragma: no-cache +Upgrade-Insecure-Requests: 1 +If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT +Cache-Control: max-age=315576000 +Origin: https://www.privoxy.org/ +Bum: Tschaka +Blafasel: Die da + + + + diff --git a/tests/cts/client-header-order/privoxy.conf b/tests/cts/client-header-order/privoxy.conf new file mode 100644 index 00000000..fb15ffd8 --- /dev/null +++ b/tests/cts/client-header-order/privoxy.conf @@ -0,0 +1,20 @@ +listen-address 127.0.0.1:9119 + +client-header-order Host User-Agent Accept Accept-Language Accept-Encoding \ + Proxy-Connection Referer Cookie DNT Connection \ + Pragma Upgrade-Insecure-Requests If-Modified-Since \ + Cache-Control Content-Length Origin Content-Type + +debug 1 # Log the destination for each request Privoxy let through. See also debug 1024. +debug 2 # show each connection status +debug 4 # show tagging-related messages +debug 8 # show header parsing +debug 32 # debug force feature +debug 64 # debug regular expression filters +debug 128 # debug redirects +debug 256 # debug GIF de-animation +debug 512 # Common Log Format +debug 1024 # Log the destination for requests Privoxy didn't let through, and the reason why. +debug 4096 # Startup banner and warnings. +debug 8192 # Non-fatal errors + -- 2.35.2 From 80675a85ccfd9ea94f883aa009bb82eedccb8b05 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 Jan 2021 19:05:53 +0100 Subject: [PATCH 09/33] Add tests for client-header-order with https-inspection enabled --- .../cts/ca-directory/privoxy-test-cacert.crt | 20 ++++++ tests/cts/ca-directory/privoxy-test-cakey.pem | 30 +++++++++ tests/cts/certs/.gitignore | 3 + .../client-header-order-https/data/test332 | 66 +++++++++++++++++++ .../client-header-order-https/data/test333 | 64 ++++++++++++++++++ .../client-header-order-https/data/test334 | 56 ++++++++++++++++ .../client-header-order-https/data/test335 | 66 +++++++++++++++++++ .../https-inspection.action | 2 + .../client-header-order-https/privoxy.conf | 31 +++++++++ 9 files changed, 338 insertions(+) create mode 100644 tests/cts/ca-directory/privoxy-test-cacert.crt create mode 100644 tests/cts/ca-directory/privoxy-test-cakey.pem create mode 100644 tests/cts/certs/.gitignore create mode 100644 tests/cts/client-header-order-https/data/test332 create mode 100644 tests/cts/client-header-order-https/data/test333 create mode 100644 tests/cts/client-header-order-https/data/test334 create mode 100644 tests/cts/client-header-order-https/data/test335 create mode 100644 tests/cts/client-header-order-https/https-inspection.action create mode 100644 tests/cts/client-header-order-https/privoxy.conf diff --git a/tests/cts/ca-directory/privoxy-test-cacert.crt b/tests/cts/ca-directory/privoxy-test-cacert.crt new file mode 100644 index 00000000..49d29636 --- /dev/null +++ b/tests/cts/ca-directory/privoxy-test-cacert.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDVTCCAj2gAwIBAgIJAMn580TY7tn4MA0GCSqGSIb3DQEBCwUAMEExCzAJBgNV +BAYTAkRFMRMwEQYDVQQIDApTb21lLVN0YXRlMR0wGwYDVQQKDBRQcml2b3h5IFRl +c3QgQ29tcGFueTAeFw0yMTAxMDkxNjQxMzNaFw0zMTAxMDcxNjQxMzNaMEExCzAJ +BgNVBAYTAkRFMRMwEQYDVQQIDApTb21lLVN0YXRlMR0wGwYDVQQKDBRQcml2b3h5 +IFRlc3QgQ29tcGFueTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMCG +dyCmLX+UmlP0eN+4kWQsP0bVATlmfycSjgXwhdMfQ7pHrP5RUM+Mzy60vYfXVLCn +O6evxqx2XcZnEk6QxXRaUZfmd4Zw9F1n39x9ERFXgDNseRyoebqfmgl05STxe3Co +NSSvYRz6iX4grzx+LWikwxvMHvPZyntPXgFhwuo6j9mBm6XfN8zIghwlQQbcgb9m +WDC/XbFDCQcrEnGaQNJ5T2406EZptm2MZ4xU4P5M7LmQws8p7VgLiYdPyouZ2YN7 +PkCcVGzVY3YiCg7W2ETZHHurn4XsRWXdr7tgH2RzzC1Eok8QtEyn9C3XtUNWQv3J +LeRn5LAiF7CxJKUtI3sCAwEAAaNQME4wHQYDVR0OBBYEFB7YVII5luGzJXJKgYZh +fVH8D7CXMB8GA1UdIwQYMBaAFB7YVII5luGzJXJKgYZhfVH8D7CXMAwGA1UdEwQF +MAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAoEtgyL3paPULfconAkzRRk74XDjsz8 +sz+eSsbf0pVS4CNEWa1jrgv5PfKMqquYkDJ7IY8dD9Kqmzd/82KNZSlYACwOSbPV +1t98wNWvvVg7eMJD3IdKcSR3HAvR75eQFI7JVy0EG62o5fGplfAZxauBa5cPhgGd +I1DJrp/XFTOvv/SfNBMXgLZq8b6SMwyqomflNSFEfiC8IO7gEcE7CBbkHN+Hd15W +YvAbJraAeP0YR+r2HKCqUbIVxbBWZlkjS1alvwsgDgamTVyQvNiAPzVsNY0G7lWJ +f10kaB7Gd8NoER7jKBs6nbQTvb5UapQa0G4i3RHlYBUyg8+znz2OXK0= +-----END CERTIFICATE----- diff --git a/tests/cts/ca-directory/privoxy-test-cakey.pem b/tests/cts/ca-directory/privoxy-test-cakey.pem new file mode 100644 index 00000000..612e9ed8 --- /dev/null +++ b/tests/cts/ca-directory/privoxy-test-cakey.pem @@ -0,0 +1,30 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFDjBABgkqhkiG9w0BBQ0wMzAbBgkqhkiG9w0BBQwwDgQI3FrXU0ecRrECAggA +MBQGCCqGSIb3DQMHBAhSvq1gGYC2RQSCBMihUnjghKpYieBD+F25LE8xbT+hJn1D +n91Ax+q/0zpGCiX8CCLVbfK9i6rCmnO1qEa7YNzKSUdEmWoMrpH7wdJG5AHgRv4g +9IiFG8HOCNFit75cQX8Fu2XrkfCSzKa0vjKZLylSrjf3UPPD+Oiifq/FD6zZtKIv +hDM9tq0+JXQBrM2YME6yyKhueg6g1LFs6Spk2EoOW5er6ztzuM5Tv0g7P2BtPnS5 +Gi/xIGUCA0Nm8UyzjumDNtmet2xq2viKs49Ditlubu6gAJq/wHiyY/SpyBnNVbKT +jO4VJVlXLBc0qracLLPIvL894Sc3mTe4RGDpZ2qGLgkGISzvWK9BHJjK/C37KAbQ +/0zJFkRqJ4MOgiVlYzsTqixsw0T5qbGbtZkjV/iLwoeGKR+pVezqmlt6PvYgrRBg +rlVnDLHuR3UKXstww0/Iyy8yON0jQYfzvSTFhsMEwv9eph/M9vwbND2oxrHY4SgY +3Dm/alM3Bg/ZT4i7zwTMFTBC9SsJZvhAZ09Jvp1sStCmdKh1OQzeKDXZwjfbQKBC +QYFJkXsTZkCANoHNHVVWH31KUGMc4YAR+JMYSMJFre1IG1zxGITZ3T3oULwd/Ym7 +1cWifrLWqXvZnm+xJFuvKR+pZxkPPeCmRcFA6K9U6CFEBpDHNFxYQv7Dbsv8GBpn +PqYDegtB8v2ZLr50u2p4sdQxsqKwiTvO8Ok9DU7cKx0JqGti/owi+1e2kJAj6b++ +dwwrfExrfy50BiAQeTtDx/5Urq7N+U1gfGNkVvXLO7UBCgy4irkOZtTy+akETrkG +gZiZz+d/4n/B5x6PP6JxWSX/FhRA4BCywTKV2WQqFNjVEx9QTe6dZPvJ+f1qUEUY +yavQZTisiOnyCTFb/oDRAOcRBcUKhZYlDhfujAoiMQi9YHPmwywNiLcEJgVZcM/j +OJmAzlGR1RA/9IjcJAp63OyfqwIUwMGx61shGv0yuApiKib/tBhQdc6MGZ/kqKMd +qMy2/YK0IeAtq+E5yuHwh5XcoU8zff2+WWKdiygA6fiBHAp7ct5HD8HvpvM5cuyz +yBkZKrd2vxrADEw14cCKeZ++8MoHGgAENtnwOEzZZzH0QLNgB2FiNsBCF/YFIQAZ +RXPp/ltox/fQcT2XNiZK7B818Dsu2JD7TIXdI/cQPCToYwCKOvSGWsh7lLml86wE +cXj07Rn8asWJUan/UlTHJLydHjMa0eQm9pCBwhpZLOOmvxXrBRlPAQdcly9uNCso +SwjyY4hb8F6L2BY61mmC84F5wShFdxEtyhF+icYjj2wdiA6PWAHacHpuDN/pivEX +oWM4m9NQTHqO+vujLuLOIyZ4Kz8uK5QkMveSdYP8Hitc8DfCGwatpbDbrZ/dh/jC +8znnrmkglkNb/C7DMyJLFRWU4QAjAa6HCGCzzH2QEU7DgTYPB+q+/umr5/WcVu5N +av2mfyRwTWvMiYQvbG7R9lOd8pHMP6NEaeeKJvjuDfdDv/7klu/wMEngns+JNlS+ +uqhLYlvDVkOph9423radHtirk97fdzn1fQuWAt0AphQKtvExNEmVs3dQEKcwHJmh +KGKprXEq1xVAkhv+MA7H5XuYAXsJ+MOc7V2SlA/ccavWRFjPUuHQu08ksc7rcJUa +vGw= +-----END ENCRYPTED PRIVATE KEY----- diff --git a/tests/cts/certs/.gitignore b/tests/cts/certs/.gitignore new file mode 100644 index 00000000..b9913c10 --- /dev/null +++ b/tests/cts/certs/.gitignore @@ -0,0 +1,3 @@ +# Ignore the certificates and keys in this directory +*.crt +*.pem diff --git a/tests/cts/client-header-order-https/data/test332 b/tests/cts/client-header-order-https/data/test332 new file mode 100644 index 00000000..2e0287f9 --- /dev/null +++ b/tests/cts/client-header-order-https/data/test332 @@ -0,0 +1,66 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 +Connection: close + +Blafasel + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 +Connection: close + +Blafasel + + + + + + +https + + + +HTTP client header order through https + + +--insecure -H "Origin: https://www.privoxy.org/" -H "Cache-Control: max-age=315576000" -H "If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT" -H "Upgrade-Insecure-Requests: 1" -H "Pragma: no-cache" -H "DNT: 1" -H "Cookie: yolo=123" -H "Referer: https://www.example.org/" -H "Proxy-Connection: keep-alive" -H "Accept-Encoding: gzip" -H "Accept: */*" -H "Host: %HOSTIP:%HTTPSPORT" -H "Accept-Language: en-CA" --user-agent "Client with unsorted headers" https://%HOSTIP:%HTTPSPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPSPORT +User-Agent: Client with unsorted headers +Accept: */* +Accept-Language: en-CA +Accept-Encoding: gzip +Referer: https://www.example.org/ +Cookie: yolo=123 +DNT: 1 +Pragma: no-cache +Upgrade-Insecure-Requests: 1 +If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT +Cache-Control: max-age=315576000 +Origin: https://www.privoxy.org/ + + + + diff --git a/tests/cts/client-header-order-https/data/test333 b/tests/cts/client-header-order-https/data/test333 new file mode 100644 index 00000000..38d4ea82 --- /dev/null +++ b/tests/cts/client-header-order-https/data/test333 @@ -0,0 +1,64 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 +Connection: close + +Blafasel + + + + + + +https + + +HTTP client header order through https 2 + + +--insecure -H "Origin: https://www.privoxy.org/" -H "Cache-Control: max-age=315576000" -H "If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT" -H "Upgrade-Insecure-Requests: 1" -H "DNT: 1" -H "Pragma: no-cache" -H "Cookie: yolo=123" -H "Referer: https://www.example.org/" -H "Accept-Encoding: gzip" -H "Proxy-Connection: keep-alive" -H "Accept: */*" -H "Accept-Language: en-CA" -H "Host: %HOSTIP:%HTTPSPORT" --user-agent "Client with unsorted headers" https://%HOSTIP:%HTTPSPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPSPORT +User-Agent: Client with unsorted headers +Accept: */* +Accept-Language: en-CA +Accept-Encoding: gzip +Referer: https://www.example.org/ +Cookie: yolo=123 +DNT: 1 +Pragma: no-cache +Upgrade-Insecure-Requests: 1 +If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT +Cache-Control: max-age=315576000 +Origin: https://www.privoxy.org/ + + + + diff --git a/tests/cts/client-header-order-https/data/test334 b/tests/cts/client-header-order-https/data/test334 new file mode 100644 index 00000000..21e9fdee --- /dev/null +++ b/tests/cts/client-header-order-https/data/test334 @@ -0,0 +1,56 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 +Connection: close + +Blafasel + + + + + +SSL + + +https + + +HTTP client header order 3 through https. Only a few headers to sort. + + +--insecure https://%HOSTIP:%HTTPSPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPSPORT +User-Agent: curl/%VERSION +Accept: */* + + + + diff --git a/tests/cts/client-header-order-https/data/test335 b/tests/cts/client-header-order-https/data/test335 new file mode 100644 index 00000000..03315aa7 --- /dev/null +++ b/tests/cts/client-header-order-https/data/test335 @@ -0,0 +1,66 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 +Connection: close + +Blafasel + + + + + + +https + + +HTTP client header order 4. Left over headers. + + +--insecure -H "Bum: Tschaka" -H "Blafasel: Die da" -H "Origin: https://www.privoxy.org/" -H "Cache-Control: max-age=315576000" -H "If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT" -H "Upgrade-Insecure-Requests: 1" -H "DNT: 1" -H "Pragma: no-cache" -H "Cookie: yolo=123" -H "Referer: https://www.example.org/" -H "Accept-Encoding: gzip" -H "Proxy-Connection: keep-alive" -H "Accept: */*" -H "Accept-Language: en-CA" -H "Host: %HOSTIP:%HTTPSPORT" --user-agent "Client with unsorted headers" https://%HOSTIP:%HTTPSPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPSPORT +User-Agent: Client with unsorted headers +Accept: */* +Accept-Language: en-CA +Accept-Encoding: gzip +Referer: https://www.example.org/ +Cookie: yolo=123 +DNT: 1 +Pragma: no-cache +Upgrade-Insecure-Requests: 1 +If-Modified-Since: Thu, 07 Jan 2021 22:05:31 GMT +Cache-Control: max-age=315576000 +Origin: https://www.privoxy.org/ +Bum: Tschaka +Blafasel: Die da + + + + diff --git a/tests/cts/client-header-order-https/https-inspection.action b/tests/cts/client-header-order-https/https-inspection.action new file mode 100644 index 00000000..01b2181d --- /dev/null +++ b/tests/cts/client-header-order-https/https-inspection.action @@ -0,0 +1,2 @@ +{+https-inspection +ignore-certificate-errors} +/ diff --git a/tests/cts/client-header-order-https/privoxy.conf b/tests/cts/client-header-order-https/privoxy.conf new file mode 100644 index 00000000..2df6cceb --- /dev/null +++ b/tests/cts/client-header-order-https/privoxy.conf @@ -0,0 +1,31 @@ +listen-address 127.0.0.1:9119 + +ca-directory ../ca-directory +ca-cert-file privoxy-test-cacert.crt +ca-key-file privoxy-test-cakey.pem +ca-password blafasel +# We don't check certificate anyway +trusted-cas-file privoxy-test-cacert.crt + +certificate-directory ../certs + +client-header-order Host User-Agent Accept Accept-Language Accept-Encoding \ + Proxy-Connection Referer Cookie DNT Connection \ + Pragma Upgrade-Insecure-Requests If-Modified-Since \ + Cache-Control Content-Length Origin Content-Type + +actionsfile ./https-inspection.action + +debug 1 # Log the destination for each request Privoxy let through. See also debug 1024. +debug 2 # show each connection status +debug 4 # show tagging-related messages +debug 8 # show header parsing +debug 32 # debug force feature +debug 64 # debug regular expression filters +debug 128 # debug redirects +debug 256 # debug GIF de-animation +debug 512 # Common Log Format +debug 1024 # Log the destination for requests Privoxy didn't let through, and the reason why. +debug 4096 # Startup banner and warnings. +debug 8192 # Non-fatal errors + -- 2.35.2 From 8f569c5ab5166d4a0ba7521764bac33099bbd5c6 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 Jan 2021 04:37:16 +0100 Subject: [PATCH 10/33] Add tests for rewrite behind the client's back with https --- .../data/test1 | 48 ++++++++++++++++ .../data/test2 | 54 ++++++++++++++++++ .../data/test3 | 55 +++++++++++++++++++ .../data/test4 | 54 ++++++++++++++++++ .../data/test5 | 47 ++++++++++++++++ .../privoxy.conf | 26 +++++++++ .../rewrites.action | 17 ++++++ .../rewrites.filter | 16 ++++++ 8 files changed, 317 insertions(+) create mode 100644 tests/cts/rewrite-behind-client-back-https/data/test1 create mode 100644 tests/cts/rewrite-behind-client-back-https/data/test2 create mode 100644 tests/cts/rewrite-behind-client-back-https/data/test3 create mode 100644 tests/cts/rewrite-behind-client-back-https/data/test4 create mode 100644 tests/cts/rewrite-behind-client-back-https/data/test5 create mode 100644 tests/cts/rewrite-behind-client-back-https/privoxy.conf create mode 100644 tests/cts/rewrite-behind-client-back-https/rewrites.action create mode 100644 tests/cts/rewrite-behind-client-back-https/rewrites.filter diff --git a/tests/cts/rewrite-behind-client-back-https/data/test1 b/tests/cts/rewrite-behind-client-back-https/data/test1 new file mode 100644 index 00000000..53ee6789 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back-https/data/test1 @@ -0,0 +1,48 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 18 + +Here's your data. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 18 + +Here's your data. + + + + + + +http + + +Rewrite behind the client's back: Downgrade from https to http + + +--insecure https://%HOSTIP/%TESTNUMBER-downgrade-to-http-%HOSTIP:%HTTPPORT + + + + + + diff --git a/tests/cts/rewrite-behind-client-back-https/data/test2 b/tests/cts/rewrite-behind-client-back-https/data/test2 new file mode 100644 index 00000000..a8f10943 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back-https/data/test2 @@ -0,0 +1,54 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 18 + +Here's your data. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 18 + +Here's your data. + + + + + +https + + +Rewrite path behind the client's back + + +--insecure https://%HOSTIP:%HTTPSPORT/%TESTNUMBER-remove-this + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPSPORT +User-Agent: curl/%VERSION +Accept: */* + + + + diff --git a/tests/cts/rewrite-behind-client-back-https/data/test3 b/tests/cts/rewrite-behind-client-back-https/data/test3 new file mode 100644 index 00000000..6c49f64e --- /dev/null +++ b/tests/cts/rewrite-behind-client-back-https/data/test3 @@ -0,0 +1,55 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 18 + +Here's your data. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 18 + +Here's your data. + + + + + + +https + + +Rewrite host behind the client's back + + +--insecure https://www.example.org/%TESTNUMBER-%HOSTIP:%HTTPSPORT + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: www.example.org +User-Agent: curl/%VERSION +Accept: */* + + + + diff --git a/tests/cts/rewrite-behind-client-back-https/data/test4 b/tests/cts/rewrite-behind-client-back-https/data/test4 new file mode 100644 index 00000000..23752718 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back-https/data/test4 @@ -0,0 +1,54 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 18 + +Here's your data. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 18 + +Here's your data. + + + + + +https + + +Client header filter that doesn't match anything. According to Valgrind it triggers a "Conditional jump or move depends on uninitialised value(s)" + + +--insecure https:///%HOSTIP:%HTTPSPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPSPORT +User-Agent: curl/%VERSION +Accept: */* + + + + diff --git a/tests/cts/rewrite-behind-client-back-https/data/test5 b/tests/cts/rewrite-behind-client-back-https/data/test5 new file mode 100644 index 00000000..2e6bd11b --- /dev/null +++ b/tests/cts/rewrite-behind-client-back-https/data/test5 @@ -0,0 +1,47 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 18 + +Here's your data. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 400 Malformed request after rewriting +Content-Type: text/plain +Connection: close + +Bad request. Messed up with header filters. + + + + + + +none + + +Rewrite behind the client's back: unsupporte http version + + +--insecure https://%HOSTIP/%TESTNUMBER + + + + + + diff --git a/tests/cts/rewrite-behind-client-back-https/privoxy.conf b/tests/cts/rewrite-behind-client-back-https/privoxy.conf new file mode 100644 index 00000000..8fb261e9 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back-https/privoxy.conf @@ -0,0 +1,26 @@ +listen-address 127.0.0.1:9119 + +ca-directory ../ca-directory +ca-cert-file privoxy-test-cacert.crt +ca-key-file privoxy-test-cakey.pem +ca-password blafasel +# We don't check certificate anyway +trusted-cas-file privoxy-test-cacert.crt + +certificate-directory ../certs + +debug 1 # Log the destination for each request Privoxy let through. See also debug 1024. +debug 2 # show each connection status +debug 4 # show tagging-related messages +debug 8 # show header parsing +debug 32 # debug force feature +debug 64 # debug regular expression filters +debug 128 # debug redirects +debug 256 # debug GIF de-animation +debug 512 # Common Log Format +debug 1024 # Log the destination for requests Privoxy didn't let through, and the reason why. +debug 4096 # Startup banner and warnings. +debug 8192 # Non-fatal errors + +actionsfile rewrites.action +filterfile rewrites.filter diff --git a/tests/cts/rewrite-behind-client-back-https/rewrites.action b/tests/cts/rewrite-behind-client-back-https/rewrites.action new file mode 100644 index 00000000..26235521 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back-https/rewrites.action @@ -0,0 +1,17 @@ +{+https-inspection +ignore-certificate-errors} +/ + +{+client-header-filter{https-to-http}} +/1-downgrade-to-http + +{+client-header-filter{truncate-path}} +/2-remove-this + +{+client-header-filter{rewrite-host}} +/3 + +{+client-header-filter{non-matching-rewriter}} +/4 + +{+client-header-filter{invalid-http-version}} +/5 diff --git a/tests/cts/rewrite-behind-client-back-https/rewrites.filter b/tests/cts/rewrite-behind-client-back-https/rewrites.filter new file mode 100644 index 00000000..a6442551 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back-https/rewrites.filter @@ -0,0 +1,16 @@ +CLIENT-HEADER-FILTER: https-to-http Downgrade protocol to http +s@^(\w+) (/\d-downgrade-to-http)-(\d+\.\d+\.\d+\.\d+\:\d+)@$1 http://$3$2@i + +CLIENT-HEADER-FILTER: truncate-path Removes '-remove-this' from the path +s@(/\d)-remove-this (HTTP/1.1)@$1 $2@i + +CLIENT-HEADER-FILTER: rewrite-host Replaces the host with the one specified in the path +s@GET /(\d)-(\d+\.\d+\.\d+\.\d+\:\d+) (HTTP/1.1)@GET https://$2/$1 $3@i + +# Results in Conditional jump or move depends on uninitialised value(s)? +CLIENT-HEADER-FILTER: non-matching-rewriter Does not actually match +s@/-@@ + +CLIENT-HEADER-FILTER: invalid-http-version Rewrites the request line with an invalid HTTP version +s@HTTP/1.1@HTTP/9000@ + -- 2.35.2 From 8512866f9c68418f84816512a73976662d7b084e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 9 Jan 2021 19:54:09 +0100 Subject: [PATCH 11/33] Add tests for destination rewrite --- .../cts/rewrite-behind-client-back/data/test1 | 45 +++++++++++++++++++ .../cts/rewrite-behind-client-back/data/test2 | 41 +++++++++++++++++ .../cts/rewrite-behind-client-back/data/test3 | 42 +++++++++++++++++ .../cts/rewrite-behind-client-back/data/test4 | 42 +++++++++++++++++ .../cts/rewrite-behind-client-back/data/test5 | 44 ++++++++++++++++++ .../rewrite-behind-client-back/privoxy.conf | 17 +++++++ .../rewrites.action | 14 ++++++ .../rewrites.filter | 16 +++++++ 8 files changed, 261 insertions(+) create mode 100644 tests/cts/rewrite-behind-client-back/data/test1 create mode 100644 tests/cts/rewrite-behind-client-back/data/test2 create mode 100644 tests/cts/rewrite-behind-client-back/data/test3 create mode 100644 tests/cts/rewrite-behind-client-back/data/test4 create mode 100644 tests/cts/rewrite-behind-client-back/data/test5 create mode 100644 tests/cts/rewrite-behind-client-back/privoxy.conf create mode 100644 tests/cts/rewrite-behind-client-back/rewrites.action create mode 100644 tests/cts/rewrite-behind-client-back/rewrites.filter diff --git a/tests/cts/rewrite-behind-client-back/data/test1 b/tests/cts/rewrite-behind-client-back/data/test1 new file mode 100644 index 00000000..dd0175bc --- /dev/null +++ b/tests/cts/rewrite-behind-client-back/data/test1 @@ -0,0 +1,45 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 400 Malformed request after rewriting +Content-Type: text/plain +Connection: close + +Bad request. Messed up with header filters. + + + + + + +none + + +Rewrite behind the client's back: invalid upgrade to https + + +http://%HOSTIP/%TESTNUMBER-upgrade-to-https + + + + + + diff --git a/tests/cts/rewrite-behind-client-back/data/test2 b/tests/cts/rewrite-behind-client-back/data/test2 new file mode 100644 index 00000000..b145d732 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back/data/test2 @@ -0,0 +1,41 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + + +http + + +Rewrite path behind the client's back + + +http://%HOSTIP:%HTTPPORT/%TESTNUMBER-remove-this + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* + + + + diff --git a/tests/cts/rewrite-behind-client-back/data/test3 b/tests/cts/rewrite-behind-client-back/data/test3 new file mode 100644 index 00000000..61e35577 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back/data/test3 @@ -0,0 +1,42 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + + +http + + +Rewrite host behind the client's back + + +http://www.example.org/%TESTNUMBER-%HOSTIP:%HTTPPORT + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: www.example.org +User-Agent: curl/%VERSION +Accept: */* +Connection: close + + + + diff --git a/tests/cts/rewrite-behind-client-back/data/test4 b/tests/cts/rewrite-behind-client-back/data/test4 new file mode 100644 index 00000000..d7c10f89 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back/data/test4 @@ -0,0 +1,42 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Content-Length: 9 + +Blafasel + + + + + + +http + + +Client header filter that doesn't match anything. According to Valgrind it triggers a "Conditional jump or move depends on uninitialised value(s)" + + +http:///%HOSTIP:%HTTPPORT/%TESTNUMBER + + + + + +GET /%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Connection: close + + + + diff --git a/tests/cts/rewrite-behind-client-back/data/test5 b/tests/cts/rewrite-behind-client-back/data/test5 new file mode 100644 index 00000000..82da79df --- /dev/null +++ b/tests/cts/rewrite-behind-client-back/data/test5 @@ -0,0 +1,44 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 400 Malformed request after rewriting +Content-Type: text/plain +Connection: close + +Bad request. Messed up with header filters. + + + + + +none + + +Rewrite behind the client's back: unsupported http version + + +http://%HOSTIP/%TESTNUMBER + + + + + + diff --git a/tests/cts/rewrite-behind-client-back/privoxy.conf b/tests/cts/rewrite-behind-client-back/privoxy.conf new file mode 100644 index 00000000..06304307 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back/privoxy.conf @@ -0,0 +1,17 @@ +listen-address 127.0.0.1:9119 + +debug 1 # Log the destination for each request Privoxy let through. See also debug 1024. +debug 2 # show each connection status +debug 4 # show tagging-related messages +debug 8 # show header parsing +debug 32 # debug force feature +debug 64 # debug regular expression filters +debug 128 # debug redirects +debug 256 # debug GIF de-animation +debug 512 # Common Log Format +debug 1024 # Log the destination for requests Privoxy didn't let through, and the reason why. +debug 4096 # Startup banner and warnings. +debug 8192 # Non-fatal errors + +actionsfile rewrites.action +filterfile rewrites.filter diff --git a/tests/cts/rewrite-behind-client-back/rewrites.action b/tests/cts/rewrite-behind-client-back/rewrites.action new file mode 100644 index 00000000..78d3b5f5 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back/rewrites.action @@ -0,0 +1,14 @@ +{+client-header-filter{http-to-https}} +/1-upgrade-to-https + +{+client-header-filter{truncate-path}} +/2-remove-this + +{+client-header-filter{rewrite-host}} +/3 + +{+client-header-filter{non-matching-rewriter}} +/4 + +{+client-header-filter{invalid-http-version}} +/5 diff --git a/tests/cts/rewrite-behind-client-back/rewrites.filter b/tests/cts/rewrite-behind-client-back/rewrites.filter new file mode 100644 index 00000000..30f59de3 --- /dev/null +++ b/tests/cts/rewrite-behind-client-back/rewrites.filter @@ -0,0 +1,16 @@ +CLIENT-HEADER-FILTER: http-to-https Upgrade http requests to https behind the client's back. Not actually supported. +s@^(\w+)\s+http://@$1 https://@i + +CLIENT-HEADER-FILTER: truncate-path Removes '-remove-this' from the path +s@(/\d)-remove-this (HTTP/1.1)@$1 $2@i + +CLIENT-HEADER-FILTER: rewrite-host Replaces the host with the one specified in the path +s@GET .*/(\d)-(\d+\.\d+\.\d+\.\d+\:\d+) (HTTP/1.1)@GET http://$2/$1 $3@i + +# Results in Conditional jump or move depends on uninitialised value(s)? +CLIENT-HEADER-FILTER: non-matching-rewriter Does not actually match +s@/-@@ + +CLIENT-HEADER-FILTER: invalid-http-version Rewrites the request line with an invalid HTTP version +s@HTTP/1.1@HTTP/9000@ + -- 2.35.2 From 0650e8f80be987c0f9168b7420d79138c2ef8a60 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 Jan 2021 09:48:35 +0100 Subject: [PATCH 12/33] Add redirect tests --- tests/cts/redirects/data/test1 | 45 ++++++++++++++++++++++++++++ tests/cts/redirects/data/test2 | 45 ++++++++++++++++++++++++++++ tests/cts/redirects/data/test3 | 45 ++++++++++++++++++++++++++++ tests/cts/redirects/data/test4 | 45 ++++++++++++++++++++++++++++ tests/cts/redirects/privoxy.conf | 16 ++++++++++ tests/cts/redirects/redirects.action | 11 +++++++ 6 files changed, 207 insertions(+) create mode 100644 tests/cts/redirects/data/test1 create mode 100644 tests/cts/redirects/data/test2 create mode 100644 tests/cts/redirects/data/test3 create mode 100644 tests/cts/redirects/data/test4 create mode 100644 tests/cts/redirects/privoxy.conf create mode 100644 tests/cts/redirects/redirects.action diff --git a/tests/cts/redirects/data/test1 b/tests/cts/redirects/data/test1 new file mode 100644 index 00000000..3bcd690a --- /dev/null +++ b/tests/cts/redirects/data/test1 @@ -0,0 +1,45 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 302 Local Redirect from Privoxy +Location: http://www.privoxy.org +Content-Length: 0 +X-Ignore-Header: Date +Date: Sun, 10 Jan 2021 08:13:02 GMT + + + + + + +none + + +Redirect to http://www.privoxy.org + + +http://%HOSTIP/%TESTNUMBER-redirect-me-to-www.privoxy.org + + + + + + diff --git a/tests/cts/redirects/data/test2 b/tests/cts/redirects/data/test2 new file mode 100644 index 00000000..245b5bcf --- /dev/null +++ b/tests/cts/redirects/data/test2 @@ -0,0 +1,45 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 302 Local Redirect from Privoxy +Location: https://www.privoxy.org +Content-Length: 0 +X-Ignore-Header: Date +Date: Sun, 10 Jan 2021 08:13:02 GMT + + + + + + +none + + +Redirect to https://www.privoxy.org + + +http://%HOSTIP/%TESTNUMBER-redirect-me-to-www.privoxy.org + + + + + + diff --git a/tests/cts/redirects/data/test3 b/tests/cts/redirects/data/test3 new file mode 100644 index 00000000..625131b5 --- /dev/null +++ b/tests/cts/redirects/data/test3 @@ -0,0 +1,45 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 302 Local Redirect from Privoxy +Location: http://www.privoxy.org/ +Content-Length: 0 +X-Ignore-Header: Date +Date: Sun, 10 Jan 2021 08:13:02 GMT + + + + + + +none + + +Redirect to http://www.privoxy.org using a pcrs command + + +http://www.example.org/%TESTNUMBER-redirect-me-to-www.privoxy.org + + + + + + diff --git a/tests/cts/redirects/data/test4 b/tests/cts/redirects/data/test4 new file mode 100644 index 00000000..f9c632b7 --- /dev/null +++ b/tests/cts/redirects/data/test4 @@ -0,0 +1,45 @@ + + + +HTTP +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 302 Local Redirect from Privoxy +Location: http://www.privoxy.org/path-replaced +Content-Length: 0 +X-Ignore-Header: Date +Date: Sun, 10 Jan 2021 08:13:02 GMT + + + + + + +none + + +Redirect to http://www.privoxy.org/path-replaced using a pcrs command + + +http://www.privoxy.org/%TESTNUMBER-replace-me + + + + + + diff --git a/tests/cts/redirects/privoxy.conf b/tests/cts/redirects/privoxy.conf new file mode 100644 index 00000000..777fb322 --- /dev/null +++ b/tests/cts/redirects/privoxy.conf @@ -0,0 +1,16 @@ +listen-address 127.0.0.1:9119 + +debug 1 # Log the destination for each request Privoxy let through. See also debug 1024. +debug 2 # show each connection status +debug 4 # show tagging-related messages +debug 8 # show header parsing +debug 32 # debug force feature +debug 64 # debug regular expression filters +debug 128 # debug redirects +debug 256 # debug GIF de-animation +debug 512 # Common Log Format +debug 1024 # Log the destination for requests Privoxy didn't let through, and the reason why. +debug 4096 # Startup banner and warnings. +debug 8192 # Non-fatal errors + +actionsfile redirects.action diff --git a/tests/cts/redirects/redirects.action b/tests/cts/redirects/redirects.action new file mode 100644 index 00000000..b95ea2ea --- /dev/null +++ b/tests/cts/redirects/redirects.action @@ -0,0 +1,11 @@ +{+redirect{http://www.privoxy.org}} +/1-redirect-me-to-www.privoxy.org + +{+redirect{https://www.privoxy.org}} +/2-redirect-me-to-www.privoxy.org + +{+redirect{s@http://www.example.org/.*@http://www.privoxy.org/@}} +/3-redirect-me-to-www.privoxy.org + +{+redirect{s@http://www.privoxy.org/\d-replace-me@http://www.privoxy.org/path-replaced@}} +/4-replace-me -- 2.35.2 From e0b35ac57ee0cbcee929b9987e7bc6154f36a49e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 Jan 2021 10:17:30 +0100 Subject: [PATCH 13/33] Add tests for redirects for https-inspected requests --- tests/cts/redirects-https/data/test1 | 47 ++++++++++++++++++++++ tests/cts/redirects-https/data/test2 | 47 ++++++++++++++++++++++ tests/cts/redirects-https/data/test3 | 47 ++++++++++++++++++++++ tests/cts/redirects-https/data/test4 | 47 ++++++++++++++++++++++ tests/cts/redirects-https/data/test5 | 47 ++++++++++++++++++++++ tests/cts/redirects-https/privoxy.conf | 25 ++++++++++++ tests/cts/redirects-https/redirects.action | 17 ++++++++ 7 files changed, 277 insertions(+) create mode 100644 tests/cts/redirects-https/data/test1 create mode 100644 tests/cts/redirects-https/data/test2 create mode 100644 tests/cts/redirects-https/data/test3 create mode 100644 tests/cts/redirects-https/data/test4 create mode 100644 tests/cts/redirects-https/data/test5 create mode 100644 tests/cts/redirects-https/privoxy.conf create mode 100644 tests/cts/redirects-https/redirects.action diff --git a/tests/cts/redirects-https/data/test1 b/tests/cts/redirects-https/data/test1 new file mode 100644 index 00000000..a1a69ada --- /dev/null +++ b/tests/cts/redirects-https/data/test1 @@ -0,0 +1,47 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 302 Local Redirect from Privoxy +Location: http://www.privoxy.org/ +Content-Length: 0 +X-Ignore-Header: Date +Date: Sun, 10 Jan 2021 08:13:02 GMT + + + + + + +none + + +Redirect to http://www.privoxy.org + + +--insecure https://%HOSTIP/%TESTNUMBER-redirect-me-to-www.privoxy.org + + + + + + diff --git a/tests/cts/redirects-https/data/test2 b/tests/cts/redirects-https/data/test2 new file mode 100644 index 00000000..60e815f6 --- /dev/null +++ b/tests/cts/redirects-https/data/test2 @@ -0,0 +1,47 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 302 Local Redirect from Privoxy +Location: https://www.privoxy.org/ +Content-Length: 0 +X-Ignore-Header: Date +Date: Sun, 10 Jan 2021 08:13:02 GMT + + + + + + +none + + +Redirect to https://www.privoxy.org + + +--insecure https://%HOSTIP/%TESTNUMBER-redirect-me-to-www.privoxy.org + + + + + + diff --git a/tests/cts/redirects-https/data/test3 b/tests/cts/redirects-https/data/test3 new file mode 100644 index 00000000..3e7368b2 --- /dev/null +++ b/tests/cts/redirects-https/data/test3 @@ -0,0 +1,47 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 302 Local Redirect from Privoxy +Location: http://www.privoxy.org/ +Content-Length: 0 +X-Ignore-Header: Date +Date: Sun, 10 Jan 2021 08:13:02 GMT + + + + + + +none + + +Redirect to http://www.privoxy.org/ using a pcrs command + + +--insecure https://www.example.org/%TESTNUMBER-redirect-me-to-www.privoxy.org + + + + + + diff --git a/tests/cts/redirects-https/data/test4 b/tests/cts/redirects-https/data/test4 new file mode 100644 index 00000000..47491e6b --- /dev/null +++ b/tests/cts/redirects-https/data/test4 @@ -0,0 +1,47 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 302 Local Redirect from Privoxy +Location: https://www.privoxy.org/ +Content-Length: 0 +X-Ignore-Header: Date +Date: Sun, 10 Jan 2021 08:13:02 GMT + + + + + + +none + + +Redirect to https://www.privoxy.org/ using a pcrs command + + +--insecure https://www.example.org/%TESTNUMBER-redirect-me-to-www.privoxy.org + + + + + + diff --git a/tests/cts/redirects-https/data/test5 b/tests/cts/redirects-https/data/test5 new file mode 100644 index 00000000..9bcc1bfa --- /dev/null +++ b/tests/cts/redirects-https/data/test5 @@ -0,0 +1,47 @@ + + + +HTTPS +HTTP GET + + + + + +HTTP/1.1 200 OK +Connection: close +Content-Type: text/html +Content-Length: 29 + +Not actually used. + + + + + +HTTP/1.1 200 Connection established + +HTTP/1.1 302 Local Redirect from Privoxy +Location: https://www.privoxy.org/path-replaced +Content-Length: 0 +X-Ignore-Header: Date +Date: Sun, 10 Jan 2021 08:13:02 GMT + + + + + + +none + + +Redirect to https://www.privoxy.org/path-replaced using a pcrs command + + +--insecure https://www.privoxy.org/%TESTNUMBER-replace-me + + + + + + diff --git a/tests/cts/redirects-https/privoxy.conf b/tests/cts/redirects-https/privoxy.conf new file mode 100644 index 00000000..5bc00dab --- /dev/null +++ b/tests/cts/redirects-https/privoxy.conf @@ -0,0 +1,25 @@ +listen-address 127.0.0.1:9119 + +ca-directory ../ca-directory +ca-cert-file privoxy-test-cacert.crt +ca-key-file privoxy-test-cakey.pem +ca-password blafasel +# We don't check certificate anyway +trusted-cas-file privoxy-test-cacert.crt + +certificate-directory ../certs + +debug 1 # Log the destination for each request Privoxy let through. See also debug 1024. +debug 2 # show each connection status +debug 4 # show tagging-related messages +debug 8 # show header parsing +debug 32 # debug force feature +debug 64 # debug regular expression filters +debug 128 # debug redirects +debug 256 # debug GIF de-animation +debug 512 # Common Log Format +debug 1024 # Log the destination for requests Privoxy didn't let through, and the reason why. +debug 4096 # Startup banner and warnings. +debug 8192 # Non-fatal errors + +actionsfile redirects.action diff --git a/tests/cts/redirects-https/redirects.action b/tests/cts/redirects-https/redirects.action new file mode 100644 index 00000000..4c5017d8 --- /dev/null +++ b/tests/cts/redirects-https/redirects.action @@ -0,0 +1,17 @@ +{+https-inspection +ignore-certificate-errors} +/ + +{+redirect{http://www.privoxy.org/}} +/1-redirect-me-to-www.privoxy.org + +{+redirect{https://www.privoxy.org/}} +/2-redirect-me-to-www.privoxy.org + +{+redirect{s@https://www.example.org/.*@http://www.privoxy.org/@}} +/3-redirect-me-to-www.privoxy.org + +{+redirect{s@https://www.example.org/.*@https://www.privoxy.org/@}} +/4-redirect-me-to-www.privoxy.org + +{+redirect{s@https://www.privoxy.org/\d-replace-me@https://www.privoxy.org/path-replaced@}} +/5-replace-me -- 2.35.2 From 961807e23c69397cfb7b5b207e19bf1a67187cd4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 10 Jan 2021 16:18:17 +0100 Subject: [PATCH 14/33] Add upstream-tests scenario --- tests/cts/upstream-tests/privoxy.conf | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/cts/upstream-tests/privoxy.conf diff --git a/tests/cts/upstream-tests/privoxy.conf b/tests/cts/upstream-tests/privoxy.conf new file mode 100644 index 00000000..2bbef63c --- /dev/null +++ b/tests/cts/upstream-tests/privoxy.conf @@ -0,0 +1,25 @@ +listen-address 127.0.0.1:9119 + +debug 1 # Log the destination for each request Privoxy let through. See also debug 1024. +debug 2 # show each connection status +debug 4 # show tagging-related messages +debug 8 # show header parsing +debug 32 # debug force feature +debug 64 # debug regular expression filters +debug 128 # debug redirects +debug 256 # debug GIF de-animation +debug 512 # Common Log Format +debug 1024 # Log the destination for requests Privoxy didn't let through, and the reason why. +debug 4096 # Startup banner and warnings. +debug 8192 # Non-fatal errors + +#debug 16 # log all data written to the network +#debug 32768 # log all data read from the network + +actionsfile ../../../default.action.master +filterfile ../../../default.filter +templdir ../../../templates + +keep-alive-timeout 5 +socket-timeout 5 +default-server-timeout 5 -- 2.35.2 From 1d6c92d5125196aea32c4023072858f173ebd904 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 19 Jan 2021 18:54:50 +0100 Subject: [PATCH 15/33] Add test scenario content-filters --- .../content-filters/content-filters.action | 95 +++++++ tests/cts/content-filters/data/test1 | 229 +++++++++++++++++ tests/cts/content-filters/data/test10 | 108 ++++++++ tests/cts/content-filters/data/test11 | 102 ++++++++ tests/cts/content-filters/data/test12 | 92 +++++++ tests/cts/content-filters/data/test13 | 84 +++++++ tests/cts/content-filters/data/test14 | 118 +++++++++ tests/cts/content-filters/data/test15 | 80 ++++++ tests/cts/content-filters/data/test16 | 216 ++++++++++++++++ tests/cts/content-filters/data/test17 | 138 ++++++++++ tests/cts/content-filters/data/test18 | 76 ++++++ tests/cts/content-filters/data/test19 | 116 +++++++++ tests/cts/content-filters/data/test2 | 129 ++++++++++ tests/cts/content-filters/data/test200 | 62 +++++ tests/cts/content-filters/data/test201 | 65 +++++ tests/cts/content-filters/data/test202 | 61 +++++ tests/cts/content-filters/data/test203 | 61 +++++ tests/cts/content-filters/data/test204 | 61 +++++ tests/cts/content-filters/data/test21 | 60 +++++ tests/cts/content-filters/data/test22 | 88 +++++++ tests/cts/content-filters/data/test23 | 124 +++++++++ tests/cts/content-filters/data/test24 | 132 ++++++++++ tests/cts/content-filters/data/test25 | 100 ++++++++ tests/cts/content-filters/data/test3 | 121 +++++++++ tests/cts/content-filters/data/test300 | 62 +++++ tests/cts/content-filters/data/test301 | 62 +++++ tests/cts/content-filters/data/test302 | 62 +++++ tests/cts/content-filters/data/test310 | 62 +++++ tests/cts/content-filters/data/test311 | 62 +++++ tests/cts/content-filters/data/test312 | 62 +++++ tests/cts/content-filters/data/test320 | 62 +++++ tests/cts/content-filters/data/test321 | 62 +++++ tests/cts/content-filters/data/test322 | 62 +++++ tests/cts/content-filters/data/test33 | 99 ++++++++ tests/cts/content-filters/data/test34 | 134 ++++++++++ tests/cts/content-filters/data/test35 | 80 ++++++ tests/cts/content-filters/data/test36 | 84 +++++++ tests/cts/content-filters/data/test37 | 74 ++++++ tests/cts/content-filters/data/test38 | 84 +++++++ tests/cts/content-filters/data/test390 | 115 +++++++++ tests/cts/content-filters/data/test391 | 63 +++++ tests/cts/content-filters/data/test392 | 64 +++++ tests/cts/content-filters/data/test393 | 63 +++++ tests/cts/content-filters/data/test4 | 81 ++++++ tests/cts/content-filters/data/test5 | 89 +++++++ tests/cts/content-filters/data/test6 | 97 +++++++ tests/cts/content-filters/data/test7 | 89 +++++++ tests/cts/content-filters/data/test8 | 105 ++++++++ tests/cts/content-filters/data/test9 | 236 ++++++++++++++++++ tests/cts/content-filters/privoxy.conf | 17 ++ 50 files changed, 4650 insertions(+) create mode 100644 tests/cts/content-filters/content-filters.action create mode 100644 tests/cts/content-filters/data/test1 create mode 100644 tests/cts/content-filters/data/test10 create mode 100644 tests/cts/content-filters/data/test11 create mode 100644 tests/cts/content-filters/data/test12 create mode 100644 tests/cts/content-filters/data/test13 create mode 100644 tests/cts/content-filters/data/test14 create mode 100644 tests/cts/content-filters/data/test15 create mode 100644 tests/cts/content-filters/data/test16 create mode 100644 tests/cts/content-filters/data/test17 create mode 100644 tests/cts/content-filters/data/test18 create mode 100644 tests/cts/content-filters/data/test19 create mode 100644 tests/cts/content-filters/data/test2 create mode 100644 tests/cts/content-filters/data/test200 create mode 100644 tests/cts/content-filters/data/test201 create mode 100644 tests/cts/content-filters/data/test202 create mode 100644 tests/cts/content-filters/data/test203 create mode 100644 tests/cts/content-filters/data/test204 create mode 100644 tests/cts/content-filters/data/test21 create mode 100644 tests/cts/content-filters/data/test22 create mode 100644 tests/cts/content-filters/data/test23 create mode 100644 tests/cts/content-filters/data/test24 create mode 100644 tests/cts/content-filters/data/test25 create mode 100644 tests/cts/content-filters/data/test3 create mode 100644 tests/cts/content-filters/data/test300 create mode 100644 tests/cts/content-filters/data/test301 create mode 100644 tests/cts/content-filters/data/test302 create mode 100644 tests/cts/content-filters/data/test310 create mode 100644 tests/cts/content-filters/data/test311 create mode 100644 tests/cts/content-filters/data/test312 create mode 100644 tests/cts/content-filters/data/test320 create mode 100644 tests/cts/content-filters/data/test321 create mode 100644 tests/cts/content-filters/data/test322 create mode 100644 tests/cts/content-filters/data/test33 create mode 100644 tests/cts/content-filters/data/test34 create mode 100644 tests/cts/content-filters/data/test35 create mode 100644 tests/cts/content-filters/data/test36 create mode 100644 tests/cts/content-filters/data/test37 create mode 100644 tests/cts/content-filters/data/test38 create mode 100644 tests/cts/content-filters/data/test390 create mode 100644 tests/cts/content-filters/data/test391 create mode 100644 tests/cts/content-filters/data/test392 create mode 100644 tests/cts/content-filters/data/test393 create mode 100644 tests/cts/content-filters/data/test4 create mode 100644 tests/cts/content-filters/data/test5 create mode 100644 tests/cts/content-filters/data/test6 create mode 100644 tests/cts/content-filters/data/test7 create mode 100644 tests/cts/content-filters/data/test8 create mode 100644 tests/cts/content-filters/data/test9 create mode 100644 tests/cts/content-filters/privoxy.conf diff --git a/tests/cts/content-filters/content-filters.action b/tests/cts/content-filters/content-filters.action new file mode 100644 index 00000000..25de2530 --- /dev/null +++ b/tests/cts/content-filters/content-filters.action @@ -0,0 +1,95 @@ +{+filter{js-annoyances}} +/js-annoyances/ + +{+filter{js-events}} +/js-events/ + +{+filter{html-annoyances}} +/html-annoyances/ + +{+filter{content-cookies}} +/content-cookies/ + +{+filter{refresh-tags}} +/refresh-tags/ + +{+filter{unsolicited-popups}} +/unsolicited-popups/ + +{+filter{all-popups}} +/all-popups/ + +{+filter{img-reorder}} +/img-reorder/ + +{+filter{banners-by-size}} +/banners-by-size/ + +{+filter{banners-by-link}} +/banners-by-link/ + +{+filter{webbugs}} +/webbugs/ + +{+filter{tiny-textforms}} +/tiny-textforms/ + +{+filter{jumping-windows}} +/jumping-windows/ + +{+filter{frameset-borders}} +/frameset-borders/ + +{+filter{iframes}} +/iframes/ + +{+filter{demoronizer}} +/demoronizer/ + +{+filter{shockwave-flash}} +/shockwave-flash/ + +{+filter{quicktime-kioskmode}} +/quicktime-kioskmode/ + +{+filter{fun}} +/fun/ + +{+filter{crude-parental}} +/crude-parental/ + +{+filter{ie-exploits}} +/ie-exploits/ + +{+filter{site-specifics}} +/site-specifics/ + +{+filter{no-ping}} +/no-ping/ + +{+filter{allow-autocompletion}} +/allow-autocompletion/ + +{+filter{github}} +/github/ + +{+filter{imdb}} +/imdb/ + +{+filter{google}} +/google/ + +{+filter{google}} +/google/ + +{+filter{yahoo}} +/yahoo/ + +{+filter{msn}} +/msn/ + +{+filter{blogspot}} +/blogspot/ + +{+filter{sourceforge}} +/sourceforge/ diff --git a/tests/cts/content-filters/data/test1 b/tests/cts/content-filters/data/test1 new file mode 100644 index 00000000..e9713072 --- /dev/null +++ b/tests/cts/content-filters/data/test1 @@ -0,0 +1,229 @@ + + + +HTTP +HTTP GET +filter js-annoyances + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Connection: close +Content-Type: text/html +X-Control: swsclose + +# Here are some strings the js-annoyances filter should modify: + +# pcrs command 1: + +blafasel.referrer + +# pcrs command 2: + +bla bla bla this.defaultstatus = "hoho" x y z +bla bla bla window.status = "whatever" x y z + +# pcrs command 3: + +this.status = blafasel; +window.defaultstatus= dieda; + +# pcrs command 4: + + + +# pcrs command 5: + + + +# pcrs command 6: + +open (blafasel resizable="yes"); +open(blafasel resizable='yes'); + +# pcrs command 7: + +open(blafasel location="yes") +open(blafasel location='yes') + +# pcrs command 8: + +open(blafasel status="yes") +open(blafasel status='yes') + +# pcrs command 9: + +open(blafasel scrolling="auto") +open(blafasel scrollbars='auto') + +# pcrs command 10: + +open(blafasel menubar="yes") +open(blafasel menubar='yes') + +# pcrs command 11: + +open(blafasel toolbar="yes") +open(blafasel toolbar='yes') + +# pcrs command 12: + +open(blafasel directories="yes") +open(blafasel directories='yes') + +# pcrs command 13: + +open(blafasel fullscreen="no") +open(blafasel fullscreen='0') + +# pcrs command 14: + +open(blafasel alwaysraised="no") +open(blafasel alwayslowered='no') + +# pcrs command 15: + +open(blafasel z-lock="no") +open(blafasel zlock='no') + +# pcrs command 16: + +open(blafasel hotkeys="no") +open(blafasel hotkeys='no') + +# pcrs command 17: + +open(blafasel titlebar="yes") +open(blafasel titlebar='yes') + + + + + + +http + + ++filter{js-annoyances} + + +proxy + + +http://%HOSTIP:%HTTPPORT/js-annoyances/%TESTNUMBER + + + + + +GET /js-annoyances/%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Connection: close + + + + diff --git a/tests/cts/content-filters/data/test10 b/tests/cts/content-filters/data/test10 new file mode 100644 index 00000000..e5c35643 --- /dev/null +++ b/tests/cts/content-filters/data/test10 @@ -0,0 +1,108 @@ + + + +HTTP +HTTP GET +filter banners-by-link + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Connection: close +Content-Type: text/html +X-Control: swsclose + +# Here is some HTML that the banners-by-link filter should modify: + +# pcrs command 1: + + + + + + + + + + + +# pcrs command 2: + + + + + + + + + + + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Connection: close +Content-Type: text/html +X-Control: swsclose +Content-Length: 3256 + +# Here is some HTML that the banners-by-link filter should modify: + +# pcrs command 1: + + + + + + + + + + + +# pcrs command 2: + + + + + + + + + + + + + + + +http + + ++filter{banners-by-link} + + +proxy + + +http://%HOSTIP:%HTTPPORT/banners-by-link/%TESTNUMBER + + + + + +GET /banners-by-link/%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Connection: close + + + + diff --git a/tests/cts/content-filters/data/test11 b/tests/cts/content-filters/data/test11 new file mode 100644 index 00000000..277ea886 --- /dev/null +++ b/tests/cts/content-filters/data/test11 @@ -0,0 +1,102 @@ + + + +HTTP +HTTP GET +filter webbugs + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Connection: close +Content-Type: text/html +X-Control: swsclose + +# Here is some HTML that the webbugs filter should squash: + + + + + + + + + + + + + + + + + + + + + + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Connection: close +Content-Type: text/html +X-Control: swsclose +Content-Length: 80 + +# Here is some HTML that the webbugs filter should squash: + + + + + + + + + + + + + + + + + + + + + + + + + + +http + + ++filter{webbugs} + + +proxy + + +http://%HOSTIP:%HTTPPORT/webbugs/%TESTNUMBER + + + + + +GET /webbugs/%TESTNUMBER HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +User-Agent: curl/%VERSION +Accept: */* +Connection: close + + + + diff --git a/tests/cts/content-filters/data/test12 b/tests/cts/content-filters/data/test12 new file mode 100644 index 00000000..a7a7fd65 --- /dev/null +++ b/tests/cts/content-filters/data/test12 @@ -0,0 +1,92 @@ + + + +HTTP +HTTP GET +filter tiny-textforms + + + + + +HTTP/1.1 200 OK +Date: Thu, 22 Jul 2010 11:22:33 GMT +Connection: close +Content-Type: text/html +X-Control: swsclose + +# Here are some tiny textforms the tiny-textforms filter should filter: + +# pcrs command 1: + +