From 3badc3fd57ff76f8b7dce3424ee2ce30b18b9e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 00:56:08 +0000 Subject: [PATCH 01/12] Add IpClassifier with explicit IPv4/IPv6 reserved ranges Match reserved networks with inet_pton and a mask so PHP 7.4 through 8.5 agree, without FILTER_FLAG_GLOBAL_RANGE. Also canonicalize the inet_aton spellings libcurl connects to without consulting DNS. --- src/Http/IpClassifier.php | 355 +++++++++++++++++++++++++++++++++++++ tests/IpClassifierTest.php | 120 +++++++++++++ 2 files changed, 475 insertions(+) create mode 100644 src/Http/IpClassifier.php create mode 100644 tests/IpClassifierTest.php diff --git a/src/Http/IpClassifier.php b/src/Http/IpClassifier.php new file mode 100644 index 00000000..11871e27 --- /dev/null +++ b/src/Http/IpClassifier.php @@ -0,0 +1,355 @@ + + */ + private const IPV4_NETWORKS = [ + ['0.0.0.0', 8], + ['10.0.0.0', 8], + ['100.64.0.0', 10], + ['127.0.0.0', 8], + ['169.254.0.0', 16], + ['172.16.0.0', 12], + ['192.0.0.0', 24], + ['192.0.2.0', 24], + ['192.88.99.0', 24], + ['192.168.0.0', 16], + ['198.18.0.0', 15], + ['198.51.100.0', 24], + ['203.0.113.0', 24], + ['224.0.0.0', 4], + ['240.0.0.0', 4], + ['255.255.255.255', 32], + ]; + + /** + * IPv6 networks that must not be contacted. + * + * @var array + */ + private const IPV6_NETWORKS = [ + ['::', 128], + ['::1', 128], + ['fc00::', 7], + ['fe80::', 10], + ['fec0::', 10], + ['ff00::', 8], + ['2001:db8::', 32], + ['100::', 64], + ['2001::', 32], + ['64:ff9b:1::', 48], + ]; + + public static function isPublic(string $ip): bool + { + $packed = self::pack($ip); + if ($packed === null) { + return false; + } + + if (strlen($packed) === 4) { + return !self::inNetworks($packed, self::IPV4_NETWORKS); + } + + $embedded = self::embeddedIpv4($packed); + if ($embedded !== null) { + return !self::inNetworks($embedded, self::IPV4_NETWORKS); + } + + return !self::inNetworks($packed, self::IPV6_NETWORKS); + } + + /** + * Canonical textual form of an IP literal, including inet_aton spellings + * (hex, octal, and shortened IPv4). Returns null when $value is a hostname. + */ + public static function canonical(string $value): ?string + { + $value = self::unwrap($value); + if ($value === '') { + return null; + } + + if (filter_var($value, FILTER_VALIDATE_IP) !== false) { + return self::format($value); + } + + return self::inetAton($value); + } + + /** + * True when both values are the same address. IPv4 and its IPv4-mapped + * IPv6 form compare equal. + */ + public static function equals(string $left, string $right): bool + { + $a = self::pack($left); + $b = self::pack($right); + if ($a === null || $b === null) { + return false; + } + + if ($a === $b) { + return true; + } + + $a4 = self::asIpv4($a); + $b4 = self::asIpv4($b); + + return $a4 !== null && $a4 === $b4; + } + + private static function pack(string $ip): ?string + { + $canonical = self::canonical($ip); + if ($canonical === null) { + return null; + } + + $packed = inet_pton($canonical); + if ($packed === false) { + return null; + } + + return $packed; + } + + private static function format(string $ip): ?string + { + $packed = inet_pton($ip); + if ($packed === false) { + return null; + } + + $text = inet_ntop($packed); + + return $text === false ? null : $text; + } + + private static function unwrap(string $value): string + { + if (strlen($value) >= 2 && $value[0] === '[' && substr($value, -1) === ']') { + return substr($value, 1, -1); + } + + return $value; + } + + /** + * glibc inet_aton / libcurl accept these spellings and connect to the + * resulting address without consulting CURLOPT_RESOLVE. + */ + private static function inetAton(string $host): ?string + { + if (preg_match('/^(?:0x[0-9a-f]+|\d+)(?:\.(?:0x[0-9a-f]+|\d+)){0,3}$/i', $host) !== 1) { + return null; + } + + $parts = explode('.', $host); + $count = count($parts); + $numbers = []; + + foreach ($parts as $part) { + $number = self::parseComponent($part); + if ($number === null) { + return null; + } + $numbers[] = $number; + } + + $bytes = self::atonBytes($numbers, $count); + if ($bytes === null) { + return null; + } + + return $bytes[0].'.'.$bytes[1].'.'.$bytes[2].'.'.$bytes[3]; + } + + private static function parseComponent(string $part): ?float + { + if ($part === '') { + return null; + } + + if (stripos($part, '0x') === 0) { + $hex = substr($part, 2); + if ($hex === '' || strlen($hex) > 8 || !ctype_xdigit($hex)) { + return null; + } + + return (float) hexdec($hex); + } + + if (strlen($part) > 1 && $part[0] === '0') { + if (strlen($part) > 11 || strspn($part, '01234567') !== strlen($part)) { + return null; + } + + return (float) octdec($part); + } + + if (!ctype_digit($part) || strlen($part) > 10) { + return null; + } + + return (float) $part; + } + + /** + * @param array $numbers + * + * @return array{0: int, 1: int, 2: int, 3: int}|null + */ + private static function atonBytes(array $numbers, int $count): ?array + { + if ($count === 1) { + return self::split($numbers[0], 4); + } + + if ($count === 2) { + if ($numbers[0] > 255) { + return null; + } + $rest = self::split($numbers[1], 3); + if ($rest === null) { + return null; + } + + return [(int) $numbers[0], $rest[1], $rest[2], $rest[3]]; + } + + if ($count === 3) { + if ($numbers[0] > 255 || $numbers[1] > 255) { + return null; + } + $rest = self::split($numbers[2], 2); + if ($rest === null) { + return null; + } + + return [(int) $numbers[0], (int) $numbers[1], $rest[2], $rest[3]]; + } + + if ($count !== 4 || $numbers[0] > 255 || $numbers[1] > 255 || $numbers[2] > 255 || $numbers[3] > 255) { + return null; + } + + return [(int) $numbers[0], (int) $numbers[1], (int) $numbers[2], (int) $numbers[3]]; + } + + /** + * @return array{0: int, 1: int, 2: int, 3: int}|null + */ + private static function split(float $value, int $width): ?array + { + $limit = [1 => 255.0, 2 => 65535.0, 3 => 16777215.0, 4 => 4294967295.0]; + if ($value < 0 || $value > $limit[$width]) { + return null; + } + + $bytes = [0, 0, 0, 0]; + $remaining = $value; + for ($i = 3; $i >= 4 - $width; --$i) { + $bytes[$i] = (int) fmod($remaining, 256.0); + $remaining = floor($remaining / 256.0); + } + + return [$bytes[0], $bytes[1], $bytes[2], $bytes[3]]; + } + + private static function embeddedIpv4(string $packed): ?string + { + if (strlen($packed) !== 16) { + return null; + } + + // IPv4-mapped (::ffff:0:0/96), including the hexadecimal form. + if (substr($packed, 0, 12) === "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff") { + return substr($packed, 12, 4); + } + + // NAT64 well-known prefix 64:ff9b::/96, dotted or hex. + $nat64 = inet_pton('64:ff9b::'); + if ($nat64 !== false && substr($packed, 0, 12) === substr($nat64, 0, 12)) { + return substr($packed, 12, 4); + } + + // 6to4, 2002::/16. The IPv4 address occupies bits 16-47. + if (substr($packed, 0, 2) === "\x20\x02") { + return substr($packed, 2, 4); + } + + // Deprecated IPv4-compatible addresses, ::/96. + if (substr($packed, 0, 12) === str_repeat("\x00", 12)) { + return substr($packed, 12, 4); + } + + return null; + } + + private static function asIpv4(string $packed): ?string + { + if (strlen($packed) === 4) { + return $packed; + } + + if (strlen($packed) === 16 && substr($packed, 0, 12) === "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff") { + return substr($packed, 12, 4); + } + + return null; + } + + /** + * @param array $networks + */ + private static function inNetworks(string $packed, array $networks): bool + { + foreach ($networks as $network) { + if (self::inCidr($packed, $network[0], $network[1])) { + return true; + } + } + + return false; + } + + private static function inCidr(string $packed, string $network, int $bits): bool + { + $networkPacked = inet_pton($network); + if ($networkPacked === false || strlen($networkPacked) !== strlen($packed)) { + return false; + } + + $fullBytes = intdiv($bits, 8); + if ($fullBytes > 0 && substr($packed, 0, $fullBytes) !== substr($networkPacked, 0, $fullBytes)) { + return false; + } + + $remainder = $bits % 8; + if ($remainder === 0) { + return true; + } + + $mask = (0xFF << (8 - $remainder)) & 0xFF; + + return (ord($packed[$fullBytes]) & $mask) === (ord($networkPacked[$fullBytes]) & $mask); + } +} diff --git a/tests/IpClassifierTest.php b/tests/IpClassifierTest.php new file mode 100644 index 00000000..3fafdef3 --- /dev/null +++ b/tests/IpClassifierTest.php @@ -0,0 +1,120 @@ + + */ + public function addressesProvider(): array + { + return [ + 'unspecified v4' => ['0.0.0.0', false], + 'zero network end' => ['0.255.255.255', false], + 'private 10 start' => ['10.0.0.0', false], + 'private 10 end' => ['10.255.255.255', false], + 'just outside 10' => ['11.0.0.1', true], + 'cgnat start' => ['100.64.0.0', false], + 'cgnat end' => ['100.127.255.255', false], + 'below cgnat' => ['100.63.255.255', true], + 'above cgnat' => ['100.128.0.0', true], + 'loopback' => ['127.0.0.1', false], + 'loopback end' => ['127.255.255.255', false], + 'link local start' => ['169.254.0.0', false], + 'metadata' => ['169.254.169.254', false], + 'below private 172' => ['172.15.255.255', true], + 'private 172 start' => ['172.16.0.0', false], + 'private 172 end' => ['172.31.255.255', false], + 'above private 172' => ['172.32.0.0', true], + 'ietf protocol' => ['192.0.0.0', false], + 'ietf protocol end' => ['192.0.0.255', false], + 'outside ietf protocol' => ['192.0.1.1', true], + 'test net 1' => ['192.0.2.1', false], + '6to4 relay anycast' => ['192.88.99.1', false], + 'rfc1918 192.168' => ['192.168.1.1', false], + 'benchmark start' => ['198.18.0.1', false], + 'benchmark end' => ['198.19.255.255', false], + 'below benchmark' => ['198.17.255.255', true], + 'above benchmark' => ['198.20.0.1', true], + 'test net 2' => ['198.51.100.1', false], + 'test net 3' => ['203.0.113.1', false], + 'below multicast' => ['223.255.255.255', true], + 'multicast start' => ['224.0.0.1', false], + 'multicast end' => ['239.255.255.255', false], + 'reserved start' => ['240.0.0.1', false], + 'broadcast' => ['255.255.255.255', false], + 'google dns' => ['8.8.8.8', true], + 'cloudflare dns' => ['1.1.1.1', true], + 'unspecified v6' => ['::', false], + 'loopback v6' => ['::1', false], + 'unique local fc' => ['fc00::1', false], + 'unique local fd' => ['fd12:3456:789a::1', false], + 'link local v6' => ['fe80::1', false], + 'link local v6 end' => ['febf::1', false], + 'site local v6' => ['fec0::1', false], + 'multicast v6' => ['ff02::1', false], + 'documentation v6' => ['2001:db8::1', false], + 'discard prefix' => ['100::', false], + 'discard prefix host' => ['100::1', false], + 'teredo' => ['2001:0:4136:e378::1', false], + 'nat64 local use' => ['64:ff9b:1::1', false], + 'google dns v6' => ['2001:4860:4860::8888', true], + 'cloudflare dns v6' => ['2606:4700:4700::1111', true], + 'outside discard prefix' => ['100:0:0:1::', true], + 'mapped public' => ['::ffff:8.8.8.8', true], + 'mapped public hex' => ['::ffff:808:808', true], + 'mapped loopback' => ['::ffff:127.0.0.1', false], + 'mapped loopback hex' => ['::ffff:7f00:1', false], + 'compatible public' => ['::8.8.8.8', true], + 'compatible loopback' => ['::7f00:1', false], + 'nat64 public' => ['64:ff9b::8.8.8.8', true], + 'nat64 loopback' => ['64:ff9b::127.0.0.1', false], + 'nat64 loopback hex' => ['64:ff9b::7f00:1', false], + '6to4 public' => ['2002:808:808::', true], + '6to4 loopback' => ['2002:7f00:1::', false], + '6to4 rfc1918' => ['2002:c0a8:1::', false], + 'decimal loopback' => ['2130706433', false], + 'short loopback' => ['127.1', false], + 'octal loopback' => ['0177.0.0.1', false], + 'hex loopback' => ['0x7f000001', false], + 'dotted hex loopback' => ['0x7f.0.0.1', false], + 'decimal public' => ['134744072', true], + 'not an ip' => ['example.com', false], + 'empty' => ['', false], + 'five octets' => ['8.8.8.8.8', false], + ]; + } + + /** + * @dataProvider addressesProvider + */ + public function testIsPublic(string $ip, bool $public): void + { + $this->assertSame($public, IpClassifier::isPublic($ip)); + } + + public function testCanonicalNormalizesAlternativeSpellings(): void + { + $this->assertSame('127.0.0.1', IpClassifier::canonical('2130706433')); + $this->assertSame('127.0.0.1', IpClassifier::canonical('127.1')); + $this->assertSame('127.0.0.1', IpClassifier::canonical('0177.0.0.1')); + $this->assertSame('127.0.0.1', IpClassifier::canonical('0x7f000001')); + $this->assertSame('8.8.8.8', IpClassifier::canonical('134744072')); + $this->assertSame('8.8.8.8', IpClassifier::canonical('8.8.8.8')); + $this->assertNull(IpClassifier::canonical('example.com')); + $this->assertNull(IpClassifier::canonical('0x7f.example.com')); + } + + public function testEqualsTreatsIpv4MappedAsTheSameAddress(): void + { + $this->assertTrue(IpClassifier::equals('127.0.0.1', '::ffff:127.0.0.1')); + $this->assertTrue(IpClassifier::equals('8.8.8.8', '::ffff:808:808')); + $this->assertFalse(IpClassifier::equals('8.8.8.8', '1.1.1.1')); + $this->assertFalse(IpClassifier::equals('8.8.8.8', '2002:808:808::')); + } +} From 84343021fa2dbebc6ca43afc5891f564dd287982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 00:56:08 +0000 Subject: [PATCH 02/12] Add UrlPolicy and BlockedRequestException Block non-public addresses by default. One private address rejects the whole lookup, including when another returned address is allowlisted. Internationalized names need ext-intl; otherwise they are rejected. --- src/Http/BlockedRequestException.php | 45 ++++ src/Http/UrlPolicy.php | 323 +++++++++++++++++++++++++++ tests/UrlPolicyTest.php | 299 +++++++++++++++++++++++++ 3 files changed, 667 insertions(+) create mode 100644 src/Http/BlockedRequestException.php create mode 100644 src/Http/UrlPolicy.php create mode 100644 tests/UrlPolicyTest.php diff --git a/src/Http/BlockedRequestException.php b/src/Http/BlockedRequestException.php new file mode 100644 index 00000000..d54f1957 --- /dev/null +++ b/src/Http/BlockedRequestException.php @@ -0,0 +1,45 @@ +request = $request; + $this->reason = $reason; + } + + public function getRequest(): RequestInterface + { + return $this->request; + } + + /** + * @return 'scheme'|'host'|'non_public_address'|'port'|'redirect' + */ + public function getReason(): string + { + return $this->reason; + } +} diff --git a/src/Http/UrlPolicy.php b/src/Http/UrlPolicy.php new file mode 100644 index 00000000..ed5e1d75 --- /dev/null +++ b/src/Http/UrlPolicy.php @@ -0,0 +1,323 @@ + */ + private array $allowedHosts; + + /** @var array|null */ + private ?array $allowedPorts; + + /** + * @var callable(string): array|null + */ + private $resolver; + + /** + * @param array $allowedHosts + * @param array|null $allowedPorts + * @param callable(string): array|null $resolver + */ + private function __construct(bool $allowPrivate, array $allowedHosts, ?array $allowedPorts, $resolver) + { + $this->allowPrivate = $allowPrivate; + $this->allowedHosts = $allowedHosts; + $this->allowedPorts = $allowedPorts; + $this->resolver = $resolver; + } + + public static function default(): self + { + return new self(false, [], null, null); + } + + /** + * Allow addresses that are not public (loopback, RFC 1918, link-local, ...). + * Use this for intranet installs. + */ + public function allowPrivateNetworks(): self + { + $copy = clone $this; + $copy->allowPrivate = true; + + return $copy; + } + + /** + * Hosts that may resolve to a non-public address. + * A pattern "*.corp" matches any name ending in ".corp". + */ + public function withAllowedHosts(string ...$hosts): self + { + $copy = clone $this; + $copy->allowedHosts = array_values($hosts); + + return $copy; + } + + /** + * Restrict the effective port (80 and 443 when the URI omits one). + * Passing no restriction is the default: every port is allowed. + * + * @param array $ports + */ + public function withAllowedPorts(array $ports): self + { + $normalized = []; + foreach ($ports as $port) { + if (!is_int($port)) { + throw new \InvalidArgumentException('Allowed ports must be a list of integers.'); + } + $normalized[] = $port; + } + + $copy = clone $this; + $copy->allowedPorts = $normalized; + + return $copy; + } + + /** + * Replace DNS. The callable receives an ASCII hostname and returns IPv4 strings. + * An empty list is treated as a DNS failure. + * + * @param callable(string): array $resolver + */ + public function withResolver(callable $resolver): self + { + $copy = clone $this; + $copy->resolver = $resolver; + + return $copy; + } + + /** + * @throws BlockedRequestException + * @throws NetworkException when the hostname cannot be resolved + * @return array addresses that were accepted, in lookup order + */ + public function validate(RequestInterface $request): array + { + $uri = $request->getUri(); + $scheme = strtolower($uri->getScheme()); + + if ($scheme !== 'http' && $scheme !== 'https') { + throw $this->blocked($request, 'scheme'); + } + + if (strpos((string) $uri, '\\') !== false || $uri->getUserInfo() !== '') { + throw $this->blocked($request, 'host'); + } + + $host = $this->host($uri->getHost()); + if ($host === null) { + throw $this->blocked($request, 'host'); + } + + $port = $uri->getPort(); + if ($port === 0) { + throw $this->blocked($request, 'port'); + } + + if ($this->allowedPorts !== null) { + $effective = $port ?? ($scheme === 'https' ? 443 : 80); + if (!in_array($effective, $this->allowedPorts, true)) { + throw $this->blocked($request, 'port'); + } + } + + $literal = IpClassifier::canonical($host); + if ($literal !== null) { + $ips = [$literal]; + $lookup = $host; + } else { + $lookup = $this->asciiHost($request, $host); + $ips = $this->resolve($request, $lookup); + } + + // One non-public address blocks the whole lookup. An allowlisted + // address does not exempt the others returned alongside it. + if (!$this->allowPrivate && !$this->hostAllowed($host) && !$this->hostAllowed($lookup)) { + foreach ($ips as $ip) { + if (!IpClassifier::isPublic($ip) && !$this->hostAllowed($ip)) { + throw $this->blocked($request, 'non_public_address'); + } + } + } + + return $ips; + } + + /** + * Host libcurl will look up, so a CURLOPT_RESOLVE entry matches it. + * Brackets are removed and internationalized names are punycode. + * A trailing dot is preserved: curl keeps it. + */ + public function curlResolveHost(string $host): string + { + if (strlen($host) >= 2 && $host[0] === '[' && substr($host, -1) === ']') { + $host = substr($host, 1, -1); + } + + if (preg_match('/[^\x00-\x7f]/', $host) !== 1 || !function_exists('idn_to_ascii')) { + return $host; + } + + $ascii = idn_to_ascii($host, IDNA_DEFAULT); + if (!is_string($ascii) || $ascii === '') { + return $host; + } + + return $ascii; + } + + private function host(string $host): ?string + { + if (strlen($host) >= 2 && $host[0] === '[' && substr($host, -1) === ']') { + $host = substr($host, 1, -1); + } + + $host = rtrim($host, '.'); + if ($host === '') { + return null; + } + + if (preg_match('/[\x00-\x20\x7f\\\\@\[\]]/', $host) === 1) { + return null; + } + + return $host; + } + + private function asciiHost(RequestInterface $request, string $host): string + { + if (preg_match('/[^\x00-\x7f]/', $host) !== 1) { + return $host; + } + + if (!function_exists('idn_to_ascii')) { + throw $this->blocked( + $request, + 'host', + 'Internationalized host names require the intl extension.' + ); + } + + $ascii = idn_to_ascii($host, IDNA_DEFAULT); + if ($ascii === false || $ascii === '') { + throw $this->blocked($request, 'host'); + } + + return $ascii; + } + + /** + * @return array + */ + private function resolve(RequestInterface $request, string $host): array + { + if ($this->resolver !== null) { + $resolver = $this->resolver; + $records = $resolver($host); + } else { + $lookedUp = gethostbynamel($host); + $records = $lookedUp === false ? [] : $lookedUp; + } + + $ips = []; + foreach ($records as $record) { + $canonical = IpClassifier::canonical($record); + if ($canonical === null) { + $ips = []; + break; + } + if (!in_array($canonical, $ips, true)) { + $ips[] = $canonical; + } + } + + if ($ips === []) { + throw new NetworkException('Could not resolve host: '.$host, CURLE_COULDNT_RESOLVE_HOST, $request); + } + + return $ips; + } + + private function hostAllowed(string $host): bool + { + $host = mb_strtolower($host, 'UTF-8'); + + foreach ($this->allowedHosts as $pattern) { + $pattern = mb_strtolower(rtrim($pattern, '.'), 'UTF-8'); + if ($pattern === '') { + continue; + } + + if (strlen($pattern) > 2 && $pattern[0] === '*' && $pattern[1] === '.') { + $suffix = substr($pattern, 1); + if (!is_string($suffix) || $suffix === '') { + continue; + } + $hostSuffix = substr($host, -strlen($suffix)); + if (strlen($host) > strlen($suffix) && is_string($hostSuffix) && $hostSuffix === $suffix) { + return true; + } + continue; + } + + if ($host === $pattern) { + return true; + } + } + + return false; + } + + /** + * @param 'scheme'|'host'|'non_public_address'|'port'|'redirect' $reason + */ + private function blocked(RequestInterface $request, string $reason, ?string $message = null): BlockedRequestException + { + if ($message === null) { + $message = self::message($reason); + } + + return new BlockedRequestException($message, $request, $reason); + } + + /** + * @param 'scheme'|'host'|'non_public_address'|'port'|'redirect' $reason + */ + private static function message(string $reason): string + { + if ($reason === 'scheme') { + return 'The URL scheme is not allowed.'; + } + if ($reason === 'host') { + return 'The URL host is not allowed.'; + } + if ($reason === 'non_public_address') { + return 'The URL resolves to a non-public address.'; + } + if ($reason === 'port') { + return 'The URL port is not allowed.'; + } + + return 'The redirect target is not allowed.'; + } +} diff --git a/tests/UrlPolicyTest.php b/tests/UrlPolicyTest.php new file mode 100644 index 00000000..172a3621 --- /dev/null +++ b/tests/UrlPolicyTest.php @@ -0,0 +1,299 @@ +policy(['8.8.8.8', '1.1.1.1']); + + $ips = $policy->validate($this->request('https://example.com/a')); + + $this->assertSame(['8.8.8.8', '1.1.1.1'], $ips); + } + + public function testUnderscoreInHostnameIsAllowed(): void + { + $seen = null; + $policy = UrlPolicy::default()->withResolver(function (string $host) use (&$seen): array { + $seen = $host; + + return ['1.1.1.1']; + }); + + $ips = $policy->validate($this->request('http://my_host.example/path')); + + $this->assertSame('my_host.example', $seen); + $this->assertSame(['1.1.1.1'], $ips); + } + + public function testInternationalizedNameIsConvertedBeforeLookup(): void + { + if (!function_exists('idn_to_ascii')) { + $this->markTestSkipped('The intl extension is required to resolve IDN hosts.'); + } + + $seen = null; + $policy = UrlPolicy::default()->withResolver(function (string $host) use (&$seen): array { + $seen = $host; + + return ['1.1.1.1']; + }); + + $policy->validate($this->request('http://bücher.example/')); + + $this->assertSame('xn--bcher-kva.example', $seen); + } + + public function testMixedPublicAndPrivateAddressesAreBlocked(): void + { + $policy = $this->policy(['8.8.8.8', '10.0.0.1']); + + $exception = $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://example.com/')); + }, 'non_public_address'); + + $this->assertStringNotContainsString('10.0.0.1', $exception->getMessage()); + $this->assertStringNotContainsString('8.8.8.8', $exception->getMessage()); + } + + public function testDnsFailureIsANetworkException(): void + { + $policy = $this->policy([]); + + try { + $policy->validate($this->request('http://missing.test/')); + $this->fail('A DNS failure must not be reported as a successful validation.'); + } catch (BlockedRequestException $exception) { + $this->fail('A DNS failure must stay a network error.'); + } catch (NetworkException $exception) { + $this->assertSame(6, $exception->getCode()); + $this->assertInstanceOf(NetworkExceptionInterface::class, $exception); + $this->assertStringNotContainsString('10.', $exception->getMessage()); + } + } + + public function testAllowPrivateNetworksKeepsTheResolvedAddresses(): void + { + $policy = $this->policy(['10.1.2.3'])->allowPrivateNetworks(); + + $this->assertSame(['10.1.2.3'], $policy->validate($this->request('http://intranet.test/'))); + } + + public function testAllowedHostsAndWildcardsSkipThePublicAddressCheck(): void + { + $policy = $this->policy(['10.0.0.5'])->withAllowedHosts('wiki.corp', '*.corp'); + + $this->assertSame(['10.0.0.5'], $policy->validate($this->request('http://wiki.corp/a'))); + $this->assertSame(['10.0.0.5'], $policy->validate($this->request('http://a.b.corp/a'))); + + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://corp/a')); + }, 'non_public_address'); + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://evil.com/')); + }, 'non_public_address'); + } + + public function testOneAllowedAddressDoesNotExemptAnother(): void + { + $policy = UrlPolicy::default() + ->withAllowedHosts('8.8.8.8') + ->withResolver(static function (string $host): array { + return ['8.8.8.8', '10.0.0.1']; + }); + + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://evil.example/')); + }, 'non_public_address'); + + $only = UrlPolicy::default() + ->withAllowedHosts('10.0.0.1') + ->withResolver(static function (string $host): array { + return ['10.0.0.1']; + }); + + $this->assertSame(['10.0.0.1'], $only->validate($this->request('http://evil.example/'))); + } + + public function testAllowedHostMatchesTheUnicodeNameAndItsPunycode(): void + { + if (!function_exists('idn_to_ascii')) { + $this->markTestSkipped('The intl extension is required to resolve IDN hosts.'); + } + + $policy = UrlPolicy::default() + ->withAllowedHosts('Bücher.Example') + ->withResolver(static function (string $host): array { + return ['10.0.0.8']; + }); + + $this->assertSame(['10.0.0.8'], $policy->validate($this->request('http://bücher.example/'))); + + $punycode = UrlPolicy::default() + ->withAllowedHosts('xn--bcher-kva.example') + ->withResolver(static function (string $host): array { + return ['10.0.0.8']; + }); + + $this->assertSame(['10.0.0.8'], $punycode->validate($this->request('http://bücher.example/'))); + $this->assertSame('xn--bcher-kva.example.', $policy->curlResolveHost('bücher.example.')); + } + + public function testAllowedPortsUseTheSchemeDefault(): void + { + $policy = $this->policy(['1.1.1.1'])->withAllowedPorts([80, 443]); + + $this->assertSame(['1.1.1.1'], $policy->validate($this->request('http://example.com/'))); + $this->assertSame(['1.1.1.1'], $policy->validate($this->request('https://example.com/'))); + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://example.com:8080/')); + }, 'port'); + $this->assertSame(['1.1.1.1'], $policy->validate($this->request('http://example.com:443/'))); + } + + public function testLiteralLoopbackSpellingsAreBlockedWithoutDns(): void + { + $policy = UrlPolicy::default()->withResolver(static function (string $host): array { + throw new \RuntimeException('DNS was used for '.$host); + }); + + foreach ([ + 'http://127.0.0.1/', + 'http://2130706433/', + 'http://127.1/', + 'http://0177.0.0.1/', + 'http://0x7f000001/', + 'http://0x7f.0.0.1/', + 'http://[::1]/', + 'http://[::ffff:127.0.0.1]/', + 'http://[::ffff:7f00:1]/', + 'http://[64:ff9b::7f00:1]/', + 'http://[64:ff9b::127.0.0.1]/', + 'http://[2002:7f00:1::]/', + 'http://169.254.169.254/', + ] as $url) { + $this->assertBlocked(function () use ($policy, $url): void { + $policy->validate($this->request($url)); + }, 'non_public_address'); + } + } + + public function testPublicNumericLiteralDoesNotUseDns(): void + { + $policy = UrlPolicy::default()->withResolver(static function (string $host): array { + throw new \RuntimeException('DNS was used for '.$host); + }); + + $this->assertSame(['8.8.8.8'], $policy->validate($this->request('http://134744072/path'))); + $this->assertSame(['::ffff:8.8.8.8'], $policy->validate($this->request('http://[::ffff:8.8.8.8]/'))); + } + + /** + * @return array + */ + public function rejectedUrlsProvider(): array + { + return [ + 'ftp' => ['ftp://example.com/', 'scheme'], + 'file' => ['file:///etc/passwd', 'scheme'], + 'gopher' => ['gopher://127.0.0.1/', 'scheme'], + 'relative' => ['/x', 'scheme'], + 'protocol relative' => ['//example.com/x', 'scheme'], + 'userinfo' => ['http://user:pass@example.com/', 'host'], + 'backslash' => ['http://example.com\\@127.0.0.1/', 'host'], + 'space in host' => ['http://ex ample.com/', 'host'], + 'empty host' => ['http://', 'host'], + ]; + } + + /** + * @dataProvider rejectedUrlsProvider + */ + public function testRejectedUrls(string $url, string $reason): void + { + try { + $this->request($url); + } catch (InvalidArgumentException $exception) { + $this->assertContains($reason, ['scheme', 'host']); + + return; + } + + $this->assertBlocked(function () use ($url): void { + $this->policy(['8.8.8.8'])->validate($this->request($url)); + }, $reason); + } + + public function testBlockedExceptionCarriesTheRequest(): void + { + $request = $this->request('http://127.0.0.1/secret'); + + try { + UrlPolicy::default()->validate($request); + $this->fail('A loopback URL must be blocked.'); + } catch (BlockedRequestException $exception) { + $this->assertSame($request, $exception->getRequest()); + $this->assertSame('non_public_address', $exception->getReason()); + $this->assertInstanceOf(InvalidArgumentException::class, $exception); + $this->assertInstanceOf(RequestExceptionInterface::class, $exception); + } + } + + public function testWithersDoNotMutateTheOriginalPolicy(): void + { + $policy = UrlPolicy::default(); + $relaxed = $policy->allowPrivateNetworks(); + + $this->assertNotSame($policy, $relaxed); + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://10.0.0.1/')); + }, 'non_public_address'); + $this->assertSame(['10.0.0.1'], $relaxed->validate($this->request('http://10.0.0.1/'))); + } + + /** + * @param array $ips + */ + private function policy(array $ips): UrlPolicy + { + return UrlPolicy::default()->withResolver(static function (string $host) use ($ips): array { + return $ips; + }); + } + + private function request(string $uri): RequestInterface + { + return (new Psr17Factory())->createRequest('GET', $uri); + } + + /** + * @param callable(): void $check + */ + private function assertBlocked(callable $check, string $reason): BlockedRequestException + { + try { + $check(); + } catch (BlockedRequestException $exception) { + $this->assertSame($reason, $exception->getReason()); + $this->assertInstanceOf(InvalidArgumentException::class, $exception); + $this->assertInstanceOf(RequestExceptionInterface::class, $exception); + + return $exception; + } + + $this->fail('Expected BlockedRequestException with reason '.$reason); + } +} From a9e4643544e4941dc0918f240f59643b52eab4eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 00:56:08 +0000 Subject: [PATCH 03/12] Fix resolveUri() dropping the base port A relative reference resolved against a base with an explicit port kept the host and dropped the port, so a later manual redirect would leave it. --- src/functions.php | 2 +- tests/FunctionsTest.php | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/functions.php b/src/functions.php index 9965ee63..09569e05 100644 --- a/src/functions.php +++ b/src/functions.php @@ -61,7 +61,7 @@ function resolveUri(UriInterface $base, UriInterface $uri): UriInterface $uri = $uri->withPath(resolvePath($base->getPath(), $uri->getPath())); if ($uri->getHost() === '') { - $uri = $uri->withHost($base->getHost()); + $uri = $uri->withHost($base->getHost())->withPort($base->getPort()); } if ($uri->getScheme() === '') { diff --git a/tests/FunctionsTest.php b/tests/FunctionsTest.php index bb5e959d..a6a5b800 100644 --- a/tests/FunctionsTest.php +++ b/tests/FunctionsTest.php @@ -4,6 +4,8 @@ namespace Embed\Tests; use function Embed\isHttp; +use function Embed\resolveUri; +use Nyholm\Psr7\Factory\Psr17Factory; use PHPUnit\Framework\TestCase; class FunctionsTest extends TestCase @@ -32,4 +34,22 @@ public function testIsHttp(string $url, bool $expected) $result = isHttp($url); $this->assertSame($expected, $result); } + + public function testResolveUriKeepsTheBasePort(): void + { + $factory = new Psr17Factory(); + $base = $factory->createUri('http://h:8080/a/b'); + + $absolute = resolveUri($base, $factory->createUri('/x')); + $this->assertSame('http://h:8080/x', (string) $absolute); + + $relative = resolveUri($base, $factory->createUri('c')); + $this->assertSame('http://h:8080/a/c', (string) $relative); + + $protocolRelative = resolveUri($base, $factory->createUri('//cdn.example/favicon.ico')); + $this->assertSame('http://cdn.example/favicon.ico', (string) $protocolRelative); + + $other = resolveUri($base, $factory->createUri('https://other.example:9/z')); + $this->assertSame('https://other.example:9/z', (string) $other); + } } From f01f17ed3c8fcff20a90ff1f96533d534b9894a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 00:56:08 +0000 Subject: [PATCH 04/12] Enforce UrlPolicy in CurlDispatcher with pinned DNS and manual redirects Pin the addresses that were validated, using the punycode host libcurl looks up. Follow redirects in the dispatcher so every hop is checked, and read every curl_multi_info_read message. --- src/Http/CurlClient.php | 27 +- src/Http/CurlDispatcher.php | 741 +++++++++++++++++++++++++++++++----- tests/CurlClientTest.php | 413 ++++++++++++++++++++ tests/LocalHttpServer.php | 161 ++++++++ tests/server/router.php | 106 ++++++ 5 files changed, 1346 insertions(+), 102 deletions(-) create mode 100644 tests/CurlClientTest.php create mode 100644 tests/LocalHttpServer.php create mode 100644 tests/server/router.php diff --git a/src/Http/CurlClient.php b/src/Http/CurlClient.php index 9794d718..26a43938 100644 --- a/src/Http/CurlClient.php +++ b/src/Http/CurlClient.php @@ -14,12 +14,14 @@ final class CurlClient implements ClientInterface { private ResponseFactoryInterface $responseFactory; + private UrlPolicy $urlPolicy; /** @var array */ private array $settings = []; public function __construct(?ResponseFactoryInterface $responseFactory = null) { $this->responseFactory = $responseFactory !== null ? $responseFactory : FactoryDiscovery::getResponseFactory(); + $this->urlPolicy = UrlPolicy::default(); } /** @@ -30,9 +32,19 @@ public function setSettings(array $settings): void $this->settings = $settings + $this->settings; } + public function setUrlPolicy(UrlPolicy $urlPolicy): void + { + $this->urlPolicy = $urlPolicy; + } + + public function getUrlPolicy(): UrlPolicy + { + return $this->urlPolicy; + } + public function sendRequest(RequestInterface $request): ResponseInterface { - $responses = CurlDispatcher::fetch($this->settings, $this->responseFactory, $request); + $responses = CurlDispatcher::fetch($this->dispatchSettings(), $this->responseFactory, $request); return $responses[0]; } @@ -42,6 +54,17 @@ public function sendRequest(RequestInterface $request): ResponseInterface */ public function sendRequests(RequestInterface ...$request): array { - return CurlDispatcher::fetch($this->settings, $this->responseFactory, ...$request); + return CurlDispatcher::fetch($this->dispatchSettings(), $this->responseFactory, ...$request); + } + + /** + * @return array + */ + private function dispatchSettings(): array + { + $settings = $this->settings; + $settings['url_policy'] = $this->urlPolicy; + + return $settings; } } diff --git a/src/Http/CurlDispatcher.php b/src/Http/CurlDispatcher.php index 0658c0fd..d9ee279a 100644 --- a/src/Http/CurlDispatcher.php +++ b/src/Http/CurlDispatcher.php @@ -4,27 +4,31 @@ namespace Embed\Http; use Composer\CaBundle\CaBundle; +use function Embed\resolveUri; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseFactoryInterface; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\StreamFactoryInterface; use Psr\Http\Message\StreamInterface; +use Psr\Http\Message\UriFactoryInterface; /** - * Class to fetch html pages + * Class to fetch html pages. * - * @phpstan-type CurlResource resource|\CurlHandle + * Redirects are followed here, not by libcurl, so every hop is checked against + * the URL policy and pinned to the addresses that were validated. */ final class CurlDispatcher { + private const REDIRECT_STATUSES = [301, 302, 303, 307, 308]; + private static int $contentLengthThreshold = 5000000; private RequestInterface $request; private StreamFactoryInterface $streamFactory; - /** - * @var resource|\CurlHandle - * @phpstan-ignore property.unusedType (resource type needed for PHP 7.4 compatibility) - */ + private UriFactoryInterface $uriFactory; + private UrlPolicy $policy; + /** @var \CurlHandle */ private $curl; /** @var array */ private array $headers = []; @@ -33,153 +37,590 @@ final class CurlDispatcher private ?int $error = null; /** @var array */ private array $settings; + /** @var array */ + private array $pinnedIps; + private bool $proxy; + private bool $follow; + private int $maxRedirs; + private bool $prereqBlocked = false; + private bool $closed = false; + private int $followed = 0; /** - * @param array $settings + * @param array $settings * @return ResponseInterface[] */ public static function fetch(array $settings, ResponseFactoryInterface $responseFactory, RequestInterface ...$requests): array { + if ($requests === []) { + return []; + } + + $policy = self::policy($settings); + $proxy = self::environmentProxyEnabled(); + $pins = []; + foreach ($requests as $request) { + $pins[] = $policy->validate($request); + } + if (count($requests) === 1) { - $connection = new static($settings, $requests[0]); - /** @var resource|\CurlHandle $curlHandle */ - $curlHandle = $connection->curl; - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ - curl_exec($curlHandle); - return [$connection->getResponse($responseFactory)]; + $connection = new self($settings, $requests[0], $pins[0], $policy, $proxy); + try { + $connection->execute(); + + return [$connection->getResponse($responseFactory)]; + } catch (\Throwable $exception) { + $connection->closeCurl(); + throw $exception; + } + } + + return self::fetchMulti($settings, $responseFactory, array_values($requests), $pins, $policy, $proxy); + } + + /** + * @param array $settings + */ + private static function policy(array $settings): UrlPolicy + { + $policy = $settings['url_policy'] ?? null; + if ($policy instanceof UrlPolicy) { + return $policy; } - //Init connections + return UrlPolicy::default(); + } + + private static function environmentProxyEnabled(): bool + { + foreach (['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'] as $name) { + $value = getenv($name); + if (is_string($value) && trim($value) !== '') { + return true; + } + } + + return false; + } + + /** + * @param array $settings + * @param array $requests + * @param array> $pins + * @return ResponseInterface[] + */ + private static function fetchMulti( + array $settings, + ResponseFactoryInterface $responseFactory, + array $requests, + array $pins, + UrlPolicy $policy, + bool $proxy + ): array { $multi = curl_multi_init(); $connections = []; - foreach ($requests as $request) { - $connection = new static($settings, $request); - /** @var resource|\CurlHandle $curlHandle */ - $curlHandle = $connection->curl; - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ - curl_multi_add_handle($multi, $curlHandle); + try { + foreach ($requests as $index => $request) { + $connection = new self($settings, $request, $pins[$index], $policy, $proxy); + $connections[] = $connection; + $curlHandle = $connection->curl; + curl_multi_add_handle($multi, $curlHandle); + } + + self::pump($multi, $connections); + } catch (\Throwable $exception) { + self::releaseMulti($multi, $connections, true); + throw $exception; + } + + self::releaseMulti($multi, $connections, false); - $connections[] = $connection; + $responses = []; + foreach ($connections as $connection) { + $responses[] = $connection->getResponse($responseFactory); } - //Run - $active = null; + return $responses; + } + + /** + * @param \CurlMultiHandle $multi + * @param array $connections + */ + private static function pump($multi, array $connections): void + { + $active = 0; + $status = CURLM_OK; + do { - $status = curl_multi_exec($multi, $active); + do { + $status = curl_multi_exec($multi, $active); + } while ($status === CURLM_CALL_MULTI_PERFORM); + + $requeued = false; + while (true) { + $info = curl_multi_info_read($multi); + if (!is_array($info)) { + break; + } - if ($active) { - curl_multi_select($multi); - } + $result = $info['result'] ?? null; + $handle = $info['handle'] ?? null; + if (!is_int($result)) { + continue; + } - $info = curl_multi_info_read($multi); - - if (is_array($info) && isset($info['handle'], $info['result'])) { - $result = $info['result']; - // Validate and cast result to int, only set if it's a non-success error code - if (is_numeric($result)) { - $errorCode = (int) $result; - if ($errorCode !== CURLE_OK) { - foreach ($connections as $connection) { - if ($connection->curl === $info['handle']) { - $connection->error = $errorCode; - break; - } - } + foreach ($connections as $connection) { + if ($connection->curl !== $handle) { + continue; + } + if (!$connection->finishTransfer($result)) { + break; } + + $curlHandle = $connection->curl; + curl_multi_remove_handle($multi, $curlHandle); + curl_multi_add_handle($multi, $curlHandle); + $requeued = true; + break; + } + } + + if ($requeued) { + $active = 1; + } + + if ($active !== 0 && $status === CURLM_OK) { + $selected = curl_multi_select($multi, 1.0); + if ($selected === -1) { + usleep(10000); } } - } while ($active && $status === CURLM_OK); + } while ($active !== 0 && $status === CURLM_OK); + } - //Close connections + /** + * @param \CurlMultiHandle $multi + * @param array $connections + */ + private static function releaseMulti($multi, array $connections, bool $closeHandles): void + { foreach ($connections as $connection) { - /** @var resource|\CurlHandle $curlHandle */ $curlHandle = $connection->curl; - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ curl_multi_remove_handle($multi, $curlHandle); + if ($closeHandles) { + $connection->closeCurl(); + } } if (PHP_VERSION_ID < 80000) { curl_multi_close($multi); } - - return array_map( - fn ($connection) => $connection->getResponse($responseFactory), - $connections - ); } /** * @param array $settings + * @param array $pinnedIps */ - private function __construct(array $settings, RequestInterface $request, ?StreamFactoryInterface $streamFactory = null) + private function __construct(array $settings, RequestInterface $request, array $pinnedIps, UrlPolicy $policy, bool $proxy) { + $curl = curl_init(); + if ($curl === false) { + throw new NetworkException('Unable to initialize curl', 0, $request); + } + $this->request = $request; - $this->curl = curl_init((string) $request->getUri()); $this->settings = $settings; - $this->streamFactory = $streamFactory ?? FactoryDiscovery::getStreamFactory(); - - $cookies = $settings['cookies_path'] ?? str_replace('//', '/', sys_get_temp_dir().'/embed-cookies.txt'); - - curl_setopt_array($this->curl, [ - CURLOPT_HTTPHEADER => $this->getRequestHeaders(), - CURLOPT_POST => strtoupper($request->getMethod()) === 'POST', - CURLOPT_MAXREDIRS => $settings['max_redirs'] ?? 10, - CURLOPT_CONNECTTIMEOUT => $settings['connect_timeout'] ?? 10, - CURLOPT_TIMEOUT => $settings['timeout'] ?? 10, - CURLOPT_RETURNTRANSFER => true, - CURLOPT_SSL_VERIFYHOST => $settings['ssl_verify_host'] ?? 0, - CURLOPT_SSL_VERIFYPEER => $settings['ssl_verify_peer'] ?? false, - CURLOPT_ENCODING => '', - CURLOPT_CAINFO => CaBundle::getSystemCaRootBundlePath(), - CURLOPT_AUTOREFERER => true, - CURLOPT_FOLLOWLOCATION => $settings['follow_location'] ?? true, - CURLOPT_IPRESOLVE => CURL_IPRESOLVE_V4, - CURLOPT_USERAGENT => $settings['user_agent'] ?? $request->getHeaderLine('User-Agent'), - CURLOPT_COOKIEJAR => $cookies, - CURLOPT_COOKIEFILE => $cookies, - CURLOPT_HEADERFUNCTION => [$this, 'writeHeader'], - CURLOPT_WRITEFUNCTION => [$this, 'writeBody'], - ]); + $this->pinnedIps = $pinnedIps; + $this->policy = $policy; + $this->proxy = $proxy; + $this->follow = $this->settingBool('follow_location', true); + $this->maxRedirs = max(0, $this->settingInt('max_redirs', 10)); + $this->streamFactory = FactoryDiscovery::getStreamFactory(); + $this->uriFactory = FactoryDiscovery::getUriFactory(); + $this->curl = $curl; + + $this->applyBaseOptions(); + $this->applyToHandle(); + } + + private function execute(): void + { + while (true) { + $this->resetExchange(); + $this->execOnce(); + $this->guardConnection(); + + if (!$this->shouldFollowRedirect()) { + return; + } + + if ($this->followed >= $this->maxRedirs) { + $this->error = CURLE_TOO_MANY_REDIRECTS; + + return; + } + + $next = $this->redirectRequest(); + if ($next === null) { + return; + } + + $this->request = $next; + $this->pinnedIps = $this->policy->validate($next); + $this->applyToHandle(); + ++$this->followed; + } + } + + /** + * @return bool true when the handle was updated and must be run again + */ + private function finishTransfer(int $result): bool + { + if ($this->prereqBlocked) { + throw new BlockedRequestException( + 'The URL resolves to a non-public address.', + $this->request, + 'non_public_address' + ); + } + + if ($result !== CURLE_OK && !($this->isBinary && $result === CURLE_WRITE_ERROR)) { + $this->error = $result; + + return false; + } + + $this->guardPrimaryIp(); + + if (!$this->shouldFollowRedirect()) { + return false; + } + + if ($this->followed >= $this->maxRedirs) { + $this->error = CURLE_TOO_MANY_REDIRECTS; + + return false; + } + + $next = $this->redirectRequest(); + if ($next === null) { + return false; + } + + $this->request = $next; + $this->pinnedIps = $this->policy->validate($next); + $this->applyToHandle(); + $this->resetExchange(); + ++$this->followed; + + return true; + } + + private function execOnce(): void + { + $curlHandle = $this->curl; + curl_exec($curlHandle); + } + + private function guardConnection(): void + { + if ($this->prereqBlocked) { + throw new BlockedRequestException( + 'The URL resolves to a non-public address.', + $this->request, + 'non_public_address' + ); + } + + $this->guardPrimaryIp(); + + $curlHandle = $this->curl; + $errno = curl_errno($curlHandle); + if ($errno === 0 || ($this->isBinary && $errno === CURLE_WRITE_ERROR)) { + return; + } + + $this->error(curl_error($curlHandle), $errno); + } + + private function guardPrimaryIp(): void + { + if ($this->proxy) { + return; + } + + $curlHandle = $this->curl; + $ip = curl_getinfo($curlHandle, CURLINFO_PRIMARY_IP); + if ($ip === '') { + return; + } + + if ($this->addressIsPinned($ip)) { + return; + } + + throw new BlockedRequestException( + 'The URL resolves to a non-public address.', + $this->request, + 'non_public_address' + ); + } + + private function shouldFollowRedirect(): bool + { + if (!$this->follow) { + return false; + } + + return in_array($this->httpStatus(), self::REDIRECT_STATUSES, true); + } + + private function redirectRequest(): ?RequestInterface + { + $location = null; + foreach ($this->headers as $header) { + if ($header[0] === 'location') { + $location = $header[1]; + } + } + + if ($location === null) { + return null; + } + + $location = trim($location); + if ($location === '') { + return null; + } + + if (preg_match('/[\x00-\x1f\x7f]/', $location) === 1) { + throw new BlockedRequestException( + 'The redirect target is not allowed.', + $this->request, + 'redirect' + ); + } + + try { + $target = $this->uriFactory->createUri($location); + } catch (\InvalidArgumentException $exception) { + throw new BlockedRequestException( + 'The redirect target is not allowed.', + $this->request, + 'redirect' + ); + } + + $current = $this->request->getUri(); + $resolved = resolveUri($current, $target); + $method = $this->methodAfterRedirect($this->httpStatus()); + $request = $this->request->withMethod($method)->withUri($resolved); + + if ($method === 'GET' || $method === 'HEAD') { + $request = $request + ->withBody($this->streamFactory->createStream('')) + ->withoutHeader('Content-Type') + ->withoutHeader('Content-Length'); + } + + $currentHost = strtolower($this->bareHost($current->getHost())); + $nextHost = strtolower($this->bareHost($resolved->getHost())); + if ($currentHost !== $nextHost) { + $request = $request->withoutHeader('Authorization')->withoutHeader('Cookie'); + } + + return $request->withHeader('Referer', (string) $current); + } + + private function methodAfterRedirect(int $status): string + { + $method = strtoupper($this->request->getMethod()); + if ($status === 303) { + return $method === 'HEAD' ? 'HEAD' : 'GET'; + } + + if (($status === 301 || $status === 302) && $method === 'POST') { + return 'GET'; + } + + return $method; + } + + private function bareHost(string $host): string + { + if (strlen($host) >= 2 && $host[0] === '[' && substr($host, -1) === ']') { + return substr($host, 1, -1); + } + + return $host; + } + + private function addressIsPinned(string $ip): bool + { + foreach ($this->pinnedIps as $pinned) { + if (IpClassifier::equals($pinned, $ip)) { + return true; + } + } + + return false; + } + + private function applyBaseOptions(): void + { + $cookies = $this->cookiesPath(); + $caBundle = CaBundle::getSystemCaRootBundlePath(); + + $this->setopt(CURLOPT_CONNECTTIMEOUT, $this->settingInt('connect_timeout', 10)); + $this->setopt(CURLOPT_TIMEOUT, $this->settingInt('timeout', 10)); + $this->setopt(CURLOPT_RETURNTRANSFER, true); + $this->setopt(CURLOPT_SSL_VERIFYHOST, $this->sslVerifyHost()); + $this->setopt(CURLOPT_SSL_VERIFYPEER, $this->sslVerifyPeer()); + $this->setopt(CURLOPT_ENCODING, ''); + if ($caBundle !== '') { + $this->setopt(CURLOPT_CAINFO, $caBundle); + } + $this->setopt(CURLOPT_FOLLOWLOCATION, false); + $this->setopt(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + $this->setopt(CURLOPT_USERAGENT, $this->userAgent()); + $this->setopt(CURLOPT_COOKIEJAR, $cookies); + $this->setopt(CURLOPT_COOKIEFILE, $cookies); + $this->setopt(CURLOPT_HEADERFUNCTION, [$this, 'writeHeader']); + $this->setopt(CURLOPT_WRITEFUNCTION, [$this, 'writeBody']); + $this->applyProtocolLimits(); + $this->applyPrereq(); + } + + private function applyProtocolLimits(): void + { + if (defined('CURLOPT_PROTOCOLS_STR')) { + $this->setopt(CURLOPT_PROTOCOLS_STR, 'HTTP,HTTPS'); + $this->setopt(CURLOPT_REDIR_PROTOCOLS_STR, 'HTTP,HTTPS'); + + return; + } + + $this->setopt(CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + $this->setopt(CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + } + + private function applyPrereq(): void + { + if (PHP_VERSION_ID < 80400 || !defined('CURLOPT_PREREQFUNCTION')) { + return; + } + + $this->setopt(CURLOPT_PREREQFUNCTION, [$this, 'approveConnection']); + } + + /** + * Called by libcurl on PHP 8.4+ before the request is sent. + * + * @param resource|\CurlHandle $curl + */ + public function approveConnection($curl, string $primaryIp, string $localIp, int $primaryPort, int $localPort): int + { + if ($this->proxy || $this->addressIsPinned($primaryIp)) { + return CURL_PREREQFUNC_OK; + } + + $this->prereqBlocked = true; + + return CURL_PREREQFUNC_ABORT; + } + + private function applyToHandle(): void + { + $this->setopt(CURLOPT_URL, (string) $this->request->getUri()); + $this->setopt(CURLOPT_HTTPHEADER, $this->getRequestHeaders()); + + $method = strtoupper($this->request->getMethod()); + if ($method === 'POST') { + $this->setopt(CURLOPT_POST, true); + $this->setopt(CURLOPT_POSTFIELDS, (string) $this->request->getBody()); + } elseif ($method === 'HEAD') { + $this->setopt(CURLOPT_NOBODY, true); + $this->setopt(CURLOPT_HTTPGET, true); + } else { + $this->setopt(CURLOPT_HTTPGET, true); + $this->setopt(CURLOPT_POST, false); + } + + $this->applyResolve(); + } + + private function applyResolve(): void + { + if ($this->proxy || $this->pinnedIps === []) { + return; + } + + $uri = $this->request->getUri(); + $host = $this->policy->curlResolveHost($uri->getHost()); + $port = $uri->getPort(); + if ($port === null) { + $port = strtolower($uri->getScheme()) === 'https' ? 443 : 80; + } + + $ips = $this->pinnedIps; + $version = curl_version(); + $curlVersion = is_array($version) && isset($version['version']) && is_string($version['version']) + ? $version['version'] + : '0'; + if (version_compare($curlVersion, '7.59.0', '<')) { + $ips = [$ips[0]]; + } + + $formatted = []; + foreach ($ips as $ip) { + $formatted[] = strpos($ip, ':') !== false ? '['.$ip.']' : $ip; + } + + $resolveHost = strpos($host, ':') !== false ? '['.$host.']' : $host; + $this->setopt(CURLOPT_RESOLVE, [$resolveHost.':'.$port.':'.implode(',', $formatted)]); } private function getResponse(ResponseFactoryInterface $responseFactory): ResponseInterface { - /** @var resource|\CurlHandle $curlHandle */ + try { + return $this->buildResponse($responseFactory); + } finally { + $this->closeCurl(); + } + } + + private function buildResponse(ResponseFactoryInterface $responseFactory): ResponseInterface + { $curlHandle = $this->curl; - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $info = curl_getinfo($curlHandle); if ($this->error !== null && $this->error !== 0) { - /** @phpstan-ignore argument.type (curl_strerror returns string|null in some versions) */ - $this->error(curl_strerror($this->error), $this->error); + $message = curl_strerror($this->error); + $this->error($message === null ? 'curl error' : $message, $this->error); } - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $errno = curl_errno($curlHandle); - if ($errno !== 0) { - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + if ($errno !== 0 && !($this->isBinary && $errno === CURLE_WRITE_ERROR)) { $this->error(curl_error($curlHandle), $errno); } - if (PHP_VERSION_ID < 80000) { - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ - curl_close($curlHandle); - } - $response = $responseFactory->createResponse($info['http_code']); foreach ($this->headers as $header) { - list($name, $value) = $header; + [$name, $value] = $header; $response = $response->withAddedHeader($name, $value); } + $effective = $info['url'] !== '' ? $info['url'] : (string) $this->request->getUri(); + $response = $response - ->withAddedHeader('Content-Location', $info['url']) + ->withAddedHeader('Content-Location', $effective) ->withAddedHeader('X-Request-Time', sprintf('%.3f ms', $info['total_time'])); if ($this->body !== null) { - //5Mb max $this->body->rewind(); $response = $response->withBody($this->body); $this->body = null; @@ -197,27 +638,127 @@ private function error(string $message, int $code): void } if ($this->isBinary && $code === CURLE_WRITE_ERROR) { - // The write callback aborted the request to prevent a download of the binary file return; } throw new NetworkException($message, $code, $this->request); } + private function closeCurl(): void + { + if ($this->closed || PHP_VERSION_ID >= 80000) { + $this->closed = true; + + return; + } + + $this->closed = true; + $curlHandle = $this->curl; + curl_close($curlHandle); + } + + private function resetExchange(): void + { + $this->headers = []; + $this->body = null; + $this->isBinary = false; + $this->error = null; + $this->prereqBlocked = false; + } + + private function httpStatus(): int + { + $curlHandle = $this->curl; + return curl_getinfo($curlHandle, CURLINFO_HTTP_CODE); + } + /** - * @return array + * @param mixed $value + */ + private function setopt(int $option, $value): void + { + $curlHandle = $this->curl; + curl_setopt($curlHandle, $option, $value); + } + + private function settingInt(string $key, int $default): int + { + $value = $this->settings[$key] ?? $default; + if (is_int($value)) { + return $value; + } + if (is_float($value) || (is_string($value) && is_numeric($value))) { + return (int) $value; + } + + return $default; + } + + private function settingBool(string $key, bool $default): bool + { + $value = $this->settings[$key] ?? $default; + if (is_bool($value)) { + return $value; + } + if ($value === 1 || $value === '1') { + return true; + } + if ($value === 0 || $value === '0') { + return false; + } + + return $default; + } + + /** @return 0|2 */ + private function sslVerifyHost(): int + { + $value = $this->settings['ssl_verify_host'] ?? 0; + if ($value === 2 || $value === '2') { + return 2; + } + + return 0; + } + + private function sslVerifyPeer(): bool + { + return $this->settingBool('ssl_verify_peer', false); + } + + private function userAgent(): string + { + $value = $this->settings['user_agent'] ?? $this->request->getHeaderLine('User-Agent'); + if (is_string($value) && $value !== '') { + return $value; + } + + return 'Embed/curl'; + } + + private function cookiesPath(): string + { + $value = $this->settings['cookies_path'] ?? null; + if (is_string($value) && $value !== '') { + return $value; + } + + return str_replace('//', '/', sys_get_temp_dir().'/embed-cookies.txt'); + } + + /** + * @return array */ private function getRequestHeaders(): array { $headers = []; foreach ($this->request->getHeaders() as $name => $values) { - switch (strtolower($name)) { - case 'user-agent': - break; - default: - $headers[] = $name . ':' . implode(', ', $values); + if (strtolower($name) === 'user-agent') { + continue; } + + $headers[] = $name.':'.implode(', ', $values); } return $headers; @@ -225,7 +766,7 @@ private function getRequestHeaders(): array /** * @param resource|\CurlHandle $curl - * @param mixed $string + * @param mixed $string */ private function writeHeader($curl, $string): int { @@ -251,7 +792,7 @@ private function writeHeader($curl, $string): int /** * @param resource|\CurlHandle $curl - * @param mixed $string + * @param mixed $string */ private function writeBody($curl, $string): int { diff --git a/tests/CurlClientTest.php b/tests/CurlClientTest.php new file mode 100644 index 00000000..cf85fcf8 --- /dev/null +++ b/tests/CurlClientTest.php @@ -0,0 +1,413 @@ +stop(); + self::clearProxyEnvironment(); + if (isset(self::$cookies) && is_file(self::$cookies)) { + unlink(self::$cookies); + } + } + + protected function setUp(): void + { + self::clearProxyEnvironment(); + self::$server->clearLog(); + if (is_file(self::$cookies)) { + file_put_contents(self::$cookies, ''); + } + } + + public function testInternationalizedHostIsPinnedInPunycode(): void + { + if (!function_exists('idn_to_ascii')) { + $this->markTestSkipped('The intl extension is required to resolve IDN hosts.'); + } + + $seen = null; + $policy = UrlPolicy::default() + ->withAllowedHosts('bücher.example') + ->withResolver(static function (string $host) use (&$seen): array { + $seen = $host; + + return ['127.0.0.1']; + }); + + $response = $this->client([], $policy)->sendRequest( + $this->request('http://bücher.example:'.self::$server->port().'/landed') + ); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('xn--bcher-kva.example', $seen); + $this->assertSame(['/landed'], self::$server->paths()); + } + + public function testPinnedHostReachesTheLocalServer(): void + { + $response = $this->client()->sendRequest($this->request($this->url('/landed'))); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('ok', (string) $response->getBody()); + $this->assertSame($this->url('/landed'), $response->getHeaderLine('Content-Location')); + $this->assertSame(['/landed'], self::$server->paths()); + } + + /** + * @return array + */ + public function redirectStatusProvider(): array + { + return [ + '301' => [301], + '302' => [302], + '303' => [303], + '307' => [307], + '308' => [308], + ]; + } + + /** + * @dataProvider redirectStatusProvider + */ + public function testRedirectToLoopbackIsBlockedBeforeItIsRequested(int $status): void + { + $target = 'http://127.0.0.1:'.self::$server->port().'/secret'; + + $exception = $this->captureBlocked(function () use ($status, $target): void { + $this->client()->sendRequest($this->request($this->url('/go?code='.$status.'&to='.rawurlencode($target)))); + }); + + $this->assertSame('non_public_address', $exception->getReason()); + $this->assertStringNotContainsString('127.0.0.1', $exception->getMessage()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testRedirectToPrivateNameIsBlockedWithoutLeakingTheAddress(): void + { + $target = 'http://internal.test:'.self::$server->port().'/secret'; + + $exception = $this->captureBlocked(function () use ($target): void { + $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode($target)))); + }); + + $this->assertSame('non_public_address', $exception->getReason()); + $this->assertStringNotContainsString('10.0.0.1', $exception->getMessage()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testRelativeRedirectKeepsTheBasePort(): void + { + $response = $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('/landed')))); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame($this->url('/landed'), $response->getHeaderLine('Content-Location')); + $this->assertSame(['/go', '/landed'], self::$server->paths()); + } + + public function testProtocolRelativeLoopbackIsBlocked(): void + { + $this->captureBlocked(function (): void { + $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('//127.0.0.1/secret')))); + }); + + $this->assertSame(['/go'], self::$server->paths()); + } + + /** + * @return array + */ + public function forbiddenSchemeProvider(): array + { + return [ + 'file' => ['file:///etc/passwd', 'scheme'], + 'gopher' => ['gopher://127.0.0.1/secret', 'scheme'], + ]; + } + + /** + * @dataProvider forbiddenSchemeProvider + */ + public function testRedirectToForbiddenSchemeIsBlocked(string $target, string $reason): void + { + $exception = $this->captureBlocked(function () use ($target): void { + $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode($target)))); + }); + + $this->assertSame($reason, $exception->getReason()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testUnparseableRedirectIsBlocked(): void + { + $exception = $this->captureBlocked(function (): void { + $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('http://')))); + }); + + $this->assertSame('redirect', $exception->getReason()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testThreeHopChainUsesTheFinalUrl(): void + { + $response = $this->client()->sendRequest($this->request($this->url('/chain?i=0&n=3'))); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('done', (string) $response->getBody()); + $this->assertSame($this->url('/chain?i=3&n=3'), $response->getHeaderLine('Content-Location')); + $this->assertSame( + ['/chain', '/chain', '/chain', '/chain'], + self::$server->paths() + ); + $queries = array_map(static function (array $row): string { + return $row['query'] ?? ''; + }, self::$server->requests()); + $this->assertSame(['i=0&n=3', 'i=1&n=3', 'i=2&n=3', 'i=3&n=3'], $queries); + } + + public function testTooManyRedirectsUsesCurlErrorCode(): void + { + try { + $this->client()->sendRequest($this->request($this->url('/chain?i=0&n=11'))); + $this->fail('The eleventh redirect must fail.'); + } catch (NetworkException $exception) { + $this->assertSame(47, $exception->getCode()); + } + + $queries = array_map(static function (array $row): string { + return $row['query'] ?? ''; + }, self::$server->requests()); + $this->assertNotContains('i=11&n=11', $queries); + $this->assertContains('i=10&n=11', $queries); + $this->assertCount(11, $queries); + } + + public function testFollowLocationFalseReturnsTheRedirect(): void + { + $response = $this->client(['follow_location' => false])->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('/landed')))); + + $this->assertSame(302, $response->getStatusCode()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testCookieFromTheRedirectIsSentOnTheNextHop(): void + { + $response = $this->client()->sendRequest($this->request($this->url('/set-cookie'))); + + $this->assertSame(200, $response->getStatusCode()); + $rows = self::$server->requests(); + $this->assertCount(2, $rows); + $this->assertSame('/read-cookie', $rows[1]['path']); + $this->assertStringContainsString('consent=yes', $rows[1]['cookie']); + } + + public function testAuthorizationAndCookieAreStrippedWhenTheHostChanges(): void + { + $factory = new Psr17Factory(); + $request = $factory->createRequest('GET', $this->url('/to-other?port='.self::$server->port())) + ->withHeader('Authorization', 'Bearer secret') + ->withHeader('Cookie', 'a=b'); + + $response = $this->client()->sendRequest($request); + + $this->assertSame(200, $response->getStatusCode()); + $rows = self::$server->requests(); + $this->assertCount(2, $rows); + $this->assertSame('Bearer secret', $rows[0]['authorization']); + $this->assertSame('', $rows[1]['authorization']); + $this->assertSame('', $rows[1]['cookie']); + $this->assertStringContainsString('other.test', $rows[1]['host']); + } + + public function testPostRedirectsToGetExceptFor307And308(): void + { + $factory = new Psr17Factory(); + $body = $factory->createStream('payload'); + $request = $factory->createRequest('POST', $this->url('/switch?code=302'))->withBody($body); + + $this->client()->sendRequest($request); + + $rows = self::$server->requests(); + $this->assertSame('POST', $rows[0]['method']); + $this->assertSame('GET', $rows[1]['method']); + $this->assertSame('', $rows[1]['body']); + + self::$server->clearLog(); + $again = $factory->createRequest('POST', $this->url('/switch?code=307')) + ->withBody($factory->createStream('payload')); + $this->client()->sendRequest($again); + $rows = self::$server->requests(); + $this->assertSame('POST', $rows[0]['method']); + $this->assertSame('POST', $rows[1]['method']); + $this->assertSame('payload', $rows[1]['body']); + } + + public function testDnsRebindingDoesNotMoveTheConnection(): void + { + $calls = 0; + $policy = UrlPolicy::default() + ->withAllowedHosts('rebind.test') + ->withResolver(static function (string $host) use (&$calls): array { + ++$calls; + if ($host === 'rebind.test' && $calls > 1) { + return ['10.0.0.1']; + } + + return ['127.0.0.1']; + }); + + $response = $this->client([], $policy)->sendRequest( + $this->request('http://rebind.test:'.self::$server->port().'/landed') + ); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame(1, $calls); + $this->assertSame(['/landed'], self::$server->paths()); + } + + public function testParallelRequestsFollowRedirects(): void + { + $factory = new Psr17Factory(); + $responses = $this->client()->sendRequests( + $factory->createRequest('GET', $this->url('/chain?i=0&n=1')), + $factory->createRequest('GET', $this->url('/landed')) + ); + + $this->assertCount(2, $responses); + $this->assertSame(200, $responses[0]->getStatusCode()); + $this->assertSame(200, $responses[1]->getStatusCode()); + $this->assertSame($this->url('/chain?i=1&n=1'), $responses[0]->getHeaderLine('Content-Location')); + $this->assertContains('/landed', self::$server->paths()); + $this->assertContains('/chain', self::$server->paths()); + } + + public function testParallelBatchIsRejectedBeforeAnyRequestWhenOneUrlIsBlocked(): void + { + $factory = new Psr17Factory(); + + $exception = $this->captureBlocked(function () use ($factory): void { + $this->client()->sendRequests( + $factory->createRequest('GET', $this->url('/landed')), + $factory->createRequest('GET', 'http://127.0.0.1/secret') + ); + }); + + $this->assertSame('non_public_address', $exception->getReason()); + $this->assertSame([], self::$server->paths()); + } + + public function testEnvironmentProxyKeepsPreValidationAndDoesNotConnectDirectly(): void + { + putenv('http_proxy=http://127.0.0.1:9'); + + try { + $blocked = $this->captureBlocked(function (): void { + $this->client()->sendRequest($this->request('http://127.0.0.1/secret')); + }); + $this->assertSame('non_public_address', $blocked->getReason()); + $this->assertSame([], self::$server->paths()); + + try { + $this->client()->sendRequest($this->request($this->url('/landed'))); + $this->fail('A configured proxy must be used for the connection.'); + } catch (NetworkException $exception) { + $this->assertNotSame(0, $exception->getCode()); + } + $this->assertSame([], self::$server->paths()); + } finally { + putenv('http_proxy'); + } + } + + /** + * @param array $settings + */ + private function client(array $settings = [], ?UrlPolicy $policy = null): CurlClient + { + $client = new CurlClient(); + $client->setUrlPolicy($policy ?? $this->policy()); + $client->setSettings($settings + [ + 'timeout' => 3, + 'connect_timeout' => 3, + 'cookies_path' => self::$cookies, + 'follow_location' => true, + 'max_redirs' => 10, + ]); + + return $client; + } + + private function policy(): UrlPolicy + { + return UrlPolicy::default() + ->withAllowedHosts('allowed.test', 'other.test') + ->withResolver(static function (string $host): array { + if ($host === 'allowed.test' || $host === 'other.test') { + return ['127.0.0.1']; + } + if ($host === 'internal.test') { + return ['10.0.0.1']; + } + + return []; + }); + } + + private function url(string $path): string + { + return 'http://allowed.test:'.self::$server->port().$path; + } + + private function request(string $uri): RequestInterface + { + return (new Psr17Factory())->createRequest('GET', $uri); + } + + /** + * @param callable(): void $check + */ + private function captureBlocked(callable $check): BlockedRequestException + { + try { + $check(); + } catch (BlockedRequestException $exception) { + return $exception; + } + + $this->fail('Expected BlockedRequestException'); + } + + private static function clearProxyEnvironment(): void + { + foreach (['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'] as $name) { + putenv($name); + unset($_ENV[$name], $_SERVER[$name]); + } + } +} diff --git a/tests/LocalHttpServer.php b/tests/LocalHttpServer.php new file mode 100644 index 00000000..f61320b4 --- /dev/null +++ b/tests/LocalHttpServer.php @@ -0,0 +1,161 @@ + ['pipe', 'r'], + 1 => ['file', $log.'.stdout', 'w'], + 2 => ['file', $stderr, 'w'], + ]; + + $env = []; + foreach (array_merge($_SERVER, $_ENV) as $name => $value) { + if (is_string($name) && is_string($value)) { + $env[$name] = $value; + } + } + $env['EMBED_TEST_LOG'] = $log; + + $process = proc_open($command, $spec, $pipes, null, $env); + if (!is_resource($process)) { + throw new RuntimeException('Unable to start the test server'); + } + fclose($pipes[0]); + + $server = new self($process, $log, $port, $stderr); + $server->waitUntilListening(); + + return $server; + } + + /** + * @param resource $process + */ + private function __construct($process, string $logFile, int $port, string $stderr) + { + $this->process = $process; + $this->logFile = $logFile; + $this->port = $port; + $this->stderr = $stderr; + } + + public function port(): int + { + return $this->port; + } + + public function clearLog(): void + { + file_put_contents($this->logFile, ''); + } + + /** + * @return array> + */ + public function requests(): array + { + $raw = is_file($this->logFile) ? (string) file_get_contents($this->logFile) : ''; + $rows = []; + foreach (preg_split('/\r?\n/', trim($raw)) ?: [] as $line) { + if ($line === '') { + continue; + } + $decoded = json_decode($line, true); + if (!is_array($decoded)) { + continue; + } + $row = []; + foreach ($decoded as $key => $value) { + if (is_string($key)) { + $row[$key] = is_string($value) ? $value : ''; + } + } + $rows[] = $row; + } + + return $rows; + } + + /** + * @return array + */ + public function paths(): array + { + $paths = []; + foreach ($this->requests() as $request) { + $paths[] = $request['path'] ?? ''; + } + + return $paths; + } + + public function stop(): void + { + proc_terminate($this->process); + proc_close($this->process); + } + + private function waitUntilListening(): void + { + $deadline = microtime(true) + 5; + while (microtime(true) < $deadline) { + $socket = @fsockopen('127.0.0.1', $this->port, $errno, $error, 0.2); + if (is_resource($socket)) { + fclose($socket); + + return; + } + usleep(50000); + } + + $details = is_file($this->stderr) ? (string) file_get_contents($this->stderr) : ''; + $this->stop(); + + throw new RuntimeException('Test server did not start: '.$details); + } + + private static function allocatePort(): int + { + $socket = stream_socket_server('tcp://127.0.0.1:0', $errno, $error); + if ($socket === false) { + throw new RuntimeException('Unable to allocate a port: '.$error); + } + + $name = stream_socket_get_name($socket, false); + fclose($socket); + if (!is_string($name)) { + throw new RuntimeException('Unable to read the allocated port'); + } + + $colon = strrpos($name, ':'); + if ($colon === false) { + throw new RuntimeException('Unable to read the allocated port'); + } + + return (int) substr($name, $colon + 1); + } +} diff --git a/tests/server/router.php b/tests/server/router.php new file mode 100644 index 00000000..7150acc3 --- /dev/null +++ b/tests/server/router.php @@ -0,0 +1,106 @@ + $value) { + $headers[strtolower((string) $name)] = (string) $value; + } +} + +$entry = [ + 'path' => $path, + 'query' => $queryString, + 'method' => (string) ($_SERVER['REQUEST_METHOD'] ?? ''), + 'authorization' => $headers['authorization'] ?? '', + 'cookie' => $headers['cookie'] ?? '', + 'host' => $headers['host'] ?? '', + 'referer' => $headers['referer'] ?? '', + 'body' => (string) file_get_contents('php://input'), +]; + +if (is_string($log) && $log !== '') { + $encoded = json_encode($entry); + if (is_string($encoded)) { + file_put_contents($log, $encoded."\n", FILE_APPEND | LOCK_EX); + } +} + +$params = []; +parse_str($queryString, $params); + +if ($path === '/go') { + $code = isset($params['code']) ? (int) $params['code'] : 302; + if (!in_array($code, [301, 302, 303, 307, 308], true)) { + $code = 302; + } + $to = isset($params['to']) && is_string($params['to']) ? $params['to'] : '/secret'; + header('Location: '.$to, true, $code); + echo 'redirect'; + + return; +} + +if ($path === '/chain') { + $i = isset($params['i']) ? (int) $params['i'] : 0; + $n = isset($params['n']) ? (int) $params['n'] : 0; + if ($i >= $n) { + http_response_code(200); + echo 'done'; + + return; + } + header('Location: /chain?i='.($i + 1).'&n='.$n, true, 302); + echo 'redirect'; + + return; +} + +if ($path === '/set-cookie') { + header('Set-Cookie: consent=yes; Path=/'); + header('Location: /read-cookie', true, 302); + echo 'redirect'; + + return; +} + +if ($path === '/to-other') { + $port = isset($params['port']) ? (string) $params['port'] : ''; + header('Location: http://other.test:'.$port.'/landed', true, 302); + echo 'redirect'; + + return; +} + +if ($path === '/switch') { + $code = isset($params['code']) ? (int) $params['code'] : 302; + if (!in_array($code, [301, 302, 303, 307, 308], true)) { + $code = 302; + } + header('Location: /landed', true, $code); + echo 'redirect'; + + return; +} + +if ($path === '/secret') { + http_response_code(200); + echo 'LEAKED'; + + return; +} + +http_response_code(200); +echo 'ok'; From 8b485c2b0b090440bc055a8af0eb9ca5ad79e268 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 00:56:08 +0000 Subject: [PATCH 05/12] Apply UrlPolicy in Crawler for custom PSR-18 clients CurlClient still validates inside the dispatcher, so this path does not resolve twice. Cached page tests use a fake resolver that returns a public address, otherwise dead hosts fail closed as NXDOMAIN. --- src/Http/Crawler.php | 30 ++++++++++++++++++++++++++++++ tests/DocumentTest.php | 7 ++++++- tests/PagesTestCase.php | 11 +++++++++-- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/Http/Crawler.php b/src/Http/Crawler.php index 2c233636..01dee3c7 100644 --- a/src/Http/Crawler.php +++ b/src/Http/Crawler.php @@ -15,6 +15,7 @@ class Crawler implements ClientInterface, RequestFactoryInterface, UriFactoryInt private RequestFactoryInterface $requestFactory; private UriFactoryInterface $uriFactory; private ClientInterface $client; + private UrlPolicy $urlPolicy; /** @var array */ private array $defaultHeaders = [ 'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:73.0) Gecko/20100101 Firefox/73.0', @@ -26,6 +27,20 @@ public function __construct(?ClientInterface $client = null, ?RequestFactoryInte $this->client = $client !== null ? $client : new CurlClient(); $this->requestFactory = $requestFactory !== null ? $requestFactory : FactoryDiscovery::getRequestFactory(); $this->uriFactory = $uriFactory !== null ? $uriFactory : FactoryDiscovery::getUriFactory(); + $this->urlPolicy = $this->client instanceof CurlClient ? $this->client->getUrlPolicy() : UrlPolicy::default(); + } + + public function setUrlPolicy(UrlPolicy $urlPolicy): void + { + $this->urlPolicy = $urlPolicy; + if ($this->client instanceof CurlClient) { + $this->client->setUrlPolicy($urlPolicy); + } + } + + public function getUrlPolicy(): UrlPolicy + { + return $this->urlPolicy; } /** @@ -57,6 +72,8 @@ public function createUri(string $uri = ''): UriInterface public function sendRequest(RequestInterface $request): ResponseInterface { + $this->validateCustomClient($request); + return $this->client->sendRequest($request); } @@ -69,12 +86,25 @@ public function sendRequests(RequestInterface ...$requests): array return $this->client->sendRequests(...$requests); } + foreach ($requests as $request) { + $this->urlPolicy->validate($request); + } + return array_map( fn ($request) => $this->client->sendRequest($request), $requests ); } + private function validateCustomClient(RequestInterface $request): void + { + if ($this->client instanceof CurlClient) { + return; + } + + $this->urlPolicy->validate($request); + } + public function getResponseUri(ResponseInterface $response): ?UriInterface { $location = $response->getHeaderLine('Content-Location'); diff --git a/tests/DocumentTest.php b/tests/DocumentTest.php index aa516acd..171b0aba 100644 --- a/tests/DocumentTest.php +++ b/tests/DocumentTest.php @@ -5,6 +5,7 @@ use Embed\Embed; use Embed\Http\Crawler; +use Embed\Http\UrlPolicy; use PHPUnit\Framework\TestCase; class DocumentTest extends TestCase @@ -19,8 +20,12 @@ private static function getEmbed(): Embed $dispatcher = new FileClient(__DIR__.'/cache'); $dispatcher->setMode(0); + $crawler = new Crawler($dispatcher); + $crawler->setUrlPolicy(UrlPolicy::default()->withResolver(static function (string $host): array { + return ['8.8.8.8']; + })); - return self::$embed = new Embed(new Crawler($dispatcher)); + return self::$embed = new Embed($crawler); } public function testSelectors() diff --git a/tests/PagesTestCase.php b/tests/PagesTestCase.php index 46db1772..1f3b5402 100644 --- a/tests/PagesTestCase.php +++ b/tests/PagesTestCase.php @@ -9,6 +9,7 @@ use Embed\Extractor; use Embed\ExtractorFactory; use Embed\Http\Crawler; +use Embed\Http\UrlPolicy; use JsonSerializable; use PHPUnit\Framework\TestCase; use Psr\Http\Message\UriInterface; @@ -109,8 +110,14 @@ private static function getEmbed(): Embed $dispatcher = new FileClient(__DIR__.'/cache'); $dispatcher->setMode(self::getCacheMode()); - - return self::$embed = new Embed(new Crawler($dispatcher), self::getExtractorFactory()); + $crawler = new Crawler($dispatcher); + // Cached fixtures must not depend on live DNS. Dead hosts such as + // ustream and viddler would otherwise fail closed as NXDOMAIN. + $crawler->setUrlPolicy(UrlPolicy::default()->withResolver(static function (string $host): array { + return ['8.8.8.8']; + })); + + return self::$embed = new Embed($crawler, self::getExtractorFactory()); } protected function assertEmbed(string $url) From 98dfd6856ecf028044253fc2ec7d2f1fafb6cdaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 00:56:08 +0000 Subject: [PATCH 06/12] Skip blocked secondary requests A page-declared oEmbed endpoint, adapter API, or meta-refresh target that the policy rejects is ignored. The URL passed to get() and getMulti() still throws, and getMulti rejects the batch before any request is sent. --- src/Embed.php | 7 +- src/HttpApiTrait.php | 6 +- src/OEmbed.php | 7 +- tests/RecordingClient.php | 33 +++++++++ tests/SsrfEmbedTest.php | 143 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 193 insertions(+), 3 deletions(-) create mode 100644 tests/RecordingClient.php create mode 100644 tests/SsrfEmbedTest.php diff --git a/src/Embed.php b/src/Embed.php index 6f2d583a..43c619e0 100644 --- a/src/Embed.php +++ b/src/Embed.php @@ -3,6 +3,7 @@ namespace Embed; +use Embed\Http\BlockedRequestException; use Embed\Http\Crawler; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; @@ -88,7 +89,11 @@ private function extract(RequestInterface $request, ResponseInterface $response, } $request = $this->crawler->createRequest('GET', (string) $redirectUri); - $response = $this->crawler->sendRequest($request); + try { + $response = $this->crawler->sendRequest($request); + } catch (BlockedRequestException $exception) { + return $extractor; + } return $this->extract($request, $response, false); } diff --git a/src/HttpApiTrait.php b/src/HttpApiTrait.php index 8e4180bb..419fbdc5 100644 --- a/src/HttpApiTrait.php +++ b/src/HttpApiTrait.php @@ -24,7 +24,11 @@ private function fetchJSON(UriInterface $uri): array { $crawler = $this->extractor->getCrawler(); $request = $crawler->createRequest('GET', $uri); - $response = $crawler->sendRequest($request); + try { + $response = $crawler->sendRequest($request); + } catch (\Embed\Http\BlockedRequestException $exception) { + return []; + } try { $data = json_decode((string) $response->getBody(), true); diff --git a/src/OEmbed.php b/src/OEmbed.php index b530591f..37e05127 100644 --- a/src/OEmbed.php +++ b/src/OEmbed.php @@ -3,6 +3,7 @@ namespace Embed; +use Embed\Http\BlockedRequestException; use Exception; use Psr\Http\Message\UriInterface; use SimpleXMLElement; @@ -58,7 +59,11 @@ protected function fetchData(): array $crawler = $this->extractor->getCrawler(); $request = $crawler->createRequest('GET', $this->endpoint); - $response = $crawler->sendRequest($request); + try { + $response = $crawler->sendRequest($request); + } catch (BlockedRequestException $exception) { + return []; + } if (self::isXML($request->getUri())) { return $this->extractXML((string) $response->getBody()); diff --git a/tests/RecordingClient.php b/tests/RecordingClient.php new file mode 100644 index 00000000..ca885eab --- /dev/null +++ b/tests/RecordingClient.php @@ -0,0 +1,33 @@ + */ + public array $requests = []; + + /** @var callable(RequestInterface): ResponseInterface */ + private $handler; + + /** + * @param callable(RequestInterface): ResponseInterface $handler + */ + public function __construct(callable $handler) + { + $this->handler = $handler; + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $this->requests[] = $request; + $handler = $this->handler; + + return $handler($request); + } +} diff --git a/tests/SsrfEmbedTest.php b/tests/SsrfEmbedTest.php new file mode 100644 index 00000000..7d3ed1e1 --- /dev/null +++ b/tests/SsrfEmbedTest.php @@ -0,0 +1,143 @@ + + + + +Page title + + + + + + + +HTML; + $client = $this->client(static function (RequestInterface $request) use ($html): ResponseInterface { + $factory = new Psr17Factory(); + if ($request->getUri()->getPath() === '/oembed.json') { + return $factory->createResponse(200) + ->withHeader('Content-Type', 'application/json') + ->withBody($factory->createStream('{"title":"From oEmbed","type":"link"}')); + } + + return $factory->createResponse(200) + ->withHeader('Content-Type', 'text/html') + ->withBody($factory->createStream($html)); + }); + + $info = $this->embed($client)->get('http://public.test/article'); + + $this->assertSame('From oEmbed', $info->title); + $this->assertSame('http://public.test/images/pic.jpg', (string) $info->image); + $this->assertSame('http://cdn.example/favicon.ico', (string) $info->favicon); + $this->assertSame('http://public.test/es', (string) $info->languages['es']); + + $paths = array_map(static function (RequestInterface $request): string { + return $request->getUri()->getHost().$request->getUri()->getPath(); + }, $client->requests); + $this->assertSame(['public.test/article', 'public.test/oembed.json'], $paths); + } + + public function testBlockedOEmbedEndpointIsSkipped(): void + { + $html = 'Stay here' + .'' + .''; + $client = $this->htmlClient($html); + + $info = $this->embed($client)->get('http://public.test/page'); + + $this->assertSame('Stay here', $info->title); + $this->assertSame([], $info->getOEmbed()->all()); + $this->assertCount(1, $client->requests); + $this->assertSame('public.test', $client->requests[0]->getUri()->getHost()); + } + + public function testBlockedMetaRefreshIsIgnored(): void + { + $html = 'Stay' + .'' + .''; + $client = $this->htmlClient($html); + + $info = $this->embed($client)->get('http://public.test/refresh'); + + $this->assertSame('Stay', $info->title); + $this->assertCount(1, $client->requests); + } + + public function testCustomClientDoesNotReceiveABlockedUrl(): void + { + $client = $this->htmlClient(''); + + try { + $this->embed($client)->get('http://127.0.0.1/secret'); + $this->fail('The main URL must be rejected.'); + } catch (BlockedRequestException $exception) { + $this->assertSame('non_public_address', $exception->getReason()); + } + + $this->assertSame([], $client->requests); + } + + public function testCustomClientValidatesEveryParallelUrlBeforeSending(): void + { + $client = $this->htmlClient(''); + + try { + $this->embed($client)->getMulti('http://public.test/a', 'http://127.0.0.1/secret'); + $this->fail('One blocked URL must reject the batch.'); + } catch (BlockedRequestException $exception) { + $this->assertSame('non_public_address', $exception->getReason()); + } + + $this->assertSame([], $client->requests); + } + + private function embed(RecordingClient $client): Embed + { + $crawler = new Crawler($client); + $crawler->setUrlPolicy(UrlPolicy::default()->withResolver(static function (string $host): array { + return ['1.1.1.1']; + })); + + return new Embed($crawler); + } + + private function htmlClient(string $html): RecordingClient + { + return $this->client(static function (RequestInterface $request) use ($html): ResponseInterface { + $factory = new Psr17Factory(); + + return $factory->createResponse(200) + ->withHeader('Content-Type', 'text/html') + ->withBody($factory->createStream($html)); + }); + } + + /** + * @param callable(RequestInterface): ResponseInterface $handler + */ + private function client(callable $handler): RecordingClient + { + return new RecordingClient($handler); + } +} From 89f34dcf99b63b838c62688fde42736f0323349e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 00:56:08 +0000 Subject: [PATCH 07/12] Document SSRF protection Describe the default block, the intranet opt-out, custom PSR-18 clients, and environment proxies. Record the change under Unreleased for 4.5.0. --- CHANGELOG.md | 12 ++++++++ README.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++++-- composer.json | 1 + 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5845778..3437ee8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [Unreleased] +### Security +- Block requests to non-public addresses before they are sent, including redirects, page-declared oEmbed endpoints, adapter APIs, and meta-refresh targets. Intended for 4.5.0. Private and loopback URLs are rejected unless `UrlPolicy::allowPrivateNetworks()` or `withAllowedHosts()` is used [#573] + +### Added +- `Embed\Http\UrlPolicy` and `Embed\Http\BlockedRequestException`, configurable from `Crawler::setUrlPolicy()` + +### Fixed +- `resolveUri()` dropped the port of the base URL +- Read every `curl_multi_info_read` message in the parallel client instead of only one per iteration + ## [4.4.19] - 2026-07-08 ### Changed - Access tokens are now optional with Meta (Facebook & Instagram) [#577] @@ -301,6 +312,7 @@ Full library refactoring. [#572]: https://github.com/oscarotero/Embed/issues/572 [#574]: https://github.com/oscarotero/Embed/issues/574 [#577]: https://github.com/oscarotero/Embed/issues/577 +[#573]: https://github.com/php-embed/Embed/issues/573 [4.4.19]: https://github.com/oscarotero/Embed/compare/v4.4.18...v4.4.19 [4.4.18]: https://github.com/oscarotero/Embed/compare/v4.4.17...v4.4.18 diff --git a/README.md b/README.md index b610ee32..08f79d67 100644 --- a/README.md +++ b/README.md @@ -329,12 +329,12 @@ $client = new CurlClient(); $client->setSettings([ 'cookies_path' => $cookies_path, 'ignored_errors' => [18], - 'max_redirs' => 3, // see CURLOPT_MAXREDIRS + 'max_redirs' => 3, // maximum number of redirects to follow 'connect_timeout' => 2, // see CURLOPT_CONNECTTIMEOUT 'timeout' => 2, // see CURLOPT_TIMEOUT 'ssl_verify_host' => 2, // see CURLOPT_SSL_VERIFYHOST 'ssl_verify_peer' => 1, // see CURLOPT_SSL_VERIFYPEER - 'follow_location' => true, // see CURLOPT_FOLLOWLOCATION + 'follow_location' => true, // follow redirects; every hop is checked 'user_agent' => 'Mozilla', // see CURLOPT_USERAGENT ]); @@ -359,6 +359,85 @@ $info = $embed->get($url); Note: The built-in detectors does not require settings. This feature is only for convenience if you create a specific detector that requires settings. +## Security / SSRF protection + +The default HTTP client refuses to connect to non-public addresses. That covers the URL you pass to `get()` / `getMulti()` and every later request Embed makes itself (redirects, oEmbed endpoints declared by the page, adapter APIs, and meta-refresh targets). A hostname is allowed only when every IPv4 address it resolves to is public. Loopback, private, link-local, CGNAT, documentation, and multicast ranges are rejected, including the addresses embedded in IPv4-mapped IPv6, NAT64, 6to4, and Teredo. Only `http` and `https` are allowed, and redirects are followed inside Embed so each hop is checked the same way. `follow_location` and `max_redirs` still mean what they did before. + +This is a behavior change: a URL that used to be fetched from an intranet now throws `Embed\Http\BlockedRequestException`. To opt back in: + +```php +use Embed\Http\UrlPolicy; + +$embed->getCrawler()->setUrlPolicy( + UrlPolicy::default()->allowPrivateNetworks() +); +``` + +You can allow specific names without opening every private network. `*.corp` matches any host that ends in `.corp`. Allowing one resolved address does not allow the other addresses returned for that name: + +```php +$embed->getCrawler()->setUrlPolicy( + UrlPolicy::default()->withAllowedHosts('wiki.corp', '*.corp') +); +``` + +`withAllowedPorts([80, 443])` is optional. When it is set, an omitted port counts as 80 or 443. Internationalized host names are converted with `idn_to_ascii()`, which needs the `intl` extension; without it those hosts are rejected. The built-in client pins the punycode form, because that is the name libcurl looks up. + +Values such as `image` or `favicon` are URLs extracted from the page. Embed does not download them, so it does not apply this policy to them. Check one before you fetch it yourself: + +```php +$policy = $embed->getCrawler()->getUrlPolicy(); +$image = $info->image; + +if ($image !== null) { + $policy->validate($embed->getCrawler()->createRequest('GET', (string) $image)); +} +``` + +A page-declared oEmbed endpoint, adapter API, or meta-refresh target that the policy rejects is skipped, and extraction continues with the rest of the page. The URL passed to `get()` or `getMulti()` is not skipped: that request throws. In `getMulti()`, one rejected URL rejects the whole batch before any request is sent. + +### Custom PSR-18 clients + +If you pass your own PSR-18 client, Embed validates the URI before calling it and does not validate again inside that client. Two things are then outside Embed's control: + +* DNS can change between the check and the connection (rebinding). +* Redirects followed by the client are not checked, unless the client gives you a hook. + +The built-in curl client pins the validated addresses with `CURLOPT_RESOLVE` and follows redirects itself, so those gaps do not apply to it. With Guzzle you can repeat the check on each redirect: + +```php +use Embed\Http\UrlPolicy; +use GuzzleHttp\Client; +use GuzzleHttp\HandlerStack; +use GuzzleHttp\Middleware; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; +use Psr\Http\Message\UriInterface; + +$policy = UrlPolicy::default(); +$stack = HandlerStack::create(); +$stack->push(Middleware::mapRequest(function (RequestInterface $request) use ($policy) { + $policy->validate($request); + + return $request; +})); + +$client = new Client([ + 'handler' => $stack, + 'allow_redirects' => [ + 'on_redirect' => function (RequestInterface $request, ResponseInterface $response, UriInterface $uri) use ($policy): void { + $policy->validate($request->withUri($uri)); + }, + ], +]); +``` + +Even that hook cannot see the address the client actually connected to. Prefer the built-in client when the pages are untrusted. + +### Proxies + +If `http_proxy`, `https_proxy`, or `all_proxy` is set, curl connects to the proxy and the proxy resolves the origin. Embed still validates each URL before requesting it, and still checks redirects, but it cannot pin DNS or verify the address the proxy used. Point those variables only at a proxy you trust. + --- ## Testing diff --git a/composer.json b/composer.json index 993f65e5..eea29c00 100644 --- a/composer.json +++ b/composer.json @@ -47,6 +47,7 @@ "phpstan/phpstan-strict-rules": "^2.0" }, "suggest": { + "ext-intl": "To resolve internationalized domain names when protecting against SSRF", "symfony/css-selector": "If you want to get elements using css selectors" }, "autoload": { From 0c72582386d8988b1243aadb3c394bd4a7ce5eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 01:18:12 +0000 Subject: [PATCH 08/12] Match curl's host verification, redirect limit, and timeout ssl_verify_host now sends 2 for any truthy value, so true and 1 still check the hostname. A negative max_redirs stays unlimited, capped at 50. timeout is the budget for the whole redirect chain. Curl handle annotations keep the PHP 7.4 resource type, with the comments and phpstan-ignore notes that make that union analyse cleanly. --- src/Http/CurlDispatcher.php | 145 ++++++++++++++++++++++++----- tests/CurlClientTest.php | 178 ++++++++++++++++++++++++++++++++++++ tests/server/router.php | 18 ++++ 3 files changed, 316 insertions(+), 25 deletions(-) diff --git a/src/Http/CurlDispatcher.php b/src/Http/CurlDispatcher.php index d9ee279a..a0fbcdab 100644 --- a/src/Http/CurlDispatcher.php +++ b/src/Http/CurlDispatcher.php @@ -13,10 +13,12 @@ use Psr\Http\Message\UriFactoryInterface; /** - * Class to fetch html pages. + * Class to fetch html pages * * Redirects are followed here, not by libcurl, so every hop is checked against * the URL policy and pinned to the addresses that were validated. + * + * @phpstan-type CurlResource resource|\CurlHandle */ final class CurlDispatcher { @@ -28,7 +30,10 @@ final class CurlDispatcher private StreamFactoryInterface $streamFactory; private UriFactoryInterface $uriFactory; private UrlPolicy $policy; - /** @var \CurlHandle */ + /** + * @var resource|\CurlHandle + * @phpstan-ignore property.unusedType (resource type needed for PHP 7.4 compatibility) + */ private $curl; /** @var array */ private array $headers = []; @@ -42,12 +47,13 @@ final class CurlDispatcher private bool $proxy; private bool $follow; private int $maxRedirs; + private ?float $deadline; private bool $prereqBlocked = false; private bool $closed = false; private int $followed = 0; /** - * @param array $settings + * @param array $settings * @return ResponseInterface[] */ public static function fetch(array $settings, ResponseFactoryInterface $responseFactory, RequestInterface ...$requests): array @@ -104,9 +110,9 @@ private static function environmentProxyEnabled(): bool } /** - * @param array $settings - * @param array $requests - * @param array> $pins + * @param array $settings + * @param array $requests + * @param array> $pins * @return ResponseInterface[] */ private static function fetchMulti( @@ -118,14 +124,20 @@ private static function fetchMulti( bool $proxy ): array { $multi = curl_multi_init(); + // curl_multi_init() can fail on PHP 7.4. Current stubs type it as CurlMultiHandle only. + /** @phpstan-ignore identical.alwaysFalse */ + if ($multi === false) { + throw new NetworkException('Unable to initialize curl', 0, $requests[0]); + } $connections = []; try { foreach ($requests as $index => $request) { $connection = new self($settings, $request, $pins[$index], $policy, $proxy); $connections[] = $connection; - $curlHandle = $connection->curl; - curl_multi_add_handle($multi, $curlHandle); + // PHP 7.4: $multi and the easy handle are resources. + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + curl_multi_add_handle($multi, $connection->curl); } self::pump($multi, $connections); @@ -145,7 +157,9 @@ private static function fetchMulti( } /** - * @param \CurlMultiHandle $multi + * On PHP 7.4 $multi is a resource. PHP 8 uses CurlMultiHandle. + * + * @param resource|\CurlMultiHandle $multi * @param array $connections */ private static function pump($multi, array $connections): void @@ -155,11 +169,13 @@ private static function pump($multi, array $connections): void do { do { + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $status = curl_multi_exec($multi, $active); } while ($status === CURLM_CALL_MULTI_PERFORM); $requeued = false; while (true) { + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $info = curl_multi_info_read($multi); if (!is_array($info)) { break; @@ -180,7 +196,9 @@ private static function pump($multi, array $connections): void } $curlHandle = $connection->curl; + /** @phpstan-ignore argument.type, argument.type (PHP 7.4/8.0 compatibility) */ curl_multi_remove_handle($multi, $curlHandle); + /** @phpstan-ignore argument.type, argument.type (PHP 7.4/8.0 compatibility) */ curl_multi_add_handle($multi, $curlHandle); $requeued = true; break; @@ -192,6 +210,7 @@ private static function pump($multi, array $connections): void } if ($active !== 0 && $status === CURLM_OK) { + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $selected = curl_multi_select($multi, 1.0); if ($selected === -1) { usleep(10000); @@ -201,13 +220,16 @@ private static function pump($multi, array $connections): void } /** - * @param \CurlMultiHandle $multi + * On PHP 7.4 $multi is a resource. PHP 8 uses CurlMultiHandle. + * + * @param resource|\CurlMultiHandle $multi * @param array $connections */ private static function releaseMulti($multi, array $connections, bool $closeHandles): void { foreach ($connections as $connection) { $curlHandle = $connection->curl; + /** @phpstan-ignore argument.type, argument.type (PHP 7.4/8.0 compatibility) */ curl_multi_remove_handle($multi, $curlHandle); if ($closeHandles) { $connection->closeCurl(); @@ -215,13 +237,14 @@ private static function releaseMulti($multi, array $connections, bool $closeHand } if (PHP_VERSION_ID < 80000) { + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ curl_multi_close($multi); } } /** * @param array $settings - * @param array $pinnedIps + * @param array $pinnedIps */ private function __construct(array $settings, RequestInterface $request, array $pinnedIps, UrlPolicy $policy, bool $proxy) { @@ -236,7 +259,11 @@ private function __construct(array $settings, RequestInterface $request, array $ $this->policy = $policy; $this->proxy = $proxy; $this->follow = $this->settingBool('follow_location', true); - $this->maxRedirs = max(0, $this->settingInt('max_redirs', 10)); + $this->maxRedirs = $this->redirectLimit(); + $timeout = $this->settingInt('timeout', 10); + // libcurl's CURLOPT_TIMEOUT covers the whole transfer, including + // redirects. Each hop receives only the time that is still left. + $this->deadline = $timeout > 0 ? microtime(true) + $timeout : null; $this->streamFactory = FactoryDiscovery::getStreamFactory(); $this->uriFactory = FactoryDiscovery::getUriFactory(); $this->curl = $curl; @@ -249,6 +276,12 @@ private function execute(): void { while (true) { $this->resetExchange(); + if ($this->timeoutExpired()) { + $this->error = CURLE_OPERATION_TIMEDOUT; + + return; + } + $this->applyTimeout(); $this->execOnce(); $this->guardConnection(); @@ -305,12 +338,20 @@ private function finishTransfer(int $result): bool return false; } + if ($this->timeoutExpired()) { + $this->error = CURLE_OPERATION_TIMEDOUT; + + return false; + } + $next = $this->redirectRequest(); if ($next === null) { return false; } $this->request = $next; + // DNS for this hop runs here, on the multi thread, so the other + // transfers in the batch wait until validate() returns. $this->pinnedIps = $this->policy->validate($next); $this->applyToHandle(); $this->resetExchange(); @@ -322,6 +363,7 @@ private function finishTransfer(int $result): bool private function execOnce(): void { $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ curl_exec($curlHandle); } @@ -338,11 +380,13 @@ private function guardConnection(): void $this->guardPrimaryIp(); $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $errno = curl_errno($curlHandle); if ($errno === 0 || ($this->isBinary && $errno === CURLE_WRITE_ERROR)) { return; } + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $this->error(curl_error($curlHandle), $errno); } @@ -353,6 +397,7 @@ private function guardPrimaryIp(): void } $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $ip = curl_getinfo($curlHandle, CURLINFO_PRIMARY_IP); if ($ip === '') { return; @@ -475,7 +520,6 @@ private function applyBaseOptions(): void $caBundle = CaBundle::getSystemCaRootBundlePath(); $this->setopt(CURLOPT_CONNECTTIMEOUT, $this->settingInt('connect_timeout', 10)); - $this->setopt(CURLOPT_TIMEOUT, $this->settingInt('timeout', 10)); $this->setopt(CURLOPT_RETURNTRANSFER, true); $this->setopt(CURLOPT_SSL_VERIFYHOST, $this->sslVerifyHost()); $this->setopt(CURLOPT_SSL_VERIFYPEER, $this->sslVerifyPeer()); @@ -537,6 +581,8 @@ private function applyToHandle(): void $this->setopt(CURLOPT_URL, (string) $this->request->getUri()); $this->setopt(CURLOPT_HTTPHEADER, $this->getRequestHeaders()); + $this->applyTimeout(); + $method = strtoupper($this->request->getMethod()); if ($method === 'POST') { $this->setopt(CURLOPT_POST, true); @@ -595,6 +641,7 @@ private function getResponse(ResponseFactoryInterface $responseFactory): Respons private function buildResponse(ResponseFactoryInterface $responseFactory): ResponseInterface { $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $info = curl_getinfo($curlHandle); if ($this->error !== null && $this->error !== 0) { @@ -602,15 +649,17 @@ private function buildResponse(ResponseFactoryInterface $responseFactory): Respo $this->error($message === null ? 'curl error' : $message, $this->error); } + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $errno = curl_errno($curlHandle); if ($errno !== 0 && !($this->isBinary && $errno === CURLE_WRITE_ERROR)) { + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $this->error(curl_error($curlHandle), $errno); } $response = $responseFactory->createResponse($info['http_code']); foreach ($this->headers as $header) { - [$name, $value] = $header; + list($name, $value) = $header; $response = $response->withAddedHeader($name, $value); } @@ -621,6 +670,7 @@ private function buildResponse(ResponseFactoryInterface $responseFactory): Respo ->withAddedHeader('X-Request-Time', sprintf('%.3f ms', $info['total_time'])); if ($this->body !== null) { + //5Mb max $this->body->rewind(); $response = $response->withBody($this->body); $this->body = null; @@ -638,6 +688,7 @@ private function error(string $message, int $code): void } if ($this->isBinary && $code === CURLE_WRITE_ERROR) { + // The write callback aborted the request to prevent a download of the binary file return; } @@ -654,6 +705,7 @@ private function closeCurl(): void $this->closed = true; $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ curl_close($curlHandle); } @@ -669,6 +721,7 @@ private function resetExchange(): void private function httpStatus(): int { $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ return curl_getinfo($curlHandle, CURLINFO_HTTP_CODE); } @@ -678,6 +731,7 @@ private function httpStatus(): int private function setopt(int $option, $value): void { $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ curl_setopt($curlHandle, $option, $value); } @@ -710,15 +764,55 @@ private function settingBool(string $key, bool $default): bool return $default; } - /** @return 0|2 */ + /** + * libcurl accepts only 0 and 2. PHP turns 1 into 2 and emits a notice, + * which is what a truthy setting used to do when it was passed through. + * + * @return 0|2 + */ private function sslVerifyHost(): int { $value = $this->settings['ssl_verify_host'] ?? 0; - if ($value === 2 || $value === '2') { - return 2; + if ($value === false || $value === 0 || $value === 0.0 || $value === '' || $value === '0') { + return 0; } - return 0; + return 2; + } + + /** + * A negative max_redirs is curl's "unlimited". Cap it so a redirect loop + * cannot run without a bound now that we follow hops ourselves. + */ + private function redirectLimit(): int + { + $limit = $this->settingInt('max_redirs', 10); + if ($limit < 0) { + return 50; + } + + return $limit; + } + + private function timeoutExpired(): bool + { + return $this->deadline !== null && microtime(true) >= $this->deadline; + } + + private function applyTimeout(): void + { + if ($this->deadline === null) { + $this->setopt(CURLOPT_TIMEOUT, 0); + + return; + } + + $remainingMs = (int) floor(($this->deadline - microtime(true)) * 1000); + if ($remainingMs < 1) { + $remainingMs = 1; + } + + $this->setopt(CURLOPT_TIMEOUT_MS, $remainingMs); } private function sslVerifyPeer(): bool @@ -747,18 +841,19 @@ private function cookiesPath(): string } /** - * @return array + * @return array */ private function getRequestHeaders(): array { $headers = []; foreach ($this->request->getHeaders() as $name => $values) { - if (strtolower($name) === 'user-agent') { - continue; + switch (strtolower($name)) { + case 'user-agent': + break; + default: + $headers[] = $name . ':' . implode(', ', $values); } - - $headers[] = $name.':'.implode(', ', $values); } return $headers; @@ -766,7 +861,7 @@ private function getRequestHeaders(): array /** * @param resource|\CurlHandle $curl - * @param mixed $string + * @param mixed $string */ private function writeHeader($curl, $string): int { @@ -792,7 +887,7 @@ private function writeHeader($curl, $string): int /** * @param resource|\CurlHandle $curl - * @param mixed $string + * @param mixed $string */ private function writeBody($curl, $string): int { diff --git a/tests/CurlClientTest.php b/tests/CurlClientTest.php index cf85fcf8..3f547e30 100644 --- a/tests/CurlClientTest.php +++ b/tests/CurlClientTest.php @@ -198,6 +198,7 @@ public function testTooManyRedirectsUsesCurlErrorCode(): void $this->fail('The eleventh redirect must fail.'); } catch (NetworkException $exception) { $this->assertSame(47, $exception->getCode()); + $this->assertSame('Number of redirects hit maximum amount', $exception->getMessage()); } $queries = array_map(static function (array $row): string { @@ -208,6 +209,91 @@ public function testTooManyRedirectsUsesCurlErrorCode(): void $this->assertCount(11, $queries); } + public function testNegativeMaxRedirsFollowsUpToTheCap(): void + { + $response = $this->client(['max_redirs' => -1])->sendRequest($this->request($this->url('/chain?i=0&n=3'))); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame(['/chain', '/chain', '/chain', '/chain'], self::$server->paths()); + + self::$server->clearLog(); + + try { + $this->client(['max_redirs' => -1])->sendRequest($this->request($this->url('/chain?i=0&n=51'))); + $this->fail('A negative max_redirs is capped at 50.'); + } catch (NetworkException $exception) { + $this->assertSame(47, $exception->getCode()); + } + + $queries = array_map(static function (array $row): string { + return $row['query'] ?? ''; + }, self::$server->requests()); + $this->assertContains('i=50&n=51', $queries); + $this->assertNotContains('i=51&n=51', $queries); + $this->assertCount(51, $queries); + } + + public function testTimeoutIsABudgetAcrossRedirects(): void + { + $next = rawurlencode('/pause?ms=700'); + + try { + $this->client(['timeout' => 1])->sendRequest($this->request($this->url('/pause?ms=700&to='.$next))); + $this->fail('Two pauses must exceed the total timeout.'); + } catch (NetworkException $exception) { + $this->assertSame(28, $exception->getCode()); + } + + $within = $this->client(['timeout' => 2])->sendRequest($this->request($this->url('/pause?ms=300'))); + $this->assertSame(200, $within->getStatusCode()); + } + + public function testSslVerifyHostTruthyValuesKeepHostnameChecks(): void + { + $tls = $this->startTlsServer(); + + try { + $url = 'https://allowed.test:'.$tls['port'].'/'; + foreach ([0, false, '0'] as $disabled) { + $response = $this->client([ + 'ssl_verify_host' => $disabled, + 'ssl_verify_peer' => false, + ])->sendRequest($this->request($url)); + $this->assertSame(200, $response->getStatusCode(), 'disabled '.var_export($disabled, true)); + } + + $notices = []; + set_error_handler(static function (int $severity, string $message) use (&$notices): bool { + if (stripos($message, 'SSL_VERIFYHOST') !== false) { + $notices[] = $message; + } + + return true; + }); + + try { + foreach ([true, 1, '1', 2, '2'] as $enabled) { + try { + $this->client([ + 'ssl_verify_host' => $enabled, + 'ssl_verify_peer' => false, + ])->sendRequest($this->request($url)); + $this->fail('Hostname verification must stay on for '.var_export($enabled, true)); + } catch (NetworkException $exception) { + $this->assertSame(60, $exception->getCode(), var_export($enabled, true)); + $this->assertStringContainsString('does not match target host name', $exception->getMessage()); + } + } + } finally { + restore_error_handler(); + } + + $this->assertSame([], $notices); + } finally { + $this->stopTlsServer($tls); + } + } + public function testFollowLocationFalseReturnsTheRedirect(): void { $response = $this->client(['follow_location' => false])->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('/landed')))); @@ -403,6 +489,98 @@ private function captureBlocked(callable $check): BlockedRequestException $this->fail('Expected BlockedRequestException'); } + /** + * @return array{port: int, process: resource, paths: array} + */ + private function startTlsServer(): array + { + $directory = sys_get_temp_dir().'/embed-tls-'.bin2hex(random_bytes(4)); + if (!mkdir($directory) && !is_dir($directory)) { + $this->fail('Unable to create a certificate directory'); + } + + $key = $directory.'/key.pem'; + $cert = $directory.'/cert.pem'; + $command = 'openssl req -x509 -newkey rsa:2048 -keyout '.escapeshellarg($key).' -out '.escapeshellarg($cert).' -days 1 -nodes -subj /CN=mismatch.test 2>/dev/null'; + exec($command, $output, $exitCode); + if ($exitCode !== 0 || !is_file($cert)) { + $this->fail('Unable to create a TLS certificate'); + } + + $port = self::allocatePort(); + $stderr = $directory.'/stderr'; + $process = proc_open( + ['openssl', 's_server', '-accept', '127.0.0.1:'.$port, '-cert', $cert, '-key', $key, '-www'], + [ + 0 => ['pipe', 'r'], + 1 => ['file', $directory.'/stdout', 'w'], + 2 => ['file', $stderr, 'w'], + ], + $pipes + ); + if (!is_resource($process)) { + $this->fail('Unable to start the TLS server'); + } + fclose($pipes[0]); + + $deadline = microtime(true) + 5; + while (microtime(true) < $deadline) { + $socket = @fsockopen('127.0.0.1', $port, $errno, $error, 0.2); + if (is_resource($socket)) { + fclose($socket); + + return ['port' => $port, 'process' => $process, 'paths' => [$directory, $key, $cert, $stderr, $directory.'/stdout']]; + } + usleep(50000); + } + + proc_terminate($process); + proc_close($process); + $details = is_file($stderr) ? (string) file_get_contents($stderr) : ''; + $this->fail('TLS server did not start: '.$details); + } + + /** + * @param array{port: int, process: resource, paths: array} $tls + */ + private function stopTlsServer(array $tls): void + { + proc_terminate($tls['process']); + proc_close($tls['process']); + $directories = []; + foreach ($tls['paths'] as $path) { + if (is_file($path)) { + unlink($path); + } elseif (is_dir($path)) { + $directories[] = $path; + } + } + foreach ($directories as $directory) { + rmdir($directory); + } + } + + private static function allocatePort(): int + { + $socket = stream_socket_server('tcp://127.0.0.1:0', $errno, $error); + if ($socket === false) { + throw new \RuntimeException('Unable to allocate a port: '.$error); + } + + $name = stream_socket_get_name($socket, false); + fclose($socket); + if (!is_string($name)) { + throw new \RuntimeException('Unable to read the allocated port'); + } + + $colon = strrpos($name, ':'); + if ($colon === false) { + throw new \RuntimeException('Unable to read the allocated port'); + } + + return (int) substr($name, $colon + 1); + } + private static function clearProxyEnvironment(): void { foreach (['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'] as $name) { diff --git a/tests/server/router.php b/tests/server/router.php index 7150acc3..cbcb8fd4 100644 --- a/tests/server/router.php +++ b/tests/server/router.php @@ -95,6 +95,24 @@ return; } +if ($path === '/pause') { + $ms = isset($params['ms']) ? (int) $params['ms'] : 0; + if ($ms > 0) { + usleep(min($ms, 5000) * 1000); + } + $to = isset($params['to']) && is_string($params['to']) ? $params['to'] : ''; + if ($to !== '') { + header('Location: '.$to, true, 302); + echo 'redirect'; + + return; + } + http_response_code(200); + echo 'ok'; + + return; +} + if ($path === '/secret') { http_response_code(200); echo 'LEAKED'; From c04d1f37fc51330e66d95b3be36011b621f0b5fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 01:18:12 +0000 Subject: [PATCH 09/12] Document the SSRF review notes Drop the version from the changelog entry and point #573 at the same repository as the other links. Describe Teredo as a whole prefix, setUrlPolicy() only forwarding to a real CurlClient, and synchronous redirect checks inside getMulti. Record the max_redirs cap and the redirect-limit message. --- CHANGELOG.md | 8 ++++++-- README.md | 18 +++++++++++------- src/Http/Crawler.php | 5 +++++ src/Http/IpClassifier.php | 2 +- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3437ee8f..3ca89eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Security -- Block requests to non-public addresses before they are sent, including redirects, page-declared oEmbed endpoints, adapter APIs, and meta-refresh targets. Intended for 4.5.0. Private and loopback URLs are rejected unless `UrlPolicy::allowPrivateNetworks()` or `withAllowedHosts()` is used [#573] +- Block requests to non-public addresses before they are sent, including redirects, page-declared oEmbed endpoints, adapter APIs, and meta-refresh targets. Private and loopback URLs are rejected unless `UrlPolicy::allowPrivateNetworks()` or `withAllowedHosts()` is used [#573] ### Added - `Embed\Http\UrlPolicy` and `Embed\Http\BlockedRequestException`, configurable from `Crawler::setUrlPolicy()` +### Changed +- A negative `max_redirs`, which curl treated as unlimited, now follows at most 50 redirects +- Reaching that limit is still curl error 47. The message is `Number of redirects hit maximum amount` rather than curl's `Maximum (N) redirects followed` + ### Fixed - `resolveUri()` dropped the port of the base URL - Read every `curl_multi_info_read` message in the parallel client instead of only one per iteration @@ -310,9 +314,9 @@ Full library refactoring. [#569]: https://github.com/oscarotero/Embed/issues/569 [#571]: https://github.com/oscarotero/Embed/issues/571 [#572]: https://github.com/oscarotero/Embed/issues/572 +[#573]: https://github.com/oscarotero/Embed/issues/573 [#574]: https://github.com/oscarotero/Embed/issues/574 [#577]: https://github.com/oscarotero/Embed/issues/577 -[#573]: https://github.com/php-embed/Embed/issues/573 [4.4.19]: https://github.com/oscarotero/Embed/compare/v4.4.18...v4.4.19 [4.4.18]: https://github.com/oscarotero/Embed/compare/v4.4.17...v4.4.18 diff --git a/README.md b/README.md index 08f79d67..5cc913a1 100644 --- a/README.md +++ b/README.md @@ -329,12 +329,12 @@ $client = new CurlClient(); $client->setSettings([ 'cookies_path' => $cookies_path, 'ignored_errors' => [18], - 'max_redirs' => 3, // maximum number of redirects to follow - 'connect_timeout' => 2, // see CURLOPT_CONNECTTIMEOUT - 'timeout' => 2, // see CURLOPT_TIMEOUT - 'ssl_verify_host' => 2, // see CURLOPT_SSL_VERIFYHOST + 'max_redirs' => 3, // redirects to follow; a negative value means up to 50 + 'connect_timeout' => 2, // per connection, see CURLOPT_CONNECTTIMEOUT + 'timeout' => 2, // total seconds for the request, including redirects + 'ssl_verify_host' => 2, // 0 disables hostname checks; any other value enables them 'ssl_verify_peer' => 1, // see CURLOPT_SSL_VERIFYPEER - 'follow_location' => true, // follow redirects; every hop is checked + 'follow_location' => true, // false returns the 3xx; each hop is checked 'user_agent' => 'Mozilla', // see CURLOPT_USERAGENT ]); @@ -361,7 +361,9 @@ Note: The built-in detectors does not require settings. This feature is only for ## Security / SSRF protection -The default HTTP client refuses to connect to non-public addresses. That covers the URL you pass to `get()` / `getMulti()` and every later request Embed makes itself (redirects, oEmbed endpoints declared by the page, adapter APIs, and meta-refresh targets). A hostname is allowed only when every IPv4 address it resolves to is public. Loopback, private, link-local, CGNAT, documentation, and multicast ranges are rejected, including the addresses embedded in IPv4-mapped IPv6, NAT64, 6to4, and Teredo. Only `http` and `https` are allowed, and redirects are followed inside Embed so each hop is checked the same way. `follow_location` and `max_redirs` still mean what they did before. +The default HTTP client refuses to connect to non-public addresses. That covers the URL you pass to `get()` / `getMulti()` and every later request Embed makes itself (redirects, oEmbed endpoints declared by the page, adapter APIs, and meta-refresh targets). A hostname is allowed only when every IPv4 address it resolves to is public. Loopback, private, link-local, CGNAT, documentation, and multicast ranges are rejected. IPv4-mapped IPv6, NAT64, and 6to4 are rejected when the embedded IPv4 address is not public. The Teredo prefix `2001::/32` is rejected as a whole, not by reading an address out of it. Only `http` and `https` are allowed, and redirects are followed inside Embed so each hop is checked the same way. + +`follow_location` set to false still returns the 3xx response. `max_redirs` is still how many redirects are followed (10 by default). A negative value, which curl treated as unlimited, follows at most 50. `timeout` is the total time for the request and its redirects; `connect_timeout` is still per connection. `ssl_verify_host` of `0` disables hostname checks, and any other value enables them, as curl does when it receives `1` or `true`. Reaching the redirect limit still fails with curl error 47. The message is `Number of redirects hit maximum amount`. This is a behavior change: a URL that used to be fetched from an intranet now throws `Embed\Http\BlockedRequestException`. To opt back in: @@ -394,7 +396,9 @@ if ($image !== null) { } ``` -A page-declared oEmbed endpoint, adapter API, or meta-refresh target that the policy rejects is skipped, and extraction continues with the rest of the page. The URL passed to `get()` or `getMulti()` is not skipped: that request throws. In `getMulti()`, one rejected URL rejects the whole batch before any request is sent. +A page-declared oEmbed endpoint, adapter API, or meta-refresh target that the policy rejects is skipped, and extraction continues with the rest of the page. The URL passed to `get()` or `getMulti()` is not skipped: that request throws. In `getMulti()`, one rejected URL rejects the whole batch before any request is sent. A redirect discovered after the batch has started is checked on the same thread, including its DNS lookup, so the other transfers in that batch wait until the check returns. + +`Crawler::setUrlPolicy()` copies the policy onto the client only when that client is exactly `Embed\Http\CurlClient`. A decorator or other wrapper around `CurlClient` is not updated. Call `setUrlPolicy()` on the inner client as well. Otherwise the wrapper is pre-checked with the crawler's policy, and the inner client keeps the policy it already had. ### Custom PSR-18 clients diff --git a/src/Http/Crawler.php b/src/Http/Crawler.php index 01dee3c7..cffafb96 100644 --- a/src/Http/Crawler.php +++ b/src/Http/Crawler.php @@ -30,6 +30,11 @@ public function __construct(?ClientInterface $client = null, ?RequestFactoryInte $this->urlPolicy = $this->client instanceof CurlClient ? $this->client->getUrlPolicy() : UrlPolicy::default(); } + /** + * The policy is copied onto the HTTP client only when that client is + * exactly CurlClient. A decorator around CurlClient is not detected; + * set the policy on the inner client as well. + */ public function setUrlPolicy(UrlPolicy $urlPolicy): void { $this->urlPolicy = $urlPolicy; diff --git a/src/Http/IpClassifier.php b/src/Http/IpClassifier.php index 11871e27..7014d187 100644 --- a/src/Http/IpClassifier.php +++ b/src/Http/IpClassifier.php @@ -53,7 +53,7 @@ final class IpClassifier ['ff00::', 8], ['2001:db8::', 32], ['100::', 64], - ['2001::', 32], + ['2001::', 32], // Teredo: the whole prefix, not the embedded IPv4 ['64:ff9b:1::', 48], ]; From 29f2988c8261811b0a1d5d09edb488c6c6f0af2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 01:32:16 +0000 Subject: [PATCH 10/12] Keep certificate checks on for non-falsy ssl_verify_peer Only a falsy ssl_verify_peer disables CURLOPT_SSL_VERIFYPEER, matching the value curl used to receive. The hostname assertion accepts both "host name" and "hostname". Restore new static, the multi-loop comments, and the resource|CurlHandle locals. --- README.md | 4 ++-- src/Http/CurlDispatcher.php | 30 +++++++++++++++++++++++------- tests/CurlClientTest.php | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5cc913a1..e8173ef5 100644 --- a/README.md +++ b/README.md @@ -333,7 +333,7 @@ $client->setSettings([ 'connect_timeout' => 2, // per connection, see CURLOPT_CONNECTTIMEOUT 'timeout' => 2, // total seconds for the request, including redirects 'ssl_verify_host' => 2, // 0 disables hostname checks; any other value enables them - 'ssl_verify_peer' => 1, // see CURLOPT_SSL_VERIFYPEER + 'ssl_verify_peer' => 1, // falsy disables certificate checks; any other value enables them 'follow_location' => true, // false returns the 3xx; each hop is checked 'user_agent' => 'Mozilla', // see CURLOPT_USERAGENT ]); @@ -363,7 +363,7 @@ Note: The built-in detectors does not require settings. This feature is only for The default HTTP client refuses to connect to non-public addresses. That covers the URL you pass to `get()` / `getMulti()` and every later request Embed makes itself (redirects, oEmbed endpoints declared by the page, adapter APIs, and meta-refresh targets). A hostname is allowed only when every IPv4 address it resolves to is public. Loopback, private, link-local, CGNAT, documentation, and multicast ranges are rejected. IPv4-mapped IPv6, NAT64, and 6to4 are rejected when the embedded IPv4 address is not public. The Teredo prefix `2001::/32` is rejected as a whole, not by reading an address out of it. Only `http` and `https` are allowed, and redirects are followed inside Embed so each hop is checked the same way. -`follow_location` set to false still returns the 3xx response. `max_redirs` is still how many redirects are followed (10 by default). A negative value, which curl treated as unlimited, follows at most 50. `timeout` is the total time for the request and its redirects; `connect_timeout` is still per connection. `ssl_verify_host` of `0` disables hostname checks, and any other value enables them, as curl does when it receives `1` or `true`. Reaching the redirect limit still fails with curl error 47. The message is `Number of redirects hit maximum amount`. +`follow_location` set to false still returns the 3xx response. `max_redirs` is still how many redirects are followed (10 by default). A negative value, which curl treated as unlimited, follows at most 50. `timeout` is the total time for the request and its redirects; `connect_timeout` is still per connection. `ssl_verify_host` of `0` disables hostname checks, and any other value enables them, as curl does when it receives `1` or `true`. `ssl_verify_peer` is off only for a falsy value (`0`, `false`, `'0'`, `''`); `1`, `2`, `'true'` and `'yes'` keep certificate checks on. Reaching the redirect limit still fails with curl error 47. The message is `Number of redirects hit maximum amount`. This is a behavior change: a URL that used to be fetched from an intranet now throws `Embed\Http\BlockedRequestException`. To opt back in: diff --git a/src/Http/CurlDispatcher.php b/src/Http/CurlDispatcher.php index a0fbcdab..7f9eec3b 100644 --- a/src/Http/CurlDispatcher.php +++ b/src/Http/CurlDispatcher.php @@ -70,7 +70,7 @@ public static function fetch(array $settings, ResponseFactoryInterface $response } if (count($requests) === 1) { - $connection = new self($settings, $requests[0], $pins[0], $policy, $proxy); + $connection = new static($settings, $requests[0], $pins[0], $policy, $proxy); try { $connection->execute(); @@ -123,6 +123,7 @@ private static function fetchMulti( UrlPolicy $policy, bool $proxy ): array { + //Init connections $multi = curl_multi_init(); // curl_multi_init() can fail on PHP 7.4. Current stubs type it as CurlMultiHandle only. /** @phpstan-ignore identical.alwaysFalse */ @@ -133,19 +134,22 @@ private static function fetchMulti( try { foreach ($requests as $index => $request) { - $connection = new self($settings, $request, $pins[$index], $policy, $proxy); + $connection = new static($settings, $request, $pins[$index], $policy, $proxy); $connections[] = $connection; - // PHP 7.4: $multi and the easy handle are resources. + /** @var resource|\CurlHandle $curlHandle */ + $curlHandle = $connection->curl; /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ - curl_multi_add_handle($multi, $connection->curl); + curl_multi_add_handle($multi, $curlHandle); } + //Run self::pump($multi, $connections); } catch (\Throwable $exception) { self::releaseMulti($multi, $connections, true); throw $exception; } + //Close connections self::releaseMulti($multi, $connections, false); $responses = []; @@ -228,6 +232,7 @@ private static function pump($multi, array $connections): void private static function releaseMulti($multi, array $connections, bool $closeHandles): void { foreach ($connections as $connection) { + /** @var resource|\CurlHandle $curlHandle */ $curlHandle = $connection->curl; /** @phpstan-ignore argument.type, argument.type (PHP 7.4/8.0 compatibility) */ curl_multi_remove_handle($multi, $curlHandle); @@ -362,6 +367,7 @@ private function finishTransfer(int $result): bool private function execOnce(): void { + /** @var resource|\CurlHandle $curlHandle */ $curlHandle = $this->curl; /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ curl_exec($curlHandle); @@ -640,13 +646,14 @@ private function getResponse(ResponseFactoryInterface $responseFactory): Respons private function buildResponse(ResponseFactoryInterface $responseFactory): ResponseInterface { + /** @var resource|\CurlHandle $curlHandle */ $curlHandle = $this->curl; /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $info = curl_getinfo($curlHandle); if ($this->error !== null && $this->error !== 0) { - $message = curl_strerror($this->error); - $this->error($message === null ? 'curl error' : $message, $this->error); + /** @phpstan-ignore argument.type (curl_strerror returns string|null in some versions) */ + $this->error(curl_strerror($this->error), $this->error); } /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ @@ -815,9 +822,18 @@ private function applyTimeout(): void $this->setopt(CURLOPT_TIMEOUT_MS, $remainingMs); } + /** + * Same rule as a value passed straight to curl: only a falsy setting + * disables certificate verification. 2, "true" and "yes" stay enabled. + */ private function sslVerifyPeer(): bool { - return $this->settingBool('ssl_verify_peer', false); + $value = $this->settings['ssl_verify_peer'] ?? false; + if ($value === false || $value === 0 || $value === 0.0 || $value === '' || $value === '0' || $value === []) { + return false; + } + + return true; } private function userAgent(): string diff --git a/tests/CurlClientTest.php b/tests/CurlClientTest.php index 3f547e30..3b8fe605 100644 --- a/tests/CurlClientTest.php +++ b/tests/CurlClientTest.php @@ -281,7 +281,7 @@ public function testSslVerifyHostTruthyValuesKeepHostnameChecks(): void $this->fail('Hostname verification must stay on for '.var_export($enabled, true)); } catch (NetworkException $exception) { $this->assertSame(60, $exception->getCode(), var_export($enabled, true)); - $this->assertStringContainsString('does not match target host name', $exception->getMessage()); + $this->assertMatchesRegularExpression('/target host ?name/i', $exception->getMessage()); } } } finally { @@ -294,6 +294,37 @@ public function testSslVerifyHostTruthyValuesKeepHostnameChecks(): void } } + public function testSslVerifyPeerFalsyValuesAreTheOnlyOnesDisabled(): void + { + $tls = $this->startTlsServer(); + + try { + $url = 'https://allowed.test:'.$tls['port'].'/'; + foreach ([0, false, '0', '', null] as $disabled) { + $response = $this->client([ + 'ssl_verify_host' => 0, + 'ssl_verify_peer' => $disabled, + ])->sendRequest($this->request($url)); + $this->assertSame(200, $response->getStatusCode(), 'disabled '.var_export($disabled, true)); + } + + foreach ([true, 1, 2, '2', 'true', 'yes'] as $enabled) { + try { + $this->client([ + 'ssl_verify_host' => 0, + 'ssl_verify_peer' => $enabled, + ])->sendRequest($this->request($url)); + $this->fail('Certificate verification must stay on for '.var_export($enabled, true)); + } catch (NetworkException $exception) { + $this->assertSame(60, $exception->getCode(), var_export($enabled, true)); + $this->assertStringContainsString('certificate', $exception->getMessage()); + } + } + } finally { + $this->stopTlsServer($tls); + } + } + public function testFollowLocationFalseReturnsTheRedirect(): void { $response = $this->client(['follow_location' => false])->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('/landed')))); From 00195039968a950031adcf68e167f13fcf14531b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Fri, 9 Oct 2026 01:32:22 +0000 Subject: [PATCH 11/12] Stop MultipleRequestsTest from calling live DNS Serve the three pages from the cache and resolve them to a public address, so a resolver that answers 198.18.0.1 does not reject the test. --- tests/MultipleRequestsTest.php | 12 +++++++++++- tests/cache/github.com.oscarotero.php | 13 +++++++++++++ tests/cache/oscarotero.com..php | 16 ++++++++++++++++ ...m.oembed.58bc2b4ee41b094b713af915ade4710c.php | 13 +++++++++++++ tests/cache/twitter.com.misteroom.php | 13 +++++++++++++ 5 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tests/cache/github.com.oscarotero.php create mode 100644 tests/cache/oscarotero.com..php create mode 100644 tests/cache/publish.twitter.com.oembed.58bc2b4ee41b094b713af915ade4710c.php create mode 100644 tests/cache/twitter.com.misteroom.php diff --git a/tests/MultipleRequestsTest.php b/tests/MultipleRequestsTest.php index 42348449..a4943374 100644 --- a/tests/MultipleRequestsTest.php +++ b/tests/MultipleRequestsTest.php @@ -4,13 +4,23 @@ namespace Embed\Tests; use Embed\Embed; +use Embed\Http\Crawler; +use Embed\Http\UrlPolicy; use PHPUnit\Framework\TestCase; class MultipleRequestsTest extends TestCase { public function testParallel() { - $embed = new Embed(); + $dispatcher = new FileClient(__DIR__.'/cache'); + $dispatcher->setMode(-1); + $crawler = new Crawler($dispatcher); + // Some resolvers answer public names with 198.18.0.1, which the policy + // rejects. Cached responses must not depend on that lookup. + $crawler->setUrlPolicy(UrlPolicy::default()->withResolver(static function (string $host): array { + return ['8.8.8.8']; + })); + $embed = new Embed($crawler); $infos = $embed->getMulti( 'https://oscarotero.com', 'https://github.com/oscarotero', diff --git a/tests/cache/github.com.oscarotero.php b/tests/cache/github.com.oscarotero.php new file mode 100644 index 00000000..876e41fd --- /dev/null +++ b/tests/cache/github.com.oscarotero.php @@ -0,0 +1,13 @@ + [ + 'content-type' => [ + 'text/html; charset=utf-8', + ], + ], + 'statusCode' => 200, + 'reasonPhrase' => 'OK', + 'body' => 'oscarotero - Overview', +]; diff --git a/tests/cache/oscarotero.com..php b/tests/cache/oscarotero.com..php new file mode 100644 index 00000000..37e2c0dc --- /dev/null +++ b/tests/cache/oscarotero.com..php @@ -0,0 +1,16 @@ + [ + 'content-type' => [ + 'text/html; charset=utf-8', + ], + 'content-location' => [ + 'https://oscarotero.com/', + ], + ], + 'statusCode' => 200, + 'reasonPhrase' => 'OK', + 'body' => 'Óscar Otero - Digital designer and developer', +]; diff --git a/tests/cache/publish.twitter.com.oembed.58bc2b4ee41b094b713af915ade4710c.php b/tests/cache/publish.twitter.com.oembed.58bc2b4ee41b094b713af915ade4710c.php new file mode 100644 index 00000000..24b3ac23 --- /dev/null +++ b/tests/cache/publish.twitter.com.oembed.58bc2b4ee41b094b713af915ade4710c.php @@ -0,0 +1,13 @@ + [ + 'content-type' => [ + 'application/json', + ], + ], + 'statusCode' => 200, + 'reasonPhrase' => 'OK', + 'body' => '{"url":"https://x.com/misteroom"}', +]; diff --git a/tests/cache/twitter.com.misteroom.php b/tests/cache/twitter.com.misteroom.php new file mode 100644 index 00000000..5c10d870 --- /dev/null +++ b/tests/cache/twitter.com.misteroom.php @@ -0,0 +1,13 @@ + [ + 'content-type' => [ + 'text/html; charset=utf-8', + ], + ], + 'statusCode' => 200, + 'reasonPhrase' => 'OK', + 'body' => 'misteroom', +]; From 4c3b3f185eea44f0c5dd5d4651941e5306e1c92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20de=20Oliveira?= Date: Wed, 14 Oct 2026 00:18:08 +0000 Subject: [PATCH 12/12] Stop page tests from fetching the schema.org context ML\JsonLD downloads that context while expanding JSON-LD, so Tumblr's publishedTime depended on the network. Serve a minimal vocabulary instead and record the date the page actually publishes. --- phpunit.xml.dist | 2 +- tests/SchemaOrgContext.php | 137 ++++++++++++++++++ tests/bootstrap.php | 6 + ...rather-ok.tumblr.com.post-165326273724.php | 2 +- 4 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 tests/SchemaOrgContext.php create mode 100644 tests/bootstrap.php diff --git a/phpunit.xml.dist b/phpunit.xml.dist index d8e22e25..8bc80df3 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -2,7 +2,7 @@ isSchemaHost($path)) { + return false; + } + + $this->body = self::BODY; + $this->position = 0; + $this->notifyMimeType(); + + return true; + } + + /** + * @param int $count + */ + public function stream_read($count): string + { + $chunk = substr($this->body, $this->position, $count); + $this->position += strlen($chunk); + + return $chunk; + } + + public function stream_eof(): bool + { + return $this->position >= strlen($this->body); + } + + /** + * @return int + */ + public function stream_tell() + { + return $this->position; + } + + /** + * @return array + */ + public function stream_stat(): array + { + $size = strlen($this->body); + + return [ + 'size' => $size, + 7 => $size, + ]; + } + + public function stream_close(): void + { + // The context body is a string, so there is no handle to close. + } + + private function notifyMimeType(): void + { + if (!isset($this->context)) { + return; + } + + $params = stream_context_get_params($this->context); + if (!isset($params['notification']) || !is_callable($params['notification'])) { + return; + } + + $params['notification']( + STREAM_NOTIFY_MIME_TYPE_IS, + STREAM_NOTIFY_SEVERITY_INFO, + 'application/ld+json', + 0, + 0, + 0 + ); + } + + /** + * @param mixed $url + */ + private function isSchemaHost($url): bool + { + if (!is_string($url) || $url === '') { + return false; + } + + $host = parse_url($url, PHP_URL_HOST); + if (!is_string($host) || $host === '') { + return false; + } + + return in_array(strtolower($host), [ + 'schema.org', + 'www.schema.org', + 'schema.googleapis.com', + ], true); + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php new file mode 100644 index 00000000..b93a76cd --- /dev/null +++ b/tests/bootstrap.php @@ -0,0 +1,6 @@ + null, 'providerName' => 'Tumblr', 'providerUrl' => 'https://www.tumblr.com/', - 'publishedTime' => null, + 'publishedTime' => '2017-09-14 10:20:35', 'redirect' => null, 'title' => 'He-who-photographs-rather-OK', 'url' => 'https://www.tumblr.com/he-who-photographs-rather-ok/165326273724',