diff --git a/CHANGELOG.md b/CHANGELOG.md index c5845778..3ca89eaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [Unreleased] +### Security +- Block requests to non-public addresses before they are sent, including redirects, page-declared oEmbed endpoints, adapter APIs, and meta-refresh targets. Private and loopback URLs are rejected unless `UrlPolicy::allowPrivateNetworks()` or `withAllowedHosts()` is used [#573] + +### Added +- `Embed\Http\UrlPolicy` and `Embed\Http\BlockedRequestException`, configurable from `Crawler::setUrlPolicy()` + +### Changed +- A negative `max_redirs`, which curl treated as unlimited, now follows at most 50 redirects +- Reaching that limit is still curl error 47. The message is `Number of redirects hit maximum amount` rather than curl's `Maximum (N) redirects followed` + +### Fixed +- `resolveUri()` dropped the port of the base URL +- Read every `curl_multi_info_read` message in the parallel client instead of only one per iteration + ## [4.4.19] - 2026-07-08 ### Changed - Access tokens are now optional with Meta (Facebook & Instagram) [#577] @@ -299,6 +314,7 @@ Full library refactoring. [#569]: https://github.com/oscarotero/Embed/issues/569 [#571]: https://github.com/oscarotero/Embed/issues/571 [#572]: https://github.com/oscarotero/Embed/issues/572 +[#573]: https://github.com/oscarotero/Embed/issues/573 [#574]: https://github.com/oscarotero/Embed/issues/574 [#577]: https://github.com/oscarotero/Embed/issues/577 diff --git a/README.md b/README.md index b610ee32..e8173ef5 100644 --- a/README.md +++ b/README.md @@ -329,12 +329,12 @@ $client = new CurlClient(); $client->setSettings([ 'cookies_path' => $cookies_path, 'ignored_errors' => [18], - 'max_redirs' => 3, // see CURLOPT_MAXREDIRS - 'connect_timeout' => 2, // see CURLOPT_CONNECTTIMEOUT - 'timeout' => 2, // see CURLOPT_TIMEOUT - 'ssl_verify_host' => 2, // see CURLOPT_SSL_VERIFYHOST - 'ssl_verify_peer' => 1, // see CURLOPT_SSL_VERIFYPEER - 'follow_location' => true, // see CURLOPT_FOLLOWLOCATION + 'max_redirs' => 3, // redirects to follow; a negative value means up to 50 + 'connect_timeout' => 2, // per connection, see CURLOPT_CONNECTTIMEOUT + 'timeout' => 2, // total seconds for the request, including redirects + 'ssl_verify_host' => 2, // 0 disables hostname checks; any other value enables them + 'ssl_verify_peer' => 1, // falsy disables certificate checks; any other value enables them + 'follow_location' => true, // false returns the 3xx; each hop is checked 'user_agent' => 'Mozilla', // see CURLOPT_USERAGENT ]); @@ -359,6 +359,89 @@ $info = $embed->get($url); Note: The built-in detectors does not require settings. This feature is only for convenience if you create a specific detector that requires settings. +## Security / SSRF protection + +The default HTTP client refuses to connect to non-public addresses. That covers the URL you pass to `get()` / `getMulti()` and every later request Embed makes itself (redirects, oEmbed endpoints declared by the page, adapter APIs, and meta-refresh targets). A hostname is allowed only when every IPv4 address it resolves to is public. Loopback, private, link-local, CGNAT, documentation, and multicast ranges are rejected. IPv4-mapped IPv6, NAT64, and 6to4 are rejected when the embedded IPv4 address is not public. The Teredo prefix `2001::/32` is rejected as a whole, not by reading an address out of it. Only `http` and `https` are allowed, and redirects are followed inside Embed so each hop is checked the same way. + +`follow_location` set to false still returns the 3xx response. `max_redirs` is still how many redirects are followed (10 by default). A negative value, which curl treated as unlimited, follows at most 50. `timeout` is the total time for the request and its redirects; `connect_timeout` is still per connection. `ssl_verify_host` of `0` disables hostname checks, and any other value enables them, as curl does when it receives `1` or `true`. `ssl_verify_peer` is off only for a falsy value (`0`, `false`, `'0'`, `''`); `1`, `2`, `'true'` and `'yes'` keep certificate checks on. Reaching the redirect limit still fails with curl error 47. The message is `Number of redirects hit maximum amount`. + +This is a behavior change: a URL that used to be fetched from an intranet now throws `Embed\Http\BlockedRequestException`. To opt back in: + +```php +use Embed\Http\UrlPolicy; + +$embed->getCrawler()->setUrlPolicy( + UrlPolicy::default()->allowPrivateNetworks() +); +``` + +You can allow specific names without opening every private network. `*.corp` matches any host that ends in `.corp`. Allowing one resolved address does not allow the other addresses returned for that name: + +```php +$embed->getCrawler()->setUrlPolicy( + UrlPolicy::default()->withAllowedHosts('wiki.corp', '*.corp') +); +``` + +`withAllowedPorts([80, 443])` is optional. When it is set, an omitted port counts as 80 or 443. Internationalized host names are converted with `idn_to_ascii()`, which needs the `intl` extension; without it those hosts are rejected. The built-in client pins the punycode form, because that is the name libcurl looks up. + +Values such as `image` or `favicon` are URLs extracted from the page. Embed does not download them, so it does not apply this policy to them. Check one before you fetch it yourself: + +```php +$policy = $embed->getCrawler()->getUrlPolicy(); +$image = $info->image; + +if ($image !== null) { + $policy->validate($embed->getCrawler()->createRequest('GET', (string) $image)); +} +``` + +A page-declared oEmbed endpoint, adapter API, or meta-refresh target that the policy rejects is skipped, and extraction continues with the rest of the page. The URL passed to `get()` or `getMulti()` is not skipped: that request throws. In `getMulti()`, one rejected URL rejects the whole batch before any request is sent. A redirect discovered after the batch has started is checked on the same thread, including its DNS lookup, so the other transfers in that batch wait until the check returns. + +`Crawler::setUrlPolicy()` copies the policy onto the client only when that client is exactly `Embed\Http\CurlClient`. A decorator or other wrapper around `CurlClient` is not updated. Call `setUrlPolicy()` on the inner client as well. Otherwise the wrapper is pre-checked with the crawler's policy, and the inner client keeps the policy it already had. + +### Custom PSR-18 clients + +If you pass your own PSR-18 client, Embed validates the URI before calling it and does not validate again inside that client. Two things are then outside Embed's control: + +* DNS can change between the check and the connection (rebinding). +* Redirects followed by the client are not checked, unless the client gives you a hook. + +The built-in curl client pins the validated addresses with `CURLOPT_RESOLVE` and follows redirects itself, so those gaps do not apply to it. With Guzzle you can repeat the check on each redirect: + +```php +use Embed\Http\UrlPolicy; +use GuzzleHttp\Client; +use GuzzleHttp\HandlerStack; +use GuzzleHttp\Middleware; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; +use Psr\Http\Message\UriInterface; + +$policy = UrlPolicy::default(); +$stack = HandlerStack::create(); +$stack->push(Middleware::mapRequest(function (RequestInterface $request) use ($policy) { + $policy->validate($request); + + return $request; +})); + +$client = new Client([ + 'handler' => $stack, + 'allow_redirects' => [ + 'on_redirect' => function (RequestInterface $request, ResponseInterface $response, UriInterface $uri) use ($policy): void { + $policy->validate($request->withUri($uri)); + }, + ], +]); +``` + +Even that hook cannot see the address the client actually connected to. Prefer the built-in client when the pages are untrusted. + +### Proxies + +If `http_proxy`, `https_proxy`, or `all_proxy` is set, curl connects to the proxy and the proxy resolves the origin. Embed still validates each URL before requesting it, and still checks redirects, but it cannot pin DNS or verify the address the proxy used. Point those variables only at a proxy you trust. + --- ## Testing diff --git a/composer.json b/composer.json index 993f65e5..eea29c00 100644 --- a/composer.json +++ b/composer.json @@ -47,6 +47,7 @@ "phpstan/phpstan-strict-rules": "^2.0" }, "suggest": { + "ext-intl": "To resolve internationalized domain names when protecting against SSRF", "symfony/css-selector": "If you want to get elements using css selectors" }, "autoload": { diff --git a/phpunit.xml.dist b/phpunit.xml.dist index d8e22e25..8bc80df3 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -2,7 +2,7 @@ crawler->createRequest('GET', (string) $redirectUri); - $response = $this->crawler->sendRequest($request); + try { + $response = $this->crawler->sendRequest($request); + } catch (BlockedRequestException $exception) { + return $extractor; + } return $this->extract($request, $response, false); } diff --git a/src/Http/BlockedRequestException.php b/src/Http/BlockedRequestException.php new file mode 100644 index 00000000..d54f1957 --- /dev/null +++ b/src/Http/BlockedRequestException.php @@ -0,0 +1,45 @@ +request = $request; + $this->reason = $reason; + } + + public function getRequest(): RequestInterface + { + return $this->request; + } + + /** + * @return 'scheme'|'host'|'non_public_address'|'port'|'redirect' + */ + public function getReason(): string + { + return $this->reason; + } +} diff --git a/src/Http/Crawler.php b/src/Http/Crawler.php index 2c233636..cffafb96 100644 --- a/src/Http/Crawler.php +++ b/src/Http/Crawler.php @@ -15,6 +15,7 @@ class Crawler implements ClientInterface, RequestFactoryInterface, UriFactoryInt private RequestFactoryInterface $requestFactory; private UriFactoryInterface $uriFactory; private ClientInterface $client; + private UrlPolicy $urlPolicy; /** @var array */ private array $defaultHeaders = [ 'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:73.0) Gecko/20100101 Firefox/73.0', @@ -26,6 +27,25 @@ public function __construct(?ClientInterface $client = null, ?RequestFactoryInte $this->client = $client !== null ? $client : new CurlClient(); $this->requestFactory = $requestFactory !== null ? $requestFactory : FactoryDiscovery::getRequestFactory(); $this->uriFactory = $uriFactory !== null ? $uriFactory : FactoryDiscovery::getUriFactory(); + $this->urlPolicy = $this->client instanceof CurlClient ? $this->client->getUrlPolicy() : UrlPolicy::default(); + } + + /** + * The policy is copied onto the HTTP client only when that client is + * exactly CurlClient. A decorator around CurlClient is not detected; + * set the policy on the inner client as well. + */ + public function setUrlPolicy(UrlPolicy $urlPolicy): void + { + $this->urlPolicy = $urlPolicy; + if ($this->client instanceof CurlClient) { + $this->client->setUrlPolicy($urlPolicy); + } + } + + public function getUrlPolicy(): UrlPolicy + { + return $this->urlPolicy; } /** @@ -57,6 +77,8 @@ public function createUri(string $uri = ''): UriInterface public function sendRequest(RequestInterface $request): ResponseInterface { + $this->validateCustomClient($request); + return $this->client->sendRequest($request); } @@ -69,12 +91,25 @@ public function sendRequests(RequestInterface ...$requests): array return $this->client->sendRequests(...$requests); } + foreach ($requests as $request) { + $this->urlPolicy->validate($request); + } + return array_map( fn ($request) => $this->client->sendRequest($request), $requests ); } + private function validateCustomClient(RequestInterface $request): void + { + if ($this->client instanceof CurlClient) { + return; + } + + $this->urlPolicy->validate($request); + } + public function getResponseUri(ResponseInterface $response): ?UriInterface { $location = $response->getHeaderLine('Content-Location'); diff --git a/src/Http/CurlClient.php b/src/Http/CurlClient.php index 9794d718..26a43938 100644 --- a/src/Http/CurlClient.php +++ b/src/Http/CurlClient.php @@ -14,12 +14,14 @@ final class CurlClient implements ClientInterface { private ResponseFactoryInterface $responseFactory; + private UrlPolicy $urlPolicy; /** @var array */ private array $settings = []; public function __construct(?ResponseFactoryInterface $responseFactory = null) { $this->responseFactory = $responseFactory !== null ? $responseFactory : FactoryDiscovery::getResponseFactory(); + $this->urlPolicy = UrlPolicy::default(); } /** @@ -30,9 +32,19 @@ public function setSettings(array $settings): void $this->settings = $settings + $this->settings; } + public function setUrlPolicy(UrlPolicy $urlPolicy): void + { + $this->urlPolicy = $urlPolicy; + } + + public function getUrlPolicy(): UrlPolicy + { + return $this->urlPolicy; + } + public function sendRequest(RequestInterface $request): ResponseInterface { - $responses = CurlDispatcher::fetch($this->settings, $this->responseFactory, $request); + $responses = CurlDispatcher::fetch($this->dispatchSettings(), $this->responseFactory, $request); return $responses[0]; } @@ -42,6 +54,17 @@ public function sendRequest(RequestInterface $request): ResponseInterface */ public function sendRequests(RequestInterface ...$request): array { - return CurlDispatcher::fetch($this->settings, $this->responseFactory, ...$request); + return CurlDispatcher::fetch($this->dispatchSettings(), $this->responseFactory, ...$request); + } + + /** + * @return array + */ + private function dispatchSettings(): array + { + $settings = $this->settings; + $settings['url_policy'] = $this->urlPolicy; + + return $settings; } } diff --git a/src/Http/CurlDispatcher.php b/src/Http/CurlDispatcher.php index 0658c0fd..7f9eec3b 100644 --- a/src/Http/CurlDispatcher.php +++ b/src/Http/CurlDispatcher.php @@ -4,23 +4,32 @@ namespace Embed\Http; use Composer\CaBundle\CaBundle; +use function Embed\resolveUri; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseFactoryInterface; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\StreamFactoryInterface; use Psr\Http\Message\StreamInterface; +use Psr\Http\Message\UriFactoryInterface; /** * Class to fetch html pages * + * Redirects are followed here, not by libcurl, so every hop is checked against + * the URL policy and pinned to the addresses that were validated. + * * @phpstan-type CurlResource resource|\CurlHandle */ final class CurlDispatcher { + private const REDIRECT_STATUSES = [301, 302, 303, 307, 308]; + private static int $contentLengthThreshold = 5000000; private RequestInterface $request; private StreamFactoryInterface $streamFactory; + private UriFactoryInterface $uriFactory; + private UrlPolicy $policy; /** * @var resource|\CurlHandle * @phpstan-ignore property.unusedType (resource type needed for PHP 7.4 compatibility) @@ -33,6 +42,15 @@ final class CurlDispatcher private ?int $error = null; /** @var array */ private array $settings; + /** @var array */ + private array $pinnedIps; + private bool $proxy; + private bool $follow; + private int $maxRedirs; + private ?float $deadline; + private bool $prereqBlocked = false; + private bool $closed = false; + private int $followed = 0; /** * @param array $settings @@ -40,110 +58,593 @@ final class CurlDispatcher */ public static function fetch(array $settings, ResponseFactoryInterface $responseFactory, RequestInterface ...$requests): array { + if ($requests === []) { + return []; + } + + $policy = self::policy($settings); + $proxy = self::environmentProxyEnabled(); + $pins = []; + foreach ($requests as $request) { + $pins[] = $policy->validate($request); + } + if (count($requests) === 1) { - $connection = new static($settings, $requests[0]); - /** @var resource|\CurlHandle $curlHandle */ - $curlHandle = $connection->curl; - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ - curl_exec($curlHandle); - return [$connection->getResponse($responseFactory)]; + $connection = new static($settings, $requests[0], $pins[0], $policy, $proxy); + try { + $connection->execute(); + + return [$connection->getResponse($responseFactory)]; + } catch (\Throwable $exception) { + $connection->closeCurl(); + throw $exception; + } + } + + return self::fetchMulti($settings, $responseFactory, array_values($requests), $pins, $policy, $proxy); + } + + /** + * @param array $settings + */ + private static function policy(array $settings): UrlPolicy + { + $policy = $settings['url_policy'] ?? null; + if ($policy instanceof UrlPolicy) { + return $policy; + } + + return UrlPolicy::default(); + } + + private static function environmentProxyEnabled(): bool + { + foreach (['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'] as $name) { + $value = getenv($name); + if (is_string($value) && trim($value) !== '') { + return true; + } } + return false; + } + + /** + * @param array $settings + * @param array $requests + * @param array> $pins + * @return ResponseInterface[] + */ + private static function fetchMulti( + array $settings, + ResponseFactoryInterface $responseFactory, + array $requests, + array $pins, + UrlPolicy $policy, + bool $proxy + ): array { //Init connections $multi = curl_multi_init(); + // curl_multi_init() can fail on PHP 7.4. Current stubs type it as CurlMultiHandle only. + /** @phpstan-ignore identical.alwaysFalse */ + if ($multi === false) { + throw new NetworkException('Unable to initialize curl', 0, $requests[0]); + } $connections = []; - foreach ($requests as $request) { - $connection = new static($settings, $request); - /** @var resource|\CurlHandle $curlHandle */ - $curlHandle = $connection->curl; - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ - curl_multi_add_handle($multi, $curlHandle); + try { + foreach ($requests as $index => $request) { + $connection = new static($settings, $request, $pins[$index], $policy, $proxy); + $connections[] = $connection; + /** @var resource|\CurlHandle $curlHandle */ + $curlHandle = $connection->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + curl_multi_add_handle($multi, $curlHandle); + } - $connections[] = $connection; + //Run + self::pump($multi, $connections); + } catch (\Throwable $exception) { + self::releaseMulti($multi, $connections, true); + throw $exception; } - //Run - $active = null; + //Close connections + self::releaseMulti($multi, $connections, false); + + $responses = []; + foreach ($connections as $connection) { + $responses[] = $connection->getResponse($responseFactory); + } + + return $responses; + } + + /** + * On PHP 7.4 $multi is a resource. PHP 8 uses CurlMultiHandle. + * + * @param resource|\CurlMultiHandle $multi + * @param array $connections + */ + private static function pump($multi, array $connections): void + { + $active = 0; + $status = CURLM_OK; + do { - $status = curl_multi_exec($multi, $active); + do { + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + $status = curl_multi_exec($multi, $active); + } while ($status === CURLM_CALL_MULTI_PERFORM); + + $requeued = false; + while (true) { + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + $info = curl_multi_info_read($multi); + if (!is_array($info)) { + break; + } - if ($active) { - curl_multi_select($multi); - } + $result = $info['result'] ?? null; + $handle = $info['handle'] ?? null; + if (!is_int($result)) { + continue; + } - $info = curl_multi_info_read($multi); - - if (is_array($info) && isset($info['handle'], $info['result'])) { - $result = $info['result']; - // Validate and cast result to int, only set if it's a non-success error code - if (is_numeric($result)) { - $errorCode = (int) $result; - if ($errorCode !== CURLE_OK) { - foreach ($connections as $connection) { - if ($connection->curl === $info['handle']) { - $connection->error = $errorCode; - break; - } - } + foreach ($connections as $connection) { + if ($connection->curl !== $handle) { + continue; + } + if (!$connection->finishTransfer($result)) { + break; } + + $curlHandle = $connection->curl; + /** @phpstan-ignore argument.type, argument.type (PHP 7.4/8.0 compatibility) */ + curl_multi_remove_handle($multi, $curlHandle); + /** @phpstan-ignore argument.type, argument.type (PHP 7.4/8.0 compatibility) */ + curl_multi_add_handle($multi, $curlHandle); + $requeued = true; + break; } } - } while ($active && $status === CURLM_OK); - //Close connections + if ($requeued) { + $active = 1; + } + + if ($active !== 0 && $status === CURLM_OK) { + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + $selected = curl_multi_select($multi, 1.0); + if ($selected === -1) { + usleep(10000); + } + } + } while ($active !== 0 && $status === CURLM_OK); + } + + /** + * On PHP 7.4 $multi is a resource. PHP 8 uses CurlMultiHandle. + * + * @param resource|\CurlMultiHandle $multi + * @param array $connections + */ + private static function releaseMulti($multi, array $connections, bool $closeHandles): void + { foreach ($connections as $connection) { /** @var resource|\CurlHandle $curlHandle */ $curlHandle = $connection->curl; - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + /** @phpstan-ignore argument.type, argument.type (PHP 7.4/8.0 compatibility) */ curl_multi_remove_handle($multi, $curlHandle); + if ($closeHandles) { + $connection->closeCurl(); + } } if (PHP_VERSION_ID < 80000) { + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ curl_multi_close($multi); } - - return array_map( - fn ($connection) => $connection->getResponse($responseFactory), - $connections - ); } /** * @param array $settings + * @param array $pinnedIps */ - private function __construct(array $settings, RequestInterface $request, ?StreamFactoryInterface $streamFactory = null) + private function __construct(array $settings, RequestInterface $request, array $pinnedIps, UrlPolicy $policy, bool $proxy) { + $curl = curl_init(); + if ($curl === false) { + throw new NetworkException('Unable to initialize curl', 0, $request); + } + $this->request = $request; - $this->curl = curl_init((string) $request->getUri()); $this->settings = $settings; - $this->streamFactory = $streamFactory ?? FactoryDiscovery::getStreamFactory(); - - $cookies = $settings['cookies_path'] ?? str_replace('//', '/', sys_get_temp_dir().'/embed-cookies.txt'); - - curl_setopt_array($this->curl, [ - CURLOPT_HTTPHEADER => $this->getRequestHeaders(), - CURLOPT_POST => strtoupper($request->getMethod()) === 'POST', - CURLOPT_MAXREDIRS => $settings['max_redirs'] ?? 10, - CURLOPT_CONNECTTIMEOUT => $settings['connect_timeout'] ?? 10, - CURLOPT_TIMEOUT => $settings['timeout'] ?? 10, - CURLOPT_RETURNTRANSFER => true, - CURLOPT_SSL_VERIFYHOST => $settings['ssl_verify_host'] ?? 0, - CURLOPT_SSL_VERIFYPEER => $settings['ssl_verify_peer'] ?? false, - CURLOPT_ENCODING => '', - CURLOPT_CAINFO => CaBundle::getSystemCaRootBundlePath(), - CURLOPT_AUTOREFERER => true, - CURLOPT_FOLLOWLOCATION => $settings['follow_location'] ?? true, - CURLOPT_IPRESOLVE => CURL_IPRESOLVE_V4, - CURLOPT_USERAGENT => $settings['user_agent'] ?? $request->getHeaderLine('User-Agent'), - CURLOPT_COOKIEJAR => $cookies, - CURLOPT_COOKIEFILE => $cookies, - CURLOPT_HEADERFUNCTION => [$this, 'writeHeader'], - CURLOPT_WRITEFUNCTION => [$this, 'writeBody'], - ]); + $this->pinnedIps = $pinnedIps; + $this->policy = $policy; + $this->proxy = $proxy; + $this->follow = $this->settingBool('follow_location', true); + $this->maxRedirs = $this->redirectLimit(); + $timeout = $this->settingInt('timeout', 10); + // libcurl's CURLOPT_TIMEOUT covers the whole transfer, including + // redirects. Each hop receives only the time that is still left. + $this->deadline = $timeout > 0 ? microtime(true) + $timeout : null; + $this->streamFactory = FactoryDiscovery::getStreamFactory(); + $this->uriFactory = FactoryDiscovery::getUriFactory(); + $this->curl = $curl; + + $this->applyBaseOptions(); + $this->applyToHandle(); + } + + private function execute(): void + { + while (true) { + $this->resetExchange(); + if ($this->timeoutExpired()) { + $this->error = CURLE_OPERATION_TIMEDOUT; + + return; + } + $this->applyTimeout(); + $this->execOnce(); + $this->guardConnection(); + + if (!$this->shouldFollowRedirect()) { + return; + } + + if ($this->followed >= $this->maxRedirs) { + $this->error = CURLE_TOO_MANY_REDIRECTS; + + return; + } + + $next = $this->redirectRequest(); + if ($next === null) { + return; + } + + $this->request = $next; + $this->pinnedIps = $this->policy->validate($next); + $this->applyToHandle(); + ++$this->followed; + } + } + + /** + * @return bool true when the handle was updated and must be run again + */ + private function finishTransfer(int $result): bool + { + if ($this->prereqBlocked) { + throw new BlockedRequestException( + 'The URL resolves to a non-public address.', + $this->request, + 'non_public_address' + ); + } + + if ($result !== CURLE_OK && !($this->isBinary && $result === CURLE_WRITE_ERROR)) { + $this->error = $result; + + return false; + } + + $this->guardPrimaryIp(); + + if (!$this->shouldFollowRedirect()) { + return false; + } + + if ($this->followed >= $this->maxRedirs) { + $this->error = CURLE_TOO_MANY_REDIRECTS; + + return false; + } + + if ($this->timeoutExpired()) { + $this->error = CURLE_OPERATION_TIMEDOUT; + + return false; + } + + $next = $this->redirectRequest(); + if ($next === null) { + return false; + } + + $this->request = $next; + // DNS for this hop runs here, on the multi thread, so the other + // transfers in the batch wait until validate() returns. + $this->pinnedIps = $this->policy->validate($next); + $this->applyToHandle(); + $this->resetExchange(); + ++$this->followed; + + return true; + } + + private function execOnce(): void + { + /** @var resource|\CurlHandle $curlHandle */ + $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + curl_exec($curlHandle); + } + + private function guardConnection(): void + { + if ($this->prereqBlocked) { + throw new BlockedRequestException( + 'The URL resolves to a non-public address.', + $this->request, + 'non_public_address' + ); + } + + $this->guardPrimaryIp(); + + $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + $errno = curl_errno($curlHandle); + if ($errno === 0 || ($this->isBinary && $errno === CURLE_WRITE_ERROR)) { + return; + } + + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + $this->error(curl_error($curlHandle), $errno); + } + + private function guardPrimaryIp(): void + { + if ($this->proxy) { + return; + } + + $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + $ip = curl_getinfo($curlHandle, CURLINFO_PRIMARY_IP); + if ($ip === '') { + return; + } + + if ($this->addressIsPinned($ip)) { + return; + } + + throw new BlockedRequestException( + 'The URL resolves to a non-public address.', + $this->request, + 'non_public_address' + ); + } + + private function shouldFollowRedirect(): bool + { + if (!$this->follow) { + return false; + } + + return in_array($this->httpStatus(), self::REDIRECT_STATUSES, true); + } + + private function redirectRequest(): ?RequestInterface + { + $location = null; + foreach ($this->headers as $header) { + if ($header[0] === 'location') { + $location = $header[1]; + } + } + + if ($location === null) { + return null; + } + + $location = trim($location); + if ($location === '') { + return null; + } + + if (preg_match('/[\x00-\x1f\x7f]/', $location) === 1) { + throw new BlockedRequestException( + 'The redirect target is not allowed.', + $this->request, + 'redirect' + ); + } + + try { + $target = $this->uriFactory->createUri($location); + } catch (\InvalidArgumentException $exception) { + throw new BlockedRequestException( + 'The redirect target is not allowed.', + $this->request, + 'redirect' + ); + } + + $current = $this->request->getUri(); + $resolved = resolveUri($current, $target); + $method = $this->methodAfterRedirect($this->httpStatus()); + $request = $this->request->withMethod($method)->withUri($resolved); + + if ($method === 'GET' || $method === 'HEAD') { + $request = $request + ->withBody($this->streamFactory->createStream('')) + ->withoutHeader('Content-Type') + ->withoutHeader('Content-Length'); + } + + $currentHost = strtolower($this->bareHost($current->getHost())); + $nextHost = strtolower($this->bareHost($resolved->getHost())); + if ($currentHost !== $nextHost) { + $request = $request->withoutHeader('Authorization')->withoutHeader('Cookie'); + } + + return $request->withHeader('Referer', (string) $current); + } + + private function methodAfterRedirect(int $status): string + { + $method = strtoupper($this->request->getMethod()); + if ($status === 303) { + return $method === 'HEAD' ? 'HEAD' : 'GET'; + } + + if (($status === 301 || $status === 302) && $method === 'POST') { + return 'GET'; + } + + return $method; + } + + private function bareHost(string $host): string + { + if (strlen($host) >= 2 && $host[0] === '[' && substr($host, -1) === ']') { + return substr($host, 1, -1); + } + + return $host; + } + + private function addressIsPinned(string $ip): bool + { + foreach ($this->pinnedIps as $pinned) { + if (IpClassifier::equals($pinned, $ip)) { + return true; + } + } + + return false; + } + + private function applyBaseOptions(): void + { + $cookies = $this->cookiesPath(); + $caBundle = CaBundle::getSystemCaRootBundlePath(); + + $this->setopt(CURLOPT_CONNECTTIMEOUT, $this->settingInt('connect_timeout', 10)); + $this->setopt(CURLOPT_RETURNTRANSFER, true); + $this->setopt(CURLOPT_SSL_VERIFYHOST, $this->sslVerifyHost()); + $this->setopt(CURLOPT_SSL_VERIFYPEER, $this->sslVerifyPeer()); + $this->setopt(CURLOPT_ENCODING, ''); + if ($caBundle !== '') { + $this->setopt(CURLOPT_CAINFO, $caBundle); + } + $this->setopt(CURLOPT_FOLLOWLOCATION, false); + $this->setopt(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + $this->setopt(CURLOPT_USERAGENT, $this->userAgent()); + $this->setopt(CURLOPT_COOKIEJAR, $cookies); + $this->setopt(CURLOPT_COOKIEFILE, $cookies); + $this->setopt(CURLOPT_HEADERFUNCTION, [$this, 'writeHeader']); + $this->setopt(CURLOPT_WRITEFUNCTION, [$this, 'writeBody']); + $this->applyProtocolLimits(); + $this->applyPrereq(); + } + + private function applyProtocolLimits(): void + { + if (defined('CURLOPT_PROTOCOLS_STR')) { + $this->setopt(CURLOPT_PROTOCOLS_STR, 'HTTP,HTTPS'); + $this->setopt(CURLOPT_REDIR_PROTOCOLS_STR, 'HTTP,HTTPS'); + + return; + } + + $this->setopt(CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + $this->setopt(CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + } + + private function applyPrereq(): void + { + if (PHP_VERSION_ID < 80400 || !defined('CURLOPT_PREREQFUNCTION')) { + return; + } + + $this->setopt(CURLOPT_PREREQFUNCTION, [$this, 'approveConnection']); + } + + /** + * Called by libcurl on PHP 8.4+ before the request is sent. + * + * @param resource|\CurlHandle $curl + */ + public function approveConnection($curl, string $primaryIp, string $localIp, int $primaryPort, int $localPort): int + { + if ($this->proxy || $this->addressIsPinned($primaryIp)) { + return CURL_PREREQFUNC_OK; + } + + $this->prereqBlocked = true; + + return CURL_PREREQFUNC_ABORT; + } + + private function applyToHandle(): void + { + $this->setopt(CURLOPT_URL, (string) $this->request->getUri()); + $this->setopt(CURLOPT_HTTPHEADER, $this->getRequestHeaders()); + + $this->applyTimeout(); + + $method = strtoupper($this->request->getMethod()); + if ($method === 'POST') { + $this->setopt(CURLOPT_POST, true); + $this->setopt(CURLOPT_POSTFIELDS, (string) $this->request->getBody()); + } elseif ($method === 'HEAD') { + $this->setopt(CURLOPT_NOBODY, true); + $this->setopt(CURLOPT_HTTPGET, true); + } else { + $this->setopt(CURLOPT_HTTPGET, true); + $this->setopt(CURLOPT_POST, false); + } + + $this->applyResolve(); + } + + private function applyResolve(): void + { + if ($this->proxy || $this->pinnedIps === []) { + return; + } + + $uri = $this->request->getUri(); + $host = $this->policy->curlResolveHost($uri->getHost()); + $port = $uri->getPort(); + if ($port === null) { + $port = strtolower($uri->getScheme()) === 'https' ? 443 : 80; + } + + $ips = $this->pinnedIps; + $version = curl_version(); + $curlVersion = is_array($version) && isset($version['version']) && is_string($version['version']) + ? $version['version'] + : '0'; + if (version_compare($curlVersion, '7.59.0', '<')) { + $ips = [$ips[0]]; + } + + $formatted = []; + foreach ($ips as $ip) { + $formatted[] = strpos($ip, ':') !== false ? '['.$ip.']' : $ip; + } + + $resolveHost = strpos($host, ':') !== false ? '['.$host.']' : $host; + $this->setopt(CURLOPT_RESOLVE, [$resolveHost.':'.$port.':'.implode(',', $formatted)]); } private function getResponse(ResponseFactoryInterface $responseFactory): ResponseInterface + { + try { + return $this->buildResponse($responseFactory); + } finally { + $this->closeCurl(); + } + } + + private function buildResponse(ResponseFactoryInterface $responseFactory): ResponseInterface { /** @var resource|\CurlHandle $curlHandle */ $curlHandle = $this->curl; @@ -157,16 +658,11 @@ private function getResponse(ResponseFactoryInterface $responseFactory): Respons /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $errno = curl_errno($curlHandle); - if ($errno !== 0) { + if ($errno !== 0 && !($this->isBinary && $errno === CURLE_WRITE_ERROR)) { /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ $this->error(curl_error($curlHandle), $errno); } - if (PHP_VERSION_ID < 80000) { - /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ - curl_close($curlHandle); - } - $response = $responseFactory->createResponse($info['http_code']); foreach ($this->headers as $header) { @@ -174,8 +670,10 @@ private function getResponse(ResponseFactoryInterface $responseFactory): Respons $response = $response->withAddedHeader($name, $value); } + $effective = $info['url'] !== '' ? $info['url'] : (string) $this->request->getUri(); + $response = $response - ->withAddedHeader('Content-Location', $info['url']) + ->withAddedHeader('Content-Location', $effective) ->withAddedHeader('X-Request-Time', sprintf('%.3f ms', $info['total_time'])); if ($this->body !== null) { @@ -204,6 +702,160 @@ private function error(string $message, int $code): void throw new NetworkException($message, $code, $this->request); } + private function closeCurl(): void + { + if ($this->closed || PHP_VERSION_ID >= 80000) { + $this->closed = true; + + return; + } + + $this->closed = true; + $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + curl_close($curlHandle); + } + + private function resetExchange(): void + { + $this->headers = []; + $this->body = null; + $this->isBinary = false; + $this->error = null; + $this->prereqBlocked = false; + } + + private function httpStatus(): int + { + $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + return curl_getinfo($curlHandle, CURLINFO_HTTP_CODE); + } + + /** + * @param mixed $value + */ + private function setopt(int $option, $value): void + { + $curlHandle = $this->curl; + /** @phpstan-ignore argument.type (PHP 7.4/8.0 compatibility) */ + curl_setopt($curlHandle, $option, $value); + } + + private function settingInt(string $key, int $default): int + { + $value = $this->settings[$key] ?? $default; + if (is_int($value)) { + return $value; + } + if (is_float($value) || (is_string($value) && is_numeric($value))) { + return (int) $value; + } + + return $default; + } + + private function settingBool(string $key, bool $default): bool + { + $value = $this->settings[$key] ?? $default; + if (is_bool($value)) { + return $value; + } + if ($value === 1 || $value === '1') { + return true; + } + if ($value === 0 || $value === '0') { + return false; + } + + return $default; + } + + /** + * libcurl accepts only 0 and 2. PHP turns 1 into 2 and emits a notice, + * which is what a truthy setting used to do when it was passed through. + * + * @return 0|2 + */ + private function sslVerifyHost(): int + { + $value = $this->settings['ssl_verify_host'] ?? 0; + if ($value === false || $value === 0 || $value === 0.0 || $value === '' || $value === '0') { + return 0; + } + + return 2; + } + + /** + * A negative max_redirs is curl's "unlimited". Cap it so a redirect loop + * cannot run without a bound now that we follow hops ourselves. + */ + private function redirectLimit(): int + { + $limit = $this->settingInt('max_redirs', 10); + if ($limit < 0) { + return 50; + } + + return $limit; + } + + private function timeoutExpired(): bool + { + return $this->deadline !== null && microtime(true) >= $this->deadline; + } + + private function applyTimeout(): void + { + if ($this->deadline === null) { + $this->setopt(CURLOPT_TIMEOUT, 0); + + return; + } + + $remainingMs = (int) floor(($this->deadline - microtime(true)) * 1000); + if ($remainingMs < 1) { + $remainingMs = 1; + } + + $this->setopt(CURLOPT_TIMEOUT_MS, $remainingMs); + } + + /** + * Same rule as a value passed straight to curl: only a falsy setting + * disables certificate verification. 2, "true" and "yes" stay enabled. + */ + private function sslVerifyPeer(): bool + { + $value = $this->settings['ssl_verify_peer'] ?? false; + if ($value === false || $value === 0 || $value === 0.0 || $value === '' || $value === '0' || $value === []) { + return false; + } + + return true; + } + + private function userAgent(): string + { + $value = $this->settings['user_agent'] ?? $this->request->getHeaderLine('User-Agent'); + if (is_string($value) && $value !== '') { + return $value; + } + + return 'Embed/curl'; + } + + private function cookiesPath(): string + { + $value = $this->settings['cookies_path'] ?? null; + if (is_string($value) && $value !== '') { + return $value; + } + + return str_replace('//', '/', sys_get_temp_dir().'/embed-cookies.txt'); + } + /** * @return array */ diff --git a/src/Http/IpClassifier.php b/src/Http/IpClassifier.php new file mode 100644 index 00000000..7014d187 --- /dev/null +++ b/src/Http/IpClassifier.php @@ -0,0 +1,355 @@ + + */ + private const IPV4_NETWORKS = [ + ['0.0.0.0', 8], + ['10.0.0.0', 8], + ['100.64.0.0', 10], + ['127.0.0.0', 8], + ['169.254.0.0', 16], + ['172.16.0.0', 12], + ['192.0.0.0', 24], + ['192.0.2.0', 24], + ['192.88.99.0', 24], + ['192.168.0.0', 16], + ['198.18.0.0', 15], + ['198.51.100.0', 24], + ['203.0.113.0', 24], + ['224.0.0.0', 4], + ['240.0.0.0', 4], + ['255.255.255.255', 32], + ]; + + /** + * IPv6 networks that must not be contacted. + * + * @var array + */ + private const IPV6_NETWORKS = [ + ['::', 128], + ['::1', 128], + ['fc00::', 7], + ['fe80::', 10], + ['fec0::', 10], + ['ff00::', 8], + ['2001:db8::', 32], + ['100::', 64], + ['2001::', 32], // Teredo: the whole prefix, not the embedded IPv4 + ['64:ff9b:1::', 48], + ]; + + public static function isPublic(string $ip): bool + { + $packed = self::pack($ip); + if ($packed === null) { + return false; + } + + if (strlen($packed) === 4) { + return !self::inNetworks($packed, self::IPV4_NETWORKS); + } + + $embedded = self::embeddedIpv4($packed); + if ($embedded !== null) { + return !self::inNetworks($embedded, self::IPV4_NETWORKS); + } + + return !self::inNetworks($packed, self::IPV6_NETWORKS); + } + + /** + * Canonical textual form of an IP literal, including inet_aton spellings + * (hex, octal, and shortened IPv4). Returns null when $value is a hostname. + */ + public static function canonical(string $value): ?string + { + $value = self::unwrap($value); + if ($value === '') { + return null; + } + + if (filter_var($value, FILTER_VALIDATE_IP) !== false) { + return self::format($value); + } + + return self::inetAton($value); + } + + /** + * True when both values are the same address. IPv4 and its IPv4-mapped + * IPv6 form compare equal. + */ + public static function equals(string $left, string $right): bool + { + $a = self::pack($left); + $b = self::pack($right); + if ($a === null || $b === null) { + return false; + } + + if ($a === $b) { + return true; + } + + $a4 = self::asIpv4($a); + $b4 = self::asIpv4($b); + + return $a4 !== null && $a4 === $b4; + } + + private static function pack(string $ip): ?string + { + $canonical = self::canonical($ip); + if ($canonical === null) { + return null; + } + + $packed = inet_pton($canonical); + if ($packed === false) { + return null; + } + + return $packed; + } + + private static function format(string $ip): ?string + { + $packed = inet_pton($ip); + if ($packed === false) { + return null; + } + + $text = inet_ntop($packed); + + return $text === false ? null : $text; + } + + private static function unwrap(string $value): string + { + if (strlen($value) >= 2 && $value[0] === '[' && substr($value, -1) === ']') { + return substr($value, 1, -1); + } + + return $value; + } + + /** + * glibc inet_aton / libcurl accept these spellings and connect to the + * resulting address without consulting CURLOPT_RESOLVE. + */ + private static function inetAton(string $host): ?string + { + if (preg_match('/^(?:0x[0-9a-f]+|\d+)(?:\.(?:0x[0-9a-f]+|\d+)){0,3}$/i', $host) !== 1) { + return null; + } + + $parts = explode('.', $host); + $count = count($parts); + $numbers = []; + + foreach ($parts as $part) { + $number = self::parseComponent($part); + if ($number === null) { + return null; + } + $numbers[] = $number; + } + + $bytes = self::atonBytes($numbers, $count); + if ($bytes === null) { + return null; + } + + return $bytes[0].'.'.$bytes[1].'.'.$bytes[2].'.'.$bytes[3]; + } + + private static function parseComponent(string $part): ?float + { + if ($part === '') { + return null; + } + + if (stripos($part, '0x') === 0) { + $hex = substr($part, 2); + if ($hex === '' || strlen($hex) > 8 || !ctype_xdigit($hex)) { + return null; + } + + return (float) hexdec($hex); + } + + if (strlen($part) > 1 && $part[0] === '0') { + if (strlen($part) > 11 || strspn($part, '01234567') !== strlen($part)) { + return null; + } + + return (float) octdec($part); + } + + if (!ctype_digit($part) || strlen($part) > 10) { + return null; + } + + return (float) $part; + } + + /** + * @param array $numbers + * + * @return array{0: int, 1: int, 2: int, 3: int}|null + */ + private static function atonBytes(array $numbers, int $count): ?array + { + if ($count === 1) { + return self::split($numbers[0], 4); + } + + if ($count === 2) { + if ($numbers[0] > 255) { + return null; + } + $rest = self::split($numbers[1], 3); + if ($rest === null) { + return null; + } + + return [(int) $numbers[0], $rest[1], $rest[2], $rest[3]]; + } + + if ($count === 3) { + if ($numbers[0] > 255 || $numbers[1] > 255) { + return null; + } + $rest = self::split($numbers[2], 2); + if ($rest === null) { + return null; + } + + return [(int) $numbers[0], (int) $numbers[1], $rest[2], $rest[3]]; + } + + if ($count !== 4 || $numbers[0] > 255 || $numbers[1] > 255 || $numbers[2] > 255 || $numbers[3] > 255) { + return null; + } + + return [(int) $numbers[0], (int) $numbers[1], (int) $numbers[2], (int) $numbers[3]]; + } + + /** + * @return array{0: int, 1: int, 2: int, 3: int}|null + */ + private static function split(float $value, int $width): ?array + { + $limit = [1 => 255.0, 2 => 65535.0, 3 => 16777215.0, 4 => 4294967295.0]; + if ($value < 0 || $value > $limit[$width]) { + return null; + } + + $bytes = [0, 0, 0, 0]; + $remaining = $value; + for ($i = 3; $i >= 4 - $width; --$i) { + $bytes[$i] = (int) fmod($remaining, 256.0); + $remaining = floor($remaining / 256.0); + } + + return [$bytes[0], $bytes[1], $bytes[2], $bytes[3]]; + } + + private static function embeddedIpv4(string $packed): ?string + { + if (strlen($packed) !== 16) { + return null; + } + + // IPv4-mapped (::ffff:0:0/96), including the hexadecimal form. + if (substr($packed, 0, 12) === "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff") { + return substr($packed, 12, 4); + } + + // NAT64 well-known prefix 64:ff9b::/96, dotted or hex. + $nat64 = inet_pton('64:ff9b::'); + if ($nat64 !== false && substr($packed, 0, 12) === substr($nat64, 0, 12)) { + return substr($packed, 12, 4); + } + + // 6to4, 2002::/16. The IPv4 address occupies bits 16-47. + if (substr($packed, 0, 2) === "\x20\x02") { + return substr($packed, 2, 4); + } + + // Deprecated IPv4-compatible addresses, ::/96. + if (substr($packed, 0, 12) === str_repeat("\x00", 12)) { + return substr($packed, 12, 4); + } + + return null; + } + + private static function asIpv4(string $packed): ?string + { + if (strlen($packed) === 4) { + return $packed; + } + + if (strlen($packed) === 16 && substr($packed, 0, 12) === "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff") { + return substr($packed, 12, 4); + } + + return null; + } + + /** + * @param array $networks + */ + private static function inNetworks(string $packed, array $networks): bool + { + foreach ($networks as $network) { + if (self::inCidr($packed, $network[0], $network[1])) { + return true; + } + } + + return false; + } + + private static function inCidr(string $packed, string $network, int $bits): bool + { + $networkPacked = inet_pton($network); + if ($networkPacked === false || strlen($networkPacked) !== strlen($packed)) { + return false; + } + + $fullBytes = intdiv($bits, 8); + if ($fullBytes > 0 && substr($packed, 0, $fullBytes) !== substr($networkPacked, 0, $fullBytes)) { + return false; + } + + $remainder = $bits % 8; + if ($remainder === 0) { + return true; + } + + $mask = (0xFF << (8 - $remainder)) & 0xFF; + + return (ord($packed[$fullBytes]) & $mask) === (ord($networkPacked[$fullBytes]) & $mask); + } +} diff --git a/src/Http/UrlPolicy.php b/src/Http/UrlPolicy.php new file mode 100644 index 00000000..ed5e1d75 --- /dev/null +++ b/src/Http/UrlPolicy.php @@ -0,0 +1,323 @@ + */ + private array $allowedHosts; + + /** @var array|null */ + private ?array $allowedPorts; + + /** + * @var callable(string): array|null + */ + private $resolver; + + /** + * @param array $allowedHosts + * @param array|null $allowedPorts + * @param callable(string): array|null $resolver + */ + private function __construct(bool $allowPrivate, array $allowedHosts, ?array $allowedPorts, $resolver) + { + $this->allowPrivate = $allowPrivate; + $this->allowedHosts = $allowedHosts; + $this->allowedPorts = $allowedPorts; + $this->resolver = $resolver; + } + + public static function default(): self + { + return new self(false, [], null, null); + } + + /** + * Allow addresses that are not public (loopback, RFC 1918, link-local, ...). + * Use this for intranet installs. + */ + public function allowPrivateNetworks(): self + { + $copy = clone $this; + $copy->allowPrivate = true; + + return $copy; + } + + /** + * Hosts that may resolve to a non-public address. + * A pattern "*.corp" matches any name ending in ".corp". + */ + public function withAllowedHosts(string ...$hosts): self + { + $copy = clone $this; + $copy->allowedHosts = array_values($hosts); + + return $copy; + } + + /** + * Restrict the effective port (80 and 443 when the URI omits one). + * Passing no restriction is the default: every port is allowed. + * + * @param array $ports + */ + public function withAllowedPorts(array $ports): self + { + $normalized = []; + foreach ($ports as $port) { + if (!is_int($port)) { + throw new \InvalidArgumentException('Allowed ports must be a list of integers.'); + } + $normalized[] = $port; + } + + $copy = clone $this; + $copy->allowedPorts = $normalized; + + return $copy; + } + + /** + * Replace DNS. The callable receives an ASCII hostname and returns IPv4 strings. + * An empty list is treated as a DNS failure. + * + * @param callable(string): array $resolver + */ + public function withResolver(callable $resolver): self + { + $copy = clone $this; + $copy->resolver = $resolver; + + return $copy; + } + + /** + * @throws BlockedRequestException + * @throws NetworkException when the hostname cannot be resolved + * @return array addresses that were accepted, in lookup order + */ + public function validate(RequestInterface $request): array + { + $uri = $request->getUri(); + $scheme = strtolower($uri->getScheme()); + + if ($scheme !== 'http' && $scheme !== 'https') { + throw $this->blocked($request, 'scheme'); + } + + if (strpos((string) $uri, '\\') !== false || $uri->getUserInfo() !== '') { + throw $this->blocked($request, 'host'); + } + + $host = $this->host($uri->getHost()); + if ($host === null) { + throw $this->blocked($request, 'host'); + } + + $port = $uri->getPort(); + if ($port === 0) { + throw $this->blocked($request, 'port'); + } + + if ($this->allowedPorts !== null) { + $effective = $port ?? ($scheme === 'https' ? 443 : 80); + if (!in_array($effective, $this->allowedPorts, true)) { + throw $this->blocked($request, 'port'); + } + } + + $literal = IpClassifier::canonical($host); + if ($literal !== null) { + $ips = [$literal]; + $lookup = $host; + } else { + $lookup = $this->asciiHost($request, $host); + $ips = $this->resolve($request, $lookup); + } + + // One non-public address blocks the whole lookup. An allowlisted + // address does not exempt the others returned alongside it. + if (!$this->allowPrivate && !$this->hostAllowed($host) && !$this->hostAllowed($lookup)) { + foreach ($ips as $ip) { + if (!IpClassifier::isPublic($ip) && !$this->hostAllowed($ip)) { + throw $this->blocked($request, 'non_public_address'); + } + } + } + + return $ips; + } + + /** + * Host libcurl will look up, so a CURLOPT_RESOLVE entry matches it. + * Brackets are removed and internationalized names are punycode. + * A trailing dot is preserved: curl keeps it. + */ + public function curlResolveHost(string $host): string + { + if (strlen($host) >= 2 && $host[0] === '[' && substr($host, -1) === ']') { + $host = substr($host, 1, -1); + } + + if (preg_match('/[^\x00-\x7f]/', $host) !== 1 || !function_exists('idn_to_ascii')) { + return $host; + } + + $ascii = idn_to_ascii($host, IDNA_DEFAULT); + if (!is_string($ascii) || $ascii === '') { + return $host; + } + + return $ascii; + } + + private function host(string $host): ?string + { + if (strlen($host) >= 2 && $host[0] === '[' && substr($host, -1) === ']') { + $host = substr($host, 1, -1); + } + + $host = rtrim($host, '.'); + if ($host === '') { + return null; + } + + if (preg_match('/[\x00-\x20\x7f\\\\@\[\]]/', $host) === 1) { + return null; + } + + return $host; + } + + private function asciiHost(RequestInterface $request, string $host): string + { + if (preg_match('/[^\x00-\x7f]/', $host) !== 1) { + return $host; + } + + if (!function_exists('idn_to_ascii')) { + throw $this->blocked( + $request, + 'host', + 'Internationalized host names require the intl extension.' + ); + } + + $ascii = idn_to_ascii($host, IDNA_DEFAULT); + if ($ascii === false || $ascii === '') { + throw $this->blocked($request, 'host'); + } + + return $ascii; + } + + /** + * @return array + */ + private function resolve(RequestInterface $request, string $host): array + { + if ($this->resolver !== null) { + $resolver = $this->resolver; + $records = $resolver($host); + } else { + $lookedUp = gethostbynamel($host); + $records = $lookedUp === false ? [] : $lookedUp; + } + + $ips = []; + foreach ($records as $record) { + $canonical = IpClassifier::canonical($record); + if ($canonical === null) { + $ips = []; + break; + } + if (!in_array($canonical, $ips, true)) { + $ips[] = $canonical; + } + } + + if ($ips === []) { + throw new NetworkException('Could not resolve host: '.$host, CURLE_COULDNT_RESOLVE_HOST, $request); + } + + return $ips; + } + + private function hostAllowed(string $host): bool + { + $host = mb_strtolower($host, 'UTF-8'); + + foreach ($this->allowedHosts as $pattern) { + $pattern = mb_strtolower(rtrim($pattern, '.'), 'UTF-8'); + if ($pattern === '') { + continue; + } + + if (strlen($pattern) > 2 && $pattern[0] === '*' && $pattern[1] === '.') { + $suffix = substr($pattern, 1); + if (!is_string($suffix) || $suffix === '') { + continue; + } + $hostSuffix = substr($host, -strlen($suffix)); + if (strlen($host) > strlen($suffix) && is_string($hostSuffix) && $hostSuffix === $suffix) { + return true; + } + continue; + } + + if ($host === $pattern) { + return true; + } + } + + return false; + } + + /** + * @param 'scheme'|'host'|'non_public_address'|'port'|'redirect' $reason + */ + private function blocked(RequestInterface $request, string $reason, ?string $message = null): BlockedRequestException + { + if ($message === null) { + $message = self::message($reason); + } + + return new BlockedRequestException($message, $request, $reason); + } + + /** + * @param 'scheme'|'host'|'non_public_address'|'port'|'redirect' $reason + */ + private static function message(string $reason): string + { + if ($reason === 'scheme') { + return 'The URL scheme is not allowed.'; + } + if ($reason === 'host') { + return 'The URL host is not allowed.'; + } + if ($reason === 'non_public_address') { + return 'The URL resolves to a non-public address.'; + } + if ($reason === 'port') { + return 'The URL port is not allowed.'; + } + + return 'The redirect target is not allowed.'; + } +} diff --git a/src/HttpApiTrait.php b/src/HttpApiTrait.php index 8e4180bb..419fbdc5 100644 --- a/src/HttpApiTrait.php +++ b/src/HttpApiTrait.php @@ -24,7 +24,11 @@ private function fetchJSON(UriInterface $uri): array { $crawler = $this->extractor->getCrawler(); $request = $crawler->createRequest('GET', $uri); - $response = $crawler->sendRequest($request); + try { + $response = $crawler->sendRequest($request); + } catch (\Embed\Http\BlockedRequestException $exception) { + return []; + } try { $data = json_decode((string) $response->getBody(), true); diff --git a/src/OEmbed.php b/src/OEmbed.php index b530591f..37e05127 100644 --- a/src/OEmbed.php +++ b/src/OEmbed.php @@ -3,6 +3,7 @@ namespace Embed; +use Embed\Http\BlockedRequestException; use Exception; use Psr\Http\Message\UriInterface; use SimpleXMLElement; @@ -58,7 +59,11 @@ protected function fetchData(): array $crawler = $this->extractor->getCrawler(); $request = $crawler->createRequest('GET', $this->endpoint); - $response = $crawler->sendRequest($request); + try { + $response = $crawler->sendRequest($request); + } catch (BlockedRequestException $exception) { + return []; + } if (self::isXML($request->getUri())) { return $this->extractXML((string) $response->getBody()); diff --git a/src/functions.php b/src/functions.php index 9965ee63..09569e05 100644 --- a/src/functions.php +++ b/src/functions.php @@ -61,7 +61,7 @@ function resolveUri(UriInterface $base, UriInterface $uri): UriInterface $uri = $uri->withPath(resolvePath($base->getPath(), $uri->getPath())); if ($uri->getHost() === '') { - $uri = $uri->withHost($base->getHost()); + $uri = $uri->withHost($base->getHost())->withPort($base->getPort()); } if ($uri->getScheme() === '') { diff --git a/tests/CurlClientTest.php b/tests/CurlClientTest.php new file mode 100644 index 00000000..3b8fe605 --- /dev/null +++ b/tests/CurlClientTest.php @@ -0,0 +1,622 @@ +stop(); + self::clearProxyEnvironment(); + if (isset(self::$cookies) && is_file(self::$cookies)) { + unlink(self::$cookies); + } + } + + protected function setUp(): void + { + self::clearProxyEnvironment(); + self::$server->clearLog(); + if (is_file(self::$cookies)) { + file_put_contents(self::$cookies, ''); + } + } + + public function testInternationalizedHostIsPinnedInPunycode(): void + { + if (!function_exists('idn_to_ascii')) { + $this->markTestSkipped('The intl extension is required to resolve IDN hosts.'); + } + + $seen = null; + $policy = UrlPolicy::default() + ->withAllowedHosts('bücher.example') + ->withResolver(static function (string $host) use (&$seen): array { + $seen = $host; + + return ['127.0.0.1']; + }); + + $response = $this->client([], $policy)->sendRequest( + $this->request('http://bücher.example:'.self::$server->port().'/landed') + ); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('xn--bcher-kva.example', $seen); + $this->assertSame(['/landed'], self::$server->paths()); + } + + public function testPinnedHostReachesTheLocalServer(): void + { + $response = $this->client()->sendRequest($this->request($this->url('/landed'))); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('ok', (string) $response->getBody()); + $this->assertSame($this->url('/landed'), $response->getHeaderLine('Content-Location')); + $this->assertSame(['/landed'], self::$server->paths()); + } + + /** + * @return array + */ + public function redirectStatusProvider(): array + { + return [ + '301' => [301], + '302' => [302], + '303' => [303], + '307' => [307], + '308' => [308], + ]; + } + + /** + * @dataProvider redirectStatusProvider + */ + public function testRedirectToLoopbackIsBlockedBeforeItIsRequested(int $status): void + { + $target = 'http://127.0.0.1:'.self::$server->port().'/secret'; + + $exception = $this->captureBlocked(function () use ($status, $target): void { + $this->client()->sendRequest($this->request($this->url('/go?code='.$status.'&to='.rawurlencode($target)))); + }); + + $this->assertSame('non_public_address', $exception->getReason()); + $this->assertStringNotContainsString('127.0.0.1', $exception->getMessage()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testRedirectToPrivateNameIsBlockedWithoutLeakingTheAddress(): void + { + $target = 'http://internal.test:'.self::$server->port().'/secret'; + + $exception = $this->captureBlocked(function () use ($target): void { + $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode($target)))); + }); + + $this->assertSame('non_public_address', $exception->getReason()); + $this->assertStringNotContainsString('10.0.0.1', $exception->getMessage()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testRelativeRedirectKeepsTheBasePort(): void + { + $response = $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('/landed')))); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame($this->url('/landed'), $response->getHeaderLine('Content-Location')); + $this->assertSame(['/go', '/landed'], self::$server->paths()); + } + + public function testProtocolRelativeLoopbackIsBlocked(): void + { + $this->captureBlocked(function (): void { + $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('//127.0.0.1/secret')))); + }); + + $this->assertSame(['/go'], self::$server->paths()); + } + + /** + * @return array + */ + public function forbiddenSchemeProvider(): array + { + return [ + 'file' => ['file:///etc/passwd', 'scheme'], + 'gopher' => ['gopher://127.0.0.1/secret', 'scheme'], + ]; + } + + /** + * @dataProvider forbiddenSchemeProvider + */ + public function testRedirectToForbiddenSchemeIsBlocked(string $target, string $reason): void + { + $exception = $this->captureBlocked(function () use ($target): void { + $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode($target)))); + }); + + $this->assertSame($reason, $exception->getReason()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testUnparseableRedirectIsBlocked(): void + { + $exception = $this->captureBlocked(function (): void { + $this->client()->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('http://')))); + }); + + $this->assertSame('redirect', $exception->getReason()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testThreeHopChainUsesTheFinalUrl(): void + { + $response = $this->client()->sendRequest($this->request($this->url('/chain?i=0&n=3'))); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame('done', (string) $response->getBody()); + $this->assertSame($this->url('/chain?i=3&n=3'), $response->getHeaderLine('Content-Location')); + $this->assertSame( + ['/chain', '/chain', '/chain', '/chain'], + self::$server->paths() + ); + $queries = array_map(static function (array $row): string { + return $row['query'] ?? ''; + }, self::$server->requests()); + $this->assertSame(['i=0&n=3', 'i=1&n=3', 'i=2&n=3', 'i=3&n=3'], $queries); + } + + public function testTooManyRedirectsUsesCurlErrorCode(): void + { + try { + $this->client()->sendRequest($this->request($this->url('/chain?i=0&n=11'))); + $this->fail('The eleventh redirect must fail.'); + } catch (NetworkException $exception) { + $this->assertSame(47, $exception->getCode()); + $this->assertSame('Number of redirects hit maximum amount', $exception->getMessage()); + } + + $queries = array_map(static function (array $row): string { + return $row['query'] ?? ''; + }, self::$server->requests()); + $this->assertNotContains('i=11&n=11', $queries); + $this->assertContains('i=10&n=11', $queries); + $this->assertCount(11, $queries); + } + + public function testNegativeMaxRedirsFollowsUpToTheCap(): void + { + $response = $this->client(['max_redirs' => -1])->sendRequest($this->request($this->url('/chain?i=0&n=3'))); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame(['/chain', '/chain', '/chain', '/chain'], self::$server->paths()); + + self::$server->clearLog(); + + try { + $this->client(['max_redirs' => -1])->sendRequest($this->request($this->url('/chain?i=0&n=51'))); + $this->fail('A negative max_redirs is capped at 50.'); + } catch (NetworkException $exception) { + $this->assertSame(47, $exception->getCode()); + } + + $queries = array_map(static function (array $row): string { + return $row['query'] ?? ''; + }, self::$server->requests()); + $this->assertContains('i=50&n=51', $queries); + $this->assertNotContains('i=51&n=51', $queries); + $this->assertCount(51, $queries); + } + + public function testTimeoutIsABudgetAcrossRedirects(): void + { + $next = rawurlencode('/pause?ms=700'); + + try { + $this->client(['timeout' => 1])->sendRequest($this->request($this->url('/pause?ms=700&to='.$next))); + $this->fail('Two pauses must exceed the total timeout.'); + } catch (NetworkException $exception) { + $this->assertSame(28, $exception->getCode()); + } + + $within = $this->client(['timeout' => 2])->sendRequest($this->request($this->url('/pause?ms=300'))); + $this->assertSame(200, $within->getStatusCode()); + } + + public function testSslVerifyHostTruthyValuesKeepHostnameChecks(): void + { + $tls = $this->startTlsServer(); + + try { + $url = 'https://allowed.test:'.$tls['port'].'/'; + foreach ([0, false, '0'] as $disabled) { + $response = $this->client([ + 'ssl_verify_host' => $disabled, + 'ssl_verify_peer' => false, + ])->sendRequest($this->request($url)); + $this->assertSame(200, $response->getStatusCode(), 'disabled '.var_export($disabled, true)); + } + + $notices = []; + set_error_handler(static function (int $severity, string $message) use (&$notices): bool { + if (stripos($message, 'SSL_VERIFYHOST') !== false) { + $notices[] = $message; + } + + return true; + }); + + try { + foreach ([true, 1, '1', 2, '2'] as $enabled) { + try { + $this->client([ + 'ssl_verify_host' => $enabled, + 'ssl_verify_peer' => false, + ])->sendRequest($this->request($url)); + $this->fail('Hostname verification must stay on for '.var_export($enabled, true)); + } catch (NetworkException $exception) { + $this->assertSame(60, $exception->getCode(), var_export($enabled, true)); + $this->assertMatchesRegularExpression('/target host ?name/i', $exception->getMessage()); + } + } + } finally { + restore_error_handler(); + } + + $this->assertSame([], $notices); + } finally { + $this->stopTlsServer($tls); + } + } + + public function testSslVerifyPeerFalsyValuesAreTheOnlyOnesDisabled(): void + { + $tls = $this->startTlsServer(); + + try { + $url = 'https://allowed.test:'.$tls['port'].'/'; + foreach ([0, false, '0', '', null] as $disabled) { + $response = $this->client([ + 'ssl_verify_host' => 0, + 'ssl_verify_peer' => $disabled, + ])->sendRequest($this->request($url)); + $this->assertSame(200, $response->getStatusCode(), 'disabled '.var_export($disabled, true)); + } + + foreach ([true, 1, 2, '2', 'true', 'yes'] as $enabled) { + try { + $this->client([ + 'ssl_verify_host' => 0, + 'ssl_verify_peer' => $enabled, + ])->sendRequest($this->request($url)); + $this->fail('Certificate verification must stay on for '.var_export($enabled, true)); + } catch (NetworkException $exception) { + $this->assertSame(60, $exception->getCode(), var_export($enabled, true)); + $this->assertStringContainsString('certificate', $exception->getMessage()); + } + } + } finally { + $this->stopTlsServer($tls); + } + } + + public function testFollowLocationFalseReturnsTheRedirect(): void + { + $response = $this->client(['follow_location' => false])->sendRequest($this->request($this->url('/go?code=302&to='.rawurlencode('/landed')))); + + $this->assertSame(302, $response->getStatusCode()); + $this->assertSame(['/go'], self::$server->paths()); + } + + public function testCookieFromTheRedirectIsSentOnTheNextHop(): void + { + $response = $this->client()->sendRequest($this->request($this->url('/set-cookie'))); + + $this->assertSame(200, $response->getStatusCode()); + $rows = self::$server->requests(); + $this->assertCount(2, $rows); + $this->assertSame('/read-cookie', $rows[1]['path']); + $this->assertStringContainsString('consent=yes', $rows[1]['cookie']); + } + + public function testAuthorizationAndCookieAreStrippedWhenTheHostChanges(): void + { + $factory = new Psr17Factory(); + $request = $factory->createRequest('GET', $this->url('/to-other?port='.self::$server->port())) + ->withHeader('Authorization', 'Bearer secret') + ->withHeader('Cookie', 'a=b'); + + $response = $this->client()->sendRequest($request); + + $this->assertSame(200, $response->getStatusCode()); + $rows = self::$server->requests(); + $this->assertCount(2, $rows); + $this->assertSame('Bearer secret', $rows[0]['authorization']); + $this->assertSame('', $rows[1]['authorization']); + $this->assertSame('', $rows[1]['cookie']); + $this->assertStringContainsString('other.test', $rows[1]['host']); + } + + public function testPostRedirectsToGetExceptFor307And308(): void + { + $factory = new Psr17Factory(); + $body = $factory->createStream('payload'); + $request = $factory->createRequest('POST', $this->url('/switch?code=302'))->withBody($body); + + $this->client()->sendRequest($request); + + $rows = self::$server->requests(); + $this->assertSame('POST', $rows[0]['method']); + $this->assertSame('GET', $rows[1]['method']); + $this->assertSame('', $rows[1]['body']); + + self::$server->clearLog(); + $again = $factory->createRequest('POST', $this->url('/switch?code=307')) + ->withBody($factory->createStream('payload')); + $this->client()->sendRequest($again); + $rows = self::$server->requests(); + $this->assertSame('POST', $rows[0]['method']); + $this->assertSame('POST', $rows[1]['method']); + $this->assertSame('payload', $rows[1]['body']); + } + + public function testDnsRebindingDoesNotMoveTheConnection(): void + { + $calls = 0; + $policy = UrlPolicy::default() + ->withAllowedHosts('rebind.test') + ->withResolver(static function (string $host) use (&$calls): array { + ++$calls; + if ($host === 'rebind.test' && $calls > 1) { + return ['10.0.0.1']; + } + + return ['127.0.0.1']; + }); + + $response = $this->client([], $policy)->sendRequest( + $this->request('http://rebind.test:'.self::$server->port().'/landed') + ); + + $this->assertSame(200, $response->getStatusCode()); + $this->assertSame(1, $calls); + $this->assertSame(['/landed'], self::$server->paths()); + } + + public function testParallelRequestsFollowRedirects(): void + { + $factory = new Psr17Factory(); + $responses = $this->client()->sendRequests( + $factory->createRequest('GET', $this->url('/chain?i=0&n=1')), + $factory->createRequest('GET', $this->url('/landed')) + ); + + $this->assertCount(2, $responses); + $this->assertSame(200, $responses[0]->getStatusCode()); + $this->assertSame(200, $responses[1]->getStatusCode()); + $this->assertSame($this->url('/chain?i=1&n=1'), $responses[0]->getHeaderLine('Content-Location')); + $this->assertContains('/landed', self::$server->paths()); + $this->assertContains('/chain', self::$server->paths()); + } + + public function testParallelBatchIsRejectedBeforeAnyRequestWhenOneUrlIsBlocked(): void + { + $factory = new Psr17Factory(); + + $exception = $this->captureBlocked(function () use ($factory): void { + $this->client()->sendRequests( + $factory->createRequest('GET', $this->url('/landed')), + $factory->createRequest('GET', 'http://127.0.0.1/secret') + ); + }); + + $this->assertSame('non_public_address', $exception->getReason()); + $this->assertSame([], self::$server->paths()); + } + + public function testEnvironmentProxyKeepsPreValidationAndDoesNotConnectDirectly(): void + { + putenv('http_proxy=http://127.0.0.1:9'); + + try { + $blocked = $this->captureBlocked(function (): void { + $this->client()->sendRequest($this->request('http://127.0.0.1/secret')); + }); + $this->assertSame('non_public_address', $blocked->getReason()); + $this->assertSame([], self::$server->paths()); + + try { + $this->client()->sendRequest($this->request($this->url('/landed'))); + $this->fail('A configured proxy must be used for the connection.'); + } catch (NetworkException $exception) { + $this->assertNotSame(0, $exception->getCode()); + } + $this->assertSame([], self::$server->paths()); + } finally { + putenv('http_proxy'); + } + } + + /** + * @param array $settings + */ + private function client(array $settings = [], ?UrlPolicy $policy = null): CurlClient + { + $client = new CurlClient(); + $client->setUrlPolicy($policy ?? $this->policy()); + $client->setSettings($settings + [ + 'timeout' => 3, + 'connect_timeout' => 3, + 'cookies_path' => self::$cookies, + 'follow_location' => true, + 'max_redirs' => 10, + ]); + + return $client; + } + + private function policy(): UrlPolicy + { + return UrlPolicy::default() + ->withAllowedHosts('allowed.test', 'other.test') + ->withResolver(static function (string $host): array { + if ($host === 'allowed.test' || $host === 'other.test') { + return ['127.0.0.1']; + } + if ($host === 'internal.test') { + return ['10.0.0.1']; + } + + return []; + }); + } + + private function url(string $path): string + { + return 'http://allowed.test:'.self::$server->port().$path; + } + + private function request(string $uri): RequestInterface + { + return (new Psr17Factory())->createRequest('GET', $uri); + } + + /** + * @param callable(): void $check + */ + private function captureBlocked(callable $check): BlockedRequestException + { + try { + $check(); + } catch (BlockedRequestException $exception) { + return $exception; + } + + $this->fail('Expected BlockedRequestException'); + } + + /** + * @return array{port: int, process: resource, paths: array} + */ + private function startTlsServer(): array + { + $directory = sys_get_temp_dir().'/embed-tls-'.bin2hex(random_bytes(4)); + if (!mkdir($directory) && !is_dir($directory)) { + $this->fail('Unable to create a certificate directory'); + } + + $key = $directory.'/key.pem'; + $cert = $directory.'/cert.pem'; + $command = 'openssl req -x509 -newkey rsa:2048 -keyout '.escapeshellarg($key).' -out '.escapeshellarg($cert).' -days 1 -nodes -subj /CN=mismatch.test 2>/dev/null'; + exec($command, $output, $exitCode); + if ($exitCode !== 0 || !is_file($cert)) { + $this->fail('Unable to create a TLS certificate'); + } + + $port = self::allocatePort(); + $stderr = $directory.'/stderr'; + $process = proc_open( + ['openssl', 's_server', '-accept', '127.0.0.1:'.$port, '-cert', $cert, '-key', $key, '-www'], + [ + 0 => ['pipe', 'r'], + 1 => ['file', $directory.'/stdout', 'w'], + 2 => ['file', $stderr, 'w'], + ], + $pipes + ); + if (!is_resource($process)) { + $this->fail('Unable to start the TLS server'); + } + fclose($pipes[0]); + + $deadline = microtime(true) + 5; + while (microtime(true) < $deadline) { + $socket = @fsockopen('127.0.0.1', $port, $errno, $error, 0.2); + if (is_resource($socket)) { + fclose($socket); + + return ['port' => $port, 'process' => $process, 'paths' => [$directory, $key, $cert, $stderr, $directory.'/stdout']]; + } + usleep(50000); + } + + proc_terminate($process); + proc_close($process); + $details = is_file($stderr) ? (string) file_get_contents($stderr) : ''; + $this->fail('TLS server did not start: '.$details); + } + + /** + * @param array{port: int, process: resource, paths: array} $tls + */ + private function stopTlsServer(array $tls): void + { + proc_terminate($tls['process']); + proc_close($tls['process']); + $directories = []; + foreach ($tls['paths'] as $path) { + if (is_file($path)) { + unlink($path); + } elseif (is_dir($path)) { + $directories[] = $path; + } + } + foreach ($directories as $directory) { + rmdir($directory); + } + } + + private static function allocatePort(): int + { + $socket = stream_socket_server('tcp://127.0.0.1:0', $errno, $error); + if ($socket === false) { + throw new \RuntimeException('Unable to allocate a port: '.$error); + } + + $name = stream_socket_get_name($socket, false); + fclose($socket); + if (!is_string($name)) { + throw new \RuntimeException('Unable to read the allocated port'); + } + + $colon = strrpos($name, ':'); + if ($colon === false) { + throw new \RuntimeException('Unable to read the allocated port'); + } + + return (int) substr($name, $colon + 1); + } + + private static function clearProxyEnvironment(): void + { + foreach (['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY', 'all_proxy', 'ALL_PROXY'] as $name) { + putenv($name); + unset($_ENV[$name], $_SERVER[$name]); + } + } +} diff --git a/tests/DocumentTest.php b/tests/DocumentTest.php index aa516acd..171b0aba 100644 --- a/tests/DocumentTest.php +++ b/tests/DocumentTest.php @@ -5,6 +5,7 @@ use Embed\Embed; use Embed\Http\Crawler; +use Embed\Http\UrlPolicy; use PHPUnit\Framework\TestCase; class DocumentTest extends TestCase @@ -19,8 +20,12 @@ private static function getEmbed(): Embed $dispatcher = new FileClient(__DIR__.'/cache'); $dispatcher->setMode(0); + $crawler = new Crawler($dispatcher); + $crawler->setUrlPolicy(UrlPolicy::default()->withResolver(static function (string $host): array { + return ['8.8.8.8']; + })); - return self::$embed = new Embed(new Crawler($dispatcher)); + return self::$embed = new Embed($crawler); } public function testSelectors() diff --git a/tests/FunctionsTest.php b/tests/FunctionsTest.php index bb5e959d..a6a5b800 100644 --- a/tests/FunctionsTest.php +++ b/tests/FunctionsTest.php @@ -4,6 +4,8 @@ namespace Embed\Tests; use function Embed\isHttp; +use function Embed\resolveUri; +use Nyholm\Psr7\Factory\Psr17Factory; use PHPUnit\Framework\TestCase; class FunctionsTest extends TestCase @@ -32,4 +34,22 @@ public function testIsHttp(string $url, bool $expected) $result = isHttp($url); $this->assertSame($expected, $result); } + + public function testResolveUriKeepsTheBasePort(): void + { + $factory = new Psr17Factory(); + $base = $factory->createUri('http://h:8080/a/b'); + + $absolute = resolveUri($base, $factory->createUri('/x')); + $this->assertSame('http://h:8080/x', (string) $absolute); + + $relative = resolveUri($base, $factory->createUri('c')); + $this->assertSame('http://h:8080/a/c', (string) $relative); + + $protocolRelative = resolveUri($base, $factory->createUri('//cdn.example/favicon.ico')); + $this->assertSame('http://cdn.example/favicon.ico', (string) $protocolRelative); + + $other = resolveUri($base, $factory->createUri('https://other.example:9/z')); + $this->assertSame('https://other.example:9/z', (string) $other); + } } diff --git a/tests/IpClassifierTest.php b/tests/IpClassifierTest.php new file mode 100644 index 00000000..3fafdef3 --- /dev/null +++ b/tests/IpClassifierTest.php @@ -0,0 +1,120 @@ + + */ + public function addressesProvider(): array + { + return [ + 'unspecified v4' => ['0.0.0.0', false], + 'zero network end' => ['0.255.255.255', false], + 'private 10 start' => ['10.0.0.0', false], + 'private 10 end' => ['10.255.255.255', false], + 'just outside 10' => ['11.0.0.1', true], + 'cgnat start' => ['100.64.0.0', false], + 'cgnat end' => ['100.127.255.255', false], + 'below cgnat' => ['100.63.255.255', true], + 'above cgnat' => ['100.128.0.0', true], + 'loopback' => ['127.0.0.1', false], + 'loopback end' => ['127.255.255.255', false], + 'link local start' => ['169.254.0.0', false], + 'metadata' => ['169.254.169.254', false], + 'below private 172' => ['172.15.255.255', true], + 'private 172 start' => ['172.16.0.0', false], + 'private 172 end' => ['172.31.255.255', false], + 'above private 172' => ['172.32.0.0', true], + 'ietf protocol' => ['192.0.0.0', false], + 'ietf protocol end' => ['192.0.0.255', false], + 'outside ietf protocol' => ['192.0.1.1', true], + 'test net 1' => ['192.0.2.1', false], + '6to4 relay anycast' => ['192.88.99.1', false], + 'rfc1918 192.168' => ['192.168.1.1', false], + 'benchmark start' => ['198.18.0.1', false], + 'benchmark end' => ['198.19.255.255', false], + 'below benchmark' => ['198.17.255.255', true], + 'above benchmark' => ['198.20.0.1', true], + 'test net 2' => ['198.51.100.1', false], + 'test net 3' => ['203.0.113.1', false], + 'below multicast' => ['223.255.255.255', true], + 'multicast start' => ['224.0.0.1', false], + 'multicast end' => ['239.255.255.255', false], + 'reserved start' => ['240.0.0.1', false], + 'broadcast' => ['255.255.255.255', false], + 'google dns' => ['8.8.8.8', true], + 'cloudflare dns' => ['1.1.1.1', true], + 'unspecified v6' => ['::', false], + 'loopback v6' => ['::1', false], + 'unique local fc' => ['fc00::1', false], + 'unique local fd' => ['fd12:3456:789a::1', false], + 'link local v6' => ['fe80::1', false], + 'link local v6 end' => ['febf::1', false], + 'site local v6' => ['fec0::1', false], + 'multicast v6' => ['ff02::1', false], + 'documentation v6' => ['2001:db8::1', false], + 'discard prefix' => ['100::', false], + 'discard prefix host' => ['100::1', false], + 'teredo' => ['2001:0:4136:e378::1', false], + 'nat64 local use' => ['64:ff9b:1::1', false], + 'google dns v6' => ['2001:4860:4860::8888', true], + 'cloudflare dns v6' => ['2606:4700:4700::1111', true], + 'outside discard prefix' => ['100:0:0:1::', true], + 'mapped public' => ['::ffff:8.8.8.8', true], + 'mapped public hex' => ['::ffff:808:808', true], + 'mapped loopback' => ['::ffff:127.0.0.1', false], + 'mapped loopback hex' => ['::ffff:7f00:1', false], + 'compatible public' => ['::8.8.8.8', true], + 'compatible loopback' => ['::7f00:1', false], + 'nat64 public' => ['64:ff9b::8.8.8.8', true], + 'nat64 loopback' => ['64:ff9b::127.0.0.1', false], + 'nat64 loopback hex' => ['64:ff9b::7f00:1', false], + '6to4 public' => ['2002:808:808::', true], + '6to4 loopback' => ['2002:7f00:1::', false], + '6to4 rfc1918' => ['2002:c0a8:1::', false], + 'decimal loopback' => ['2130706433', false], + 'short loopback' => ['127.1', false], + 'octal loopback' => ['0177.0.0.1', false], + 'hex loopback' => ['0x7f000001', false], + 'dotted hex loopback' => ['0x7f.0.0.1', false], + 'decimal public' => ['134744072', true], + 'not an ip' => ['example.com', false], + 'empty' => ['', false], + 'five octets' => ['8.8.8.8.8', false], + ]; + } + + /** + * @dataProvider addressesProvider + */ + public function testIsPublic(string $ip, bool $public): void + { + $this->assertSame($public, IpClassifier::isPublic($ip)); + } + + public function testCanonicalNormalizesAlternativeSpellings(): void + { + $this->assertSame('127.0.0.1', IpClassifier::canonical('2130706433')); + $this->assertSame('127.0.0.1', IpClassifier::canonical('127.1')); + $this->assertSame('127.0.0.1', IpClassifier::canonical('0177.0.0.1')); + $this->assertSame('127.0.0.1', IpClassifier::canonical('0x7f000001')); + $this->assertSame('8.8.8.8', IpClassifier::canonical('134744072')); + $this->assertSame('8.8.8.8', IpClassifier::canonical('8.8.8.8')); + $this->assertNull(IpClassifier::canonical('example.com')); + $this->assertNull(IpClassifier::canonical('0x7f.example.com')); + } + + public function testEqualsTreatsIpv4MappedAsTheSameAddress(): void + { + $this->assertTrue(IpClassifier::equals('127.0.0.1', '::ffff:127.0.0.1')); + $this->assertTrue(IpClassifier::equals('8.8.8.8', '::ffff:808:808')); + $this->assertFalse(IpClassifier::equals('8.8.8.8', '1.1.1.1')); + $this->assertFalse(IpClassifier::equals('8.8.8.8', '2002:808:808::')); + } +} diff --git a/tests/LocalHttpServer.php b/tests/LocalHttpServer.php new file mode 100644 index 00000000..f61320b4 --- /dev/null +++ b/tests/LocalHttpServer.php @@ -0,0 +1,161 @@ + ['pipe', 'r'], + 1 => ['file', $log.'.stdout', 'w'], + 2 => ['file', $stderr, 'w'], + ]; + + $env = []; + foreach (array_merge($_SERVER, $_ENV) as $name => $value) { + if (is_string($name) && is_string($value)) { + $env[$name] = $value; + } + } + $env['EMBED_TEST_LOG'] = $log; + + $process = proc_open($command, $spec, $pipes, null, $env); + if (!is_resource($process)) { + throw new RuntimeException('Unable to start the test server'); + } + fclose($pipes[0]); + + $server = new self($process, $log, $port, $stderr); + $server->waitUntilListening(); + + return $server; + } + + /** + * @param resource $process + */ + private function __construct($process, string $logFile, int $port, string $stderr) + { + $this->process = $process; + $this->logFile = $logFile; + $this->port = $port; + $this->stderr = $stderr; + } + + public function port(): int + { + return $this->port; + } + + public function clearLog(): void + { + file_put_contents($this->logFile, ''); + } + + /** + * @return array> + */ + public function requests(): array + { + $raw = is_file($this->logFile) ? (string) file_get_contents($this->logFile) : ''; + $rows = []; + foreach (preg_split('/\r?\n/', trim($raw)) ?: [] as $line) { + if ($line === '') { + continue; + } + $decoded = json_decode($line, true); + if (!is_array($decoded)) { + continue; + } + $row = []; + foreach ($decoded as $key => $value) { + if (is_string($key)) { + $row[$key] = is_string($value) ? $value : ''; + } + } + $rows[] = $row; + } + + return $rows; + } + + /** + * @return array + */ + public function paths(): array + { + $paths = []; + foreach ($this->requests() as $request) { + $paths[] = $request['path'] ?? ''; + } + + return $paths; + } + + public function stop(): void + { + proc_terminate($this->process); + proc_close($this->process); + } + + private function waitUntilListening(): void + { + $deadline = microtime(true) + 5; + while (microtime(true) < $deadline) { + $socket = @fsockopen('127.0.0.1', $this->port, $errno, $error, 0.2); + if (is_resource($socket)) { + fclose($socket); + + return; + } + usleep(50000); + } + + $details = is_file($this->stderr) ? (string) file_get_contents($this->stderr) : ''; + $this->stop(); + + throw new RuntimeException('Test server did not start: '.$details); + } + + private static function allocatePort(): int + { + $socket = stream_socket_server('tcp://127.0.0.1:0', $errno, $error); + if ($socket === false) { + throw new RuntimeException('Unable to allocate a port: '.$error); + } + + $name = stream_socket_get_name($socket, false); + fclose($socket); + if (!is_string($name)) { + throw new RuntimeException('Unable to read the allocated port'); + } + + $colon = strrpos($name, ':'); + if ($colon === false) { + throw new RuntimeException('Unable to read the allocated port'); + } + + return (int) substr($name, $colon + 1); + } +} diff --git a/tests/MultipleRequestsTest.php b/tests/MultipleRequestsTest.php index 42348449..a4943374 100644 --- a/tests/MultipleRequestsTest.php +++ b/tests/MultipleRequestsTest.php @@ -4,13 +4,23 @@ namespace Embed\Tests; use Embed\Embed; +use Embed\Http\Crawler; +use Embed\Http\UrlPolicy; use PHPUnit\Framework\TestCase; class MultipleRequestsTest extends TestCase { public function testParallel() { - $embed = new Embed(); + $dispatcher = new FileClient(__DIR__.'/cache'); + $dispatcher->setMode(-1); + $crawler = new Crawler($dispatcher); + // Some resolvers answer public names with 198.18.0.1, which the policy + // rejects. Cached responses must not depend on that lookup. + $crawler->setUrlPolicy(UrlPolicy::default()->withResolver(static function (string $host): array { + return ['8.8.8.8']; + })); + $embed = new Embed($crawler); $infos = $embed->getMulti( 'https://oscarotero.com', 'https://github.com/oscarotero', diff --git a/tests/PagesTestCase.php b/tests/PagesTestCase.php index 46db1772..1f3b5402 100644 --- a/tests/PagesTestCase.php +++ b/tests/PagesTestCase.php @@ -9,6 +9,7 @@ use Embed\Extractor; use Embed\ExtractorFactory; use Embed\Http\Crawler; +use Embed\Http\UrlPolicy; use JsonSerializable; use PHPUnit\Framework\TestCase; use Psr\Http\Message\UriInterface; @@ -109,8 +110,14 @@ private static function getEmbed(): Embed $dispatcher = new FileClient(__DIR__.'/cache'); $dispatcher->setMode(self::getCacheMode()); - - return self::$embed = new Embed(new Crawler($dispatcher), self::getExtractorFactory()); + $crawler = new Crawler($dispatcher); + // Cached fixtures must not depend on live DNS. Dead hosts such as + // ustream and viddler would otherwise fail closed as NXDOMAIN. + $crawler->setUrlPolicy(UrlPolicy::default()->withResolver(static function (string $host): array { + return ['8.8.8.8']; + })); + + return self::$embed = new Embed($crawler, self::getExtractorFactory()); } protected function assertEmbed(string $url) diff --git a/tests/RecordingClient.php b/tests/RecordingClient.php new file mode 100644 index 00000000..ca885eab --- /dev/null +++ b/tests/RecordingClient.php @@ -0,0 +1,33 @@ + */ + public array $requests = []; + + /** @var callable(RequestInterface): ResponseInterface */ + private $handler; + + /** + * @param callable(RequestInterface): ResponseInterface $handler + */ + public function __construct(callable $handler) + { + $this->handler = $handler; + } + + public function sendRequest(RequestInterface $request): ResponseInterface + { + $this->requests[] = $request; + $handler = $this->handler; + + return $handler($request); + } +} diff --git a/tests/SchemaOrgContext.php b/tests/SchemaOrgContext.php new file mode 100644 index 00000000..17a56ed3 --- /dev/null +++ b/tests/SchemaOrgContext.php @@ -0,0 +1,137 @@ +isSchemaHost($path)) { + return false; + } + + $this->body = self::BODY; + $this->position = 0; + $this->notifyMimeType(); + + return true; + } + + /** + * @param int $count + */ + public function stream_read($count): string + { + $chunk = substr($this->body, $this->position, $count); + $this->position += strlen($chunk); + + return $chunk; + } + + public function stream_eof(): bool + { + return $this->position >= strlen($this->body); + } + + /** + * @return int + */ + public function stream_tell() + { + return $this->position; + } + + /** + * @return array + */ + public function stream_stat(): array + { + $size = strlen($this->body); + + return [ + 'size' => $size, + 7 => $size, + ]; + } + + public function stream_close(): void + { + // The context body is a string, so there is no handle to close. + } + + private function notifyMimeType(): void + { + if (!isset($this->context)) { + return; + } + + $params = stream_context_get_params($this->context); + if (!isset($params['notification']) || !is_callable($params['notification'])) { + return; + } + + $params['notification']( + STREAM_NOTIFY_MIME_TYPE_IS, + STREAM_NOTIFY_SEVERITY_INFO, + 'application/ld+json', + 0, + 0, + 0 + ); + } + + /** + * @param mixed $url + */ + private function isSchemaHost($url): bool + { + if (!is_string($url) || $url === '') { + return false; + } + + $host = parse_url($url, PHP_URL_HOST); + if (!is_string($host) || $host === '') { + return false; + } + + return in_array(strtolower($host), [ + 'schema.org', + 'www.schema.org', + 'schema.googleapis.com', + ], true); + } +} diff --git a/tests/SsrfEmbedTest.php b/tests/SsrfEmbedTest.php new file mode 100644 index 00000000..7d3ed1e1 --- /dev/null +++ b/tests/SsrfEmbedTest.php @@ -0,0 +1,143 @@ + + + + +Page title + + + + + + + +HTML; + $client = $this->client(static function (RequestInterface $request) use ($html): ResponseInterface { + $factory = new Psr17Factory(); + if ($request->getUri()->getPath() === '/oembed.json') { + return $factory->createResponse(200) + ->withHeader('Content-Type', 'application/json') + ->withBody($factory->createStream('{"title":"From oEmbed","type":"link"}')); + } + + return $factory->createResponse(200) + ->withHeader('Content-Type', 'text/html') + ->withBody($factory->createStream($html)); + }); + + $info = $this->embed($client)->get('http://public.test/article'); + + $this->assertSame('From oEmbed', $info->title); + $this->assertSame('http://public.test/images/pic.jpg', (string) $info->image); + $this->assertSame('http://cdn.example/favicon.ico', (string) $info->favicon); + $this->assertSame('http://public.test/es', (string) $info->languages['es']); + + $paths = array_map(static function (RequestInterface $request): string { + return $request->getUri()->getHost().$request->getUri()->getPath(); + }, $client->requests); + $this->assertSame(['public.test/article', 'public.test/oembed.json'], $paths); + } + + public function testBlockedOEmbedEndpointIsSkipped(): void + { + $html = 'Stay here' + .'' + .''; + $client = $this->htmlClient($html); + + $info = $this->embed($client)->get('http://public.test/page'); + + $this->assertSame('Stay here', $info->title); + $this->assertSame([], $info->getOEmbed()->all()); + $this->assertCount(1, $client->requests); + $this->assertSame('public.test', $client->requests[0]->getUri()->getHost()); + } + + public function testBlockedMetaRefreshIsIgnored(): void + { + $html = 'Stay' + .'' + .''; + $client = $this->htmlClient($html); + + $info = $this->embed($client)->get('http://public.test/refresh'); + + $this->assertSame('Stay', $info->title); + $this->assertCount(1, $client->requests); + } + + public function testCustomClientDoesNotReceiveABlockedUrl(): void + { + $client = $this->htmlClient(''); + + try { + $this->embed($client)->get('http://127.0.0.1/secret'); + $this->fail('The main URL must be rejected.'); + } catch (BlockedRequestException $exception) { + $this->assertSame('non_public_address', $exception->getReason()); + } + + $this->assertSame([], $client->requests); + } + + public function testCustomClientValidatesEveryParallelUrlBeforeSending(): void + { + $client = $this->htmlClient(''); + + try { + $this->embed($client)->getMulti('http://public.test/a', 'http://127.0.0.1/secret'); + $this->fail('One blocked URL must reject the batch.'); + } catch (BlockedRequestException $exception) { + $this->assertSame('non_public_address', $exception->getReason()); + } + + $this->assertSame([], $client->requests); + } + + private function embed(RecordingClient $client): Embed + { + $crawler = new Crawler($client); + $crawler->setUrlPolicy(UrlPolicy::default()->withResolver(static function (string $host): array { + return ['1.1.1.1']; + })); + + return new Embed($crawler); + } + + private function htmlClient(string $html): RecordingClient + { + return $this->client(static function (RequestInterface $request) use ($html): ResponseInterface { + $factory = new Psr17Factory(); + + return $factory->createResponse(200) + ->withHeader('Content-Type', 'text/html') + ->withBody($factory->createStream($html)); + }); + } + + /** + * @param callable(RequestInterface): ResponseInterface $handler + */ + private function client(callable $handler): RecordingClient + { + return new RecordingClient($handler); + } +} diff --git a/tests/UrlPolicyTest.php b/tests/UrlPolicyTest.php new file mode 100644 index 00000000..172a3621 --- /dev/null +++ b/tests/UrlPolicyTest.php @@ -0,0 +1,299 @@ +policy(['8.8.8.8', '1.1.1.1']); + + $ips = $policy->validate($this->request('https://example.com/a')); + + $this->assertSame(['8.8.8.8', '1.1.1.1'], $ips); + } + + public function testUnderscoreInHostnameIsAllowed(): void + { + $seen = null; + $policy = UrlPolicy::default()->withResolver(function (string $host) use (&$seen): array { + $seen = $host; + + return ['1.1.1.1']; + }); + + $ips = $policy->validate($this->request('http://my_host.example/path')); + + $this->assertSame('my_host.example', $seen); + $this->assertSame(['1.1.1.1'], $ips); + } + + public function testInternationalizedNameIsConvertedBeforeLookup(): void + { + if (!function_exists('idn_to_ascii')) { + $this->markTestSkipped('The intl extension is required to resolve IDN hosts.'); + } + + $seen = null; + $policy = UrlPolicy::default()->withResolver(function (string $host) use (&$seen): array { + $seen = $host; + + return ['1.1.1.1']; + }); + + $policy->validate($this->request('http://bücher.example/')); + + $this->assertSame('xn--bcher-kva.example', $seen); + } + + public function testMixedPublicAndPrivateAddressesAreBlocked(): void + { + $policy = $this->policy(['8.8.8.8', '10.0.0.1']); + + $exception = $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://example.com/')); + }, 'non_public_address'); + + $this->assertStringNotContainsString('10.0.0.1', $exception->getMessage()); + $this->assertStringNotContainsString('8.8.8.8', $exception->getMessage()); + } + + public function testDnsFailureIsANetworkException(): void + { + $policy = $this->policy([]); + + try { + $policy->validate($this->request('http://missing.test/')); + $this->fail('A DNS failure must not be reported as a successful validation.'); + } catch (BlockedRequestException $exception) { + $this->fail('A DNS failure must stay a network error.'); + } catch (NetworkException $exception) { + $this->assertSame(6, $exception->getCode()); + $this->assertInstanceOf(NetworkExceptionInterface::class, $exception); + $this->assertStringNotContainsString('10.', $exception->getMessage()); + } + } + + public function testAllowPrivateNetworksKeepsTheResolvedAddresses(): void + { + $policy = $this->policy(['10.1.2.3'])->allowPrivateNetworks(); + + $this->assertSame(['10.1.2.3'], $policy->validate($this->request('http://intranet.test/'))); + } + + public function testAllowedHostsAndWildcardsSkipThePublicAddressCheck(): void + { + $policy = $this->policy(['10.0.0.5'])->withAllowedHosts('wiki.corp', '*.corp'); + + $this->assertSame(['10.0.0.5'], $policy->validate($this->request('http://wiki.corp/a'))); + $this->assertSame(['10.0.0.5'], $policy->validate($this->request('http://a.b.corp/a'))); + + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://corp/a')); + }, 'non_public_address'); + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://evil.com/')); + }, 'non_public_address'); + } + + public function testOneAllowedAddressDoesNotExemptAnother(): void + { + $policy = UrlPolicy::default() + ->withAllowedHosts('8.8.8.8') + ->withResolver(static function (string $host): array { + return ['8.8.8.8', '10.0.0.1']; + }); + + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://evil.example/')); + }, 'non_public_address'); + + $only = UrlPolicy::default() + ->withAllowedHosts('10.0.0.1') + ->withResolver(static function (string $host): array { + return ['10.0.0.1']; + }); + + $this->assertSame(['10.0.0.1'], $only->validate($this->request('http://evil.example/'))); + } + + public function testAllowedHostMatchesTheUnicodeNameAndItsPunycode(): void + { + if (!function_exists('idn_to_ascii')) { + $this->markTestSkipped('The intl extension is required to resolve IDN hosts.'); + } + + $policy = UrlPolicy::default() + ->withAllowedHosts('Bücher.Example') + ->withResolver(static function (string $host): array { + return ['10.0.0.8']; + }); + + $this->assertSame(['10.0.0.8'], $policy->validate($this->request('http://bücher.example/'))); + + $punycode = UrlPolicy::default() + ->withAllowedHosts('xn--bcher-kva.example') + ->withResolver(static function (string $host): array { + return ['10.0.0.8']; + }); + + $this->assertSame(['10.0.0.8'], $punycode->validate($this->request('http://bücher.example/'))); + $this->assertSame('xn--bcher-kva.example.', $policy->curlResolveHost('bücher.example.')); + } + + public function testAllowedPortsUseTheSchemeDefault(): void + { + $policy = $this->policy(['1.1.1.1'])->withAllowedPorts([80, 443]); + + $this->assertSame(['1.1.1.1'], $policy->validate($this->request('http://example.com/'))); + $this->assertSame(['1.1.1.1'], $policy->validate($this->request('https://example.com/'))); + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://example.com:8080/')); + }, 'port'); + $this->assertSame(['1.1.1.1'], $policy->validate($this->request('http://example.com:443/'))); + } + + public function testLiteralLoopbackSpellingsAreBlockedWithoutDns(): void + { + $policy = UrlPolicy::default()->withResolver(static function (string $host): array { + throw new \RuntimeException('DNS was used for '.$host); + }); + + foreach ([ + 'http://127.0.0.1/', + 'http://2130706433/', + 'http://127.1/', + 'http://0177.0.0.1/', + 'http://0x7f000001/', + 'http://0x7f.0.0.1/', + 'http://[::1]/', + 'http://[::ffff:127.0.0.1]/', + 'http://[::ffff:7f00:1]/', + 'http://[64:ff9b::7f00:1]/', + 'http://[64:ff9b::127.0.0.1]/', + 'http://[2002:7f00:1::]/', + 'http://169.254.169.254/', + ] as $url) { + $this->assertBlocked(function () use ($policy, $url): void { + $policy->validate($this->request($url)); + }, 'non_public_address'); + } + } + + public function testPublicNumericLiteralDoesNotUseDns(): void + { + $policy = UrlPolicy::default()->withResolver(static function (string $host): array { + throw new \RuntimeException('DNS was used for '.$host); + }); + + $this->assertSame(['8.8.8.8'], $policy->validate($this->request('http://134744072/path'))); + $this->assertSame(['::ffff:8.8.8.8'], $policy->validate($this->request('http://[::ffff:8.8.8.8]/'))); + } + + /** + * @return array + */ + public function rejectedUrlsProvider(): array + { + return [ + 'ftp' => ['ftp://example.com/', 'scheme'], + 'file' => ['file:///etc/passwd', 'scheme'], + 'gopher' => ['gopher://127.0.0.1/', 'scheme'], + 'relative' => ['/x', 'scheme'], + 'protocol relative' => ['//example.com/x', 'scheme'], + 'userinfo' => ['http://user:pass@example.com/', 'host'], + 'backslash' => ['http://example.com\\@127.0.0.1/', 'host'], + 'space in host' => ['http://ex ample.com/', 'host'], + 'empty host' => ['http://', 'host'], + ]; + } + + /** + * @dataProvider rejectedUrlsProvider + */ + public function testRejectedUrls(string $url, string $reason): void + { + try { + $this->request($url); + } catch (InvalidArgumentException $exception) { + $this->assertContains($reason, ['scheme', 'host']); + + return; + } + + $this->assertBlocked(function () use ($url): void { + $this->policy(['8.8.8.8'])->validate($this->request($url)); + }, $reason); + } + + public function testBlockedExceptionCarriesTheRequest(): void + { + $request = $this->request('http://127.0.0.1/secret'); + + try { + UrlPolicy::default()->validate($request); + $this->fail('A loopback URL must be blocked.'); + } catch (BlockedRequestException $exception) { + $this->assertSame($request, $exception->getRequest()); + $this->assertSame('non_public_address', $exception->getReason()); + $this->assertInstanceOf(InvalidArgumentException::class, $exception); + $this->assertInstanceOf(RequestExceptionInterface::class, $exception); + } + } + + public function testWithersDoNotMutateTheOriginalPolicy(): void + { + $policy = UrlPolicy::default(); + $relaxed = $policy->allowPrivateNetworks(); + + $this->assertNotSame($policy, $relaxed); + $this->assertBlocked(function () use ($policy): void { + $policy->validate($this->request('http://10.0.0.1/')); + }, 'non_public_address'); + $this->assertSame(['10.0.0.1'], $relaxed->validate($this->request('http://10.0.0.1/'))); + } + + /** + * @param array $ips + */ + private function policy(array $ips): UrlPolicy + { + return UrlPolicy::default()->withResolver(static function (string $host) use ($ips): array { + return $ips; + }); + } + + private function request(string $uri): RequestInterface + { + return (new Psr17Factory())->createRequest('GET', $uri); + } + + /** + * @param callable(): void $check + */ + private function assertBlocked(callable $check, string $reason): BlockedRequestException + { + try { + $check(); + } catch (BlockedRequestException $exception) { + $this->assertSame($reason, $exception->getReason()); + $this->assertInstanceOf(InvalidArgumentException::class, $exception); + $this->assertInstanceOf(RequestExceptionInterface::class, $exception); + + return $exception; + } + + $this->fail('Expected BlockedRequestException with reason '.$reason); + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php new file mode 100644 index 00000000..b93a76cd --- /dev/null +++ b/tests/bootstrap.php @@ -0,0 +1,6 @@ + [ + 'content-type' => [ + 'text/html; charset=utf-8', + ], + ], + 'statusCode' => 200, + 'reasonPhrase' => 'OK', + 'body' => 'oscarotero - Overview', +]; diff --git a/tests/cache/oscarotero.com..php b/tests/cache/oscarotero.com..php new file mode 100644 index 00000000..37e2c0dc --- /dev/null +++ b/tests/cache/oscarotero.com..php @@ -0,0 +1,16 @@ + [ + 'content-type' => [ + 'text/html; charset=utf-8', + ], + 'content-location' => [ + 'https://oscarotero.com/', + ], + ], + 'statusCode' => 200, + 'reasonPhrase' => 'OK', + 'body' => 'Óscar Otero - Digital designer and developer', +]; diff --git a/tests/cache/publish.twitter.com.oembed.58bc2b4ee41b094b713af915ade4710c.php b/tests/cache/publish.twitter.com.oembed.58bc2b4ee41b094b713af915ade4710c.php new file mode 100644 index 00000000..24b3ac23 --- /dev/null +++ b/tests/cache/publish.twitter.com.oembed.58bc2b4ee41b094b713af915ade4710c.php @@ -0,0 +1,13 @@ + [ + 'content-type' => [ + 'application/json', + ], + ], + 'statusCode' => 200, + 'reasonPhrase' => 'OK', + 'body' => '{"url":"https://x.com/misteroom"}', +]; diff --git a/tests/cache/twitter.com.misteroom.php b/tests/cache/twitter.com.misteroom.php new file mode 100644 index 00000000..5c10d870 --- /dev/null +++ b/tests/cache/twitter.com.misteroom.php @@ -0,0 +1,13 @@ + [ + 'content-type' => [ + 'text/html; charset=utf-8', + ], + ], + 'statusCode' => 200, + 'reasonPhrase' => 'OK', + 'body' => 'misteroom', +]; diff --git a/tests/fixtures/he-who-photographs-rather-ok.tumblr.com.post-165326273724.php b/tests/fixtures/he-who-photographs-rather-ok.tumblr.com.post-165326273724.php index cc8f6852..fd9f2890 100644 --- a/tests/fixtures/he-who-photographs-rather-ok.tumblr.com.post-165326273724.php +++ b/tests/fixtures/he-who-photographs-rather-ok.tumblr.com.post-165326273724.php @@ -30,7 +30,7 @@ 'license' => null, 'providerName' => 'Tumblr', 'providerUrl' => 'https://www.tumblr.com/', - 'publishedTime' => null, + 'publishedTime' => '2017-09-14 10:20:35', 'redirect' => null, 'title' => 'He-who-photographs-rather-OK', 'url' => 'https://www.tumblr.com/he-who-photographs-rather-ok/165326273724', diff --git a/tests/server/router.php b/tests/server/router.php new file mode 100644 index 00000000..cbcb8fd4 --- /dev/null +++ b/tests/server/router.php @@ -0,0 +1,124 @@ + $value) { + $headers[strtolower((string) $name)] = (string) $value; + } +} + +$entry = [ + 'path' => $path, + 'query' => $queryString, + 'method' => (string) ($_SERVER['REQUEST_METHOD'] ?? ''), + 'authorization' => $headers['authorization'] ?? '', + 'cookie' => $headers['cookie'] ?? '', + 'host' => $headers['host'] ?? '', + 'referer' => $headers['referer'] ?? '', + 'body' => (string) file_get_contents('php://input'), +]; + +if (is_string($log) && $log !== '') { + $encoded = json_encode($entry); + if (is_string($encoded)) { + file_put_contents($log, $encoded."\n", FILE_APPEND | LOCK_EX); + } +} + +$params = []; +parse_str($queryString, $params); + +if ($path === '/go') { + $code = isset($params['code']) ? (int) $params['code'] : 302; + if (!in_array($code, [301, 302, 303, 307, 308], true)) { + $code = 302; + } + $to = isset($params['to']) && is_string($params['to']) ? $params['to'] : '/secret'; + header('Location: '.$to, true, $code); + echo 'redirect'; + + return; +} + +if ($path === '/chain') { + $i = isset($params['i']) ? (int) $params['i'] : 0; + $n = isset($params['n']) ? (int) $params['n'] : 0; + if ($i >= $n) { + http_response_code(200); + echo 'done'; + + return; + } + header('Location: /chain?i='.($i + 1).'&n='.$n, true, 302); + echo 'redirect'; + + return; +} + +if ($path === '/set-cookie') { + header('Set-Cookie: consent=yes; Path=/'); + header('Location: /read-cookie', true, 302); + echo 'redirect'; + + return; +} + +if ($path === '/to-other') { + $port = isset($params['port']) ? (string) $params['port'] : ''; + header('Location: http://other.test:'.$port.'/landed', true, 302); + echo 'redirect'; + + return; +} + +if ($path === '/switch') { + $code = isset($params['code']) ? (int) $params['code'] : 302; + if (!in_array($code, [301, 302, 303, 307, 308], true)) { + $code = 302; + } + header('Location: /landed', true, $code); + echo 'redirect'; + + return; +} + +if ($path === '/pause') { + $ms = isset($params['ms']) ? (int) $params['ms'] : 0; + if ($ms > 0) { + usleep(min($ms, 5000) * 1000); + } + $to = isset($params['to']) && is_string($params['to']) ? $params['to'] : ''; + if ($to !== '') { + header('Location: '.$to, true, 302); + echo 'redirect'; + + return; + } + http_response_code(200); + echo 'ok'; + + return; +} + +if ($path === '/secret') { + http_response_code(200); + echo 'LEAKED'; + + return; +} + +http_response_code(200); +echo 'ok';