diff --git a/docs/plugins.md b/docs/plugins.md index 95bd5956b..530284afb 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -81,6 +81,10 @@ Saves screenshots from the browser at points triggered by `on=`. Saves a screenshot when a test fails. +## [sitemap](/plugins/sitemap) + +Collects all pages visited while tests are running and saves them as a [sitemap][1] XML file. + ## [stepTimeout](/plugins/stepTimeout) Set timeout for test steps globally. diff --git a/docs/plugins/sitemap.md b/docs/plugins/sitemap.md new file mode 100644 index 000000000..b9923333c --- /dev/null +++ b/docs/plugins/sitemap.md @@ -0,0 +1,48 @@ +--- +permalink: /plugins/sitemap +editLink: false +sidebar: auto +title: sitemap +--- + + + +## sitemap + +Collects all pages visited while tests are running and saves them as a [sitemap][1] XML file. + +The plugin listens to browser navigation events, so it doesn't slow down test steps. +Page loads and SPA route changes (`history.pushState`) of the top-level page are recorded, iframes are ignored. +Every unique `http(s)` URL is added to the sitemap. URL fragments (`#...`) are dropped. + +Supported helpers: Playwright, Puppeteer, and WebDriver with BiDi protocol enabled (default). +The sitemap is written once all tests finish. + +#### Configuration + +```js +"plugins": { + "sitemap": { + "enabled": true + } + } +``` + +Possible config options: + +* `file`: path to the sitemap file, relative to the output directory. Default: `sitemap.xml`, saved as `output/sitemap.xml`. +* `stripQuery`: remove query strings so `/posts?page=1` and `/posts?page=2` become one `/posts` entry. Set to `false` to keep them. Default: true. + +CLI examples: + + npx codeceptjs run -p sitemap + npx codeceptjs run -p sitemap:file=visited.xml + npx codeceptjs run -p sitemap:stripQuery=false + +Works with `run-workers`: URLs from all workers are merged into a single sitemap. + +### Parameters + +* `config` **any** + +[1]: https://www.sitemaps.org/protocol.html diff --git a/lib/plugin/sitemap.js b/lib/plugin/sitemap.js new file mode 100644 index 000000000..eaa589a1e --- /dev/null +++ b/lib/plugin/sitemap.js @@ -0,0 +1,228 @@ +import fs from 'fs' +import path from 'path' +import { isMainThread, threadId } from 'worker_threads' +import { mkdirp } from 'mkdirp' +import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom' + +import Container from '../container.js' +import event from '../event.js' +import recorder from '../recorder.js' +import store from '../store.js' +import output from '../output.js' +import { parsePluginArgs } from '../utils/pluginParser.js' + +const defaultConfig = { + file: 'sitemap.xml', + stripQuery: true, +} + +const SITEMAP_NS = 'http://www.sitemaps.org/schemas/sitemap/0.9' +const PARTIAL_PREFIX = '.sitemap.' + +/** + * Collects all pages visited while tests are running and saves them as a [sitemap](https://www.sitemaps.org/protocol.html) XML file. + * + * The plugin listens to browser navigation events, so it doesn't slow down test steps. + * Page loads and SPA route changes (`history.pushState`) of the top-level page are recorded, iframes are ignored. + * Every unique `http(s)` URL is added to the sitemap. URL fragments (`#...`) are dropped. + * + * Supported helpers: Playwright, Puppeteer, and WebDriver with BiDi protocol enabled (default). + * The sitemap is written once all tests finish. + * + * #### Configuration + * + * ```js + * "plugins": { + * "sitemap": { + * "enabled": true + * } + * } + * ``` + * + * Possible config options: + * + * * `file`: path to the sitemap file, relative to the output directory. Default: `sitemap.xml`, saved as `output/sitemap.xml`. + * * `stripQuery`: remove query strings so `/posts?page=1` and `/posts?page=2` become one `/posts` entry. Set to `false` to keep them. Default: true. + * + * CLI examples: + * + * ``` + * npx codeceptjs run -p sitemap + * npx codeceptjs run -p sitemap:file=visited.xml + * npx codeceptjs run -p sitemap:stripQuery=false + * ``` + * + * Works with `run-workers`: URLs from all workers are merged into a single sitemap. + * + * @param {*} config + */ +export default function (config = {}) { + config = Object.assign({}, defaultConfig, config, parsePluginArgs(config._args)) + + const urls = new Set() + + const outputDir = () => store.outputDir || process.cwd() + + const addUrl = url => { + if (!url) return + let parsed + try { + parsed = new URL(url) + } catch { + return + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return + parsed.hash = '' + if (config.stripQuery) parsed.search = '' + urls.add(parsed.toString()) + } + + const writeSitemap = list => { + if (!list.length) return + + const file = path.resolve(outputDir(), config.file) + mkdirp.sync(path.dirname(file)) + fs.writeFileSync(file, buildXml(list.sort())) + output.plugin('sitemap', `Sitemap with ${list.length} pages saved to ${file}`) + } + + const attached = new WeakSet() + + const onLoad = page => addUrl(page.url()) + + const onFrameNavigated = frame => { + if (frame.parentFrame()) return + addUrl(frame.url()) + } + + const attachPage = page => { + if (!page || attached.has(page)) return + attached.add(page) + addUrl(page.url()) + page.on('load', onLoad) + page.on('framenavigated', onFrameNavigated) + } + + const attachContext = context => { + attached.add(context) + context.on('page', attachPage) + context.pages().forEach(attachPage) + } + + const attachPlaywright = browser => { + if (!browser.contexts) return attachContext(browser) + attached.add(browser) + browser.on('context', attachContext) + browser.contexts().forEach(attachContext) + } + + const attachPuppeteer = async browser => { + attached.add(browser) + browser.on('targetcreated', async target => { + if (target.type() !== 'page') return + attachPage(await target.page()) + }) + for (const page of await browser.pages()) attachPage(page) + } + + const subscribeBidi = async browser => { + attached.add(browser) + const frames = new Set() + const onNavigation = params => { + if (frames.has(params.context)) return + addUrl(params.url) + } + browser.on('browsingContext.contextCreated', params => { + if (params.parent) frames.add(params.context) + }) + browser.on('browsingContext.load', onNavigation) + browser.on('browsingContext.historyUpdated', onNavigation) + try { + await browser.sessionSubscribe({ events: ['browsingContext.contextCreated', 'browsingContext.load'] }) + } catch (err) { + output.plugin('sitemap', `Can't subscribe to navigation events: ${err.message}`) + } + try { + await browser.sessionSubscribe({ events: ['browsingContext.historyUpdated'] }) + } catch {} + } + + event.dispatcher.on(event.test.started, () => { + const helpers = Container.helpers() + + if (helpers.Playwright) { + const browser = helpers.Playwright.browser + if (browser && !attached.has(browser)) attachPlaywright(browser) + return + } + + if (helpers.Puppeteer) { + const browser = helpers.Puppeteer.browser + if (!browser || attached.has(browser)) return + recorder.add('sitemap: listen to navigation', () => attachPuppeteer(browser), true) + return + } + + if (helpers.WebDriver) { + const browser = helpers.WebDriver.browser + if (!browser || attached.has(browser)) return + if (!browser.capabilities || !browser.capabilities.webSocketUrl) { + attached.add(browser) + output.plugin('sitemap', 'WebDriver BiDi protocol is disabled, visited pages are not collected') + return + } + recorder.add('sitemap: subscribe to navigation', () => subscribeBidi(browser), true) + } + }) + + const partialFiles = dir => { + if (!fs.existsSync(dir)) return [] + return fs + .readdirSync(dir) + .filter(name => name.startsWith(PARTIAL_PREFIX) && name.endsWith('.json')) + .map(name => path.join(dir, name)) + } + + event.dispatcher.on(event.all.result, () => { + if (isMainThread) { + if (store.workerMode) return + writeSitemap([...urls]) + return + } + if (!urls.size) return + const dir = outputDir() + mkdirp.sync(dir) + fs.writeFileSync(path.join(dir, `${PARTIAL_PREFIX}${process.pid}.${threadId}.json`), JSON.stringify([...urls])) + }) + + event.dispatcher.on(event.workers.before, () => { + for (const partial of partialFiles(outputDir())) fs.unlinkSync(partial) + }) + + event.dispatcher.on(event.workers.result, () => { + for (const partial of partialFiles(outputDir())) { + try { + for (const url of JSON.parse(fs.readFileSync(partial, 'utf8'))) urls.add(url) + } catch {} + fs.unlinkSync(partial) + } + writeSitemap([...urls]) + }) +} + +function buildXml(list) { + const doc = new DOMImplementation().createDocument(SITEMAP_NS, 'urlset', null) + const urlset = doc.documentElement + for (const url of list) { + const urlEl = doc.createElementNS(SITEMAP_NS, 'url') + const locEl = doc.createElementNS(SITEMAP_NS, 'loc') + locEl.appendChild(doc.createTextNode(url)) + urlEl.appendChild(doc.createTextNode('\n ')) + urlEl.appendChild(locEl) + urlEl.appendChild(doc.createTextNode('\n ')) + urlset.appendChild(doc.createTextNode('\n ')) + urlset.appendChild(urlEl) + } + urlset.appendChild(doc.createTextNode('\n')) + return `\n${new XMLSerializer().serializeToString(doc)}\n` +} diff --git a/test/unit/plugin/sitemap_test.js b/test/unit/plugin/sitemap_test.js new file mode 100644 index 000000000..70e86673c --- /dev/null +++ b/test/unit/plugin/sitemap_test.js @@ -0,0 +1,201 @@ +import { expect } from 'chai' +import fs from 'fs' +import { EventEmitter } from 'events' +import os from 'os' +import path from 'path' +import sitemap from '../../../lib/plugin/sitemap.js' +import container from '../../../lib/container.js' +import event from '../../../lib/event.js' +import recorder from '../../../lib/recorder.js' +import store from '../../../lib/store.js' + +let page +let outputDir +let originalOutputDir + +const createPage = () => { + const p = new EventEmitter() + p.currentUrl = 'about:blank' + p.url = () => p.currentUrl + return p +} + +const startTest = async () => { + event.dispatcher.emit(event.test.started, {}) + await recorder.promise() +} + +const visit = async url => { + page.currentUrl = url + page.emit('load', page) +} + +describe('sitemap plugin', () => { + beforeEach(() => { + recorder.reset() + recorder.start() + outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sitemap-')) + originalOutputDir = store.outputDir + store.outputDir = outputDir + page = createPage() + const browser = new EventEmitter() + browser.pages = async () => [page] + container.clear({ + Puppeteer: { + options: {}, + browser, + }, + }) + }) + + afterEach(() => { + event.dispatcher.removeAllListeners(event.test.started) + event.dispatcher.removeAllListeners(event.all.result) + event.dispatcher.removeAllListeners(event.workers.before) + event.dispatcher.removeAllListeners(event.workers.result) + store.outputDir = originalOutputDir + fs.rmSync(outputDir, { recursive: true, force: true }) + }) + + it('writes unique visited pages to sitemap.xml', async () => { + sitemap({ stripQuery: false }) + await startTest() + await visit('http://localhost:8000/') + await visit('http://localhost:8000/form/field#top') + await visit('http://localhost:8000/form/field') + await visit('about:blank') + await visit('http://localhost:8000/search?q=a&b=c') + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml).to.include('') + expect(xml.match(//g).length).to.equal(3) + expect(xml).to.include('http://localhost:8000/') + expect(xml).to.include('http://localhost:8000/form/field') + expect(xml).to.include('http://localhost:8000/search?q=a&b=c') + expect(xml).not.to.include('about:blank') + }) + + it('records SPA route changes and ignores iframe navigations', async () => { + sitemap({}) + await startTest() + await startTest() + expect(page.listenerCount('load')).to.equal(1) + page.emit('framenavigated', { url: () => 'http://localhost:8000/frame', parentFrame: () => ({}) }) + page.emit('framenavigated', { url: () => 'http://localhost:8000/spa-route', parentFrame: () => null }) + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(1) + expect(xml).to.include('http://localhost:8000/spa-route') + }) + + it('attaches to Puppeteer pages opened later', async () => { + sitemap({}) + await startTest() + const newTab = createPage() + newTab.currentUrl = 'http://localhost:8000/new-tab' + const browser = container.helpers('Puppeteer').browser + browser.emit('targetcreated', { type: () => 'page', page: async () => newTab }) + browser.emit('targetcreated', { type: () => 'service_worker', page: async () => null }) + await new Promise(resolve => setImmediate(resolve)) + newTab.currentUrl = 'http://localhost:8000/new-tab/next' + newTab.emit('load', newTab) + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml).to.include('http://localhost:8000/new-tab') + expect(xml).to.include('http://localhost:8000/new-tab/next') + }) + + it('attaches to Playwright contexts and pages once per browser', async () => { + const context = new EventEmitter() + context.pages = () => [page] + const browser = new EventEmitter() + browser.contexts = () => [context] + container.clear({ Playwright: { options: {}, browser } }) + + sitemap({}) + await startTest() + await startTest() + expect(browser.listenerCount('context')).to.equal(1) + expect(page.listenerCount('load')).to.equal(1) + + await visit('http://localhost:8000/') + const sessionContext = new EventEmitter() + sessionContext.pages = () => [] + browser.emit('context', sessionContext) + const sessionPage = createPage() + sessionContext.emit('page', sessionPage) + sessionPage.currentUrl = 'http://localhost:8000/session' + sessionPage.emit('load', sessionPage) + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(2) + expect(xml).to.include('http://localhost:8000/session') + }) + + it('collects pages from WebDriver BiDi events', async () => { + const browser = new EventEmitter() + browser.capabilities = { webSocketUrl: 'ws://localhost' } + browser.subscribed = [] + browser.sessionSubscribe = async ({ events }) => browser.subscribed.push(...events) + container.clear({ WebDriver: { options: {}, browser } }) + + sitemap({}) + event.dispatcher.emit(event.test.started, {}) + await recorder.promise() + expect(browser.subscribed).to.include('browsingContext.load') + + browser.emit('browsingContext.contextCreated', { context: 'frame-1', parent: 'top' }) + browser.emit('browsingContext.load', { context: 'top', url: 'http://localhost:8000/' }) + browser.emit('browsingContext.load', { context: 'frame-1', url: 'http://localhost:8000/frame' }) + browser.emit('browsingContext.historyUpdated', { context: 'top', url: 'http://localhost:8000/spa-route' }) + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(2) + expect(xml).not.to.include('/frame') + }) + + it('strips query strings by default and saves to custom file', async () => { + sitemap({ file: 'reports/pages.xml' }) + await startTest() + await visit('http://localhost:8000/posts?page=1') + await visit('http://localhost:8000/posts?page=2') + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'reports', 'pages.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(1) + expect(xml).to.include('http://localhost:8000/posts') + }) + + it('keeps query strings when stripQuery=false is passed from CLI', async () => { + sitemap({ _args: ['stripQuery=false'] }) + await startTest() + await visit('http://localhost:8000/posts?page=1') + await visit('http://localhost:8000/posts?page=2') + event.dispatcher.emit(event.all.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(2) + }) + + it('merges worker partials on workers.result', async () => { + sitemap({}) + fs.writeFileSync(path.join(outputDir, '.sitemap.1.1.json'), JSON.stringify(['http://localhost:8000/a'])) + fs.writeFileSync(path.join(outputDir, '.sitemap.1.2.json'), JSON.stringify(['http://localhost:8000/b', 'http://localhost:8000/a'])) + event.dispatcher.emit(event.workers.result, {}) + + const xml = fs.readFileSync(path.join(outputDir, 'sitemap.xml'), 'utf8') + expect(xml.match(//g).length).to.equal(2) + expect(fs.readdirSync(outputDir)).to.deep.equal(['sitemap.xml']) + }) + + it('does not write a file when no pages were visited', () => { + sitemap({}) + event.dispatcher.emit(event.all.result, {}) + expect(fs.existsSync(path.join(outputDir, 'sitemap.xml'))).to.equal(false) + }) +})