Skip to content

Commit 2f75cd4

Browse files
test(seo): add a deployed sitemap contract checker (#1471)
* test(seo): add a deployed sitemap contract checker * fix(seo): tighten sitemap contract validation * Reject empty sitemap preview origins and distinguish image directives
1 parent 803db81 commit 2f75cd4

6 files changed

Lines changed: 317 additions & 1 deletion

File tree

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
"docs:webhooks:plan": "tsx scripts/sync-docs-webhooks.ts --dry-run",
3939
"docs:webhooks:sync": "tsx scripts/sync-docs-webhooks.ts",
4040
"docs:links:check": "tsx scripts/check-docs-menu-links.ts",
41+
"sitemap:check": "tsx scripts/check-sitemap.ts",
4142
"husky": "pnpm run format && pnpm run test",
4243
"pretest": "pnpm run content:build",
4344
"test": "run-s test:tsc test:lint test:unit test:chat test:desktop",

‎scripts/check-sitemap.ts‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
import { load } from 'cheerio'
2+
import { parseArgs } from 'node:util'
3+
import { pathToFileURL } from 'node:url'
4+
5+
export function getSitemapUrls(xml: string) {
6+
const $ = load(xml, { xmlMode: true })
7+
return $('urlset > url > loc')
8+
.toArray()
9+
.map((node) => $(node).text().trim())
10+
}
11+
12+
export function checkSitemapPage(
13+
expectedUrl: string,
14+
response: { status: number; headers: Headers; body: string },
15+
) {
16+
const failures: Array<string> = []
17+
const expected = new URL(expectedUrl)
18+
if (
19+
/\/(?:drafts|readme(?:\.[^/]+)?)(?:\/|$)|\.draft(?:\/|$)/i.test(
20+
expected.pathname,
21+
) ||
22+
/^\/(?:account|admin|login|partners-embed|sponsors-embed)(?:\/|$)/.test(
23+
expected.pathname,
24+
)
25+
) {
26+
failures.push('disallowed sitemap path')
27+
}
28+
if (response.status !== 200) failures.push(`HTTP ${response.status}`)
29+
const $ = load(response.body)
30+
const canonical = $('head link[rel]')
31+
.toArray()
32+
.filter((node) =>
33+
($(node).attr('rel') ?? '')
34+
.toLowerCase()
35+
.split(/\s+/)
36+
.includes('canonical'),
37+
)
38+
if (canonical.length !== 1) {
39+
failures.push(`expected one canonical link, found ${canonical.length}`)
40+
} else {
41+
const href = $(canonical[0]).attr('href')
42+
try {
43+
if (!href || new URL(href, expected).href !== expected.href) {
44+
failures.push('canonical does not match sitemap URL')
45+
}
46+
} catch {
47+
failures.push('invalid canonical URL')
48+
}
49+
}
50+
const directives = [
51+
response.headers.get('x-robots-tag') ?? '',
52+
...$('meta[name]')
53+
.toArray()
54+
.filter((node) =>
55+
['robots', 'googlebot', 'bingbot'].includes(
56+
($(node).attr('name') ?? '').toLowerCase(),
57+
),
58+
)
59+
.map((node) => $(node).attr('content') ?? ''),
60+
]
61+
if (
62+
directives.some((value) =>
63+
/\b(?:noindex|none)\b/i.test(
64+
value.replace(/\bmax-image-preview\s*:\s*none\b/gi, ''),
65+
),
66+
)
67+
) {
68+
failures.push('page is noindex')
69+
}
70+
if (
71+
$('h1')
72+
.toArray()
73+
.some((node) =>
74+
/^(?:content temporarily unavailable|page not found|not found|something went wrong)$/i.test(
75+
$(node).text().trim(),
76+
),
77+
)
78+
)
79+
failures.push('error or temporary-unavailable heading')
80+
return failures
81+
}
82+
83+
async function main() {
84+
const { values } = parseArgs({
85+
options: {
86+
sitemap: { type: 'string' },
87+
'base-url': { type: 'string' },
88+
limit: { type: 'string' },
89+
},
90+
})
91+
if (!values.sitemap)
92+
throw new Error(
93+
'Pass --sitemap <URL>; optionally --base-url <preview origin> and --limit <count>.',
94+
)
95+
const sitemap = new URL(values.sitemap)
96+
const base =
97+
values['base-url'] !== undefined ? new URL(values['base-url']) : undefined
98+
const limit = values.limit === undefined ? Infinity : Number(values.limit)
99+
if (
100+
values.limit !== undefined &&
101+
(!Number.isSafeInteger(limit) || limit < 1)
102+
) {
103+
throw new Error('--limit must be a positive integer')
104+
}
105+
const response = await fetch(sitemap, {
106+
redirect: 'manual',
107+
signal: AbortSignal.timeout(30_000),
108+
})
109+
if (response.status !== 200)
110+
throw new Error(`Sitemap returned HTTP ${response.status}`)
111+
const urls = getSitemapUrls(await response.text())
112+
if (!urls.length) throw new Error('Sitemap contains no URL entries')
113+
const selected = urls.slice(0, limit)
114+
let failed = 0
115+
for (let offset = 0; offset < selected.length; offset += 4) {
116+
const results = await Promise.all(
117+
selected.slice(offset, offset + 4).map(async (url) => {
118+
try {
119+
const canonical = new URL(url)
120+
const target = base
121+
? new URL(canonical.pathname + canonical.search, base.origin)
122+
: canonical
123+
const page = await fetch(target, {
124+
redirect: 'manual',
125+
signal: AbortSignal.timeout(20_000),
126+
})
127+
return {
128+
url,
129+
failures: checkSitemapPage(url, {
130+
status: page.status,
131+
headers: page.headers,
132+
body: await page.text(),
133+
}),
134+
}
135+
} catch (error) {
136+
return {
137+
url,
138+
failures: [error instanceof Error ? error.message : String(error)],
139+
}
140+
}
141+
}),
142+
)
143+
for (const result of results) {
144+
if (result.failures.length) {
145+
failed++
146+
console.error(`${result.url}: ${result.failures.join('; ')}`)
147+
}
148+
}
149+
}
150+
console.log(
151+
`Checked ${selected.length}/${urls.length} sitemap URLs: ${failed} failed.${selected.length < urls.length ? ' Partial check only.' : ''}`,
152+
)
153+
if (failed) process.exitCode = 1
154+
}
155+
156+
if (
157+
process.argv[1] &&
158+
import.meta.url === pathToFileURL(process.argv[1]).href
159+
) {
160+
await main()
161+
}

‎sitemap-checking.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Check the sitemap contract
2+
3+
Run the full read-only check against production:
4+
5+
```sh
6+
pnpm sitemap:check --sitemap https://tanstack.com/sitemap.xml
7+
```
8+
9+
For a commit preview, use its sitemap URL and pass its origin as `--base-url`. Entries retain their production canonical URLs, while HTTP requests go to the preview:
10+
11+
```sh
12+
pnpm sitemap:check --sitemap https://YOUR-PREVIEW.workers.dev/sitemap.xml --base-url https://YOUR-PREVIEW.workers.dev
13+
```
14+
15+
The checker makes four concurrent requests, does not follow redirects, and exits nonzero for failed pages. It checks status 200, exactly one matching canonical link, robots meta/header directives, error headings, and disallowed repository-artifact or private paths. It does not run page JavaScript or prove content quality.
16+
17+
Use `--limit 20` for a quick check of the first 20 entries. The output labels this as partial; passing a sample does not verify the whole sitemap. Without a limit every entry is checked, which can take time on a large sitemap. Neither command changes deployed content or clears caches.
18+
19+
Contract regression tests run in the normal unit suite. The deployed check is available for release verification; it is not yet a mandatory deployment gate while the remaining failures in issue #1153 are being fixed.

‎src/utils/sitemap-docs.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ export function isHighValueDocsSlug(slug: string) {
1717
return (
1818
segments.length > 0 &&
1919
!segments.some((segment) =>
20-
/^(?:drafts|readme)$|\.draft$/i.test(segment),
20+
/^(?:drafts|readme(?:\.[^/]+)?)$|\.draft$/i.test(segment),
2121
) &&
2222
!segments.some((segment) => LOW_VALUE_DOCS_SITEMAP_SEGMENTS.has(segment)) &&
2323
!LOW_VALUE_DOCS_SITEMAP_SLUGS.has(slug)

‎tests/sitemap-contract.test.ts‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import assert from 'node:assert/strict'
2+
import test from 'node:test'
3+
import { spawnSync } from 'node:child_process'
4+
import { fileURLToPath } from 'node:url'
5+
import { checkSitemapPage, getSitemapUrls } from '../scripts/check-sitemap'
6+
7+
const url = 'https://tanstack.com/charts/catalog'
8+
const canonical = `<link rel="canonical" href="${url}">`
9+
function check(body = canonical, status = 200, headers = new Headers()) {
10+
return checkSitemapPage(url, { body, status, headers })
11+
}
12+
13+
test('sitemap parser decodes XML entities and ignores unrelated links', () => {
14+
assert.deepEqual(
15+
getSitemapUrls(
16+
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>https://tanstack.com/partners?a=1&amp;b=2</loc></url></urlset>',
17+
),
18+
['https://tanstack.com/partners?a=1&b=2'],
19+
)
20+
assert.deepEqual(getSitemapUrls('<html><a href="/">Home</a></html>'), [])
21+
})
22+
23+
test('contract accepts a healthy self-canonical page and rejects status/canonical failures', () => {
24+
assert.deepEqual(check(), [])
25+
assert.ok(
26+
check(`<html><head></head><body>${canonical}</body></html>`).includes(
27+
'expected one canonical link, found 0',
28+
),
29+
)
30+
assert.ok(check(canonical, 307).includes('HTTP 307'))
31+
assert.ok(check(canonical, 404).includes('HTTP 404'))
32+
assert.ok(check('').includes('expected one canonical link, found 0'))
33+
assert.ok(
34+
check(canonical + canonical).includes(
35+
'expected one canonical link, found 2',
36+
),
37+
)
38+
assert.ok(
39+
check('<link rel="canonical" href="/elsewhere">').includes(
40+
'canonical does not match sitemap URL',
41+
),
42+
)
43+
assert.ok(
44+
check('<link rel="canonical" href="http://[">').includes(
45+
'invalid canonical URL',
46+
),
47+
)
48+
})
49+
50+
test('contract rejects robots directives, placeholders and forbidden paths', () => {
51+
assert.ok(
52+
check(
53+
canonical + '<meta name="ROBOTS" content="noindex, follow">',
54+
).includes('page is noindex'),
55+
)
56+
assert.ok(
57+
check(
58+
canonical,
59+
200,
60+
new Headers({ 'X-Robots-Tag': 'googlebot: none' }),
61+
).includes('page is noindex'),
62+
)
63+
assert.ok(
64+
check(canonical + '<h1>Content temporarily unavailable</h1>').includes(
65+
'error or temporary-unavailable heading',
66+
),
67+
)
68+
for (const path of [
69+
'/router/latest/docs/how-to/drafts/test',
70+
'/db/latest/docs/README',
71+
'/db/latest/docs/README.md',
72+
'/router/latest/docs/test.draft',
73+
'/admin/users',
74+
]) {
75+
const expected = `https://tanstack.com${path}`
76+
assert.ok(
77+
checkSitemapPage(expected, {
78+
body: `<link rel="canonical" href="${expected}">`,
79+
status: 200,
80+
headers: new Headers(),
81+
}).includes('disallowed sitemap path'),
82+
)
83+
}
84+
})
85+
86+
test('image-preview restrictions do not prevent page indexing', () => {
87+
for (const directive of [
88+
'max-image-preview:none',
89+
'index, MAX-IMAGE-PREVIEW: none',
90+
'googlebot: max-image-preview: none',
91+
]) {
92+
assert.deepEqual(
93+
check(canonical + `<meta name="robots" content="${directive}">`),
94+
[],
95+
)
96+
assert.deepEqual(
97+
check(canonical, 200, new Headers({ 'X-Robots-Tag': directive })),
98+
[],
99+
)
100+
assert.ok(
101+
check(
102+
canonical,
103+
200,
104+
new Headers({ 'X-Robots-Tag': `${directive}, noindex` }),
105+
).includes('page is noindex'),
106+
)
107+
assert.ok(
108+
check(
109+
canonical,
110+
200,
111+
new Headers({ 'X-Robots-Tag': `${directive}, none` }),
112+
).includes('page is noindex'),
113+
)
114+
}
115+
})
116+
117+
test('CLI rejects an explicitly empty preview origin before fetching', () => {
118+
const result = spawnSync(
119+
process.execPath,
120+
[
121+
'--import',
122+
'tsx',
123+
fileURLToPath(new URL('../scripts/check-sitemap.ts', import.meta.url)),
124+
'--sitemap',
125+
'http://127.0.0.1:1/sitemap.xml',
126+
'--base-url',
127+
'',
128+
],
129+
{ encoding: 'utf8' },
130+
)
131+
assert.notEqual(result.status, 0)
132+
assert.match(result.stderr, /ERR_INVALID_URL/)
133+
})

‎tests/sitemap-docs.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ test('sitemap excludes repository artifacts while retaining public guides', () =
77
'how-to/drafts/README',
88
'how-to/drafts/search-params-in-forms.draft',
99
'how-to/README',
10+
'how-to/README.md',
11+
'README.mdx',
1012
'superpowers/specs/2026-07-31-tanstack-devtools-branding-design',
1113
'superpowers/plans/internal-implementation',
1214
'contributing',

0 commit comments

Comments
 (0)