diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index c4ea225fc..8db4badd0 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -311,7 +311,7 @@ current release 随包执行的 release readiness 必须追加 `--release-runtim 涉及 Nginx 模板、Pingora 路由、限流分组或路由文档时，必须同步更新 `deploy/pingora/nginx-route-parity.matrix.json`，并运行 `npm run check:pingora-route-parity`；Rust 侧 `cargo test -p pingora-gateway --manifest-path server-rs/Cargo.toml matches_nginx_route_parity_matrix` 会读取同一份矩阵验证 `classify_path` 的路由结果、body limit 和接流保护分组。 -Pingora canary 入口默认由 Server-Provision 安装但不接入主站配置。`deploy/nginx/snippets/genarrative-pingora-canary.conf` 是前缀 canary，人工 include 后用 `/__genarrative_pingora_canary/` rewrite 到 Pingora shadow；`deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 是前缀 canary 之后、direct 直连之前的真实路径 canary，只能在 Nginx `http` 上下文 include，默认监听 `127.0.0.1:18083` 并写独立 `/var/log/nginx/genarrative-pingora-realpath-canary.access.log`，不要 include 到生产 `443` server 内覆盖正式 location。启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，强制把前缀 location 片段和真实路径 server 片段一起执行 `nginx -t`。本机或 CI 可用 `npm run check:pingora-canary-docker` 启动 Docker Nginx、真实 Pingora 和 mock 上游复现两条 Nginx -> Pingora handoff，并复用 `scripts/check-pingora-canary-access-log-parity.mjs` 按 `request_id` 对账。前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`；真实路径 canary reload 后运行 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，随后用 `node scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js` 对账；真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。canary live 的 base URL、prefix、Host、额外 path、`--timeout-ms` / `GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS` 不能包含换行或 NUL，脚本会在发起请求前失败；canary access log 对账脚本的日志路径、prefix、必需路径和 `--since-lines` / `GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES` 也不能包含换行或 NUL，日志行里解析出的 URI / path 含控制字符时必须失败；timeout 和 access log 对账的 `--since-lines` / 对应 env 必须是正整数，非法值直接失败，不静默回默认值。 +Pingora canary 入口默认由 Server-Provision 安装但不接入主站配置。`deploy/nginx/snippets/genarrative-pingora-canary.conf` 是前缀 canary，人工 include 后用 `/__genarrative_pingora_canary/` rewrite 到 Pingora shadow；`deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 是前缀 canary 之后、direct 直连之前的真实路径 canary，只能在 Nginx `http` 上下文 include，默认监听 `127.0.0.1:18083` 并写独立 `/var/log/nginx/genarrative-pingora-realpath-canary.access.log`，不要 include 到生产 `443` server 内覆盖正式 location。启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，强制把前缀 location 片段和真实路径 server 片段一起执行 `nginx -t`。本机或 CI 可用 `npm run check:pingora-canary-docker` 启动 Docker Nginx、真实 Pingora 和 mock 上游复现两条 Nginx -> Pingora handoff，并复用 `scripts/check-pingora-canary-access-log-parity.mjs` 按 `request_id` 对账。需要证明 Docker Nginx -> Pingora -> 真实本地服务时，先启动真实 api-server 与 SpacetimeDB，再运行 `node scripts/check-pingora-canary-docker.mjs --require-docker --real-upstreams --api-upstream 127.0.0.1: --spacetime-upstream 127.0.0.1: --web-root dist`；该模式不会启动内置 mock 上游，会先检查真实 `/healthz` 和 `/v1/ping`，随后复用同一组 live smoke 与 access log 对账。真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，canary live 只把它作为路径路由代表，不要求该 GET 创建 identity。前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`；真实路径 canary reload 后运行 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，随后用 `node scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js` 对账；真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。canary live 的 base URL、prefix、Host、额外 path、`--timeout-ms` / `GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS` 不能包含换行或 NUL，脚本会在发起请求前失败；canary access log 对账脚本的日志路径、prefix、必需路径和 `--since-lines` / `GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES` 也不能包含换行或 NUL，日志行里解析出的 URI / path 含控制字符时必须失败；timeout 和 access log 对账的 `--since-lines` / 对应 env 必须是正整数，非法值直接失败，不静默回默认值。 Pingora 正式切换前必须额外运行聚合门禁：`node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。该门禁串起 Rust 单测、mock smoke、路由 parity、Nginx snippet 校验、Docker Nginx handoff、canary access log 对账烟测、直连入口静态预检、直连启用 / 回退 dry-run 行为检查、health patrol env 切换脚本烟测、current release 自审烟测、release readiness 计划自检、生产运维护栏、API release build 烟测、Pingora production release 真实构建烟测、API deploy release 烟测、目标机 live canary 和目标机真实 access log 对账；其中 `--require-live` 只能在目标 Nginx 已人工 include canary snippet 并 reload 后执行，并会强制要求 `--live-host`，避免只访问 `127.0.0.1` 命中 Nginx 默认 vhost。live smoke 成功后会立即读取 Nginx 与 Pingora access log 尾部记录，按 `request_id` 验证 `/__genarrative_pingora_canary/healthz` 和 `/__genarrative_pingora_canary/api/creation-entry/config` 已在两边落盘且 method/status/path 没有漂移。启用真实路径 canary 时追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，它会用独立 Nginx access log 对账真实 `/api`、`/v1` 与 `/assets` 路径。live canary、direct live 和 access log 对账的显式毫秒超时或尾部行数参数都必须是正整数，非法值应先修参数再重跑，不能把默认值兜底后的结果当成切换证据；direct live 与 release readiness 读取的直连布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，拼写错误会直接失败，避免 `REQUIRE_WSS_UPGRADE`、preflight 开关或 `SKIP_WSS` 被悄悄当成 false。若验证 Pingora 直连公网入口，还必须追加 direct preflight 参数检查目标机 env、systemd drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性和端口释放：`--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free`；该模式缺少 `--direct-http-base-url`、`--direct-host`、`--direct-redirect-host`、`--direct-pingora-access-log`、`--direct-preflight-systemd`、`--direct-preflight-check-cert-readable`、`--direct-preflight-check-service-env-file`、`--direct-preflight-check-service-user-cert-readable`、`--direct-preflight-check-service-binary-executable`、`--direct-preflight-check-ports-free` 、缺少 `--direct-health-patrol-env-file` 或缺少 `--direct-spacetime-database` 会直接失败，且会拒绝 `--direct-skip-wss`，并默认要求 WSS subscribe 返回 101，证明 HTTP redirect / ACME、正式域名 Host/SNI、redirect Location host、Pingora access log request_id + path/status 对账、health patrol direct 模式、systemd drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性、80/443 端口释放与目标 SpacetimeDB 长连接都能从直连入口透传。`check-pingora-release-readiness.mjs --help` 中的正式直连和只生成 runbook 示例也必须显式带 `--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，避免值班人员复制示例后被 `--require-direct` 自身拦住。普通本机提交前可先跑 `npm run check:pingora-release-readiness`，但该默认模式不能替代目标机切换窗口的强制门禁。直连切换前还应先运行 `npm run plan:pingora-direct-cutover -- --require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free --rollback-nginx-smoke-url http://127.0.0.1/healthz --rollback-health-patrol-public-base-url <切换前Nginx巡检入口>` 生成只读 JSON runbook；若切换参数提供 `--direct-probe-token`，启用后复核会继续透传 direct probe token 检查内部探针，runbook JSON 只显示 ``。runbook 会列出 Host 与回退巡检入口确认、current release 自包含自审、current release preflight、启用前基础门禁、direct enable dry-run、direct enable apply、用 `/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply` 切到 `pingora-direct`、启用后 health patrol env 直连复核、启用后 `--require-direct` 复核、rollback dry-run、回退前用同一脚本预置回 `nginx` 并恢复切换前 public base URL / Host、rollback apply、回退后 health patrol env Nginx 模式复核，供当班人员逐条审阅。direct enable / rollback `--apply` 都会拒绝符号链接形式的 systemd drop-in 目录或目标文件，并拒绝已存在但不是普通文件的目标，避免把低端口 capability 写入非预期位置，或在回退窗口误删 / 误判非预期 systemd 位置；health patrol env 切换脚本 `--apply` 会保留原文件权限和 owner/group，此时 `--env-file` 必须直接指向真实普通文件，不能是符号链接。如果现场 env 是链接，先确认真实目标路径后再传给脚本，避免替换链接本身或写入非预期目标。 diff --git a/scripts/check-pingora-canary-docker.mjs b/scripts/check-pingora-canary-docker.mjs index c50efc86d..e7bcae174 100644 --- a/scripts/check-pingora-canary-docker.mjs +++ b/scripts/check-pingora-canary-docker.mjs @@ -72,10 +72,35 @@ async function main() { const accessLogFile = path.join(logsRoot, 'pingora.access.log'); await mkdir(logsRoot, { recursive: true }); await chmod(logsRoot, 0o777); - await prepareStaticRoots(webRoot, acmeRoot); - const api = await startApiMock(); - const spacetime = await startSpacetimeMock(); + const gatewayWebRoot = config.webRoot || webRoot; + const gatewayAcmeRoot = config.acmeRoot || acmeRoot; + if (!config.webRoot) { + await prepareStaticWebRoot(webRoot); + } + if (!config.acmeRoot) { + await prepareAcmeRoot(acmeRoot); + } + + const realUpstreams = config.realUpstreams || config.apiUpstream || config.spacetimeUpstream; + if (realUpstreams && (!config.apiUpstream || !config.spacetimeUpstream)) { + throw new Error( + '真实上游模式必须同时提供 --api-upstream 与 --spacetime-upstream。', + ); + } + + const api = realUpstreams + ? { upstream: config.apiUpstream, state: null } + : await startApiMock(); + const spacetime = realUpstreams + ? { upstream: config.spacetimeUpstream, state: null } + : await startSpacetimeMock(); + if (realUpstreams) { + console.log( + `[pingora-canary-docker] 使用真实上游 api=${api.upstream} spacetime=${spacetime.upstream} webRoot=${gatewayWebRoot}`, + ); + await assertRealUpstreamsReady(api.upstream, spacetime.upstream); + } const pingoraPort = await getFreePort(); const nginxPort = await getFreePort(); const realpathNginxPort = await getFreePort(); @@ -97,10 +122,10 @@ async function main() { env: { ...smokeEnv(), GENARRATIVE_PINGORA_GATEWAY_LISTEN: `0.0.0.0:${pingoraPort}`, - GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM: `127.0.0.1:${api.port}`, - GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM: `127.0.0.1:${spacetime.port}`, - GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT: webRoot, - GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT: acmeRoot, + GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM: api.upstream, + GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM: spacetime.upstream, + GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT: gatewayWebRoot, + GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT: gatewayAcmeRoot, GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE: path.join( tempRoot, 'maintenance', @@ -280,27 +305,29 @@ async function main() { '/assets/app.js', ]); - ensure( - api.state.requests.some( - (request) => request.url === '/api/creation-entry/config', - ), - 'Docker Nginx canary 未把 API 代表路径交给 mock api-server', - ); - ensure( - spacetime.state.requests.some((request) => request.url === '/v1/identity'), - 'Docker Nginx canary 未把 SpacetimeDB identity 代表路径交给 mock SpacetimeDB', - ); - ensure( - api.state.requests.filter( - (request) => request.url === '/api/creation-entry/config', - ).length >= 2, - 'Docker Nginx realpath canary 未把真实 API 代表路径交给 mock api-server', - ); - ensure( - spacetime.state.requests.filter((request) => request.url === '/v1/identity') - .length >= 2, - 'Docker Nginx realpath canary 未把真实 SpacetimeDB identity 路径交给 mock SpacetimeDB', - ); + if (!realUpstreams) { + ensure( + api.state.requests.some( + (request) => request.url === '/api/creation-entry/config', + ), + 'Docker Nginx canary 未把 API 代表路径交给 mock api-server', + ); + ensure( + spacetime.state.requests.some((request) => request.url === '/v1/identity'), + 'Docker Nginx canary 未把 SpacetimeDB identity 代表路径交给 mock SpacetimeDB', + ); + ensure( + api.state.requests.filter( + (request) => request.url === '/api/creation-entry/config', + ).length >= 2, + 'Docker Nginx realpath canary 未把真实 API 代表路径交给 mock api-server', + ); + ensure( + spacetime.state.requests.filter((request) => request.url === '/v1/identity') + .length >= 2, + 'Docker Nginx realpath canary 未把真实 SpacetimeDB identity 路径交给 mock SpacetimeDB', + ); + } } function parseArgs(argv) { @@ -310,6 +337,23 @@ function parseArgs(argv) { requireDocker: false, skipBuild: false, verbose: false, + realUpstreams: false, + apiUpstream: normalizeOptionalHostPort( + process.env.GENARRATIVE_PINGORA_CANARY_REAL_API_UPSTREAM, + 'GENARRATIVE_PINGORA_CANARY_REAL_API_UPSTREAM', + ), + spacetimeUpstream: normalizeOptionalHostPort( + process.env.GENARRATIVE_PINGORA_CANARY_REAL_SPACETIME_UPSTREAM, + 'GENARRATIVE_PINGORA_CANARY_REAL_SPACETIME_UPSTREAM', + ), + webRoot: normalizeOptionalDirectory( + process.env.GENARRATIVE_PINGORA_CANARY_REAL_WEB_ROOT, + 'GENARRATIVE_PINGORA_CANARY_REAL_WEB_ROOT', + ), + acmeRoot: normalizeOptionalDirectory( + process.env.GENARRATIVE_PINGORA_CANARY_REAL_ACME_ROOT, + 'GENARRATIVE_PINGORA_CANARY_REAL_ACME_ROOT', + ), }; for (let index = 0; index < argv.length; index += 1) { @@ -325,10 +369,21 @@ Options: --pull Pull the image when it is not present locally. --require-docker Treat missing Docker/image/host-network as failure instead of skip. --skip-build Reuse an existing pingora-gateway debug binary. + --real-upstreams Require real api-server and SpacetimeDB upstream arguments. + --api-upstream + Use a real api-server upstream instead of the built-in mock. + --spacetime-upstream + Use a real SpacetimeDB upstream instead of the built-in mock. + --web-root Serve an existing web root, for example dist/. + --acme-root Serve an existing ACME root; temp root is used by default. --verbose Print Pingora and Docker Nginx process logs. Environment aliases: GENARRATIVE_PINGORA_CANARY_DOCKER_IMAGE + GENARRATIVE_PINGORA_CANARY_REAL_API_UPSTREAM + GENARRATIVE_PINGORA_CANARY_REAL_SPACETIME_UPSTREAM + GENARRATIVE_PINGORA_CANARY_REAL_WEB_ROOT + GENARRATIVE_PINGORA_CANARY_REAL_ACME_ROOT GENARRATIVE_PINGORA_GATEWAY_BINARY 默认不会拉取镜像；本机或 CI 要做强验收时建议： @@ -348,6 +403,24 @@ Environment aliases: case '--skip-build': result.skipBuild = true; break; + case '--real-upstreams': + result.realUpstreams = true; + break; + case '--api-upstream': + result.apiUpstream = normalizeHostPort(requireValue(argv, ++index, arg), arg); + break; + case '--spacetime-upstream': + result.spacetimeUpstream = normalizeHostPort( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--web-root': + result.webRoot = normalizeDirectory(requireValue(argv, ++index, arg), arg); + break; + case '--acme-root': + result.acmeRoot = normalizeDirectory(requireValue(argv, ++index, arg), arg); + break; case '--verbose': result.verbose = true; break; @@ -367,6 +440,56 @@ function requireValue(argv, index, flag) { return value; } +function normalizeOptionalHostPort(value, label) { + if (!value) { + return ''; + } + return normalizeHostPort(value, label); +} + +function normalizeHostPort(value, label) { + validateNoControlCharacters(value, label); + const raw = String(value || '').trim(); + if (!raw) { + throw new Error(`${label} 不能为空。`); + } + if (raw.includes('://') || /[\s/?#@]/u.test(raw)) { + throw new Error(`${label} 必须是 host:port，不能包含 scheme、路径、查询、片段或空白字符。`); + } + try { + const parsed = new URL(`http://${raw}`); + const port = Number.parseInt(parsed.port, 10); + if (!parsed.hostname || !Number.isInteger(port) || port <= 0 || port > 65535) { + throw new Error('invalid host:port'); + } + } catch { + throw new Error(`${label} 必须是合法 host:port。`); + } + return raw; +} + +function normalizeOptionalDirectory(value, label) { + if (!value) { + return ''; + } + return normalizeDirectory(value, label); +} + +function normalizeDirectory(value, label) { + validateNoControlCharacters(value, label); + const resolved = path.resolve(repoRoot, String(value || '').trim()); + if (!existsSync(resolved)) { + throw new Error(`${label} 不存在：${resolved}`); + } + return resolved; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value ?? ''))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + function ensureDockerReady() { const result = spawnSync( 'docker', @@ -425,12 +548,9 @@ function skipOrFail(message) { return false; } -async function prepareStaticRoots(webRoot, acmeRoot) { +async function prepareStaticWebRoot(webRoot) { await mkdir(path.join(webRoot, 'admin', 'assets'), { recursive: true }); await mkdir(path.join(webRoot, 'assets'), { recursive: true }); - await mkdir(path.join(acmeRoot, '.well-known', 'acme-challenge'), { - recursive: true, - }); await writeFile(path.join(webRoot, 'index.html'), '
site-shell
'); await writeFile( @@ -445,6 +565,12 @@ async function prepareStaticRoots(webRoot, acmeRoot) { path.join(webRoot, 'admin', 'assets', 'admin.js'), 'console.log("admin asset");', ); +} + +async function prepareAcmeRoot(acmeRoot) { + await mkdir(path.join(acmeRoot, '.well-known', 'acme-challenge'), { + recursive: true, + }); await writeFile( path.join(acmeRoot, '.well-known', 'acme-challenge', 'token'), 'acme-token', @@ -501,7 +627,7 @@ async function startApiMock() { }); const port = await listen(server); - return { port, state }; + return { upstream: `127.0.0.1:${port}`, state }; } async function startSpacetimeMock() { @@ -542,7 +668,16 @@ async function startSpacetimeMock() { }); const port = await listen(server); - return { port, state }; + return { upstream: `127.0.0.1:${port}`, state }; +} + +async function assertRealUpstreamsReady(apiUpstream, spacetimeUpstream) { + await waitForHttp(`http://${apiUpstream}/healthz`, 200, { + label: '等待真实 api-server 就绪', + }); + await waitForHttp(`http://${spacetimeUpstream}/v1/ping`, 200, { + label: '等待真实 SpacetimeDB 就绪', + }); } async function renderNginxConfig({ diff --git a/scripts/check-pingora-canary-live.mjs b/scripts/check-pingora-canary-live.mjs index 759c2810c..0527a32d0 100644 --- a/scripts/check-pingora-canary-live.mjs +++ b/scripts/check-pingora-canary-live.mjs @@ -242,7 +242,7 @@ async function main() { { name: 'spacetime-identity', path: '/v1/identity', - expectedStatuses: [200, 401, 403, 404, 503], + expectedStatuses: [200, 401, 403, 404, 405, 503], }, { name: 'web-assets',