diff --git a/.gitignore b/.gitignore index c64003e4e..8d13559b0 100644 --- a/.gitignore +++ b/.gitignore @@ -281,6 +281,9 @@ bundle-analysis/ .agents/ skills-lock.json +# Private pinned CODA checkout and conformance reports +.coda/ + plans/ # Vitest JUnit reporter output (CI uploads it as an artifact; we don't commit it) diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 3b70e6cbf..069117486 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -41,6 +41,7 @@ "ignorePatterns": [ ".azure-pipelines", ".config", + ".coda", ".cosmosdb-data", ".vscode-test", "bundle-analysis", diff --git a/.oxlintrc.jsonc b/.oxlintrc.jsonc index a117fa62c..200e785ce 100644 --- a/.oxlintrc.jsonc +++ b/.oxlintrc.jsonc @@ -24,6 +24,7 @@ "ignorePatterns": [ ".azure-pipelines", ".config", + ".coda", ".cosmosdb-data/", ".github", ".vscode-test", diff --git a/.vscodeignore b/.vscodeignore index 90eb241d4..23ddcee2f 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -1,5 +1,6 @@ # These files are excluded during vsce package .cosmosdb-data/** +.coda/** .azure-pipelines/** .github/** .husky/** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 235448058..0a1f1c432 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,43 @@ There are several ways you can contribute to the [vscode-cosmosdb repository](ht - Debug: Select the `Launch Extension` configuration in the Debug side bar and press F5 to start debugging the extension. +### CODA conformance + +`package.json.externalSpecifications.coda` pins the Git repository, tracking ref, full commit SHA and feature directory. +The SHA, not the current branch tip, determines the downloaded revision. Update the pin deliberately when adopting +another revision; normal builds and unit tests never fetch or repin CODA. + +```shell +npm run fetch-coda +npm run test:coda -- --detectors DX-001,DX-011,DX-020 +npm run test:coda +``` + +The fetch command requires Git and access to the configured private Azure DevOps repository through your existing +Git credential helper. It does not accept or store tokens in the manifest. Each commit has an isolated checkout under +`.coda`; failed downloads preserve previous checkouts, and modified caches are rejected rather than reset. +No upstream executable, Python dependency, MCP process or Azure resource collector is run. + +The test command verifies the pinned checkout and compiles the original `.feature` files with Cucumber's Gherkin +compiler. TypeScript bindings prepare fixtures and call production detector evaluations and the native registry. +Bindings cover DX-001 through DX-024, including production metric/log readers and explicit acquisition failures. +The explicit subset command selects only those features; the unfiltered command selects **every** scenario and fails on +unsupported detectors, unbound steps, unsupported arguments or failed assertions. A selected subset is not a full +conformance pass. At the current pin all 379 scenarios pass, with no unbound, skipped or unexecuted scenarios. +Successful empty consumption reads count as idle for DX-004; Monitor request and embedded metric failures do not. +DX-017 accounts for arithmetic roundoff when comparing fitted growth with the inclusive 0.1 GiB/day floor. +These synthetic contract checks do not replace live collector acceptance. + +The local `.coda/conformance.json` report records the source SHA, full scenario inventory and passed/failed/unbound/ +not-selected counts. This report and downloaded files are private, Git-ignored, excluded from formatting/linting and +excluded from VSIX packaging. Do not upload them to public logs or artifacts. Public GitHub Actions execution is refused +unless an operator explicitly sets `CODA_TRUSTED_PRIVATE_RUNNER=1` on a genuinely trusted private runner. +Ordinary contributors without private-source access can run the infrastructure's synthetic unit tests without CODA. +`npm run test:coda-tools` checks download integrity against disposable local Git repositories. +It runs as an independent yellow `coda-tools` branch alongside magenta Vitest and cyan migration-skill tests in both +`npm run vitest` and `npm run vitest:coverage`. Parser/binding regressions do not need private-source access. +Public CI against the original contracts requires an approved redistributable upstream bundle. + This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/eslint.config.mjs b/eslint.config.mjs index 1c86686b2..4c1abf42c 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -16,6 +16,7 @@ export default defineConfig([ ignores: [ '.azure-pipelines', '.config', + '.coda', '.github', '.vscode-test', 'coverage', diff --git a/package-lock.json b/package-lock.json index bf6bff345..b087bca99 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,8 @@ "zod": "~4.6.0" }, "devDependencies": { + "@cucumber/gherkin": "~42.0.1", + "@cucumber/messages": "~34.2.1", "@playwright/test": "~1.64.0", "@testing-library/jest-dom": "~7.0.0", "@testing-library/react": "~16.3.2", @@ -1281,6 +1283,23 @@ "node": ">=20.19.0" } }, + "node_modules/@cucumber/gherkin": { + "version": "42.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-42.0.1.tgz", + "integrity": "sha512-Kqg0ULhWqbXp/a1c04ND4sS4KTlIL831tQ0NhZLXWdEhT6LiNsZ5LFRzy/lBy7QQpcKhroKp/i3C2WdEnRBKNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/messages": ">=34.0.0 <35" + } + }, + "node_modules/@cucumber/messages": { + "version": "34.2.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-34.2.1.tgz", + "integrity": "sha512-aj2iCAG9ZOpMVcMxcgShifv5fbR96b7SMgdWcPXXh2QbFp7OEcAnQ5MC5YfWUDSDasmw7FU5NJEsOrXdTQOU1A==", + "dev": true, + "license": "MIT" + }, "node_modules/@emotion/hash": { "version": "0.9.2", "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz", diff --git a/package.json b/package.json index 2547dc24d..87f8d741b 100644 --- a/package.json +++ b/package.json @@ -70,8 +70,8 @@ "test:proxy": "node scripts/test-proxy.mjs", "test:migration-skill": "node --test skills/cosmosdb-relational-migration/tests/*.test.mjs", "test:shell-mcp": "cross-env COSMOSDB_SHELL_MCP_INTEGRATION=1 vitest run src/cosmosDBShell/mcpProvider.test.ts", - "vitest": "concurrently -P -n vitest,migration-skill -c magenta,cyan \"vitest run {@}\" \"npm run test:migration-skill\" --", - "vitest:coverage": "concurrently -P -n vitest,migration-skill -c magenta,cyan \"vitest run --coverage {@}\" \"npm run test:migration-skill\" --", + "vitest": "concurrently -P -n vitest,migration-skill,coda-tools -c magenta,cyan,yellow \"vitest run {@}\" \"npm run test:migration-skill\" \"npm run test:coda-tools\" --", + "vitest:coverage": "concurrently -P -n vitest,migration-skill,coda-tools -c magenta,cyan,yellow \"vitest run --coverage {@}\" \"npm run test:migration-skill\" \"npm run test:coda-tools\" --", "vitest:ui": "vitest --ui", "vite-dev": "rimraf ./dist && concurrently -n ext,views -c redBright,whiteBright \"npm run vite-dev-ext\" \"npm run vite-dev-wv\"", "vite-prod": "rimraf ./dist && concurrently -n ext,views -c redBright,whiteBright \"npm run vite-prod-ext\" \"npm run vite-prod-wv\"", @@ -91,6 +91,9 @@ "l10n:export": "npx @vscode/l10n-dev generate-xlf ./l10n/bundle.l10n.json --outFile vscode-cosmosdb.xlf", "l10n:import": "npx @vscode/l10n-dev import-xlf ./translations.xlf", "fetch-skill": "node scripts/fetch-skill.mjs", + "fetch-coda": "node scripts/coda.mjs fetch", + "test:coda": "node scripts/coda.mjs test", + "test:coda-tools": "node --test scripts/coda.test.mjs", "pre-commit": "npm run build && npm run prettier-fix && npm run lint-fix && npm run l10n && npm run vitest", "docker-up": "docker compose up -d", "docker-down": "docker compose down", @@ -118,7 +121,17 @@ ] } }, + "externalSpecifications": { + "coda": { + "repo": "https://dev.azure.com/msdata/CosmosDB/_git/CODA", + "trackingRef": "refs/heads/main", + "commit": "86549e6b118a4e88596427fb5c008a9bc0570928", + "featurePath": "specs/detectors" + } + }, "devDependencies": { + "@cucumber/gherkin": "~42.0.1", + "@cucumber/messages": "~34.2.1", "@playwright/test": "~1.64.0", "@testing-library/jest-dom": "~7.0.0", "@testing-library/react": "~16.3.2", diff --git a/scripts/coda.mjs b/scripts/coda.mjs new file mode 100644 index 000000000..34000939a --- /dev/null +++ b/scripts/coda.mjs @@ -0,0 +1,204 @@ +#!/usr/bin/env node + +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('../', import.meta.url)); + +export function readCodaPin(manifest) { + const config = manifest.externalSpecifications?.coda; + if ( + !config || + typeof config.repo !== 'string' || + !/^https:\/\/[^/@]+\/[^?#]+$/.test(config.repo) || + !/^[a-f0-9]{40}$/.test(config.commit ?? '') || + !/^refs\/heads\/[A-Za-z0-9._/-]+$/.test(config.trackingRef ?? '') || + typeof config.featurePath !== 'string' || + !/^[A-Za-z0-9_-]+(?:\/[A-Za-z0-9_-]+)*$/.test(config.featurePath) + ) { + throw new Error( + 'Invalid externalSpecifications.coda pin: expected HTTPS repository, full SHA, tracking ref and relative feature path.', + ); + } + return config; +} + +function git(directory, args) { + const result = spawnSync('git', ['--no-pager', '-C', directory, ...args], { + encoding: 'utf8', + timeout: 180_000, + maxBuffer: 8 * 1024 * 1024, + env: { ...process.env, GIT_TERMINAL_PROMPT: '0' }, + windowsHide: true, + }); + if (result.error || result.status !== 0) { + throw new Error(`CODA Git operation failed (${args[0]}): ${result.error?.message ?? result.stderr.trim()}`); + } + return result.stdout.trim(); +} + +export function featureFiles(directory) { + const result = []; + for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const file = path.join(directory, entry.name); + if (entry.isSymbolicLink()) { + throw new Error('CODA feature paths must not contain symbolic links.'); + } + if (entry.isDirectory()) { + result.push(...featureFiles(file)); + } else if (entry.isFile() && entry.name.endsWith('.feature')) { + result.push(file); + } + } + return result; +} + +function verifyCheckout(directory, pin) { + if (lstatSync(directory).isSymbolicLink()) { + throw new Error('CODA cache must not be a symbolic link.'); + } + if (git(directory, ['rev-parse', 'HEAD']) !== pin.commit) { + throw new Error('CODA cache HEAD does not match the pinned commit.'); + } + if (git(directory, ['config', '--get', 'remote.origin.url']) !== pin.repo) { + throw new Error('CODA cache origin does not match the pinned repository.'); + } + if (git(directory, ['status', '--porcelain', '--untracked-files=all', '--ignored=matching']) !== '') { + throw new Error( + 'CODA cache has local changes. Preserve them elsewhere before removing this named cache entry.', + ); + } + const parts = pin.featurePath.split('/'); + let featureRoot = directory; + for (const part of parts) { + featureRoot = path.join(featureRoot, part); + if (lstatSync(featureRoot).isSymbolicLink()) { + throw new Error('CODA feature directory must not be a symbolic link.'); + } + } + const files = featureFiles(featureRoot); + if (files.length === 0) { + throw new Error('The pinned CODA revision contains no feature files.'); + } + return featureRoot; +} + +export function fetchCoda(pin, cacheRoot = path.join(root, '.coda')) { + mkdirSync(cacheRoot, { recursive: true }); + if (lstatSync(cacheRoot).isSymbolicLink()) { + throw new Error('CODA cache root must not be a symbolic link.'); + } + const identity = createHash('sha256').update(pin.repo).digest('hex').slice(0, 12); + const destination = path.join(cacheRoot, `${identity}-${pin.commit}`); + if (existsSync(destination)) { + return verifyCheckout(destination, pin); + } + const staging = mkdtempSync(path.join(cacheRoot, 'fetch-')); + try { + git(staging, ['init', '--quiet']); + git(staging, ['config', 'core.autocrlf', 'false']); + git(staging, ['remote', 'add', 'origin', pin.repo]); + // Fetch the object, not mutable main. No submodules, hooks or upstream executables are run. + git(staging, [ + '-c', + 'credential.interactive=never', + 'fetch', + '--quiet', + '--depth=1', + '--no-tags', + 'origin', + pin.commit, + ]); + git(staging, ['-c', 'core.hooksPath=', 'checkout', '--quiet', '--detach', pin.commit]); + verifyCheckout(staging, pin); + renameSync(staging, destination); + } finally { + if (existsSync(staging)) { + rmSync(staging, { recursive: true, force: true }); + } + } + return verifyCheckout(destination, pin); +} + +export function parseArguments(args) { + const [command, ...options] = args; + if (!['fetch', 'test'].includes(command)) { + throw new Error('Usage: node scripts/coda.mjs fetch|test [--detectors DX-001,DX-011,DX-020]'); + } + let detectors; + if (options.length > 0) { + if (command !== 'test' || options.length !== 2 || options[0] !== '--detectors') { + throw new Error('Only test supports --detectors followed by a comma-separated detector list.'); + } + detectors = options[1].split(','); + if ( + detectors.length === 0 || + detectors.some((id) => !/^DX-\d{3}$/.test(id)) || + new Set(detectors).size !== detectors.length + ) { + throw new Error('Invalid or duplicate detector selection.'); + } + } + return { command, detectors }; +} + +export function main(args = process.argv.slice(2)) { + const { command, detectors } = parseArguments(args); + if (process.env.GITHUB_ACTIONS === 'true' && process.env.CODA_TRUSTED_PRIVATE_RUNNER !== '1') { + throw new Error( + 'Private CODA sources/reports must not enter public Actions logs. Use a trusted private runner with CODA_TRUSTED_PRIVATE_RUNNER=1.', + ); + } + const manifest = JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8')); + const pin = readCodaPin(manifest); + const features = fetchCoda(pin); + console.log(`CODA commit: ${pin.commit}; feature files: ${featureFiles(features).length}`); + if (command === 'fetch') { + console.log('Pinned checkout verified. No tests or upstream code executed.'); + return 0; + } + const report = path.join(root, '.coda', 'conformance.json'); + rmSync(report, { force: true }); + const result = spawnSync( + process.execPath, + [ + path.join(root, 'node_modules', 'vitest', 'vitest.mjs'), + 'run', + '--config', + path.join(root, 'vitest.coda.config.ts'), + ], + { + cwd: root, + stdio: 'inherit', + env: { + ...process.env, + CODA_FEATURE_ROOT: features, + CODA_COMMIT: pin.commit, + CODA_DETECTORS: detectors?.join(',') ?? '', + CODA_REPORT: report, + }, + windowsHide: true, + }, + ); + if (result.error) { + throw result.error; + } + return result.status ?? 1; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + process.exitCode = main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/coda.test.mjs b/scripts/coda.test.mjs new file mode 100644 index 000000000..924e14447 --- /dev/null +++ b/scripts/coda.test.mjs @@ -0,0 +1,159 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import { fetchCoda, parseArguments, readCodaPin } from './coda.mjs'; + +const pin = { + repo: 'https://example.test/team/_git/contract', + trackingRef: 'refs/heads/main', + commit: 'a'.repeat(40), + featurePath: 'specs/detectors', +}; + +test('requires immutable pin metadata without credentials or traversable paths', () => { + assert.deepEqual(readCodaPin({ externalSpecifications: { coda: pin } }), pin); + for (const change of [ + { commit: 'main' }, + { commit: 'abc123' }, + { featurePath: '../private' }, + { featurePath: 'specs\\detectors' }, + { repo: 'https://token@example.test/repo' }, + { repo: 'https://example.test/repo?token=secret' }, + { repo: 'file:///local' }, + { trackingRef: 'HEAD' }, + ]) { + assert.throws(() => readCodaPin({ externalSpecifications: { coda: { ...pin, ...change } } }), /Invalid/); + } +}); + +test('defaults to all features and requires explicit valid subset selection', () => { + assert.deepEqual(parseArguments(['test']), { command: 'test', detectors: undefined }); + assert.deepEqual(parseArguments(['test', '--detectors', 'DX-001,DX-011']), { + command: 'test', + detectors: ['DX-001', 'DX-011'], + }); + for (const args of [ + [], + ['fetch', '--detectors', 'DX-001'], + ['test', '--unknown'], + ['test', '--detectors', ''], + ['test', '--detectors', 'DX-001,DX-001'], + ['test', '--detectors', 'DX-1'], + ]) { + assert.throws(() => parseArguments(args)); + } +}); + +function git(directory, args) { + const result = spawnSync('git', ['--no-pager', '-C', directory, '-c', 'core.hooksPath=', ...args], { + encoding: 'utf8', + windowsHide: true, + }); + assert.equal(result.status, 0, result.stderr); + return result.stdout.trim(); +} + +function fixture(t) { + const directory = mkdtempSync(path.join(tmpdir(), 'coda-fetch-test-')); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const source = path.join(directory, 'source'); + const cache = path.join(directory, 'cache'); + mkdirSync(path.join(source, 'specs', 'detectors'), { recursive: true }); + git(source, ['init', '--quiet']); + git(source, ['config', 'core.autocrlf', 'false']); + const feature = 'Feature: Synthetic fetch fixture\n Scenario: Initial snapshot\n Given a fixture\n'; + writeFileSync(path.join(source, 'specs', 'detectors', 'dx900_fixture.feature'), feature); + writeFileSync(path.join(source, '.gitignore'), 'ignored.feature\n'); + git(source, ['add', '.']); + git(source, [ + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.test', + '-c', + 'commit.gpgsign=false', + 'commit', + '--quiet', + '-m', + 'fixture', + ]); + return { source, cache, feature, config: { ...pin, repo: source, commit: git(source, ['rev-parse', 'HEAD']) } }; +} + +test('fetches the exact commit instead of source HEAD and reuses a verified checkout', (t) => { + const { source, cache, feature, config } = fixture(t); + writeFileSync(path.join(source, 'specs', 'detectors', 'dx900_fixture.feature'), 'changed upstream\n'); + git(source, ['add', '.']); + git(source, [ + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.test', + '-c', + 'commit.gpgsign=false', + 'commit', + '--quiet', + '-m', + 'newer', + ]); + const downloaded = fetchCoda(config, cache); + assert.equal(readFileSync(path.join(downloaded, 'dx900_fixture.feature'), 'utf8'), feature); + assert.equal(git(path.join(downloaded, '..', '..'), ['rev-parse', 'HEAD']), config.commit); + assert.equal(fetchCoda(config, cache), downloaded); + assert.equal(readdirSync(cache).length, 1); +}); + +test('a failed fetch leaves the previous pin intact and removes only its staging directory', (t) => { + const { cache, config, feature } = fixture(t); + const previous = fetchCoda(config, cache); + assert.throws(() => fetchCoda({ ...config, commit: 'b'.repeat(40) }, cache), /Git operation failed/); + assert.equal(readFileSync(path.join(previous, 'dx900_fixture.feature'), 'utf8'), feature); + assert.equal(readdirSync(cache).length, 1); +}); + +test('rejects dirty caches without resetting or replacing local modifications', (t) => { + const { cache, config } = fixture(t); + const features = fetchCoda(config, cache); + const file = path.join(features, 'dx900_fixture.feature'); + writeFileSync(file, 'local modification\n'); + assert.throws(() => fetchCoda(config, cache), /local changes/); + assert.equal(readFileSync(file, 'utf8'), 'local modification\n'); +}); + +test('rejects injected ignored feature files as well as tracked modifications', (t) => { + const { cache, config } = fixture(t); + const features = fetchCoda(config, cache); + writeFileSync(path.join(features, 'ignored.feature'), 'injected\n'); + assert.throws(() => fetchCoda(config, cache), /local changes/); +}); + +test('rejects substituted cache origins and commits', (t) => { + const { cache, config } = fixture(t); + const features = fetchCoda(config, cache); + const checkout = path.join(features, '..', '..'); + git(checkout, ['remote', 'set-url', 'origin', 'https://example.test/other']); + assert.throws(() => fetchCoda(config, cache), /origin does not match/); + git(checkout, ['remote', 'set-url', 'origin', config.repo]); + git(checkout, [ + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.test', + '-c', + 'commit.gpgsign=false', + 'commit', + '--quiet', + '--allow-empty', + '-m', + 'substitution', + ]); + assert.throws(() => fetchCoda(config, cache), /HEAD does not match/); +}); diff --git a/specs/account-overview.md b/specs/account-overview.md index b843366cd..2d4254281 100644 --- a/specs/account-overview.md +++ b/specs/account-overview.md @@ -833,6 +833,29 @@ The collector skips DX-024 storage-metric requests for an explicitly disabled in without reporting missing metric coverage for that container. Missing or unrecognized modes do not establish disabled indexing; metric acquisition and missing/failed-evidence reporting remain unchanged for those containers. +**Conformance:** `externalSpecifications.coda` in `package.json` pins the private source revision. +`npm run test:coda` fetches/verifies an isolated Git-ignored checkout and executes its unmodified `.feature` files +through Cucumber's Gherkin compiler and TypeScript bindings for all 24 rules. Bindings prepare synthetic inputs +and call production evaluations plus the native registry. Metric/log-reader fixtures exercise partition summation, +account RU folding, storage history, latest populated index/data samples, recent throttling and acquisition failures. +Production results expose utilization bands, sizing observations, root causes and non-finding outcomes. +Diagnostic evaluation types constrain root causes and observation shapes per rule; configuration coverage and +configuration-only detectors share the same pure classifications while keeping applicability separate from missing inputs. +Bindings do not reconstruct detector decisions from the absence of a recommendation. Acquisition-error vocabulary +is normalized from actual native structured failures, not fabricated as a clean negative result. +Every selected scenario must evaluate and assert; unknown bindings fail. The private local report inventories all +scenarios, including those outside an explicitly selected subset. Original wording-sensitive guidance assertions +are retained; cross-rule `supersededBy` is additional pipeline metadata, not part of an isolated detector verdict. +Setup fidelity is checked independently of verdict assertions: the DX-008 Serverless exclusion receives otherwise +quantifiable positive manual capacity, and DX-009 migration starts from a known Periodic policy. Missing retention +tiers remain absent rather than literal strings; storage histories, query windows and configured-region normalization +retain the original bindings' inputs. +All 379 scenarios at the manifest pin pass, with zero unbound, skipped or unexecuted scenarios. +DX-004 classifies successful empty consumption reads as idle, without converting acquisition failures to zero. +DX-017 compares fitted growth against the inclusive 0.1 GiB/day floor using a roundoff estimate scaled by sample +magnitudes and regression conditioning, not an arbitrary policy epsilon or rounded fixtures. Growth measurably below +the floor remains stable. Synthetic conformance does not establish live collector acceptance. + DX-008 regional capacity observations require a complete, positive all-manual offer inventory and known provisioned account/region count. They report configured per-region RU/s and the global regional footprint, not measured usage, billable savings or permission to remove regions. Extra-region capacity is only identified for the non-production diff --git a/src/panels/accountOverview/services/advisories/conformance/additionalBindings.ts b/src/panels/accountOverview/services/advisories/conformance/additionalBindings.ts new file mode 100644 index 000000000..841c601dc --- /dev/null +++ b/src/panels/accountOverview/services/advisories/conformance/additionalBindings.ts @@ -0,0 +1,542 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { LogsQueryResultStatus } from '@azure/monitor-query-logs'; +import assert from 'node:assert/strict'; +import { getPartitionStorageSeries } from '../../partitionHealth'; +import { getThrottleRate } from '../../ruTrends'; +import { DEFAULT_ADVISORY_THRESHOLDS, DIAGNOSTIC_POLICY } from '../core/thresholds'; +import { type DerivedAdvisoryRule } from '../core/types'; +import { autoscaleCandidateDetector } from '../detectors/AutoscaleCandidateDetector'; +import { autoscaleToManualCandidateDetector } from '../detectors/AutoscaleToManualCandidateDetector'; +import { crossPartitionQueryDetector } from '../detectors/CrossPartitionQueryDetector'; +import { expensiveConsistencyDetector } from '../detectors/ExpensiveConsistencyDetector'; +import { hotPartitionRiskDetector } from '../detectors/HotPartitionRiskDetector'; +import { idleContainerDetector } from '../detectors/IdleContainerDetector'; +import { indexingCostRiskDetector } from '../detectors/IndexingCostRiskDetector'; +import { multiRegionWriteAntipatternDetector } from '../detectors/MultiRegionWriteAntipatternDetector'; +import { paidContinuousBackupDetector } from '../detectors/PaidContinuousBackupDetector'; +import { partitionMergeCandidateDetector } from '../detectors/PartitionMergeCandidateDetector'; +import { perRegionAutoscaleCandidateDetector } from '../detectors/PerRegionAutoscaleCandidateDetector'; +import { reservedCapacityCandidateDetector } from '../detectors/ReservedCapacityCandidateDetector'; +import { serverlessCandidateDetector } from '../detectors/ServerlessCandidateDetector'; +import { serverlessThrottlingDetector } from '../detectors/ServerlessThrottlingDetector'; +import { shardKeyMisalignmentDetector } from '../detectors/ShardKeyMisalignmentDetector'; +import { sharedThroughputStarvationDetector } from '../detectors/SharedThroughputStarvationDetector'; +import { storageGrowthRiskDetector } from '../detectors/StorageGrowthRiskDetector'; +import { storageSkewRiskDetector } from '../detectors/StorageSkewRiskDetector'; +import { throttlingByOperationDetector } from '../detectors/ThrottlingByOperationDetector'; +import { uncontrolledIngestionDetector } from '../detectors/UncontrolledIngestionDetector'; +import { underProvisioningDetector } from '../detectors/UnderProvisioningDetector'; +import { + fetchCrossPartitionShapes, + fetchSharedThroughputTraffic, + fetchUncontrolledIngestion, + type LogsQueryExecutor, +} from '../fetchers/logs'; +import { LogsFetchError } from '../fetchers/logsDiagnostics'; +import { + getAccountConsumedRuShape, + getAutoscaleUtilization, + getContainerIdlePeak, + getContainerIndexingUsage, +} from '../fetchers/metrics'; +import { metricResponse, withMetrics } from './metricsFixture'; +import { near, number, observation, observed, type Binding, type World } from './shared'; + +const scope = { databaseId: 'fixture-db', containerId: 'fixture-container' }; +const thresholds = DEFAULT_ADVISORY_THRESHOLDS; + +export const ADDITIONAL_RULES: Readonly> = { + 'DX-002': 'CrossPartitionQuery', + 'DX-003': 'SharedThroughputStarvation', + 'DX-004': 'IdleContainer', + 'DX-005': 'SustainedThrottlingInRegion', + 'DX-006': 'HotPartitionRisk', + 'DX-007': 'ShardKeyMisalignment', + 'DX-008': 'MultiRegionWriteAntipattern', + 'DX-009': 'PartitionMergeCandidate', + 'DX-010': 'UncontrolledIngestion', + 'DX-012': 'AutoscaleCandidate', + 'DX-013': 'AutoscaleToManualCandidate', + 'DX-014': 'ServerlessCandidate', + 'DX-015': 'StorageSkewRisk', + 'DX-016': 'ExpensiveConsistency', + 'DX-017': 'StorageGrowthRisk', + 'DX-018': 'ServerlessThrottling', + 'DX-019': 'PaidContinuousBackup', + 'DX-021': 'ThrottlingByOperation', + 'DX-022': 'PerRegionAutoscaleCandidate', + 'DX-023': 'ReservedCapacityCandidate', + 'DX-024': 'IndexingCostRisk', +}; + +async function evaluateLogs(world: World, detector: string): Promise { + let queries = 0; + const executor: LogsQueryExecutor = { + async queryResource(_resource, query) { + queries++; + if (world.logFailure) { + throw Object.assign(new Error('Synthetic log acquisition failure.'), { statusCode: 403 }); + } + let rows: Record[] = []; + if (!world.emptyLogs) { + if (detector === 'DX-002' || detector === 'DX-007') { + rows = (world.queries?.shapes ?? []).map((shape) => ({ + QueryText: shape.text, + executions: shape.executions, + avgFanout: shape.avgFanout, + maxFanout: shape.maxFanout, + })); + } else if (detector === 'DX-003') { + rows = (world.shared?.collections ?? []).map((collection) => ({ + CollectionName: collection.containerId, + requests: collection.requests, + throttled: collection.throttledRequests, + ru: collection.ruConsumed, + })); + } else if (query.includes('totalRu')) { + assert.ok(world.requests); + rows = [ + { + writeRu: world.writeRu ?? 0, + totalRu: world.totalRu ?? 0, + reqs: world.requests.total, + throttles: world.requests.throttled, + }, + ]; + } + } + const columns = Object.keys(rows[0] ?? {}); + return { + status: LogsQueryResultStatus.Success, + tables: [ + { + name: 'PrimaryResult', + columnDescriptors: columns.map((name) => ({ name, type: 'string' })), + rows: rows.map((row) => columns.map((column) => row[column])), + }, + ], + }; + }, + }; + const timespan = { + startTime: new Date(0), + endTime: new Date((world.queries?.evaluationWindowSeconds ?? 3600) * 1000), + }; + try { + if (detector === 'DX-002' || detector === 'DX-007') { + const input = await fetchCrossPartitionShapes( + executor, + 'fixture-account', + scope.databaseId, + scope.containerId, + timespan, + ); + world.inputs.crossPartition = [input]; + world.result = + detector === 'DX-002' + ? crossPartitionQueryDetector.evaluateResult(input, { + minQueries: thresholds.crossPartitionMinQueries, + fanoutThreshold: thresholds.crossPartitionFanoutThreshold, + medPct: thresholds.crossPartitionMedPct, + }) + : shardKeyMisalignmentDetector.evaluateResult( + input, + { + structuralPct: thresholds.shardKeyStructuralPct, + minPartitions: DIAGNOSTIC_POLICY.shardKey.minPartitions, + }, + thresholds.crossPartitionFanoutThreshold, + thresholds.crossPartitionMinQueries, + ); + } else if (detector === 'DX-003') { + assert.ok(world.shared); + const traffic = await fetchSharedThroughputTraffic( + executor, + 'fixture-account', + scope.databaseId, + world.shared.sharedRu, + timespan, + ); + const input = { ...traffic, sharedContainerIds: world.shared.sharedContainerIds }; + world.inputs.sharedThroughput = [input]; + world.result = sharedThroughputStarvationDetector.evaluateResult(input, { + minRequests: thresholds.sharedThroughputMinRequests, + poolThrottlePct: thresholds.sharedThroughputPoolThrottlePct, + dominancePct: thresholds.sharedThroughputDominancePct, + victimThrottlePct: thresholds.sharedThroughputVictimThrottlePct, + victimSharePct: thresholds.sharedThroughputVictimSharePct, + }); + } else { + const input = await fetchUncontrolledIngestion( + executor, + 'fixture-account', + scope.databaseId, + scope.containerId, + timespan, + ); + world.inputs.ingestion = input ? [input] : []; + world.result = uncontrolledIngestionDetector.evaluateResult(input, { + writeRuPctFloor: thresholds.ingestionWriteRuPctFloor, + throttleRatePct: thresholds.ingestionThrottleRatePct, + highPct: thresholds.ingestionHighPct, + minRequests: thresholds.ingestionMinRequests, + }); + } + } catch (error) { + if (!(error instanceof LogsFetchError)) { + throw error; + } + assert.ok(queries > 0, 'An acquisition-error scenario must execute the production fetcher.'); + // CODA and the native collector name acquisition failures differently; neither is a clean verdict. + world.result = { + rootCause: detector === 'DX-002' || detector === 'DX-007' ? 'log_analytics_unavailable' : 'query_error', + }; + } +} + +export async function evaluateAdditional(world: World, detector: string): Promise { + if (['DX-002', 'DX-003', 'DX-007', 'DX-010'].includes(detector)) { + await evaluateLogs(world, detector); + return; + } + switch (detector) { + case 'DX-004': { + assert.ok(world.idle); + const peak = world.idle.peakRuPerBucket; + const observedPeak = await withMetrics( + metricResponse('TotalRequestUnits', [peak === undefined ? [] : [peak]], 'total'), + (client) => + getContainerIdlePeak( + client, + 'fixture-account', + scope.databaseId, + world.idle?.containerId, + undefined, + world.idle?.containerId === undefined ? [scope.containerId] : undefined, + ), + ); + const input = { ...world.idle, peakRuPerBucket: observedPeak, resourceAgeDays: world.age }; + world.inputs.idleContainers = [input]; + world.result = idleContainerDetector.evaluateResult( + input, + thresholds.idlePeakRuPerBucket, + world.inputs.scopeProvisionedRuTotal, + ); + break; + } + case 'DX-005': + case 'DX-006': { + assert.ok(world.requests); + const peaks = world.partitionPeaks ?? []; + const input = { + ...scope, + maxP99: peaks.length > 0 ? Math.max(...peaks) : 0, + minP99: peaks.length > 0 ? Math.min(...peaks) : 0, + partitionCount: peaks.length, + throttleRatePercent: + world.requests.total > 0 ? (100 * world.requests.throttled) / world.requests.total : 0, + totalRequests: world.requests.total, + throughputMode: world.requests.mode, + resourceAgeDays: world.age, + }; + world.inputs.partitions = [input]; + world.result = + detector === 'DX-005' + ? underProvisioningDetector.evaluateResult( + input, + thresholds.partitionSaturationPercent, + thresholds.partitionHeadroomPercent, + thresholds.throttleRatePercent, + ) + : hotPartitionRiskDetector.evaluateResult( + input, + thresholds.partitionSaturationPercent, + thresholds.partitionHeadroomPercent, + thresholds.throttleRatePercent, + ); + break; + } + case 'DX-008': + world.inputs.accountConfig = world.account; + world.result = multiRegionWriteAntipatternDetector.evaluateResult( + world.account, + world.inputs.manualRegionalCapacityRu, + ); + break; + case 'DX-009': { + assert.ok(world.merge); + const input = { ...world.merge, backupPolicy: world.account.backupPolicy }; + world.inputs.partitionMerges = [input]; + world.result = partitionMergeCandidateDetector.evaluateResult(input); + break; + } + case 'DX-012': { + assert.ok(world.manual); + const input = { ...world.manual, resourceAgeDays: world.age }; + world.inputs.manualThroughput = [input]; + world.result = autoscaleCandidateDetector.evaluateResult(input, { + maxPercent: thresholds.autoscaleMaxPercent, + avgPercent: thresholds.autoscaleAvgPercent, + peakToAvgRatio: thresholds.autoscalePeakToAvgRatio, + }); + break; + } + case 'DX-013': { + assert.ok(world.autoscale); + const fixture = world.autoscale; + const summary = await withMetrics(metricResponse('AutoscaledRU', fixture.series ?? []), (client) => + getAutoscaleUtilization(client, 'fixture-account', scope.databaseId, scope.containerId, fixture.max), + ); + const input = { ...scope, ...summary, configuredMaxRu: fixture.max, resourceAgeDays: world.age }; + world.inputs.autoscaleUtilizations = [input]; + world.result = autoscaleToManualCandidateDetector.evaluateResult( + input, + thresholds.autoscaleToManualAvgPercent, + thresholds.autoscaleToManualPeakToAvgRatio, + ); + break; + } + case 'DX-014': { + assert.ok(world.hourlyRu); + const bucketSeconds = DIAGNOSTIC_POLICY.serverless.observation.bucketMs / 1000; + const summary = await withMetrics( + metricResponse('TotalRequestUnits', [world.hourlyRu.map((ru) => ru * bucketSeconds)], 'total'), + (client) => getAccountConsumedRuShape(client, 'fixture-account'), + ); + const input = { ...summary, isServerless: world.account.isServerless === true }; + world.inputs.serverless = input; + world.result = serverlessCandidateDetector.evaluateResult(input, { + sporadicRatio: thresholds.serverlessSporadicRatio, + peakFloorRuPerSec: thresholds.serverlessPeakFloorRuPerSec, + peakCeilingRuPerSec: thresholds.serverlessPeakCeilingRuPerSec, + }); + break; + } + case 'DX-015': + case 'DX-017': { + assert.ok(world.storage); + const response = metricResponse('PhysicalPartitionSizeInfo', []); + response.value[0].timeseries = world.storage.partitions.map((partition) => ({ + metadatavalues: [{ name: { value: 'PhysicalPartitionId' }, value: partition.partitionId }], + data: partition.samples.map((sample) => ({ + timeStamp: new Date(sample.timestamp), + maximum: sample.bytes, + })), + })); + const observation = + detector === 'DX-015' + ? DIAGNOSTIC_POLICY.storageSkew.observation + : DIAGNOSTIC_POLICY.storageGrowth.observation; + const summary = await withMetrics(response, (client) => + getPartitionStorageSeries( + client, + 'fixture-account', + observation.timeRange, + scope.databaseId, + scope.containerId, + observation, + ), + ); + const input = { ...scope, partitions: summary.partitions, resourceAgeDays: world.age }; + if (detector === 'DX-015') { + world.inputs.storageSkew = [input]; + world.result = storageSkewRiskDetector.evaluateResult(input, thresholds.storageSkewBalanceRatio); + } else { + world.inputs.storage = [input]; + world.result = storageGrowthRiskDetector.evaluateResult(input, thresholds.storageGrowthHorizonDays); + } + break; + } + case 'DX-016': + world.inputs.accountConfig = world.account; + world.result = expensiveConsistencyDetector.evaluateResult(world.account); + break; + case 'DX-018': { + assert.ok(world.requests); + const response = metricResponse( + 'TotalRequests', + [[world.requests.total - world.requests.throttled], [world.requests.throttled]], + 'count', + ); + for (const [index, series] of (response.value[0].timeseries ?? []).entries()) { + series.metadatavalues = [{ name: { value: 'StatusCode' }, value: index === 0 ? '200' : '429' }]; + } + const summary = await withMetrics(response, (client) => + getThrottleRate( + client, + 'fixture-account', + DIAGNOSTIC_POLICY.serverlessThrottling.observation.timeRange, + scope.databaseId, + scope.containerId, + DIAGNOSTIC_POLICY.serverlessThrottling.observation, + ), + ); + const input = { ...scope, totalRequests: summary.totalRequests, throttleRatePercent: summary.ratePercent }; + world.inputs.serverlessThrottling = [input]; + world.result = serverlessThrottlingDetector.evaluateResult(input); + break; + } + case 'DX-019': + world.inputs.accountConfig = world.account; + world.result = paidContinuousBackupDetector.evaluateResult(world.account); + break; + case 'DX-021': { + assert.ok(world.statuses); + const input = { statusCounts: world.statuses, throttledOperationCounts: world.operations }; + world.inputs.accountRequests = input; + world.result = throttlingByOperationDetector.evaluateResult(input); + break; + } + case 'DX-022': + world.inputs.accountConfig = world.account; + world.result = perRegionAutoscaleCandidateDetector.evaluateResult(world.account); + break; + case 'DX-023': + world.inputs.accountConfig = world.account; + world.result = reservedCapacityCandidateDetector.evaluateResult( + world.inputs.scopeProvisionedRuTotal, + world.account.isServerless, + ); + break; + case 'DX-024': { + assert.ok(world.indexing); + const summary = await withMetrics( + (options) => { + const name = options?.metricnames; + assert.ok(name === 'IndexUsage' || name === 'DataUsage'); + return metricResponse(name, [ + name === 'IndexUsage' ? (world.indexSeries ?? []) : (world.dataSeries ?? []), + ]); + }, + (client) => getContainerIndexingUsage(client, 'fixture-account', scope.databaseId, scope.containerId), + ); + const input = { ...world.indexing, ...summary }; + world.inputs.indexing = [input]; + world.result = indexingCostRiskDetector.evaluateResult(input, thresholds.indexingUsageRatio); + break; + } + default: + throw new Error(`No native binding for ${detector}.`); + } +} + +function outcome(pattern: RegExp, execute: Binding['execute']): Binding { + return { type: 'Outcome', pattern, execute }; +} + +export const additionalOutcomes: readonly Binding[] = [ + outcome(/^the recoverable capacity is (\S+) RU\/s$/, (world, match) => { + const value = observed(world, 'recoverableRu'); + assert.equal(typeof value, 'number'); + near(typeof value === 'number' ? value : undefined, match[1]); + }), + outcome(/^DX-004 applies to it$/, (world) => { + assert.ok(world.applicabilityMode); + assert.equal(idleContainerDetector.appliesTo(world.applicabilityMode), true); + world.applicabilityChecked = true; + }), + outcome(/^DX-004 is not in it$/, (world) => { + assert.equal(world.applicabilityMode, 'serverless'); + assert.equal(idleContainerDetector.appliesTo(world.applicabilityMode), false); + world.applicabilityChecked = true; + }), + outcome(/^the container is (not )?reported as under-provisioned$/, (world, match) => { + observation(world); + assert.equal(world.advisory !== undefined, !match[1]); + }), + outcome(/^the recommendation mentions the autoscale max$/, (world) => { + observation(world); + assert.ok(world.advisory?.suggestedAction.includes('autoscale max')); + }), + outcome(/^the recommendation says to convert to autoscale$/, (world) => { + observation(world); + assert.ok(world.advisory?.suggestedAction.includes('Convert to autoscale')); + }), + outcome(/^the recommendation mentions a max of (\d+) RU\/s$/, (world, match) => { + observation(world); + assert.ok(world.advisory?.suggestedAction.includes(`max ≈ ${number(match[1])} RU/s`)); + }), + outcome(/^the (monopolizer|victim) is "([^"]+)"$/, (world, match) => { + assert.equal(observed(world, match[1]), match[2]); + }), + outcome(/^the regional capacity evidence is (quantified|unavailable)$/, (world, match) => { + assert.equal(observed(world, 'regionalCapacityQuantified'), match[1] === 'quantified'); + }), + outcome( + /^the (configured per-region throughput|global regional footprint|removable extra region throughput) is (\S+) RU\/s$/, + (world, match) => { + const key = + match[1] === 'configured per-region throughput' + ? 'configuredPerRegionRu' + : match[1] === 'global regional footprint' + ? 'globalRegionalFootprintRu' + : 'removableExtraRegionRu'; + const value = observed(world, key); + assert.equal(typeof value, 'number'); + near(typeof value === 'number' ? value : undefined, match[2]); + }, + ), + outcome(/^no extra region throughput is removable$/, (world) => { + assert.equal(observed(world, 'removableExtraRegionRu'), null); + }), + outcome(/^merge eligibility is (\S+)$/, (world, match) => { + assert.equal(observed(world, 'mergeEligibility'), match[1]); + }), + outcome(/^the merge guidance contains "([^"]+)"$/, (world, match) => { + observation(world); + assert.ok(world.advisory?.suggestedAction.includes(match[1]), `Missing merge guidance: ${match[1]}`); + }), + outcome(/^no instruction to merge partitions is emitted$/, (world) => { + observation(world); + assert.ok(world.advisory); + assert.ok(!world.advisory.suggestedAction.includes('consider merging')); + assert.ok(!world.advisory.suggestedAction.includes('Merge the partitions')); + }), + outcome(/^Session guidance uses the end-user token round trip$/, (world) => { + observation(world); + assert.ok(world.advisory); + for (const required of [ + 'one default consistency level', + 'Cosmos response SessionToken', + 'end-user client in a cookie', + 'later web request', + 'Cosmos request options of whichever SDK instance', + 'no direct SDK-to-SDK synchronization', + 'opaque, partition-bound', + 'not equivalent to Strong', + ]) { + assert.ok(world.advisory.suggestedAction.includes(required), `Missing Session guidance: ${required}`); + } + }), + outcome( + /^the (?:days to the limit is|throttle rate is|index to data ratio is|user excluded path count is) (\S+)(?: percent)?$/, + (world, match) => { + const key = match[0].startsWith('the days') + ? 'daysToLimit' + : match[0].startsWith('the throttle') + ? 'throttleRatePercent' + : match[0].startsWith('the index') + ? 'indexDataRatio' + : 'userExcludedPathCount'; + const value = observed(world, key); + assert.equal(typeof value, 'number'); + near(typeof value === 'number' ? value : undefined, match[1]); + }, + ), + outcome(/^the dominant (operation|category) is (\S+)$/, (world, match) => { + assert.equal(observed(world, match[1] === 'operation' ? 'dominantOperation' : 'dominantCategory'), match[2]); + }), +]; + +export const ACTION_DETECTORS: Readonly> = { + 'classifies the shared pool': 'DX-003', + 'detects shared-throughput starvation': 'DX-003', + 'classifies the ingestion pattern': 'DX-010', + 'detects uncontrolled ingestion': 'DX-010', + 'reads the partition sizes for DX-017': 'DX-017', + 'reads the metrics for DX-018': 'DX-018', +}; diff --git a/src/panels/accountOverview/services/advisories/conformance/additionalFixtures.ts b/src/panels/accountOverview/services/advisories/conformance/additionalFixtures.ts new file mode 100644 index 000000000..d3f1702fa --- /dev/null +++ b/src/panels/accountOverview/services/advisories/conformance/additionalFixtures.ts @@ -0,0 +1,369 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'node:assert/strict'; +import { API } from '../../../../../AzureDBExperiences'; +import { countUserExcludedPaths } from '../../inventory'; +import { type StoragePartitionSeries } from '../core/types'; +import { GIB } from '../detectors/storageMath'; +import { number, shapedValues, throughputMode, type Binding, type World } from './shared'; + +const scope = { databaseId: 'fixture-db', containerId: 'fixture-container' }; +const day = 86_400_000; + +function context(pattern: RegExp, execute: Binding['execute']): Binding { + return { type: 'Context', pattern, execute }; +} + +function partition(current: number, rate = 0, id = '0'): StoragePartitionSeries { + return { + partitionId: id, + samples: Array.from({ length: 10 }, (_, index) => ({ + timestamp: Date.UTC(2026, 0, 1) + index * day, + bytes: (current - (9 - index) * rate) * GIB, + })), + }; +} + +function queries(world: World, total: number, fanout: number, partitions: number, hours: number, average = partitions) { + world.queries = { + ...scope, + evaluationWindowSeconds: hours * 3600, + shapes: [ + { text: 'SELECT * FROM c', executions: fanout, avgFanout: average, maxFanout: partitions }, + { text: 'SELECT * FROM c WHERE c.id = @id', executions: total - fanout, avgFanout: 1, maxFanout: 1 }, + ].filter((shape) => shape.executions > 0), + }; +} + +function indexing(world: World, index: (number | undefined)[], data: (number | undefined)[]) { + world.indexing = { ...scope, indexingMode: 'consistent', userExcludedPathCount: 0 }; + world.indexSeries = index; + world.dataSeries = data; +} + +export const additionalFixtures: readonly Binding[] = [ + context( + /^a container with (\d+) queries that all fan out across (\d+) physical partitions in a (\S+) hour window$/, + (world, match) => { + queries(world, number(match[1]), number(match[1]), number(match[2]), number(match[3])); + }, + ), + context( + /^a container with (\d+) queries of which (\d+) fan out across (\d+) physical partitions in a (\S+) hour window$/, + (world, match) => { + queries(world, number(match[1]), number(match[2]), number(match[3]), number(match[4])); + }, + ), + context( + /^a container with (\d+) queries averaging (\S+) partitions out of (\d+) physical partitions in a (\S+) hour window$/, + (world, match) => { + queries(world, number(match[1]), number(match[1]), number(match[3]), number(match[4]), number(match[2])); + }, + ), + context(/^a container with no query telemetry$/, (world) => { + world.queries = { ...scope, shapes: [], evaluationWindowSeconds: 24 * 3600 }; + }), + context(/^every query costs (\S+) RU$/, (_world, match) => { + // The native fan-out input deliberately has no RU gate; this fixture dimension is not a verdict. + assert.ok(number(match[1]) >= 0); + }), + context(/^(?:a Log Analytics workspace that fails the query|the Log Analytics query fails)$/, (world) => { + world.logFailure = true; + }), + context(/^the Log Analytics query returns no rows$/, (world) => { + world.emptyLogs = true; + }), + context(/^a shared database with (\S+) RU\/s$/, (world, match) => { + world.shared = { + databaseId: scope.databaseId, + sharedRu: number(match[1]), + collections: [], + sharedContainerIds: [], + }; + }), + context(/^collection "([^"]+)" has (\d+) requests, (\d+) throttled and (\S+) RU consumed$/, (world, match) => { + assert.ok(world.shared); + world.shared.collections = [ + ...world.shared.collections, + { + containerId: match[1], + requests: number(match[2]), + throttledRequests: number(match[3]), + ruConsumed: number(match[4]), + }, + ]; + world.shared.sharedContainerIds = [...(world.shared.sharedContainerIds ?? []), match[1]]; + }), + context( + /^a (manual|autoscale) container with (\S+) RU\/s that consumed at most (\S+) RU in any bucket$/, + (world, match) => { + world.idle = { + ...scope, + throughputMode: throughputMode(match[1]), + provisionedRu: number(match[2]), + peakRuPerBucket: number(match[3]), + }; + }, + ), + context(/^a manual container with (\S+) RU\/s and no consumption telemetry$/, (world, match) => { + world.idle = { ...scope, throughputMode: 'dedicated', provisionedRu: number(match[1]) }; + }), + context( + /^a shared-throughput database offer of (\S+) RU\/s that consumed at most (\S+) RU in any bucket$/, + (world, match) => { + world.idle = { + databaseId: scope.databaseId, + throughputMode: 'shared', + provisionedRu: number(match[1]), + peakRuPerBucket: number(match[2]), + }; + }, + ), + context(/^a (manual|autoscale) offer at (container|database) level$/, (world, match) => { + world.applicabilityMode = throughputMode(match[1], match[2]); + }), + context(/^the serverless detection set$/, (world) => { + world.applicabilityMode = 'serverless'; + }), + context( + /^(?:a|an) (?:(manual|autoscale|serverless) )?container with (\d+) requests of which (\d+) were throttled$/, + (world, match) => { + world.requests = { + total: number(match[2]), + throttled: number(match[3]), + mode: match[1] ? throughputMode(match[1]) : 'dedicated', + }; + }, + ), + context(/^its partitions peak at (.+) percent utilization$/, (world, match) => { + world.partitionPeaks = match[1].split(/\s+and\s+|,/).map(number); + }), + context(/^it reports no partition utilization$/, (world) => { + world.partitionPeaks = []; + }), + context(/^the container was created (\S+) hours before the window ended$/, (world, match) => { + world.age = number(match[1]) / 24; + }), + context(/^writes consumed (\S+) of (\S+) RU$/, (world, match) => { + world.writeRu = number(match[1]); + world.totalRu = number(match[2]); + }), + context( + /^a manual container with (\d+) samples averaging (\S+) percent and peaking at (\S+) percent$/, + (world, match) => { + world.manual = { + ...scope, + provisionedRu: 10_000, + ruPercents: shapedValues(number(match[1]), number(match[2]), number(match[3])), + }; + }, + ), + context(/^a manual container with no utilization data$/, (world) => { + world.manual = { ...scope, provisionedRu: 10_000, ruPercents: [] }; + }), + context( + /^an autoscale container with a (\d+) RU\/s max and (\d+) samples averaging (\S+) percent and peaking at (\S+) percent$/, + (world, match) => { + const max = number(match[1]); + const series = shapedValues(number(match[2]), number(match[3]), number(match[4])).map( + (percent) => (percent * max) / 100, + ); + world.autoscale = { max, samples: series.length, peaks: [], series: [series], shared: false }; + }, + ), + context( + /^an autoscale container with a (\d+) RU\/s max whose two partitions each provision (\d+) RU\/s for (\d+) samples$/, + (world, match) => { + const samples = number(match[3]); + world.autoscale = { + max: number(match[1]), + samples, + peaks: [], + shared: false, + series: [Array(samples).fill(number(match[2])), Array(samples).fill(number(match[2]))], + }; + }, + ), + context(/^an account with (\d+) hourly samples at (\S+) RU\/s and one peak of (\S+) RU\/s$/, (world, match) => { + world.hourlyRu = Array.from({ length: number(match[1]) }, (_, index) => + index === number(match[1]) - 1 ? number(match[3]) : number(match[2]), + ); + }), + context(/^an account with no consumed RU data$/, (world) => { + world.hourlyRu = []; + }), + context( + /^(?:the container's request metrics show|a single recent metric sample shows) (\d+) requests and (\d+) throttled responses$/, + (world, match) => { + world.requests = { total: number(match[1]), throttled: number(match[2]), mode: 'serverless' }; + }, + ), + context(/^(\d+) requests of which (\d+) were throttled$/, (world, match) => { + world.statuses = { 200: number(match[1]) - number(match[2]), 429: number(match[2]) }; + }), + context(/^(\d+) throttled (\S+) operations$/, (world, match) => { + world.operations = { ...world.operations, [match[2]]: number(match[1]) }; + }), + context(/^a (non-production|production) account$/, (world, match) => { + world.account.tags = { environment: match[1] === 'production' ? 'prod' : 'dev' }; + }), + context(/^an account with no environment signal$/, (world) => { + world.account.tags = undefined; + }), + context(/^multi-region writes are disabled$/, (world) => { + world.account.multiRegionWritesEnabled = false; + }), + context( + /^an account using the (sql|mongo|cassandra|gremlin|table) API with multi-region writes across (\d+) write regions$/, + (world, match) => { + const apis = { + sql: API.Core, + mongo: 'MongoDB', + cassandra: API.Cassandra, + gremlin: API.Graph, + table: API.Table, + } as const; + const api = Object.entries(apis).find(([name]) => name === match[1])?.[1]; + assert.ok(api); + Object.assign(world.account, { + api, + multiRegionWritesEnabled: true, + writeRegionCount: number(match[2]), + regionCount: Math.max(number(match[2]), 1), + }); + }, + ), + context(/^manual throughput offers of (\S+) and (\S+) RU\/s$/, (world, match) => { + world.inputs.manualRegionalCapacityRu = number(match[1]) + number(match[2]); + }), + context(/^(?:an autoscale offer of (\S+) RU\/s|no throughput offers)$/, (world, match) => { + if (match[1]) { + number(match[1]); + } + world.inputs.manualRegionalCapacityRu = undefined; + }), + context(/^a serverless account$/, (world) => { + world.account.isServerless = true; + world.inputs.manualRegionalCapacityRu = 1000; + }), + context(/^the account is (?:already )?serverless$/, (world) => { + world.account.isServerless = true; + }), + context(/^the account spans (\d+) regions$/, (world, match) => { + world.account.regionCount = number(match[1]); + }), + context(/^per-region provisioned autoscale is enabled$/, (world) => { + world.account.perRegionPerPartitionAutoscaleEnabled = true; + }), + context(/^an account provisioning (\S+) RU\/s$/, (world, match) => { + world.inputs.scopeProvisionedRuTotal = number(match[1]); + }), + context(/^an account with unknown provisioned throughput$/, (world) => { + world.inputs.scopeProvisionedRuTotal = undefined; + }), + context(/^the default consistency is (.+)$/, (world, match) => { + world.account.consistencyLevel = match[1]; + }), + context(/^the account does not report a consistency level$/, (world) => { + world.account.consistencyLevel = undefined; + }), + context( + /^the (?:account )?backup policy is (Continuous|Periodic|unknown)(?: with tier (\S+))?$/, + (world, match) => { + world.account.backupPolicy = { + type: match[1] === 'unknown' ? undefined : match[1], + continuousTier: match[2] === 'none' ? undefined : match[2], + }; + }, + ), + context(/^the account reports no backup policy$/, (world) => { + world.account.backupPolicy = undefined; + }), + context(/^a backup-policy migration to Continuous is in progress$/, (world) => { + world.account.backupPolicy = { + ...world.account.backupPolicy, + type: 'Periodic', + migrationState: { status: 'InProgress', targetType: 'Continuous' }, + }; + }), + context(/^all versions and deletes change feed is enabled on the account$/, (world) => { + world.account.backupPolicy = { ...world.account.backupPolicy, fullFidelityChangeFeedEnabled: true }; + }), + context( + /^a container with (\S+) RU\/s provisioned and (\S+) GiB of data on (\d+) physical partitions$/, + (world, match) => { + world.merge = { + ...scope, + provisionedRu: number(match[1]), + dataUsageBytes: number(match[2]) * GIB, + actualPartitions: number(match[3]), + }; + }, + ), + context(/^a container with no partition count data$/, (world) => { + world.merge = { ...scope, actualPartitions: 0 }; + }), + context(/^a container whose partitions hold (.+) GiB$/, (world, match) => { + world.storage = { + ...scope, + partitions: match[1].split(',').map((size, index) => partition(number(size), 0, `${index}`)), + }; + }), + context(/^a container (?:reporting no partition storage|with no partition size data)$/, (world) => { + world.storage = { ...scope, partitions: [] }; + }), + context(/^the busiest partition holds (\S+) GiB and grows (\S+) GiB per day$/, (world, match) => { + world.storage = { ...scope, partitions: [partition(number(match[1]), number(match[2]))] }; + }), + context( + /^a container with a partition at 50 GiB and another partition (growing quickly|flat at 10 GiB)$/, + (world, match) => { + world.storage = { + ...scope, + partitions: [ + partition(50), + partition(match[1] === 'growing quickly' ? 20 : 10, match[1] === 'growing quickly' ? 2 : 0, '1'), + ], + }; + }, + ), + context( + /^a container with a partition at 40 GiB growing 1 GiB per day and another partition at 20 GiB growing 1 GiB per day$/, + (world) => { + world.storage = { ...scope, partitions: [partition(40, 1), partition(20, 1, '1')] }; + }, + ), + context(/^the (?:full|faster) partition is listed (first|last)$/, (world, match) => { + assert.ok(world.storage); + if (match[1] === 'last') { + world.storage.partitions = [...world.storage.partitions].reverse(); + } + }), + context(/^a container with (\S+) GiB of index and (\S+) GiB of data$/, (world, match) => { + indexing(world, [number(match[1]) * GIB], [number(match[2]) * GIB]); + }), + context(/^a container with no index or data samples$/, (world) => { + indexing(world, [undefined], [undefined]); + }), + context(/^a container whose newest storage sample is still unpopulated$/, (world) => { + indexing(world, [10 * GIB, 60 * GIB, undefined], [100 * GIB, 100 * GIB, undefined]); + }), + context(/^the indexing policy excludes only the default etag path$/, (world) => { + assert.ok(world.indexing); + world.indexing.userExcludedPathCount = countUserExcludedPaths([{ path: '/_etag/?' }]); + }), + context(/^the indexing policy excludes (\d+) user paths$/, (world, match) => { + assert.ok(world.indexing); + world.indexing.userExcludedPathCount = countUserExcludedPaths([ + { path: '/_etag/?' }, + ...Array.from({ length: number(match[1]) }, (_, index) => ({ path: `/p${index}/*` })), + ]); + }), + context(/^the indexing mode is (\S+)$/, (world, match) => { + assert.ok(world.indexing); + world.indexing.indexingMode = match[1]; + }), +]; diff --git a/src/panels/accountOverview/services/advisories/conformance/bindingCoverage.test.ts b/src/panels/accountOverview/services/advisories/conformance/bindingCoverage.test.ts new file mode 100644 index 000000000..0a7a387d8 --- /dev/null +++ b/src/panels/accountOverview/services/advisories/conformance/bindingCoverage.test.ts @@ -0,0 +1,362 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { autoscaleMaxOverProvisionedDetector } from '../detectors/AutoscaleMaxOverProvisionedDetector'; +import { crossPartitionQueryDetector } from '../detectors/CrossPartitionQueryDetector'; +import { multiRegionWriteAntipatternDetector } from '../detectors/MultiRegionWriteAntipatternDetector'; +import { paidContinuousBackupDetector } from '../detectors/PaidContinuousBackupDetector'; +import { partitionMergeCandidateDetector } from '../detectors/PartitionMergeCandidateDetector'; +import { storageGrowthRiskDetector } from '../detectors/StorageGrowthRiskDetector'; +import { GIB } from '../detectors/storageMath'; +import { executeCase, SUPPORTED_DETECTORS } from './bindings'; +import { parseFeature } from './features'; + +const fixtures = [ + [ + '002', + 'a container with 10000 queries that all fan out across 8 physical partitions in a 2 hour window', + 'cross_partition', + 'High', + ], + [ + '003', + `a shared database with 5000 RU/s + And collection "worker" has 1200 requests, 200 throttled and 9900 RU consumed + And collection "site" has 1000 requests, 200 throttled and 100 RU consumed`, + 'starvation', + 'High', + ], + ['004', 'a manual container with 1200 RU/s that consumed at most 2 RU in any bucket', 'idle', 'Medium'], + [ + '005', + `a manual container with 12000 requests of which 3000 were throttled + And its partitions peak at 98 and 88 percent utilization`, + 'global_under_provisioning', + 'High', + ], + [ + '006', + `a container with 12000 requests of which 3000 were throttled + And its partitions peak at 98 and 20 percent utilization`, + 'hot_partition', + 'High', + ], + [ + '007', + 'a container with 10000 queries that all fan out across 8 physical partitions in a 2 hour window', + 'shard_key_misaligned', + 'High', + ], + [ + '008', + 'an account using the mongo API with multi-region writes across 2 write regions', + 'mongo_multi_region_writes', + 'High', + ], + [ + '009', + `a container with 20000 RU/s provisioned and 4 GiB of data on 9 physical partitions + And the account backup policy is Continuous`, + 'over_partitioned', + 'Medium', + ], + [ + '010', + `a container with 12000 requests of which 3000 were throttled + And writes consumed 95 of 100 RU`, + 'uncontrolled_ingestion', + 'High', + ], + [ + '012', + 'a manual container with 32 samples averaging 8 percent and peaking at 48 percent', + 'spiky_manual', + 'Medium', + ], + [ + '013', + 'an autoscale container with a 12000 RU/s max and 32 samples averaging 75 percent and peaking at 88 percent', + 'steady_autoscale', + 'Medium', + ], + ['014', 'an account with 32 hourly samples at 2 RU/s and one peak of 200 RU/s', 'serverless_candidate', 'Low'], + ['015', 'a container whose partitions hold 35,2 GiB', 'storage_skew', 'Medium'], + [ + '016', + `the default consistency is Bounded_staleness + And the account spans 4 regions`, + 'premium_multi_region', + 'Low', + ], + ['017', 'the busiest partition holds 42 GiB and grows 2 GiB per day', 'rapid_growth', 'High'], + ['018', 'a serverless container with 2000 requests of which 200 were throttled', 'serverless_throttling', 'Medium'], + [ + '019', + `a non-production account + And the backup policy is Continuous with tier Continuous35Days`, + 'nonprod_paid_continuous_backup', + 'Medium', + ], + [ + '021', + `5000 requests of which 1000 were throttled + And 900 throttled Query operations + And 100 throttled Read operations`, + 'decomposed', + 'High', + ], + ['022', 'the account spans 4 regions', 'prpa_candidate', 'Low'], + ['023', 'an account provisioning 6000 RU/s', 'reserved_candidate', 'Low'], + [ + '024', + `a container with 45 GiB of index and 90 GiB of data + And the indexing policy excludes 4 user paths`, + 'high_index_ratio', + 'Low', + ], +] as const; + +function scenario(detector: string, given: string, then: string, action = `evaluates DX-${detector}`) { + return parseFeature( + ` +Feature: Synthetic adapter regression + Scenario: Independently authored native observations + Given ${given} + When CODA ${action} + Then ${then} +`, + `dx${detector}_synthetic.feature`, + )[0]; +} + +describe('all native CODA adapters without private-source access', () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('registers precisely the 24 original detector identifiers', () => { + expect([...SUPPORTED_DETECTORS].sort()).toEqual( + Array.from({ length: 24 }, (_, index) => `DX-${String(index + 1).padStart(3, '0')}`), + ); + }); + + it.each([ + { + detector: '002', + given: 'a container with 2400 queries that all fan out across 1 physical partitions in a 2 hour window', + root: 'single_partition_container', + }, + { + detector: '017', + given: 'the busiest partition holds 0 GiB and grows 1 GiB per day', + root: 'insufficient_data', + }, + { + detector: '021', + given: '2400 requests of which 240 were throttled', + root: 'not_throttling', + }, + ])('retains the CODA negative root for DX-$detector', async ({ detector, given, root }) => { + expect(await executeCase(scenario(detector, given, `the root cause is ${root}`))).toBe(1); + }); + + it.each([ + { + given: 'an autoscale container with a 10000 RU/s max whose two partitions each provision 500 RU/s for 48 samples', + peakPercent: 10, + avgPercent: 10, + }, + { + given: 'an autoscale container with a 10000 RU/s max and 48 samples averaging 10 percent and peaking at 20 percent', + peakPercent: 20, + avgPercent: 10, + }, + ])( + 'passes explicit autoscale sample histories to the DX-011 production path', + async ({ given, peakPercent, avgPercent }) => { + const evaluate = vi.spyOn(autoscaleMaxOverProvisionedDetector, 'evaluateResult'); + const testCase = scenario( + '011', + `${given} + And the scope provisions 10000 RU/s in total`, + 'the severity is High', + ); + + expect(await executeCase(testCase)).toBe(1); + expect(evaluate).toHaveBeenCalledWith( + expect.objectContaining({ peakPercent, avgPercent: expect.closeTo(avgPercent), sampleCount: 48 }), + 30, + 10000, + ); + }, + ); + + it.each(['container', 'shared database'])( + 'uses scope-relative materiality in both DX-011 evaluation paths for an autoscale %s', + async (scope) => { + const testCase = scenario( + '011', + `an autoscale ${scope} with a 10000 RU/s max whose peak total is 1000 RU/s over 720 samples + And the container was created 60 days before the window ended + And the scope provisions 10000 RU/s in total`, + 'the severity is High', + ); + expect(await executeCase(testCase)).toBe(1); + }, + ); + + it.each(fixtures)( + 'executes DX-%s and rejects an incorrect root classification', + async (detector, given, root, severity) => { + const testCase = scenario( + detector, + given, + `the root cause is ${root}\n And the severity is ${severity}`, + ); + expect(await executeCase(testCase)).toBe(2); + const wrong = scenario(detector, given, 'the root cause is impossible_root'); + await expect(executeCase(wrong)).rejects.toThrow(); + }, + ); + + it('exercises the Serverless capacity exclusion with otherwise quantifiable manual throughput', async () => { + const evaluate = vi.spyOn(multiRegionWriteAntipatternDetector, 'evaluateResult'); + const testCase = scenario( + '008', + `an account using the sql API with multi-region writes across 2 write regions + And a serverless account`, + 'the regional capacity evidence is unavailable', + ); + + expect(await executeCase(testCase)).toBe(1); + expect(evaluate).toHaveBeenCalledWith(expect.objectContaining({ isServerless: true }), 1000); + }); + + it.each([0, 1, 3])('retains at least one configured region for %i write regions', async (writeRegions) => { + const evaluate = vi.spyOn(multiRegionWriteAntipatternDetector, 'evaluateResult'); + const regionCount = Math.max(writeRegions, 1); + const testCase = scenario( + '008', + `an account using the sql API with multi-region writes across ${writeRegions} write regions + And manual throughput offers of 600 and 400 RU/s`, + `the global regional footprint is ${1000 * regionCount} RU/s`, + ); + + expect(await executeCase(testCase)).toBe(1); + expect(evaluate).toHaveBeenCalledWith( + expect.objectContaining({ writeRegionCount: writeRegions, regionCount }), + 1000, + ); + }); + + it('exercises unsettled migration with a known Periodic policy rather than an already unknown policy', async () => { + const evaluate = vi.spyOn(partitionMergeCandidateDetector, 'evaluateResult'); + const testCase = scenario( + '009', + `a container with 24000 RU/s provisioned and 6 GiB of data on 10 physical partitions + And a backup-policy migration to Continuous is in progress`, + `merge eligibility is unknown + And no instruction to merge partitions is emitted`, + ); + + expect(await executeCase(testCase)).toBe(2); + expect(evaluate).toHaveBeenCalledWith( + expect.objectContaining({ + backupPolicy: { + type: 'Periodic', + migrationState: { status: 'InProgress', targetType: 'Continuous' }, + }, + }), + ); + }); + + it.each(['first', 'last'])('preserves both storage histories with the full partition %s', async (order) => { + const evaluate = vi.spyOn(storageGrowthRiskDetector, 'evaluateResult'); + const testCase = scenario( + '017', + `a container with a partition at 50 GiB and another partition growing quickly + And the full partition is listed ${order}`, + 'the root cause is at_or_over_limit', + 'reads the partition sizes for DX-017', + ); + + expect(await executeCase(testCase)).toBe(1); + const histories = evaluate.mock.calls[0][0].partitions.map((partition) => + partition.samples.map(({ bytes }) => bytes / GIB), + ); + const expected = [Array(10).fill(50), Array.from({ length: 10 }, (_, index) => 2 * (index + 1))]; + expect(histories).toEqual(order === 'first' ? expected : expected.reverse()); + }); + + it('distinguishes a missing continuous retention tier from an unrecognized tier string', async () => { + const evaluate = vi.spyOn(paidContinuousBackupDetector, 'evaluateResult'); + const testCase = scenario( + '019', + `a non-production account + And the backup policy is Continuous with tier none`, + 'the root cause is unknown_continuous_tier', + ); + + expect(await executeCase(testCase)).toBe(1); + expect(evaluate).toHaveBeenCalledWith( + expect.objectContaining({ backupPolicy: { type: 'Continuous', continuousTier: undefined } }), + ); + }); + + it('retains the declared daily evaluation window for empty query telemetry', async () => { + const evaluate = vi.spyOn(crossPartitionQueryDetector, 'evaluateResult'); + + expect( + await executeCase( + scenario('002', 'a container with no query telemetry', 'the root cause is insufficient_data'), + ), + ).toBe(1); + expect(evaluate).toHaveBeenCalledWith( + expect.objectContaining({ shapes: [], evaluationWindowSeconds: 86_400 }), + expect.any(Object), + ); + }); + + it.each(['002', '003', '007', '010'])('DX-%s distinguishes a genuine log acquisition failure', async (detector) => { + const given = + detector === '003' + ? 'a shared database with 5000 RU/s\n And the Log Analytics query fails' + : 'the Log Analytics query fails'; + const root = detector === '002' || detector === '007' ? 'log_analytics_unavailable' : 'query_error'; + expect(await executeCase(scenario(detector, given, `the root cause is ${root}`))).toBe(1); + }); + + it('classifies successful empty idle telemetry and measured zero as idle', async () => { + const missing = scenario( + '004', + 'a manual container with 1200 RU/s and no consumption telemetry', + 'the root cause is idle', + ); + const zero = scenario( + '004', + 'a manual container with 1200 RU/s that consumed at most 0 RU in any bucket', + 'the root cause is idle', + ); + expect(await executeCase(missing)).toBe(1); + expect(await executeCase(zero)).toBe(1); + }); + + it.each([ + ['a manual offer at database level', 'DX-004 applies to it'], + ['the serverless detection set', 'DX-004 is not in it'], + ])('evaluates actual applicability without inventing a When step: %s', async (given, then) => { + const [testCase] = parseFeature( + ` +Feature: Synthetic applicability regression + Scenario: Production applicability predicate + Given ${given} + Then ${then} +`, + 'dx004_synthetic.feature', + ); + expect(await executeCase(testCase)).toBe(1); + }); +}); diff --git a/src/panels/accountOverview/services/advisories/conformance/bindings.ts b/src/panels/accountOverview/services/advisories/conformance/bindings.ts new file mode 100644 index 000000000..9cf91000a --- /dev/null +++ b/src/panels/accountOverview/services/advisories/conformance/bindings.ts @@ -0,0 +1,364 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type PickleStep } from '@cucumber/messages'; +import assert from 'node:assert/strict'; +import { API } from '../../../../../AzureDBExperiences'; +import { DEFAULT_ADVISORY_THRESHOLDS } from '../core/thresholds'; +import { type ProvisioningEvaluation, type RequestErrorEvaluation } from '../core/types'; +import { autoscaleMaxOverProvisionedDetector } from '../detectors/AutoscaleMaxOverProvisionedDetector'; +import { nonThrottlingErrorsDetector } from '../detectors/NonThrottlingErrorsDetector'; +import { overProvisioningDetector } from '../detectors/OverProvisioningDetector'; +import { computeDerivedAdvisories } from '../engine'; +import { getAutoscaleUtilization } from '../fetchers/metrics'; +import { ACTION_DETECTORS, ADDITIONAL_RULES, additionalOutcomes, evaluateAdditional } from './additionalBindings'; +import { additionalFixtures } from './additionalFixtures'; +import { type FeatureCase } from './features'; +import { metricResponse, withMetrics } from './metricsFixture'; +import { near, number, observation, type Binding, type World } from './shared'; + +export const SUPPORTED_DETECTORS: ReadonlySet = new Set([ + 'DX-001', + 'DX-011', + 'DX-020', + ...Object.keys(ADDITIONAL_RULES), +]); + +function sizing(world: World): ProvisioningEvaluation { + const result = observation(world); + assert.ok('band' in result, 'Expected a production sizing evaluation.'); + return result; +} + +function errors(world: World): RequestErrorEvaluation { + const result = observation(world); + assert.ok('errorRatePercent' in result, 'Expected a production error evaluation.'); + return result; +} + +async function evaluate(world: World, detector: string): Promise { + assert.equal(world.result, undefined, 'A scenario may evaluate only once.'); + const threshold = DEFAULT_ADVISORY_THRESHOLDS.overProvisioningBandPercent; + if (detector === 'DX-001') { + assert.ok(world.manual, 'Missing manual offer fixture.'); + world.manual.resourceAgeDays = world.age; + world.inputs.manualThroughput = [world.manual]; + world.result = overProvisioningDetector.evaluateResult( + world.manual, + threshold, + world.inputs.scopeProvisionedRuTotal, + ); + } else if (detector === 'DX-011') { + assert.ok(world.autoscale, 'Missing autoscale fixture.'); + const fixture = world.autoscale; + const containerId = fixture.shared ? undefined : 'fixture-container'; + const summary = await withMetrics( + metricResponse( + 'AutoscaledRU', + fixture.series ?? + fixture.peaks.map((peak) => + Array.from({ length: fixture.samples }, (_, index) => + index === fixture.samples - 1 ? peak : peak / 2, + ), + ), + ), + (client) => getAutoscaleUtilization(client, 'fixture-account', 'fixture-db', containerId, fixture.max), + ); + const input = { + ...summary, + databaseId: 'fixture-db', + containerId, + configuredMaxRu: fixture.max, + resourceAgeDays: world.age, + }; + world.inputs.autoscaleUtilizations = [input]; + world.result = autoscaleMaxOverProvisionedDetector.evaluateResult( + input, + threshold, + world.inputs.scopeProvisionedRuTotal, + ); + } else if (detector === 'DX-020') { + assert.ok(world.statuses, 'Missing status-count fixture.'); + world.inputs.accountRequests = { statusCounts: world.statuses }; + world.result = nonThrottlingErrorsDetector.evaluateResult(world.inputs.accountRequests); + } else { + await evaluateAdditional(world, detector); + } + assert.ok(world.result, 'The production evaluator must return an observation.'); + world.advisory = world.result.advisory; + const rule = { + 'DX-001': 'OverProvisioning', + 'DX-011': 'AutoscaleMaxOverProvisioned', + 'DX-020': 'NonThrottlingErrors', + ...ADDITIONAL_RULES, + }[detector]; + const registered = computeDerivedAdvisories(world.inputs, DEFAULT_ADVISORY_THRESHOLDS).filter( + (finding) => finding.rule === rule, + ); + // Per-rule contracts describe the detector verdict; cross-rule supersession is additional pipeline metadata. + assert.deepEqual( + registered.map((finding) => ({ ...finding, supersededBy: undefined })), + world.advisory ? [{ ...world.advisory, supersededBy: undefined }] : [], + 'Production registry and observed evaluation must agree.', + ); +} + +const bindings: readonly Binding[] = [ + ...additionalFixtures, + ...additionalOutcomes, + { + type: 'Action', + pattern: new RegExp(`^CODA (${Object.keys(ACTION_DETECTORS).join('|')})$`), + async execute(world, match, detector) { + assert.equal(ACTION_DETECTORS[match[1]], detector, 'Evaluation step must match its feature detector.'); + await evaluate(world, detector); + }, + }, + { + type: 'Context', + pattern: /^a manual container with (\S+) RU\/s and (\d+) samples at (\S+) percent utilization$/, + execute(world, match) { + world.manual = { + databaseId: 'fixture-db', + containerId: 'fixture-container', + provisionedRu: number(match[1]), + ruPercents: Array(number(match[2])).fill(number(match[3])), + }; + }, + }, + { + type: 'Context', + pattern: /^a manual container with (\S+) RU\/s and no utilization data$/, + execute(world, match) { + world.manual = { + databaseId: 'fixture-db', + containerId: 'fixture-container', + provisionedRu: number(match[1]), + ruPercents: [], + }; + }, + }, + { + type: 'Context', + pattern: /^one more sample at (\S+) percent$/, + execute(world, match) { + assert.ok(world.manual); + world.manual.ruPercents = [...world.manual.ruPercents, number(match[1])]; + }, + }, + { + type: 'Context', + pattern: /^the scope provisions (\S+) RU\/s in total$/, + execute(world, match) { + world.inputs.scopeProvisionedRuTotal = number(match[1]); + }, + }, + { + type: 'Context', + pattern: /^the container was created (\d+) days before the window ended$/, + execute(world, match) { + world.age = number(match[1]); + }, + }, + { + type: 'Context', + pattern: + /^an autoscale (container|shared database) with a (\d+) RU\/s max whose peak total is (\d+) RU\/s over (\d+) samples$/, + execute(world, match) { + world.autoscale = { + max: number(match[2]), + peaks: [number(match[3])], + samples: number(match[4]), + shared: match[1] === 'shared database', + }; + }, + }, + { + type: 'Context', + pattern: + /^an autoscale container with a (\d+) RU\/s max whose partitions peak at (\d+) and (\d+) RU\/s over (\d+) samples$/, + execute(world, match) { + world.autoscale = { + max: number(match[1]), + peaks: [number(match[2]), number(match[3])], + samples: number(match[4]), + shared: false, + }; + }, + }, + { + type: 'Context', + pattern: /^an autoscale container with a (\d+) RU\/s max and no autoscale data$/, + execute(world, match) { + world.autoscale = { max: number(match[1]), peaks: [], samples: 0, shared: false }; + }, + }, + { + type: 'Context', + pattern: /^(\d+) requests of which (\d+) returned status (\S+)$/, + execute(world, match) { + world.statuses = { 200: number(match[1]) - number(match[2]), [match[3]]: number(match[2]) }; + }, + }, + { + type: 'Context', + pattern: /^(\d+) requests returned status (\S+)$/, + execute(world, match) { + assert.ok(world.statuses); + world.statuses[match[2]] = number(match[1]); + }, + }, + { + type: 'Action', + pattern: /^CODA evaluates (DX-\d{3})$/, + async execute(world, match, detector) { + assert.equal(match[1], detector, 'Evaluation step must match its feature detector.'); + await evaluate(world, detector); + }, + }, + { + type: 'Outcome', + pattern: /^the utilization band is (\S+)$/, + execute(world, match) { + assert.equal(sizing(world).band, match[1]); + }, + }, + { + type: 'Outcome', + pattern: /^the container is (over-provisioned|not over-provisioned)$/, + execute(world, match) { + observation(world); + assert.equal(world.advisory !== undefined, match[1] === 'over-provisioned'); + }, + }, + { + type: 'Outcome', + pattern: /^the recommended (?:throughput|max) is (\S+) RU\/s$/, + execute(world, match) { + near(sizing(world).recommendedRu, match[1]); + }, + }, + { + type: 'Outcome', + pattern: /^the wasted capacity is (\S+) RU\/s$/, + execute(world, match) { + near(sizing(world).recoverableRu, match[1]); + }, + }, + { + type: 'Outcome', + pattern: /^the severity is (\S+)$/, + execute(world, match) { + const result = observation(world); + const severity = world.advisory?.severity ?? ('severity' in result ? result.severity : undefined) ?? 'None'; + assert.equal(severity.toLowerCase(), match[1].toLowerCase()); + }, + }, + { + type: 'Outcome', + pattern: /^the root cause is (\S+)$/, + execute(world, match) { + const result = observation(world); + assert.ok('rootCause' in result); + assert.equal(result.rootCause, match[1]); + }, + }, + { + type: 'Outcome', + pattern: /^the dominant error code is (\d+)$/, + execute(world, match) { + assert.equal(errors(world).dominantStatus, number(match[1])); + }, + }, + { + type: 'Outcome', + pattern: /^the hard error rate is (\S+) percent$/, + execute(world, match) { + near(errors(world).errorRatePercent, match[1]); + }, + }, + { + type: 'Outcome', + pattern: /^the DX-020 Agent Kit rule is (\S+)$/, + execute(world, match) { + observation(world); + assert.equal(world.advisory?.agentKitRule ?? null, match[1] === 'null' ? null : match[1]); + }, + }, + { + type: 'Outcome', + pattern: /^the guidance is conditional on request-level investigation$/, + execute(world) { + observation(world); + assert.ok(world.advisory); + for (const text of [world.advisory.suggestedAction, world.advisory.evidence?.guidance]) { + assert.equal(typeof text, 'string'); + for (const required of ['status counts alone', 'operation or item', 'read-404-then-insert sequence']) { + assert.ok(String(text).includes(required), `Guidance must retain: ${required}`); + } + } + }, + }, +]; + +function resolve(step: PickleStep): Binding { + if (step.argument) { + throw new Error('No binding for step tables or doc strings.'); + } + const matches = bindings.filter( + (binding) => binding.type === step.type?.toString() && binding.pattern.test(step.text), + ); + if (matches.length !== 1) { + throw new Error(`${matches.length === 0 ? 'Unbound' : 'Ambiguous'} ${step.type} step: ${step.text}`); + } + return matches[0]; +} + +export function validateBindings(testCase: FeatureCase): void { + if (!SUPPORTED_DETECTORS.has(testCase.detector)) { + throw new Error(`Native CODA bindings are not implemented for ${testCase.detector}.`); + } + for (const step of testCase.pickle.steps) { + resolve(step); + } +} + +export async function executeCase(testCase: FeatureCase): Promise { + validateBindings(testCase); + const world: World = { + inputs: { + weeklyRuPercents: [], + hasManualThroughput: false, + manualThroughput: [], + partitions: [], + storage: [], + indexing: [], + }, + account: { + accountName: 'fixture-account', + api: API.Core, + regionCount: 1, + writeRegionCount: 1, + multiRegionWritesEnabled: false, + isServerless: false, + perRegionPerPartitionAutoscaleEnabled: false, + }, + checks: 0, + }; + for (const step of testCase.pickle.steps) { + const binding = resolve(step); + const match = binding.pattern.exec(step.text); + assert.ok(match); + await binding.execute(world, match, testCase.detector); + if (binding.type === 'Outcome') { + world.checks++; + } + } + assert.ok( + (world.result || world.applicabilityChecked) && world.checks > 0, + 'Every scenario must evaluate production code and assert at least one outcome.', + ); + return world.checks; +} diff --git a/src/panels/accountOverview/services/advisories/conformance/coda.conformance.ts b/src/panels/accountOverview/services/advisories/conformance/coda.conformance.ts new file mode 100644 index 000000000..728f5ea28 --- /dev/null +++ b/src/panels/accountOverview/services/advisories/conformance/coda.conformance.ts @@ -0,0 +1,82 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; +import { executeCase, SUPPORTED_DETECTORS, validateBindings } from './bindings'; +import { loadFeatures } from './features'; + +const featureRoot = process.env.CODA_FEATURE_ROOT; +const commit = process.env.CODA_COMMIT; +const reportFile = process.env.CODA_REPORT; +if (!featureRoot || !commit || !reportFile) { + throw new Error('Use npm run test:coda to verify the pinned checkout before executing private features.'); +} +const inventory = loadFeatures(featureRoot); +const filter = process.env.CODA_DETECTORS?.split(',').filter(Boolean) ?? []; +for (const detector of filter) { + if (!inventory.some((entry) => entry.detector === detector)) { + throw new Error(`No scenarios found for requested detector ${detector}.`); + } +} +const selected = inventory.filter((entry) => filter.length === 0 || filter.includes(entry.detector)); +const results = inventory.map((entry) => ({ + detector: entry.detector, + file: path.relative(featureRoot, entry.file), + id: entry.pickle.id, + name: entry.pickle.name, + status: selected.includes(entry) ? 'notRun' : 'notSelected', + checks: 0, +})); + +afterAll(() => { + const counts = Object.fromEntries( + ['passed', 'failed', 'unbound', 'notSelected', 'notRun'].map((status) => [ + status, + results.filter((result) => result.status === status).length, + ]), + ); + mkdirSync(path.dirname(reportFile), { recursive: true }); + writeFileSync( + reportFile, + JSON.stringify( + { + commit, + generatedAt: new Date().toISOString(), + total: inventory.length, + selected: selected.length, + supportedDetectors: [...SUPPORTED_DETECTORS], + counts, + scenarios: results, + }, + null, + 2, + ) + '\n', + ); + console.log(`CODA inventory: ${inventory.length}; selected: ${selected.length}; ${JSON.stringify(counts)}.`); +}); + +describe('pinned CODA features against native production evaluators', () => { + for (const testCase of selected) { + it(`${testCase.detector} ${path.basename(testCase.file)}:${testCase.pickle.id} ${testCase.pickle.name}`, async () => { + const result = results[inventory.indexOf(testCase)]; + try { + validateBindings(testCase); + } catch (error) { + result.status = 'unbound'; + throw error; + } + try { + result.checks = await executeCase(testCase); + expect(result.checks).toBeGreaterThan(0); + result.status = 'passed'; + } catch (error) { + result.status = 'failed'; + throw error; + } + }); + } +}); diff --git a/src/panels/accountOverview/services/advisories/conformance/features.test.ts b/src/panels/accountOverview/services/advisories/conformance/features.test.ts new file mode 100644 index 000000000..cc8164195 --- /dev/null +++ b/src/panels/accountOverview/services/advisories/conformance/features.test.ts @@ -0,0 +1,131 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { describe, expect, it, onTestFinished } from 'vitest'; +import { executeCase } from './bindings'; +import { loadFeatures, parseFeature } from './features'; + +describe('original Gherkin execution infrastructure', () => { + it('uses the official compiler for Background and every row of multiple Examples blocks', () => { + const cases = parseFeature( + ` +Feature: Synthetic parser fixture + Background: + Given a fixture + Scenario Outline: Expanded