catarrh/simple
1
0
Fork 0
Code Issues Pull Requests Actions 8 Packages Projects Releases Wiki Activity

v3.22.0: the test suites (scripts grouped, ordered, with setup/teardown)

Browse Source 
A suite is the root object: every script belongs to exactly one suite, the
suite holds the ordered member list by reference - the optional setup first,
the members, the optional teardown last - and runs them in one card session,
without resets and without resume.

- test_suites.py: the suite store (test_suites.json, --test-suites), validated
  and atomic like the script store: roles normalised (setup first, teardown
  last, at most one of each, a script appears once per suite), per-member
  on_fail stop/continue and the suite-level require_adm.
- test_scripts.py schema v2: suite_id/require_adm; list_by_suite()/orphans();
  the import reports the id_map a bundle import remaps with.  v3.21.0 scripts
  without a suite are attached once to an auto-created "Imported scripts"
  suite (migrate_scripts_to_suites, called from __main__).
- API: /api/test/suites (+ update|delete|import - delete refused while
  non-empty; the bundle import imports the scripts first, remaps the suite
  references and skips unresolvable ones); the scripts endpoints gain
  suite_id/require_adm plus move|copy (delete detaches from the suite first;
  a suite update refuses a membership change).
- runner: /api/test/run {suite_id} resolves the members and the preset, runs
  the ADM gate, then setup -> members (per-member on_fail) -> teardown.  The
  teardown always runs on a stop or a failure (skipped only when the card is
  gone; it clears the stop flag for its duration); a card reset stops the
  suite, skips the remaining members and is flagged.  Members' live steps land
  in the shared snapshot, their records (status/steps/note/log excerpt/
  counters) in _TEST_RUN['suite']; the summary carries the counts, wall time,
  preset, the SCP80 counters before/after per keyset and the session flags.
  _test_log buffers the semantic lines (bounded) for the report's per-member
  excerpts.
- ADM gate (_ensure_adm): a suite/script require_adm verifies the matched
  preset's key once per card session (the /api/verify-adm badge sets the same
  latch); a failure refuses the start, stamps the session (no automatic
  retry) and needs a manual verify.
- PWA: the Test script pill becomes four views - the suite list (default),
  the suite editor (the script list: role badges, on_fail, up/down, per-script
  Run/Edit/Move/Copy/Delete, Add script with the templates), the script editor
  (the step list; a single run shows its result under the steps and is not
  preserved) and the suite report (per-member blocks with the step table,
  notes and run log; summary; export JSON/Markdown).  Suite bundles export/
  import; the legacy script export imports into "Imported scripts".  EN/RU.
- tests: the suite store/migration, the runner orchestration (order, on_fail,
  setup failure, stop, card reset, teardown, per-member logs) and the ADM gate
  (Python); the suite HTTP paths; the PWA report/Markdown/wiring (Node).
  Tests: 833 frontend + 794 Python; sw cache simple-v397.
This commit is contained in:
Антон Трошин
2026-10-06 00:26:50 +03:00
parent e6ced47323
commit 5976ef89b4
17 changed files with 3178 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/tests/testscripts.test.js
+39 -21
View File
@@ -40,6 +40,8 @@ eval(extractFunc(html, 'testStepSummary'));
eval(extractFunc(html, 'testCommandOptions'));
globalThis.t = s => s;
globalThis.esc = s => String(s);
globalThis.testScripts = () => _testScripts || [];
globalThis.testSuites = () => _testSuites || [];
eval(extractFunc(html, 'testFormRow'));
eval(extractFunc(html, 'testFormInput'));
eval(extractFunc(html, 'testFormSelect'));
@@ -53,11 +55,14 @@ eval(extractFunc(html, 'testScriptsWriteback', true));
eval(extractFunc(html, 'testScriptsWritebackQueued'));
eval(extractFunc(html, 'testScriptsSave'));
eval(extractFunc(html, 'testRunStart', true));
eval(extractFunc(html, 'testDelete', true));
eval(extractFunc(html, 'testSuiteDeleteScript', true));
eval(extractFunc(html, 'testSuitesRefetch', true));
eval(extractFunc(html, 'testChecksCollect'));
eval('var _testEditStep = null; var _testEditChecks = []; var _testEditStepIndex = -1;'
+ ' var _testScripts = null; var _testCurrentIdx = -1; var _testRunState = null;'
+ ' var _testSaveTimer = null; var _testWriteChains = new WeakMap(); var _testLastPresetIdx = -1;');
+ ' var _testScripts = null; var _testSuites = []; var _testScriptId = null;'
+ ' var _testScriptDraft = null; var _testRunState = null; var _testSaveTimer = null;'
+ ' var _testWriteChains = new WeakMap(); var _testLastPresetIdx = -1;'
+ ' var _testRunScriptRef = null; var _testSuiteIdx = -1;');
globalThis.localStorage = { getItem: () => null, setItem: () => {}, removeItem: () => {} };
function fakeForm(values) {
@@ -443,12 +448,14 @@ test('two quick saves of a draft create one script, then update it', async () =>
};
globalThis.ioStatus = () => {};
_testScripts = [{ name: 'one', steps: [{ type: 'action', kind: 'status', params: { attempts: 1 } }] }];
_testCurrentIdx = 0;
globalThis.testCurrent = () => _testScripts[_testCurrentIdx];
_testScriptId = null;
globalThis.testCurrentScript = () => _testScripts[0];
testScriptsSave(true);
testScriptsSave(true);
await new Promise(r => setTimeout(r, 40));
assert.deepStrictEqual(calls, ['/api/test/scripts', '/api/test/scripts/update']);
// the create is followed by a suites refetch (the suite gained the member)
assert.deepStrictEqual(calls, ['/api/test/scripts', '/api/test/suites',
'/api/test/scripts/update']);
assert.strictEqual(_testScripts[0].id, 'd'.repeat(32));
});
@@ -458,15 +465,21 @@ test('deleting a script cancels a pending debounced save', async () => {
globalThis.ioStatus = () => {};
globalThis.confirm = () => true;
globalThis.testRender = () => {};
_testScripts = [{ name: 'draft', steps: [{ type: 'action', kind: 'status', params: { attempts: 1 } }] }];
_testCurrentIdx = 0;
globalThis.testCurrent = () => _testScripts[_testCurrentIdx];
globalThis.testStoresFetch = async () => {};
const script = {id: 'e'.repeat(32), name: 'edited',
steps: [{type: 'action', kind: 'status', params: {attempts: 1}}]};
_testScripts = [script];
globalThis.testCurrentScript = () => script;
globalThis.testScriptById = () => script;
globalThis.testCurrentSuite = () => ({id: 'a'.repeat(32),
scripts: [{script_id: script.id, role: 'member', on_fail: 'stop'}]});
globalThis.testSuiteEntry = () => ({script_id: script.id});
globalThis.testSuites = () => [];
testScriptsSave(); // schedules the 600 ms write
await testDelete(); // a draft: local removal only
await testSuiteDeleteScript(0); // the delete path
await new Promise(r => setTimeout(r, 700));
assert.deepStrictEqual(calls, []); // the queued save never fired
assert.strictEqual(_testScripts.length, 1);
assert.strictEqual(_testScripts[0].name, 'New test script');
assert.deepStrictEqual(calls, ['/api/test/scripts/delete']);
assert.strictEqual(script._deleted, true);
});
test('the STATUS step leaves the SW check to the server default', () => {
@@ -606,12 +619,16 @@ test('the run sends the stored preset id, not the removed flat fields', () => {
test('the run refuses without a matched or complete preset', () => {
globalThis.cards = [];
globalThis.cardsMatchedPreset = () => null;
assert.match(testRunPreset().error, /No card preset matches/);
assert.match(testRunPreset(true).error, /No card preset matches/);
const preset = { id: 'p2' };
globalThis.cards = [preset];
globalThis.cardsMatchedPreset = () => preset;
globalThis.cardsScp80Complete = () => false;
assert.match(testRunPreset().error, /incomplete SCP80/);
assert.match(testRunPreset(true).error, /incomplete SCP80/);
// an ADM-only run needs the preset but not the SCP80 completeness
assert.strictEqual(testRunPreset(false).preset_id, 'p2');
globalThis.cardsScp80Complete = () => true;
assert.strictEqual(testRunPreset(true).preset_id, 'p2');
});
test('the client-side problem check bounds the keyset number', () => {
@@ -681,10 +698,11 @@ test('a new script is created in the server store and gets its id', async () =>
assert.strictEqual(calls[0][0], '/api/test/scripts');
assert.strictEqual(calls[0][1].name, 'new');
assert.strictEqual(s.id, 'a'.repeat(32)); // the store id is adopted
assert.strictEqual(calls[1][0], '/api/test/suites'); // the member refetch
// every later write updates by id
await testScriptsWriteback(s);
assert.strictEqual(calls[1][0], '/api/test/scripts/update');
assert.strictEqual(calls[1][1].id, 'a'.repeat(32));
assert.strictEqual(calls[2][0], '/api/test/scripts/update');
assert.strictEqual(calls[2][1].id, 'a'.repeat(32));
});
test('an incomplete draft is not written to the store', async () => {
@@ -701,11 +719,11 @@ test('an immediate save writes the current script through', async () => {
globalThis.pysimFetch = async (path) => { calls.push(path); return {ok: true, script: {id: 'c'.repeat(32)}}; };
globalThis.ioStatus = () => {};
_testScripts = [{name: 'one', steps: [{type: 'action', kind: 'status', params: {attempts: 1}}]}];
_testCurrentIdx = 0;
globalThis.testCurrent = () => _testScripts[_testCurrentIdx];
_testScriptId = null;
globalThis.testCurrentScript = () => _testScripts[0];
testScriptsSave(true);
await new Promise(r => setTimeout(r, 0)); // the writeback is async
assert.deepStrictEqual(calls, ['/api/test/scripts']);
assert.deepStrictEqual(calls, ['/api/test/scripts', '/api/test/suites']);
assert.strictEqual(_testScripts[0].id, 'c'.repeat(32));
});
@@ -723,7 +741,7 @@ test('the run sends the stored script id with the edited script', async () => {
sent = {path, body};
return {running: true, name: 'saved', index: 0, total: 1, steps: [], status: null};
};
globalThis.testCurrent = () => ({id: 'b'.repeat(32), name: 'saved',
globalThis.testCurrentScript = () => ({id: 'b'.repeat(32), name: 'saved',
steps: [{type: 'action', kind: 'status', params: {attempts: 1}}]});
globalThis.testRunPreset = () => { throw new Error('no preset for a status-only script'); };
globalThis.testRunMessage = () => {};
frontend/tests/testsuites.test.js
+151
View File
@@ -0,0 +1,151 @@
// Test suite view/report tests (v3.22.0): the report renderers, the Markdown
// export and the markup wiring of the four views under the Test script pill.
const { test } = require('node:test');
const assert = require('node:assert');
const fs = require('node:fs');
const path = require('node:path');
const html = fs.readFileSync(path.join(__dirname, '..', 'index.html'), 'utf8');
function extractFunc(src, name, asyncFn) {
const re = new RegExp('(?:async\\s+)?function\\s+' + name + '\\s*\\([^)]*\\)\\s*\\{');
const m = re.exec(src);
if (!m) throw new Error('function ' + name + ' not found');
let i = m.index + m[0].length - 1;
let depth = 0;
for (; i < src.length; i++) {
if (src[i] === '{') depth++;
else if (src[i] === '}') {
depth--;
if (depth === 0) break;
}
}
return src.slice(m.index, i + 1);
}
let code = '';
for (const fn of ['testStatusBadge', 'testRunStepHtml', 'testReportMemberHtml',
'testReportMarkdown', 'testSuiteProgress', 'testRoleOptions', 'testFailOptions']) {
code += extractFunc(html, fn) + '\n';
}
eval(code);
globalThis.t = s => s;
globalThis.esc = s => String(s);
function snapshot() {
return {
name: 'regression', status: 'error', preset: 'SE test1',
started: 1, finished: 2,
suite: {
id: 'a'.repeat(32), name: 'regression', require_adm: true,
session_start: 1, session_end: 1, session_changed: false,
counters_before: {'2': '0000000010'}, counters_after: {'2': '0000000012'},
members: [
{script_id: 'b'.repeat(32), name: 'setup', role: 'setup',
on_fail: 'stop', status: 'ok',
steps: [{index: 0, type: 'action', label: 'STATUS', status: 'ok',
checks: [{label: 'SW', ok: true, expected: '9000', actual: '9000'}]}],
log: ['TEST-RUN [1/3 setup setup] step 1: STATUS 1/1 -> 9000'], log_truncated: false},
{script_id: 'c'.repeat(32), name: 'alfa-dsa', role: 'member',
on_fail: 'stop', status: 'error',
note: 'step 2 failed',
steps: [{index: 0, type: 'action', label: 'SCP80', status: 'ok', checks: []},
{index: 1, type: 'expect', label: 'REFRESH', status: 'error',
checks: [{label: 'Command', ok: false, expected: 'REFRESH',
actual: '(none)', level: 'error'}]}],
log: ['TEST-RUN [2/3 member alfa-dsa] step 1: SCP80 -> SW=9103'], log_truncated: true},
{script_id: 'd'.repeat(32), name: 'teardown', role: 'teardown',
on_fail: 'stop', status: 'skipped', steps: [], log: [], log_truncated: false},
],
summary: {ok: 1, warning: 0, error: 1, skipped: 1, stopped: false,
session_changed: false, wall_ms: 12345,
counters_before: {'2': '0000000010'}, counters_after: {'2': '0000000012'},
setup: 'ok', teardown: 'skipped'},
},
adm: {required: true, verified: true},
};
}
test('testReportMarkdown renders the members, steps, notes and logs', () => {
const md = testReportMarkdown(snapshot());
assert.match(md, /# Test suite report: regression/);
assert.match(md, /- status: error/);
assert.match(md, /- Preset: SE test1/);
assert.match(md, /passed: 1, warnings: 0, failed: 1, skipped: 1/);
assert.match(md, /wall time: 12\.3 s/);
assert.match(md, /SCP80: 2 0000000010 -> 0000000012/);
assert.match(md, /## 2\. alfa-dsa \(member, on_fail: stop\) - error/);
assert.match(md, /> step 2 failed/);
assert.match(md, /### Step 2: REFRESH - error/);
assert.match(md, /- ✗ Command: expected REFRESH, actual \(none\)/);
assert.match(md, /TEST-RUN \[2\/3 member alfa-dsa\] step 1: SCP80 -> SW=9103/);
assert.match(md, /## 3\. teardown \(teardown, on_fail: stop\) - skipped/);
});
test('testReportMarkdown tolerates an empty snapshot', () => {
assert.match(testReportMarkdown(null), /# Test suite report: /);
});
test('testReportMemberHtml carries the status, note and log details', () => {
const member = snapshot().suite.members[1];
const htmlOut = testReportMemberHtml(member, 1);
assert.match(htmlOut, /alfa-dsa/);
assert.match(htmlOut, /step 2 failed/);
assert.match(htmlOut, /Run log/);
assert.match(htmlOut, /truncated/);
assert.match(htmlOut, /Command: /);
const teardown = testReportMemberHtml(snapshot().suite.members[2], 2);
assert.match(teardown, /teardown/);
assert.match(teardown, /skipped/);
});
test('testSuiteProgress counts the finished members only', () => {
assert.strictEqual(testSuiteProgress(snapshot()), '3/3');
const snap = snapshot();
snap.suite.members[1].status = 'running';
snap.suite.members[2].status = 'pending';
assert.strictEqual(testSuiteProgress(snap), '1/3');
});
test('the role and on_fail selects mark the stored value', () => {
assert.match(testRoleOptions('setup'), /value="setup" selected/);
assert.match(testRoleOptions('member'), /value="member" selected/);
assert.doesNotMatch(testRoleOptions('member'), /value="setup" selected/);
assert.match(testFailOptions('continue'), /value="continue" selected/);
assert.match(testFailOptions('stop'), /value="stop" selected/);
});
test('the Test script pill hosts the four suite views and their wiring', () => {
for (const id of ['test-view-suites', 'test-view-suite', 'test-view-script', 'test-view-report']) {
assert.ok(html.includes('id="' + id + '"'), 'missing ' + id);
}
for (const fn of ['testSuiteNew', 'testSuiteOpen', 'testSuiteDelete', 'testSuiteExport',
'testSuiteRun', 'testSuiteAddScript', 'testSuiteDeleteScript', 'testSuiteMoveScript',
'testSuiteTargetGo', 'testOpenScript', 'testScriptBack', 'testReportExportJson',
'testReportExportMarkdown', 'testImportFile', 'testViewShow']) {
assert.ok(html.includes('function ' + fn) || html.includes('async function ' + fn),
'missing ' + fn);
}
assert.ok(/id="test-suite-name"[^>]*oninput="testSuiteNameChanged\(\)"/.test(html));
assert.ok(/id="test-suite-adm"[^>]*onchange="testSuiteAdmChanged\(\)"/.test(html));
assert.ok(/id="test-script-adm"[^>]*onchange="testScriptAdmChanged\(\)"/.test(html));
assert.ok(/id="test-suite-run-btn"[^>]*data-needs="card"/.test(html));
assert.ok(/id="test-run-btn"[^>]*data-needs="card"/.test(html));
});
test('the suite run opens the report view and names the preset', () => {
const src = extractFunc(html, 'testSuiteRun', true);
assert.match(src, /'\/api\/test\/run'/);
assert.match(src, /suite_id/);
assert.match(src, /_testView = 'report'/);
assert.match(src, /testRunPreset\(usesScp80\)/);
assert.match(src, /needsAdm/);
});
test('a single script run sends its require_adm flag and keeps its id', () => {
const src = extractFunc(html, 'testRunStart', true);
assert.match(src, /require_adm: !!cur\.require_adm/);
assert.match(src, /body\.script_id = cur\.id/);
assert.match(src, /testRenderRun\(resp\)/);
});