v3.37.0: load testing for test scripts and suites
POST /api/test/run takes an optional iterations count (the PWA's Iterations field): the run repeats, successful iterations are discarded (steps/log cleared, only load.done and the timing grow), the first error stops the loop and keeps that iteration's report, a stop ends it after the current iteration, and a card session change between iterations is an error. The SCP80 counter state is shared across iterations, so packets continue from what the previous run consumed. The status gains a load block (iteration/done/failed_iteration/ elapsed_ms/avg_ms; disabled for a normal run). The suite runner was split into a per-iteration body (_test_suite_execute_once) + the single-run finaliser, and two load workers own the loops. Review fixes: a raised runner error finalises as error (no stuck running state), a suite-internal session change is an error, a stopped member ends the suite loop, the final status keeps a warning from any iteration, avg_ms covers the attempted iterations, and each suite iteration's counters_before is read fresh. 1041 Python + 1069 frontend; simple-v475.
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
const { test } = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const html = fs.readFileSync(path.join(__dirname, '..', 'index.html'), 'utf8');
|
||||
|
||||
function extractFunc(src, name) {
|
||||
const re = new RegExp('function\\s+' + name + '\\s*\\([^)]*\\)\\s*\\{');
|
||||
const m = re.exec(src);
|
||||
if (!m) throw new Error('function ' + name + ' not found');
|
||||
let i = m.index + m[0].length - 1;
|
||||
let depth = 0;
|
||||
for (; i < src.length; i++) {
|
||||
if (src[i] === '{') depth++;
|
||||
else if (src[i] === '}') {
|
||||
depth--;
|
||||
if (depth === 0) break;
|
||||
}
|
||||
}
|
||||
return src.slice(m.index, i + 1);
|
||||
}
|
||||
|
||||
const FNS = ['esc', 'testStatusBadge', 'testRenderScriptRun', 'testRenderReport',
|
||||
'testIterationsInit', 'testIterationsChanged', 'testIterationsValue'];
|
||||
const code = FNS.map(f => extractFunc(html, f)).join('\n');
|
||||
|
||||
function fakeEl(id) {
|
||||
return { id, innerHTML: '', textContent: '', className: '', value: '',
|
||||
querySelectorAll: () => [] };
|
||||
}
|
||||
|
||||
function build(els, storage, curRef) {
|
||||
const fn = new Function('els', 'storage', 'curRef', `
|
||||
var _testRunState = null;
|
||||
var _testRunScriptRef = curRef.value;
|
||||
var TEST_ITERATIONS_MAX = 1000000;
|
||||
var _testIterationsReady = false;
|
||||
var localStorage = storage;
|
||||
var document = { getElementById: id => els[id] || null };
|
||||
var t = k => k;
|
||||
function testCurrentScript() { return curRef.value; }
|
||||
function testRunStepHtml(s) { return '<STEP ' + (s.note || '') + '>'; }
|
||||
function testReportMemberHtml(m, i) { return '<MEMBER ' + (m.name || '') + '>'; }
|
||||
${code}
|
||||
return {
|
||||
setState: v => { _testRunState = v; },
|
||||
getState: () => _testRunState,
|
||||
testRenderScriptRun, testRenderReport,
|
||||
testIterationsValue, testIterationsChanged, testIterationsInit,
|
||||
};
|
||||
`);
|
||||
return fn(els, storage, curRef);
|
||||
}
|
||||
|
||||
test('the iteration count clamps and persists (v3.37.0)', () => {
|
||||
const els = { 'test-iterations': fakeEl('test-iterations'),
|
||||
'test-suite-iterations': fakeEl('test-suite-iterations') };
|
||||
const storage = { data: {}, getItem(k) { return this.data[k] === undefined ? null : this.data[k]; },
|
||||
setItem(k, v) { this.data[k] = String(v); } };
|
||||
const app = build(els, storage, { value: null });
|
||||
|
||||
els['test-iterations'].value = '5';
|
||||
assert.strictEqual(app.testIterationsValue('test-iterations'), 5);
|
||||
assert.strictEqual(app.testIterationsValue('missing'), 1);
|
||||
els['test-iterations'].value = '99999999';
|
||||
assert.strictEqual(app.testIterationsValue('test-iterations'), 1000000);
|
||||
els['test-iterations'].value = 'nope';
|
||||
assert.strictEqual(app.testIterationsValue('test-iterations'), 1);
|
||||
|
||||
app.testIterationsChanged(els['test-iterations']);
|
||||
els['test-iterations'].value = '7';
|
||||
app.testIterationsChanged(els['test-iterations']);
|
||||
assert.strictEqual(storage.data.simple_test_iterations, '7');
|
||||
assert.strictEqual(els['test-suite-iterations'].value, '7', 'the other input follows');
|
||||
|
||||
// init restores the remembered value
|
||||
storage.data.simple_test_iterations = '12';
|
||||
app.testIterationsInit();
|
||||
assert.strictEqual(els['test-iterations'].value, '12');
|
||||
assert.strictEqual(els['test-suite-iterations'].value, '12');
|
||||
});
|
||||
|
||||
test('the run bodies carry the iterations (v3.37.0)', () => {
|
||||
const script = extractFunc(html, 'testRunStart');
|
||||
assert.match(script, /body\.iterations = testIterationsValue\('test-iterations'\)/);
|
||||
const suite = extractFunc(html, 'testSuiteRun');
|
||||
assert.match(suite, /body\.iterations = testIterationsValue\('test-suite-iterations'\)/);
|
||||
// the inputs live next to the two run buttons
|
||||
assert.match(html, /id="test-iterations" type="number" min="1" max="1000000"/);
|
||||
assert.match(html, /id="test-suite-iterations" type="number" min="1" max="1000000"/);
|
||||
});
|
||||
|
||||
test('a running load test shows the counter, not the steps (v3.37.0)', () => {
|
||||
const els = { 'test-run-status': fakeEl('test-run-status'),
|
||||
'test-run-steps': fakeEl('test-run-steps') };
|
||||
const curRef = { value: { id: 's1' } };
|
||||
const app = build(els, { getItem: () => null, setItem: () => {} }, curRef);
|
||||
app.setState({ running: true, script_id: 's1', name: 'S', steps: [],
|
||||
load: { enabled: true, iterations: 100, iteration: 3, done: 2 } });
|
||||
app.testRenderScriptRun();
|
||||
assert.match(els['test-run-status'].innerHTML, /Iteration 3 \/ 100/);
|
||||
assert.match(els['test-run-status'].innerHTML, /done: 2/);
|
||||
assert.strictEqual(els['test-run-steps'].innerHTML, '', 'steps are not tracked');
|
||||
});
|
||||
|
||||
test('a load-test failure shows the failing iteration steps (v3.37.0)', () => {
|
||||
const els = { 'test-run-status': fakeEl('test-run-status'),
|
||||
'test-run-steps': fakeEl('test-run-steps') };
|
||||
const curRef = { value: { id: 's1' } };
|
||||
const app = build(els, { getItem: () => null, setItem: () => {} }, curRef);
|
||||
app.setState({ running: false, status: 'error', script_id: 's1', name: 'S',
|
||||
steps: [{ note: 'boom' }], error: 'iteration 4 failed',
|
||||
load: { enabled: true, iterations: 100, iteration: 4, done: 3, failed_iteration: 4 } });
|
||||
app.testRenderScriptRun();
|
||||
assert.match(els['test-run-status'].innerHTML, /Iteration 4 \/ 100/);
|
||||
assert.match(els['test-run-status'].innerHTML, /iteration 4 failed/);
|
||||
assert.match(els['test-run-steps'].innerHTML, /<STEP boom>/);
|
||||
});
|
||||
|
||||
test('the suite report summarises a load run and hides successful iterations (v3.37.0)', () => {
|
||||
const els = { 'test-report-title': fakeEl('test-report-title'),
|
||||
'test-report-summary': fakeEl('test-report-summary'),
|
||||
'test-report-members': fakeEl('test-report-members') };
|
||||
const app = build(els, { getItem: () => null, setItem: () => {} }, { value: null });
|
||||
// a completed load run: no suite state, only the summary
|
||||
app.setState({ running: false, status: 'ok', name: 'Load suite',
|
||||
load: { enabled: true, iterations: 50, iteration: 50, done: 50,
|
||||
elapsed_ms: 12000, avg_ms: 240, failed_iteration: null }, suite: null });
|
||||
app.testRenderReport();
|
||||
assert.match(els['test-report-title'].innerHTML, /Load suite/);
|
||||
assert.match(els['test-report-summary'].innerHTML, /Iteration 50 \/ 50/);
|
||||
assert.match(els['test-report-summary'].innerHTML, /done: 50/);
|
||||
assert.strictEqual(els['test-report-members'].innerHTML, '');
|
||||
// a failure keeps that iteration's members
|
||||
app.setState({ running: false, status: 'error', name: 'Load suite',
|
||||
error: 'iteration 7 failed',
|
||||
load: { enabled: true, iterations: 50, iteration: 7, done: 6,
|
||||
elapsed_ms: 2000, avg_ms: 300, failed_iteration: 7 },
|
||||
suite: { name: 'Load suite', summary: { ok: 0, warning: 0, error: 1, skipped: 0, stopped: false },
|
||||
members: [{ name: 'A', status: 'error' }] } });
|
||||
app.testRenderReport();
|
||||
assert.match(els['test-report-summary'].innerHTML, /Iteration 7 \/ 50/);
|
||||
assert.match(els['test-report-members'].innerHTML, /<MEMBER A>/);
|
||||
// while running, the members are not shown (results are not tracked)
|
||||
app.setState({ running: true, status: null, name: 'Load suite',
|
||||
load: { enabled: true, iterations: 50, iteration: 2, done: 1 },
|
||||
suite: { name: 'Load suite', summary: null, members: [{ name: 'A', status: 'running' }] } });
|
||||
app.testRenderReport();
|
||||
assert.strictEqual(els['test-report-members'].innerHTML, '');
|
||||
});
|
||||
|
||||
test('a successful load run can be exported and reopened after a reload (v3.37.0)', () => {
|
||||
// the report is renderable with suite: null, so the export buttons and the
|
||||
// reload path must accept a load run too
|
||||
for (const fn of ['testReportExportJson', 'testReportExportMarkdown']) {
|
||||
const src = extractFunc(html, fn);
|
||||
assert.match(src, /_testRunState\.load && _testRunState\.load\.enabled/);
|
||||
}
|
||||
const init = extractFunc(html, 'testInit');
|
||||
assert.match(init, /st\.load && st\.load\.enabled/);
|
||||
assert.match(init, /const idx = st\.suite\s*\?/, 'a null suite must not break the reopen');
|
||||
// the banner knows a load suite even when its state is cleared
|
||||
const render = extractFunc(html, 'testRender');
|
||||
assert.match(render, /const isSuite = !!\(snap\.suite \|\| \(load && load\.suite\)\);/);
|
||||
});
|
||||
Reference in New Issue
Block a user