file manager: hide directories outside the filter's file paths

When a filter is active (not "All"), only the matching files, their
ancestor directories and the MF root stay; every other directory is
hidden.  The ancestors come from the standard file list (name -> paths):
pysimFsFilterMatchingPaths unions the registry paths of every matching
name, and pysimFsFilterShow keeps a directory iff it matches itself
(e.g. DF.GSM for svc:GPRS) or its canonical path prefixes a matching
path.  Custom files contribute their own paths; the not-loaded window
keeps directories rather than hiding everything.

Frontend 1160 tests green (+8).  (Version/sw bump still deferred.)

frontend/index.html

@@ -4417,6 +4417,11 @@ function uiccStdFilesLoad() {
 				Object.keys(_chains).forEach(id => {
 					if (document.getElementById(id + '-rows')) chainRender(id);
 				});
+				// The tree filter's directory rule needs the files' paths: a
+				// tree drawn while the list was still loading kept every
+				// directory, so redraw it now under an active filter.
+				if (pysimFsTreeRoot && pysimFsFilter.size && !pysimFsFilter.has(FS_FILTER_ALL) &&
+					document.getElementById('pysim-fs-tree')) pysimFsRenderTree();
 				return _uiccStdFiles;
 			});
 	}
@@ -16144,6 +16149,89 @@ function fsFilterIndex() {
 	return _fsFilterIndex;
 }
 
+// --- the directory rule: the canonical paths of the selected files ---
+// A directory stays in a filtered tree only when it lies on the path of a
+// file the filter selects, so the ancestors come from the standard file list
+// (frontend/uicc_files.json, the same source the SELECT picker and the
+// services editor use).  A name can live in several trees (EF.SPN is in
+// DF.GSM and in ADF.USIM), so the index keeps every path of a name.
+// The list is fetched once at startup (`_uiccStdFiles`, filled by
+// uiccStdFilesLoad); until it arrives the ancestors are unknown and the
+// directories are kept - never hide what cannot be judged.
+let _fsFilterPathIndex = null;
+let _fsFilterPathIndexSrc = null;
+let _fsFilterPaths = null;        // { key, paths } of the current selection
+let _fsFilterPathsKey = null;
+
+function fsFilterPathIndexBuild(files) {
+	const idx = new Map();
+	for (const e of (files || [])) {
+		if (!e || !e.name || !e.path) continue;
+		const name = String(e.name).toUpperCase();
+		const p = String(e.path).toUpperCase();
+		if (!idx.has(name)) idx.set(name, []);
+		if (idx.get(name).indexOf(p) < 0) idx.get(name).push(p);
+	}
+	return idx;
+}
+
+function fsFilterPathIndex() {
+	if (_fsFilterPathIndex && _fsFilterPathIndexSrc === _uiccStdFiles) return _fsFilterPathIndex;
+	_fsFilterPathIndexSrc = _uiccStdFiles;
+	_fsFilterPathIndex = fsFilterPathIndexBuild(_uiccStdFiles);
+	return _fsFilterPathIndex;
+}
+
+// The custom files' canonical paths the selection picks.  Custom files sit
+// outside the standard list (user-chosen names and DFs), so their own paths
+// are the only ancestry the filter has for them - without this the "Custom
+// files" filter would hide the DFs holding them.
+function fsFilterCustomPaths() {
+	const list = (typeof pysimCustomFiles !== 'undefined' && pysimCustomFiles) || [];
+	const out = [];
+	for (const c of list) {
+		const p = String((c && c.path) || '').toUpperCase();
+		if (!p) continue;
+		if (!pysimFsFilterMatch({ name: c.name, custom: true })) continue;
+		out.push(p);
+	}
+	return out;
+}
+
+// The selection's signature: the entries plus the custom files (that set
+// changes when the user adds or removes one).  The cache is keyed by it, so a
+// direct `pysimFsFilter` assignment recomputes as well as the setters'
+// explicit invalidation does.
+function fsFilterPathsKeyOf() {
+	return Array.from(pysimFsFilter).sort().join('|') + '#' + fsFilterCustomPaths().sort().join('|');
+}
+
+function pysimFsFilterPathsInvalidate() {
+	_fsFilterPaths = null;
+	_fsFilterPathsKey = null;
+}
+
+// The uppercased canonical paths of every file the current selection picks,
+// or null while the list is unavailable (not fetched yet, or an empty/failed
+// load): the caller keeps the directories then instead of hiding them all.
+function pysimFsFilterMatchingPaths() {
+	if (!Array.isArray(_uiccStdFiles) || !_uiccStdFiles.length) return null;
+	const key = fsFilterPathsKeyOf();
+	if (_fsFilterPaths && _fsFilterPathsKey === key) return _fsFilterPaths;
+	const out = new Set();
+	const idx = fsFilterIndex();
+	for (const [name, ids] of idx) {
+		let hit = false;
+		for (const id of pysimFsFilter) if (ids.has(id)) { hit = true; break; }
+		if (!hit) continue;
+		for (const p of (fsFilterPathIndex().get(name) || [])) out.add(p);
+	}
+	for (const p of fsFilterCustomPaths()) out.add(p);
+	_fsFilterPaths = out;
+	_fsFilterPathsKey = key;
+	return out;
+}
+
 let pysimFsFilter = new Set([FS_FILTER_ALL]);
 
 // The persisted selection, normalised: `all` is exclusive, and a missing or
@@ -16156,6 +16244,7 @@ function pysimFsFilterRestore() {
 	} catch (e) { /* ignore a broken entry */ }
 	pysimFsFilter = new Set(saved);
 	if (!pysimFsFilter.size || pysimFsFilter.has(FS_FILTER_ALL)) pysimFsFilter = new Set([FS_FILTER_ALL]);
+	pysimFsFilterPathsInvalidate();
 }
 pysimFsFilterRestore();
 
@@ -16236,6 +16325,7 @@ function pysimFsFilterSetEntry(entry, on) {
 		if (!next.size) next.add(FS_FILTER_ALL);
 	}
 	pysimFsFilter = next;
+	pysimFsFilterPathsInvalidate();
 	localStorage.setItem('simple_fs_filter', JSON.stringify(Array.from(pysimFsFilter)));
 	pysimFsFilterRender();
 	pysimFsRenderTree();
@@ -16278,6 +16368,7 @@ function pysimFsSetFilterToService(table, n) {
 	const feature = (servicesFeatureByNumber()[table] || {})[n];
 	if (!feature || !(SERVICES_FS_MAP[feature] || []).length) return;
 	pysimFsFilter = new Set(['svc:' + feature]);
+	pysimFsFilterPathsInvalidate();
 	localStorage.setItem('simple_fs_filter', JSON.stringify(Array.from(pysimFsFilter)));
 	pysimFsFilterRender();
 	pysimFsRenderTree();
@@ -16298,14 +16389,24 @@ function pysimFsFilterMatch(node) {
 	return false;
 }
 
-// Whether a node stays in the filtered tree.  A matching file shows; a
+// Whether a node stays in the filtered tree.  An empty selection and `all`
-// directory shows when it matches itself, has a matching loaded descendant,
+// show everything; otherwise a file shows when it matches, and a directory
-// or was never expanded (its children are unknown, so it cannot be pruned).
+// only when it matches itself (a service may name a DF - the GPRS feature is
+// DF.GSM) or lies on the path of a matching file (its canonical path is a
+// prefix of that file's registry path).  The MF root always stays, and while
+// the standard list is unavailable the ancestors cannot be computed, so every
+// directory is kept.
 function pysimFsFilterShow(node) {
+	if (!pysimFsFilter.size || pysimFsFilter.has(FS_FILTER_ALL)) return true;
+	if (!node) return false;
+	if (!node.isDir) return pysimFsFilterMatch(node);
 	if (pysimFsFilterMatch(node)) return true;
-	if (!node || !node.isDir) return false;
+	const path = String(pysimFsNodePath(node) || '').toUpperCase();
-	if (!node.children) return true;
+	if (!path || path === 'MF') return true;
-	return node.children.some(c => pysimFsFilterShow(c));
+	const paths = pysimFsFilterMatchingPaths();
+	if (!paths) return true;
+	for (const p of paths) if (p === path || p.indexOf(path + '/') === 0) return true;
+	return false;
 }
 
 async function pysimFsRefresh() {

frontend/tests/fs_filter.test.js

@@ -48,6 +48,15 @@ function extractStrConst(src, name) {
 // module state the extracted functions close over
 let pysimFsFilter = new Set(['all']);
 let _fsFilterIndex = null;
+// The shipped standard list stands in for the fetched one (uicc_files.json is
+// what uiccStdFilesLoad caches; the filter reads its entries synchronously).
+const stdAsset = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'uicc_files.json'), 'utf8'));
+let _uiccStdFiles = stdAsset.files;
+let _fsFilterPathIndex = null;
+let _fsFilterPathIndexSrc = null;
+let _fsFilterPaths = null;
+let _fsFilterPathsKey = null;
+let pysimCustomFiles = [];
 const _store = {};
 let _treeRenders = 0;
 
@@ -98,9 +107,11 @@ globalThis.document = {
 
 let code = '';
 for (const f of ['fsFilterServiceOptions', 'fsFilterIndexBuild', 'fsFilterIndex',
+	'fsFilterPathIndexBuild', 'fsFilterPathIndex', 'fsFilterCustomPaths',
+	'fsFilterPathsKeyOf', 'pysimFsFilterPathsInvalidate', 'pysimFsFilterMatchingPaths',
 	'pysimFsFilterMatch', 'pysimFsFilterShow', 'pysimFsFilterLabel',
 	'pysimFsFilterSummary', 'pysimFsFilterRender', 'pysimFsFilterToggle',
-	'pysimFsFilterSetEntry', 'pysimFsFilterRestore']) {
+	'pysimFsFilterSetEntry', 'pysimFsFilterRestore', 'pysimFsNodePath']) {
 	code += extractFunc(html, f) + '\n';
 }
 for (const c of ['FS_FILTER_GROUPS', 'FS_FILTER_GROUP_ORDER', 'SERVICES_FS_MAP']) code += extractConst(html, c);
@@ -254,32 +265,138 @@ test('"custom files" matches only the injected custom nodes', () => {
 	});
 });
 
-test('a directory stays while it has a matching descendant', () => {
+// A small tree with real parent links (pysimFsNodePath walks them): MF with
-	const df = children => ({ name: 'DF.GSM', fid: '7F20', isDir: true, children });
+// DF.TELECOM (EF.MSISDN), DF.GSM (EF.SPN, EF.ECC) and ADF.USIM (DF.WLAN).
+function fsTree() {
+	const root = { name: 'MF', fid: '3F00', isDir: true, children: [], parent: null, expanded: true };
+	const add = (parent, name, fid, isDir) => {
+		const n = { name, fid, isDir: !!isDir, children: isDir ? [] : null, parent, expanded: true };
+		parent.children.push(n);
+		return n;
+	};
+	const telecom = add(root, 'DF.TELECOM', '7F10', true);
+	const msisdn = add(telecom, 'EF.MSISDN', '6F40', false);
+	const gsm = add(root, 'DF.GSM', '7F20', true);
+	const spn = add(gsm, 'EF.SPN', '6F46', false);
+	const ecc = add(gsm, 'EF.ECC', '6FB7', false);
+	const usim = add(root, 'ADF.USIM', '7FFF', true);
+	const wlan = add(usim, 'DF.WLAN', '5F40', true);
+	return { root, telecom, msisdn, gsm, spn, ecc, usim, wlan };
+}
+
+test('a directory shows only on the path of a file the filter selects', () => {
+	const t = fsTree();
 	withFilter(['provider-names'], () => {
-		// a matching file shows, a non-matching one hides
+		// the root always stays; DF.GSM holds EF.SPN (MF/7F20/6F46)
-		assert.strictEqual(pysimFsFilterShow({ name: 'EF.SPN', isDir: false }), true);
+		assert.strictEqual(pysimFsFilterShow(t.root), true);
-		assert.strictEqual(pysimFsFilterShow({ name: 'EF.ECC', isDir: false }), false);
+		assert.strictEqual(pysimFsFilterShow(t.gsm), true);
-		// a DF with a matching loaded child shows, one without hides
+		// ... and ADF.USIM holds its own EF.SPN copy
-		assert.strictEqual(pysimFsFilterShow(df([{ name: 'EF.ECC', isDir: false }, { name: 'EF.SPN', isDir: false }])), true);
+		assert.strictEqual(pysimFsFilterShow(t.usim), true);
-		assert.strictEqual(pysimFsFilterShow(df([{ name: 'EF.ECC', isDir: false }])), false);
+		// neither DF.TELECOM nor DF.WLAN holds a provider-name file
-		assert.strictEqual(pysimFsFilterShow(df([])), false);
+		assert.strictEqual(pysimFsFilterShow(t.telecom), false);
-		// a collapsed DF whose children were never loaded is kept
+		assert.strictEqual(pysimFsFilterShow(t.wlan), false);
-		assert.strictEqual(pysimFsFilterShow({ name: 'DF.GSM', isDir: true, children: null, expanded: false }), true);
+		// files keep their name rule
-		// ... and a matching descendant nested one level deeper counts too
+		assert.strictEqual(pysimFsFilterShow(t.spn), true);
-		assert.strictEqual(pysimFsFilterShow(df([
+		assert.strictEqual(pysimFsFilterShow(t.ecc), false);
-			{ name: 'DF.X', isDir: true, children: [{ name: 'EF.ECC', isDir: false }, { name: 'EF.PNN', isDir: false }] },
-		])), true);
-		assert.strictEqual(pysimFsFilterShow(df([{ name: 'DF.X', isDir: true, children: null }])), true);
 	});
-	// a DF that is itself a per-service file (the GPRS feature names DF.GSM)
+	withFilter(['identifiers'], () => {
+		// EF.MSISDN is in DF.TELECOM, EF.IMSI/EF.GID1/2 in DF.GSM
+		assert.strictEqual(pysimFsFilterShow(t.telecom), true);
+		assert.strictEqual(pysimFsFilterShow(t.msisdn), true);
+		assert.strictEqual(pysimFsFilterShow(t.gsm), true);
+		assert.strictEqual(pysimFsFilterShow(t.usim), true);   // EF.IMSI/EF.MSISDN copies
+		assert.strictEqual(pysimFsFilterShow(t.wlan), false);
+	});
+	withFilter(['cb'], () => {
+		assert.strictEqual(pysimFsFilterShow(t.gsm), true);      // EF.CBMI/CBMID/CBMIR
+		assert.strictEqual(pysimFsFilterShow(t.telecom), false);
+		assert.strictEqual(pysimFsFilterShow(t.wlan), false);
+	});
+});
+
+test('a directory named by the selection itself shows', () => {
+	const t = fsTree();
+	// the GPRS feature names DF.GSM (its SST bit covers the DF, not a file)
 	withFilter(['svc:GPRS'], () => {
-		assert.strictEqual(pysimFsFilterMatch({ name: 'DF.GSM', isDir: true }), true);
+		assert.strictEqual(pysimFsFilterMatch(t.gsm), true);
+		assert.strictEqual(pysimFsFilterShow(t.gsm), true);
+		assert.strictEqual(pysimFsFilterShow(t.telecom), false);
 	});
-	// everything shows with the default
+});
-	withFilter(['all'], () => {
+
-		assert.strictEqual(pysimFsFilterShow(df([{ name: 'EF.ECC', isDir: false }])), true);
+test('"all" and an empty selection show every node', () => {
+	const t = fsTree();
+	for (const ids of [['all'], []]) {
+		withFilter(ids, () => {
+			assert.strictEqual(pysimFsFilterShow(t.root), true);
+			assert.strictEqual(pysimFsFilterShow(t.telecom), true);
+			assert.strictEqual(pysimFsFilterShow(t.wlan), true);
+			assert.strictEqual(pysimFsFilterShow(t.ecc), true);
+		});
+	}
+});
+
+test('directories are kept until the standard list is available', () => {
+	const t = fsTree();
+	const saved = _uiccStdFiles;
+	_uiccStdFiles = null;
+	try {
+		withFilter(['provider-names'], () => {
+			assert.strictEqual(pysimFsFilterMatchingPaths(), null);
+			// the ancestors cannot be judged, so nothing vanishes
+			assert.strictEqual(pysimFsFilterShow(t.telecom), true);
+			assert.strictEqual(pysimFsFilterShow(t.wlan), true);
+			// files still obey the name rule
+			assert.strictEqual(pysimFsFilterShow(t.ecc), false);
+		});
+	} finally {
+		_uiccStdFiles = saved;
+	}
+});
+
+test('the standard list gives a name every tree it lives in', () => {
+	const idx = fsFilterPathIndexBuild(_uiccStdFiles);
+	assert.deepStrictEqual(idx.get('EF.SPN').slice().sort(),
+		['ADF.USIM-NON-IMSI/6F46', 'ADF.USIM/6F46', 'MF/7F20/6F46']);
+	assert.deepStrictEqual(idx.get('EF.ICCID'), ['MF/2FE2']);
+	// the index is cached for the loaded list
+	assert.strictEqual(fsFilterPathIndex(), fsFilterPathIndex());
+});
+
+test('the selection resolves to the paths of its files', () => {
+	withFilter(['cb'], () => {
+		const paths = pysimFsFilterMatchingPaths();
+		assert.ok(paths instanceof Set, 'expected a Set');
+		assert.deepStrictEqual(Array.from(paths).sort(), [
+			'ADF.USIM-NON-IMSI/6F45', 'ADF.USIM-NON-IMSI/6F48', 'ADF.USIM-NON-IMSI/6F50',
+			'ADF.USIM/6F45', 'ADF.USIM/6F48', 'ADF.USIM/6F50',
+			'MF/7F20/6F45', 'MF/7F20/6F48', 'MF/7F20/6F50',
+		]);
+		// cached for the same selection
+		assert.strictEqual(pysimFsFilterMatchingPaths(), paths);
 	});
+	withFilter(['custom'], () => {
+		assert.deepStrictEqual(Array.from(pysimFsFilterMatchingPaths()), []);
+	});
+});
+
+test('"custom files" keeps the directories that hold them', () => {
+	const t = fsTree();
+	const custom = { name: 'EF.MYFILE', fid: '6F99', isDir: false, children: null, parent: t.gsm, custom: true };
+	t.gsm.children.push(custom);
+	pysimCustomFiles = [{ path: 'MF/7F20/6F99', name: 'EF.MYFILE', kind: 'ef' }];
+	try {
+		withFilter(['custom'], () => {
+			// custom files are outside the standard list: their own paths are
+			// the ancestry, so the holding DF stays
+			assert.strictEqual(pysimFsFilterShow(custom), true);
+			assert.strictEqual(pysimFsFilterShow(t.gsm), true);
+			assert.strictEqual(pysimFsFilterShow(t.root), true);
+			assert.strictEqual(pysimFsFilterShow(t.telecom), false);
+		});
+	} finally {
+		pysimCustomFiles = [];
+	}
 });
 
 test('the menu lists all, custom, the three headings and the services', () => {
@@ -413,6 +530,29 @@ test('the render consults the filter when walking children', () => {
 	assert.ok(!/pysimFsFilterMatch\(child\)/.test(html));
 });
 
+test('the directory rule uses the registry paths, not the loaded children', () => {
+	const fn = extractFunc(html, 'pysimFsFilterShow');
+	assert.ok(!fn.includes('node.children'), fn);
+	assert.ok(fn.indexOf('pysimFsFilterMatchingPaths()') > 0, fn);
+	assert.ok(fn.indexOf("path === 'MF'") > 0, fn);
+	// the standard list is read synchronously from the cached array
+	assert.match(html, /let _uiccStdFiles = null;/);
+	assert.ok(extractFunc(html, 'pysimFsFilterMatchingPaths').indexOf('_uiccStdFiles') > 0);
+});
+
+test('every filter change drops the path cache before the tree redraws', () => {
+	for (const name of ['pysimFsFilterSetEntry', 'pysimFsSetFilterToService', 'pysimFsFilterRestore']) {
+		const fn = extractFunc(html, name);
+		const at = fn.indexOf('pysimFsFilterPathsInvalidate()');
+		assert.ok(at > 0, name + ' does not invalidate the path cache');
+		const render = fn.indexOf('pysimFsRenderTree()');
+		if (render > 0) assert.ok(at < render, name + ': the cache is dropped before the redraw');
+	}
+	// the list arriving redraws a tree that was filtered while it loaded
+	const load = extractFunc(html, 'uiccStdFilesLoad');
+	assert.ok(load.indexOf('pysimFsRenderTree()') > 0, load);
+});
+
 test('the filter state is persisted under simple_fs_filter', () => {
 	const fn = extractFunc(html, 'pysimFsFilterSetEntry');
 	assert.ok(fn.indexOf("localStorage.setItem('simple_fs_filter'") > 0, fn);

frontend/tests/fs_service_link.test.js

@@ -45,6 +45,9 @@ function extractConst(src, name) {
 // module state the extracted functions close over
 let pysimFsFilter = new Set(['all']);
 let _servicesFeatureByNumber = null;
+// the tree filter's matching-path cache (pysimFsSetFilterToService drops it)
+let _fsFilterPaths = null;
+let _fsFilterPathsKey = null;
 const _store = {};
 let _renders = 0;
 let _treeRenders = 0;
@@ -60,7 +63,8 @@ globalThis.pysimFsFilterRender = () => { _renders++; };
 globalThis.pysimFsRenderTree = () => { _treeRenders++; };
 
 let code = '';
-for (const f of ['servicesFeatureByNumber', 'pysimFsFilterBtnHtml', 'pysimFsSetFilterToService']) {
+for (const f of ['servicesFeatureByNumber', 'pysimFsFilterBtnHtml', 'pysimFsSetFilterToService',
+	'pysimFsFilterPathsInvalidate']) {
 	code += extractFunc(html, f) + '\n';
 }
 for (const c of ['SERVICES_CROSS_REF', 'SERVICES_FS_MAP']) code += extractConst(html, c);