diff --git a/.gitignore b/.gitignore index 0cd5b14a..3f1f645f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /docs/_* /docs/generated +/docs/filesystem.rst /.cache /.local /build diff --git a/docs/conf.py b/docs/conf.py index 26e8e2aa..2e23faea 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,6 +13,7 @@ import os import sys sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('.')) # for local extensions (pysim_fs_sphinx, ...) # -- Project information ----------------------------------------------------- @@ -39,7 +40,8 @@ extensions = [ "sphinx.ext.autodoc", "sphinxarg.ext", "sphinx.ext.autosectionlabel", - "sphinx.ext.napoleon" + "sphinx.ext.napoleon", + "pysim_fs_sphinx", ] # Add any paths that contain templates here, relative to this directory. @@ -78,6 +80,7 @@ autodoc_mock_imports = ['klein', 'twisted'] # of autosectionlabel duplicate-label warnings - suppress them. autosectionlabel_maxdepth = 3 suppress_warnings = [ + 'autosectionlabel.filesystem', 'autosectionlabel.saip-tool', 'autosectionlabel.shell', 'autosectionlabel.smpp2sim', diff --git a/docs/index.rst b/docs/index.rst index a6ed7b9e..8908c4ed 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,6 +39,7 @@ pySim consists of several parts: :caption: Contents: shell + filesystem trace legacy smpp2sim diff --git a/docs/pysim_fs_sphinx.py b/docs/pysim_fs_sphinx.py new file mode 100644 index 00000000..829fc310 --- /dev/null +++ b/docs/pysim_fs_sphinx.py @@ -0,0 +1,267 @@ +""" +Sphinx extension: auto-generate docs/filesystem.rst from the pySim EF class hierarchy. + +Hooked into Sphinx's ``builder-inited`` event so the file is always regenerated +from the live Python classes before Sphinx reads any source files. + +The table of root objects to document is in SECTIONS near the top of this file. +EXCLUDED lists CardProfile/CardApplication subclasses intentionally omitted from +SECTIONS, with reasons. Both tables are read by tests/unittests/test_fs_coverage.py +to ensure every class with EF/DF content is accounted for. +""" + +import importlib +import inspect +import json +import os +import sys +import textwrap + +# Ensure pySim is importable when this module is loaded as a Sphinx extension +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from pySim.filesystem import (CardApplication, CardDF, CardMF, CardEF, # noqa: E402 + TransparentEF, TransRecEF, LinFixedEF, CyclicEF, BerTlvEF) +from pySim.profile import CardProfile # noqa: E402 + + +# Generic EF base classes whose docstrings describe the *type* of file +# (Transparent, LinFixed, ...) rather than a specific file's content. +# Suppress those boilerplate texts in the per-EF entries; they are only +# useful once, at the top of the document or in a dedicated glossary. +_EF_BASE_TYPES = frozenset([TransparentEF, + TransRecEF, + LinFixedEF, + CyclicEF, + BerTlvEF]) + + +# --------------------------------------------------------------------------- +# Sections: (heading, module, class-name) +# The class must be either a CardProfile (uses .files_in_mf) or a CardDF +# subclass (uses .children). +# --------------------------------------------------------------------------- +SECTIONS = [ + ('MF / TS 102 221 (UICC)', + 'pySim.ts_102_221', 'CardProfileUICC'), + ('ADF.USIM / TS 31.102', + 'pySim.ts_31_102', 'ADF_USIM'), + ('ADF.ISIM / TS 31.103', + 'pySim.ts_31_103', 'ADF_ISIM'), + ('ADF.HPSIM / TS 31.104', + 'pySim.ts_31_104', 'ADF_HPSIM'), + ('DF.GSM + DF.TELECOM / TS 51.011 (SIM)', + 'pySim.ts_51_011', 'CardProfileSIM'), + ('CDMA / IS-820 (RUIM)', + 'pySim.cdma_ruim', 'CardProfileRUIM'), + ('DF.EIRENE / GSM-R', + 'pySim.gsm_r', 'DF_EIRENE'), + ('DF.SYSTEM / sysmocom SJA2+SJA5', + 'pySim.sysmocom_sja2', 'DF_SYSTEM'), +] + +# --------------------------------------------------------------------------- +# Excluded: {(module, class-name)} +# CardProfile and CardApplication subclasses that have EF/DF children but are +# intentionally absent from SECTIONS. Keeping this list explicit lets +# test_fs_coverage.py detect newly added classes that the developer forgot to +# add to either table. +# --------------------------------------------------------------------------- +EXCLUDED = { + # eUICC profiles inherit files_in_mf verbatim from CardProfileUICC; the + # eUICC-specific content lives in ISD-R / ISD-P applications, not in MF. + ('pySim.euicc', 'CardProfileEuiccSGP02'), + ('pySim.euicc', 'CardProfileEuiccSGP22'), + ('pySim.euicc', 'CardProfileEuiccSGP32'), + # CardApplication* classes are thin wrappers that embed an ADF_* instance. + # The ADF contents are already documented via the corresponding ADF_* entry + # in SECTIONS above. + ('pySim.ts_31_102', 'CardApplicationUSIM'), + ('pySim.ts_31_102', 'CardApplicationUSIMnonIMSI'), + ('pySim.ts_31_103', 'CardApplicationISIM'), + ('pySim.ts_31_104', 'CardApplicationHPSIM'), +} + +# RST underline characters ordered by nesting depth +_HEADING_CHARS = ['=', '=', '-', '~', '^', '"'] +# Level 0 uses '=' with overline (page title). +# Level 1 uses '=' without overline (major sections). +# Levels 2+ use the remaining characters for DFs. + + +# --------------------------------------------------------------------------- +# RST formatting helpers +# --------------------------------------------------------------------------- + +def _heading(title: str, level: int) -> str: + """Return an RST heading string. Level 0 gets an overline.""" + char = _HEADING_CHARS[level] + rule = char * len(title) + if level == 0: + return f'{rule}\n{title}\n{rule}\n\n' + return f'{title}\n{rule}\n\n' + + +def _json_default(obj): + """Fallback serialiser: bytes -> hex, anything else -> repr.""" + if isinstance(obj, (bytes, bytearray)): + return obj.hex() + return repr(obj) + + +def _examples_block(cls) -> str: + """Return RST code-block examples (one per vector), or '' if none exist. + + Each example is rendered as a ``json5`` code-block with the hex-encoded + binary as a ``// comment`` on the first line, followed by the decoded JSON. + ``json5`` is used instead of ``json`` so that Pygments does not flag the + ``//`` comment as a syntax error. + """ + vectors = [] + for attr in ('_test_de_encode', '_test_decode'): + v = getattr(cls, attr, None) + if v: + vectors.extend(v) + if not vectors: + return '' + + lines = ['**Examples**\n\n'] + + for t in vectors: + # 2-tuple: (encoded, decoded) + # 3-tuple: (encoded, record_nr, decoded) — LinFixedEF / CyclicEF + if len(t) >= 3: + encoded, record_nr, decoded = t[0], t[1], t[2] + comment = f'record {record_nr}: {encoded.lower()}' + else: + encoded, decoded = t[0], t[1] + comment = f'file: {encoded.lower()}' + + json_str = json.dumps(decoded, default=_json_default, indent=2) + json_indented = textwrap.indent(json_str, ' ') + + lines.append('.. code-block:: json5\n\n') + lines.append(f' // {comment}\n') + lines.append(json_indented + '\n') + lines.append('\n') + + return ''.join(lines) + + +def _document_ef(ef: CardEF) -> str: + """Return RST for a single EF. Uses ``rubric`` to stay out of the TOC.""" + cls = type(ef) + + parts = [ef.fully_qualified_path_str()] + if ef.fid: + parts.append(f'({ef.fid.upper()})') + if ef.desc: + parts.append(f'\u2014 {ef.desc}') # em-dash + title = ' '.join(parts) + + lines = [f'.. rubric:: {title}\n\n'] + + # Only show a docstring if it is specific to this class. EFs that are + # direct instances of a base type (TransparentEF, LinFixedEF, ...) carry + # only the generic "what is a TransparentEF" boilerplate; named subclasses + # without their own __doc__ have cls.__dict__['__doc__'] == None. Either + # way, suppress the text here - it belongs at the document level, not + # repeated for every single EF entry. + doc = None if cls in _EF_BASE_TYPES else cls.__dict__.get('__doc__') + if doc: + lines.append(inspect.cleandoc(doc) + '\n\n') + + examples = _examples_block(cls) + if examples: + lines.append(examples) + + return ''.join(lines) + + +def _document_df(df: CardDF, level: int) -> str: + """Return RST for a DF section and all its children, recursively.""" + parts = [df.fully_qualified_path_str()] + if df.fid: + parts.append(f'({df.fid.upper()})') + if df.desc: + parts.append(f'\u2014 {df.desc}') # em-dash + title = ' '.join(parts) + + lines = [_heading(title, level)] + + cls = type(df) + doc = None if cls in (CardDF, CardMF) else cls.__dict__.get('__doc__') + if doc: + lines.append(inspect.cleandoc(doc) + '\n\n') + + for child in df.children.values(): + if isinstance(child, CardDF): + lines.append(_document_df(child, level + 1)) + elif isinstance(child, CardEF): + lines.append(_document_ef(child)) + + return ''.join(lines) + + +# --------------------------------------------------------------------------- +# Top-level generator +# --------------------------------------------------------------------------- + +def generate_filesystem_rst() -> str: + """Walk all registered sections and return the full RST document as a string.""" + out = [ + '.. This file is auto-generated by docs/pysim_fs_sphinx.py — do not edit.\n\n', + _heading('Card Filesystem Reference', 0), + 'This page documents all Elementary Files (EFs) and Dedicated Files (DFs) ' + 'implemented in pySim, organised by their location in the card filesystem.\n\n', + ] + + # Track already-documented classes so that DFs/EFs shared between profiles + # (e.g. DF.TELECOM / DF.GSM present in both CardProfileSIM and CardProfileRUIM) + # are only emitted once. + seen_types: set = set() + + for section_title, module_path, class_name in SECTIONS: + module = importlib.import_module(module_path) + cls = getattr(module, class_name) + obj = cls() + + if isinstance(obj, CardProfile): + files = obj.files_in_mf + elif isinstance(obj, CardApplication): + files = list(obj.adf.children.values()) + elif isinstance(obj, CardDF): + files = list(obj.children.values()) + else: + continue + + # Filter out files whose class was already documented in an earlier section. + files = [f for f in files if type(f) not in seen_types] + if not files: + continue + + out.append(_heading(section_title, 1)) + + for f in files: + seen_types.add(type(f)) + if isinstance(f, CardDF): + out.append(_document_df(f, level=2)) + elif isinstance(f, CardEF): + out.append(_document_ef(f)) + + return ''.join(out) + + +# --------------------------------------------------------------------------- +# Sphinx integration +# --------------------------------------------------------------------------- + +def _on_builder_inited(app): + output_path = os.path.join(app.srcdir, 'filesystem.rst') + with open(output_path, 'w') as fh: + fh.write(generate_filesystem_rst()) + + +def setup(app): + app.connect('builder-inited', _on_builder_inited) + return {'version': '0.1', 'parallel_read_safe': True} diff --git a/tests/unittests/test_fs_coverage.py b/tests/unittests/test_fs_coverage.py new file mode 100644 index 00000000..60216652 --- /dev/null +++ b/tests/unittests/test_fs_coverage.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 + +# (C) 2026 by sysmocom - s.f.m.c. GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Verify that every CardProfile / CardApplication subclass with EF/DF content, +and every standalone CardDF subclass (one not reachable as a child of any profile +or application), is either listed in docs/pysim_fs_sphinx.py::SECTIONS or +explicitly EXCLUDED.""" + +import unittest +import importlib +import inspect +import pkgutil +import sys +import os + +# Make docs/pysim_fs_sphinx.py importable without a full Sphinx build. +_DOCS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'docs') +sys.path.insert(0, os.path.abspath(_DOCS_DIR)) + +import pySim # noqa: E402 +from pySim.filesystem import CardApplication, CardDF, CardMF, CardADF # noqa: E402 +from pySim.profile import CardProfile # noqa: E402 +from pysim_fs_sphinx import EXCLUDED, SECTIONS # noqa: E402 + + +class TestFsCoverage(unittest.TestCase): + """Ensure SECTIONS + EXCLUDED together account for all classes with content.""" + + # Base CardDF types that are not concrete filesystem objects on their own. + _DF_BASE_TYPES = frozenset([CardDF, CardMF, CardADF]) + + @staticmethod + def _collect_reachable_df_types(obj) -> set: + """Return the set of all CardDF *types* reachable as children of *obj*.""" + result = set() + if isinstance(obj, CardProfile): + children = obj.files_in_mf + elif isinstance(obj, CardApplication): + result.add(type(obj.adf)) + children = list(obj.adf.children.values()) + elif isinstance(obj, CardDF): + children = list(obj.children.values()) + else: + return result + queue = list(children) + while queue: + child = queue.pop() + if isinstance(child, CardDF): + result.add(type(child)) + queue.extend(child.children.values()) + return result + + @staticmethod + def _has_content(obj) -> bool: + """Return True if *obj* owns any EFs/DFs.""" + if isinstance(obj, CardProfile): + return bool(obj.files_in_mf) + if isinstance(obj, CardApplication): + return bool(obj.adf.children) + return False + + def test_all_profiles_and_apps_covered(self): + # build a set of (module, class-name) pairs that are already accounted for + covered = {(mod, cls) for (_, mod, cls) in SECTIONS} + accounted_for = covered | EXCLUDED + + uncovered = [] + reachable_df_types = set() + loaded_modules = {} + + for modinfo in pkgutil.walk_packages(pySim.__path__, prefix='pySim.'): + modname = modinfo.name + try: + module = importlib.import_module(modname) + except Exception: # skip inport errors, if any + continue + loaded_modules[modname] = module + + for name, cls in inspect.getmembers(module, inspect.isclass): + # skip classes that are merely imported by this module + if cls.__module__ != modname: + continue + # examine only subclasses of CardProfile and CardApplication + if not issubclass(cls, (CardProfile, CardApplication)): + continue + # skip the abstract base classes themselves + if cls in (CardProfile, CardApplication): + continue + # classes that require constructor arguments cannot be probed + try: + obj = cls() + except Exception: + continue + + # collect all CardDF types reachable from this profile/application + # (used below to identify standalone DFs) + reachable_df_types |= self._collect_reachable_df_types(obj) + + if self._has_content(obj) and (modname, name) not in accounted_for: + uncovered.append((modname, name)) + + # check standalone CardDFs (such as DF.EIRENE or DF.SYSTEM) + for modname, module in loaded_modules.items(): + for name, cls in inspect.getmembers(module, inspect.isclass): + if cls.__module__ != modname: + continue + if not issubclass(cls, CardDF): + continue + if cls in self._DF_BASE_TYPES: + continue + if cls in reachable_df_types: + continue + try: + obj = cls() + except Exception: + continue + if obj.children and (modname, name) not in accounted_for: + uncovered.append((modname, name)) + + if uncovered: + lines = [ + 'The following classes have EFs/DFs, but not listed in SECTIONS or EXCLUDED:', + *(f' {modname}.{name}' for modname, name in sorted(uncovered)), + 'Please modify docs/pysim_fs_sphinx.py accordingly', + ] + self.fail('\n'.join(lines)) + + +if __name__ == '__main__': + unittest.main()