docs: auto-generate Card Filesystem Reference

Add a Sphinx extension (docs/pysim_fs_sphinx.py) that hooks into the
builder-inited event and generates docs/filesystem.rst before Sphinx
reads any source files.

The generated page contains a hierarchical listing of all implemented
EFs and DFs, organised by application/specification (UICC/TS 102 221,
ADF.USIM/TS 31.102, ADF.ISIM/TS 31.103, SIM/TS 51.011).  For each file,
the class docstring and any _test_de_encode / _test_decode vectors
are included as an encoding/decoding example table.

docs/filesystem.rst is fully generated at build time and is therefore
added to .gitignore.

Add tests/unittests/test_fs_coverage.py that walks all pySim.* modules
and verifies that every CardProfile, CardApplication, and standalone
CardDF subclass with EF/DF children is either listed in the SECTIONS
(and will appear in the docs) or explicitly EXCLUDED.

Change-Id: I06ddeefc6c11e04d7c24e116f3f39c8a6635856f
Related: OS#6316

.gitignore

@@ -3,6 +3,7 @@
 
 /docs/_*
 /docs/generated
+/docs/filesystem.rst
 /.cache
 /.local
 /build

docs/conf.py

@@ -13,6 +13,7 @@
 import os
 import sys
 sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath('.'))   # for local extensions (pysim_fs_sphinx, ...)
 
 
 # -- Project information -----------------------------------------------------
@@ -39,7 +40,8 @@ extensions = [
         "sphinx.ext.autodoc",
         "sphinxarg.ext",
         "sphinx.ext.autosectionlabel",
-        "sphinx.ext.napoleon"
+        "sphinx.ext.napoleon",
+        "pysim_fs_sphinx",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -78,6 +80,7 @@ autodoc_mock_imports = ['klein', 'twisted']
 # of autosectionlabel duplicate-label warnings - suppress them.
 autosectionlabel_maxdepth = 3
 suppress_warnings = [
+    'autosectionlabel.filesystem',
     'autosectionlabel.saip-tool',
     'autosectionlabel.shell',
     'autosectionlabel.smpp2sim',

docs/index.rst

@@ -39,6 +39,7 @@ pySim consists of several parts:
    :caption: Contents:
 
    shell
+   filesystem
    trace
    legacy
    smpp2sim

docs/pysim_fs_sphinx.py

@@ -0,0 +1,267 @@
+"""
+Sphinx extension: auto-generate docs/filesystem.rst from the pySim EF class hierarchy.
+
+Hooked into Sphinx's ``builder-inited`` event so the file is always regenerated
+from the live Python classes before Sphinx reads any source files.
+
+The table of root objects to document is in SECTIONS near the top of this file.
+EXCLUDED lists CardProfile/CardApplication subclasses intentionally omitted from
+SECTIONS, with reasons.  Both tables are read by tests/unittests/test_fs_coverage.py
+to ensure every class with EF/DF content is accounted for.
+"""
+
+import importlib
+import inspect
+import json
+import os
+import sys
+import textwrap
+
+# Ensure pySim is importable when this module is loaded as a Sphinx extension
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from pySim.filesystem import (CardApplication, CardDF, CardMF, CardEF, # noqa: E402
+                              TransparentEF, TransRecEF, LinFixedEF, CyclicEF, BerTlvEF)
+from pySim.profile import CardProfile # noqa: E402
+
+
+# Generic EF base classes whose docstrings describe the *type* of file
+# (Transparent, LinFixed, ...) rather than a specific file's content.
+# Suppress those boilerplate texts in the per-EF entries; they are only
+# useful once, at the top of the document or in a dedicated glossary.
+_EF_BASE_TYPES = frozenset([TransparentEF,
+                            TransRecEF,
+                            LinFixedEF,
+                            CyclicEF,
+                            BerTlvEF])
+
+
+# ---------------------------------------------------------------------------
+# Sections: (heading, module, class-name)
+# The class must be either a CardProfile (uses .files_in_mf) or a CardDF
+# subclass (uses .children).
+# ---------------------------------------------------------------------------
+SECTIONS = [
+    ('MF / TS 102 221 (UICC)',
+     'pySim.ts_102_221', 'CardProfileUICC'),
+    ('ADF.USIM / TS 31.102',
+     'pySim.ts_31_102',  'ADF_USIM'),
+    ('ADF.ISIM / TS 31.103',
+     'pySim.ts_31_103',  'ADF_ISIM'),
+    ('ADF.HPSIM / TS 31.104',
+     'pySim.ts_31_104',  'ADF_HPSIM'),
+    ('DF.GSM + DF.TELECOM / TS 51.011 (SIM)',
+     'pySim.ts_51_011',  'CardProfileSIM'),
+    ('CDMA / IS-820 (RUIM)',
+     'pySim.cdma_ruim',  'CardProfileRUIM'),
+    ('DF.EIRENE / GSM-R',
+     'pySim.gsm_r',      'DF_EIRENE'),
+    ('DF.SYSTEM / sysmocom SJA2+SJA5',
+     'pySim.sysmocom_sja2', 'DF_SYSTEM'),
+]
+
+# ---------------------------------------------------------------------------
+# Excluded: {(module, class-name)}
+# CardProfile and CardApplication subclasses that have EF/DF children but are
+# intentionally absent from SECTIONS.  Keeping this list explicit lets
+# test_fs_coverage.py detect newly added classes that the developer forgot to
+# add to either table.
+# ---------------------------------------------------------------------------
+EXCLUDED = {
+    # eUICC profiles inherit files_in_mf verbatim from CardProfileUICC; the
+    # eUICC-specific content lives in ISD-R / ISD-P applications, not in MF.
+    ('pySim.euicc', 'CardProfileEuiccSGP02'),
+    ('pySim.euicc', 'CardProfileEuiccSGP22'),
+    ('pySim.euicc', 'CardProfileEuiccSGP32'),
+    # CardApplication* classes are thin wrappers that embed an ADF_* instance.
+    # The ADF contents are already documented via the corresponding ADF_* entry
+    # in SECTIONS above.
+    ('pySim.ts_31_102', 'CardApplicationUSIM'),
+    ('pySim.ts_31_102', 'CardApplicationUSIMnonIMSI'),
+    ('pySim.ts_31_103', 'CardApplicationISIM'),
+    ('pySim.ts_31_104', 'CardApplicationHPSIM'),
+}
+
+# RST underline characters ordered by nesting depth
+_HEADING_CHARS = ['=', '=', '-', '~', '^', '"']
+# Level 0 uses '=' with overline (page title).
+# Level 1 uses '=' without overline (major sections).
+# Levels 2+ use the remaining characters for DFs.
+
+
+# ---------------------------------------------------------------------------
+# RST formatting helpers
+# ---------------------------------------------------------------------------
+
+def _heading(title: str, level: int) -> str:
+    """Return an RST heading string.  Level 0 gets an overline."""
+    char = _HEADING_CHARS[level]
+    rule = char * len(title)
+    if level == 0:
+        return f'{rule}\n{title}\n{rule}\n\n'
+    return f'{title}\n{rule}\n\n'
+
+
+def _json_default(obj):
+    """Fallback serialiser: bytes -> hex, anything else -> repr."""
+    if isinstance(obj, (bytes, bytearray)):
+        return obj.hex()
+    return repr(obj)
+
+
+def _examples_block(cls) -> str:
+    """Return RST code-block examples (one per vector), or '' if none exist.
+
+    Each example is rendered as a ``json5`` code-block with the hex-encoded
+    binary as a ``// comment`` on the first line, followed by the decoded JSON.
+    ``json5`` is used instead of ``json`` so that Pygments does not flag the
+    ``//`` comment as a syntax error.
+    """
+    vectors = []
+    for attr in ('_test_de_encode', '_test_decode'):
+        v = getattr(cls, attr, None)
+        if v:
+            vectors.extend(v)
+    if not vectors:
+        return ''
+
+    lines = ['**Examples**\n\n']
+
+    for t in vectors:
+        # 2-tuple: (encoded, decoded)
+        # 3-tuple: (encoded, record_nr, decoded)   — LinFixedEF / CyclicEF
+        if len(t) >= 3:
+            encoded, record_nr, decoded = t[0], t[1], t[2]
+            comment = f'record {record_nr}: {encoded.lower()}'
+        else:
+            encoded, decoded = t[0], t[1]
+            comment = f'file: {encoded.lower()}'
+
+        json_str = json.dumps(decoded, default=_json_default, indent=2)
+        json_indented = textwrap.indent(json_str, '   ')
+
+        lines.append('.. code-block:: json5\n\n')
+        lines.append(f'   // {comment}\n')
+        lines.append(json_indented + '\n')
+        lines.append('\n')
+
+    return ''.join(lines)
+
+
+def _document_ef(ef: CardEF) -> str:
+    """Return RST for a single EF.  Uses ``rubric`` to stay out of the TOC."""
+    cls = type(ef)
+
+    parts = [ef.fully_qualified_path_str()]
+    if ef.fid:
+        parts.append(f'({ef.fid.upper()})')
+    if ef.desc:
+        parts.append(f'\u2014 {ef.desc}')   # em-dash
+    title = ' '.join(parts)
+
+    lines = [f'.. rubric:: {title}\n\n']
+
+    # Only show a docstring if it is specific to this class.  EFs that are
+    # direct instances of a base type (TransparentEF, LinFixedEF, ...) carry
+    # only the generic "what is a TransparentEF" boilerplate; named subclasses
+    # without their own __doc__ have cls.__dict__['__doc__'] == None.  Either
+    # way, suppress the text here - it belongs at the document level, not
+    # repeated for every single EF entry.
+    doc = None if cls in _EF_BASE_TYPES else cls.__dict__.get('__doc__')
+    if doc:
+        lines.append(inspect.cleandoc(doc) + '\n\n')
+
+    examples = _examples_block(cls)
+    if examples:
+        lines.append(examples)
+
+    return ''.join(lines)
+
+
+def _document_df(df: CardDF, level: int) -> str:
+    """Return RST for a DF section and all its children, recursively."""
+    parts = [df.fully_qualified_path_str()]
+    if df.fid:
+        parts.append(f'({df.fid.upper()})')
+    if df.desc:
+        parts.append(f'\u2014 {df.desc}')   # em-dash
+    title = ' '.join(parts)
+
+    lines = [_heading(title, level)]
+
+    cls = type(df)
+    doc = None if cls in (CardDF, CardMF) else cls.__dict__.get('__doc__')
+    if doc:
+        lines.append(inspect.cleandoc(doc) + '\n\n')
+
+    for child in df.children.values():
+        if isinstance(child, CardDF):
+            lines.append(_document_df(child, level + 1))
+        elif isinstance(child, CardEF):
+            lines.append(_document_ef(child))
+
+    return ''.join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Top-level generator
+# ---------------------------------------------------------------------------
+
+def generate_filesystem_rst() -> str:
+    """Walk all registered sections and return the full RST document as a string."""
+    out = [
+        '.. This file is auto-generated by docs/pysim_fs_sphinx.py — do not edit.\n\n',
+        _heading('Card Filesystem Reference', 0),
+        'This page documents all Elementary Files (EFs) and Dedicated Files (DFs) '
+        'implemented in pySim, organised by their location in the card filesystem.\n\n',
+    ]
+
+    # Track already-documented classes so that DFs/EFs shared between profiles
+    # (e.g. DF.TELECOM / DF.GSM present in both CardProfileSIM and CardProfileRUIM)
+    # are only emitted once.
+    seen_types: set = set()
+
+    for section_title, module_path, class_name in SECTIONS:
+        module = importlib.import_module(module_path)
+        cls = getattr(module, class_name)
+        obj = cls()
+
+        if isinstance(obj, CardProfile):
+            files = obj.files_in_mf
+        elif isinstance(obj, CardApplication):
+            files = list(obj.adf.children.values())
+        elif isinstance(obj, CardDF):
+            files = list(obj.children.values())
+        else:
+            continue
+
+        # Filter out files whose class was already documented in an earlier section.
+        files = [f for f in files if type(f) not in seen_types]
+        if not files:
+            continue
+
+        out.append(_heading(section_title, 1))
+
+        for f in files:
+            seen_types.add(type(f))
+            if isinstance(f, CardDF):
+                out.append(_document_df(f, level=2))
+            elif isinstance(f, CardEF):
+                out.append(_document_ef(f))
+
+    return ''.join(out)
+
+
+# ---------------------------------------------------------------------------
+# Sphinx integration
+# ---------------------------------------------------------------------------
+
+def _on_builder_inited(app):
+    output_path = os.path.join(app.srcdir, 'filesystem.rst')
+    with open(output_path, 'w') as fh:
+        fh.write(generate_filesystem_rst())
+
+
+def setup(app):
+    app.connect('builder-inited', _on_builder_inited)
+    return {'version': '0.1', 'parallel_read_safe': True}

tests/unittests/test_fs_coverage.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Verify that every CardProfile / CardApplication subclass with EF/DF content,
+and every standalone CardDF subclass (one not reachable as a child of any profile
+or application), is either listed in docs/pysim_fs_sphinx.py::SECTIONS or
+explicitly EXCLUDED."""
+
+import unittest
+import importlib
+import inspect
+import pkgutil
+import sys
+import os
+
+# Make docs/pysim_fs_sphinx.py importable without a full Sphinx build.
+_DOCS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'docs')
+sys.path.insert(0, os.path.abspath(_DOCS_DIR))
+
+import pySim                                                       # noqa: E402
+from pySim.filesystem import CardApplication, CardDF, CardMF, CardADF  # noqa: E402
+from pySim.profile import CardProfile                              # noqa: E402
+from pysim_fs_sphinx import EXCLUDED, SECTIONS                     # noqa: E402
+
+
+class TestFsCoverage(unittest.TestCase):
+    """Ensure SECTIONS + EXCLUDED together account for all classes with content."""
+
+    # Base CardDF types that are not concrete filesystem objects on their own.
+    _DF_BASE_TYPES = frozenset([CardDF, CardMF, CardADF])
+
+    @staticmethod
+    def _collect_reachable_df_types(obj) -> set:
+        """Return the set of all CardDF *types* reachable as children of *obj*."""
+        result = set()
+        if isinstance(obj, CardProfile):
+            children = obj.files_in_mf
+        elif isinstance(obj, CardApplication):
+            result.add(type(obj.adf))
+            children = list(obj.adf.children.values())
+        elif isinstance(obj, CardDF):
+            children = list(obj.children.values())
+        else:
+            return result
+        queue = list(children)
+        while queue:
+            child = queue.pop()
+            if isinstance(child, CardDF):
+                result.add(type(child))
+                queue.extend(child.children.values())
+        return result
+
+    @staticmethod
+    def _has_content(obj) -> bool:
+        """Return True if *obj* owns any EFs/DFs."""
+        if isinstance(obj, CardProfile):
+            return bool(obj.files_in_mf)
+        if isinstance(obj, CardApplication):
+            return bool(obj.adf.children)
+        return False
+
+    def test_all_profiles_and_apps_covered(self):
+        # build a set of (module, class-name) pairs that are already accounted for
+        covered = {(mod, cls) for (_, mod, cls) in SECTIONS}
+        accounted_for = covered | EXCLUDED
+
+        uncovered = []
+        reachable_df_types = set()
+        loaded_modules = {}
+
+        for modinfo in pkgutil.walk_packages(pySim.__path__, prefix='pySim.'):
+            modname = modinfo.name
+            try:
+                module = importlib.import_module(modname)
+            except Exception: # skip inport errors, if any
+                continue
+            loaded_modules[modname] = module
+
+            for name, cls in inspect.getmembers(module, inspect.isclass):
+                # skip classes that are merely imported by this module
+                if cls.__module__ != modname:
+                    continue
+                # examine only subclasses of CardProfile and CardApplication
+                if not issubclass(cls, (CardProfile, CardApplication)):
+                    continue
+                # skip the abstract base classes themselves
+                if cls in (CardProfile, CardApplication):
+                    continue
+                # classes that require constructor arguments cannot be probed
+                try:
+                    obj = cls()
+                except Exception:
+                    continue
+
+                # collect all CardDF types reachable from this profile/application
+                # (used below to identify standalone DFs)
+                reachable_df_types |= self._collect_reachable_df_types(obj)
+
+                if self._has_content(obj) and (modname, name) not in accounted_for:
+                    uncovered.append((modname, name))
+
+        # check standalone CardDFs (such as DF.EIRENE or DF.SYSTEM)
+        for modname, module in loaded_modules.items():
+            for name, cls in inspect.getmembers(module, inspect.isclass):
+                if cls.__module__ != modname:
+                    continue
+                if not issubclass(cls, CardDF):
+                    continue
+                if cls in self._DF_BASE_TYPES:
+                    continue
+                if cls in reachable_df_types:
+                    continue
+                try:
+                    obj = cls()
+                except Exception:
+                    continue
+                if obj.children and (modname, name) not in accounted_for:
+                    uncovered.append((modname, name))
+
+        if uncovered:
+            lines = [
+                'The following classes have EFs/DFs, but not listed in SECTIONS or EXCLUDED:',
+                *(f'  {modname}.{name}' for modname, name in sorted(uncovered)),
+                'Please modify docs/pysim_fs_sphinx.py accordingly',
+            ]
+            self.fail('\n'.join(lines))
+
+
+if __name__ == '__main__':
+    unittest.main()