From cbeeed3608d5573bbd69d4a58a4eed7be113e3af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=A2=D1=80=D0=BE=D1=88?= =?UTF-8?q?=D0=B8=D0=BD?= Date: Sat, 17 Oct 2026 22:11:47 +0300 Subject: [PATCH] File names: the DF-scoped spec dictionary + refreshed EF/SFI tables (v2.5.0) - the file/DF/SFI dictionaries move to simtrace_analyser/ef_tables.py and are extracted from the pinned specs: TS 102 221 (MF, Annex H.1), TS 31.102 (ADF USIM 4.2 and the child DFs 4.3/4.4/4.6, the H.1-H.12 SFI annexes, the Annex L.6 Relay Node/certificate files; Annex A/E as the ADF/DF file completeness cross-check), TS 31.103 and TS 51.011 - 20 DF tables (~370 entries) and 14 SFI tables, up from ~180 flat names and 3 SFI tables - names are DF-scoped: a FID reused across DFs resolves to the file its DF context names - 4F09 is EF_SUPI_NAI under DF_5GS but EF_PROSE_GC under DF_ProSe, 6F07 is EF_IMSI in the USIM/SIM but EF_IST in the ISIM, 6F4E is EF_EXT4 in DF_TELECOM but EF_EXT5 in the USIM ADF; the flat table keeps only names that are identical everywhere and is the fallback when no DF is known (the two names the old table had wrong there are fixed: 6F0A is EF_AC_GBAUAPI, 6F53 EF_LOCIGPRS) - the SELECT-path decoder resolves the 7FFF children against the USIM ADF table, and the Files view names files DF-first - including files first seen only via their FCP (a select-only file used to keep the bare FID, e.g. ADF/6F3C is now ADF/EF_SMS (6F3C)) - tests/data/spec_files.json pins the extracted tables; tests/test_decode.py asserts the DF/ADF tables cover them, the SFI tables match them exactly and every flat name is a spec name for that FID; tests/extract_spec_tables.py regenerates the fixture from the PDFs and --check re-verifies the tables - README/README_RUS document the dictionary and its sources; PWA_VERSION and pyproject/version.py 2.5.0, SW cache simtrace-analyser-v71 --- README.md | 14 +- README_RUS.md | 13 +- frontend/index.html | 2 +- frontend/sw.js | 2 +- pyproject.toml | 2 +- simtrace_analyser/decode.py | 374 ++-------------- simtrace_analyser/ef_tables.py | 751 +++++++++++++++++++++++++++++++++ simtrace_analyser/filetrack.py | 10 +- simtrace_analyser/version.py | 2 +- tests/data/spec_files.json | 571 +++++++++++++++++++++++++ tests/extract_spec_tables.py | 330 +++++++++++++++ tests/test_decode.py | 69 ++- tests/test_filetrack.py | 28 +- tests/test_pcap_import.py | 2 +- 14 files changed, 1820 insertions(+), 350 deletions(-) create mode 100644 simtrace_analyser/ef_tables.py create mode 100644 tests/data/spec_files.json create mode 100644 tests/extract_spec_tables.py diff --git a/README.md b/README.md index c3474a4..e9db440 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,8 @@ UPDATE operations seen in the trace; card-initiated ones are inferred when a read returns bytes differing from what the image already knew — bytes never seen before just fill the image (an observation, not a change). Files reached by SFI share the entry of their path/FID accesses: the SFI ↔ FID relations come -from the normative tables (TS 102 221 Annex H, TS 31.102 Annex H) and from every +from the normative tables (TS 102 221 Annex H.1 and the TS 31.102 H.1–H.12 +annexes; the ISIM SFI rows come from TS 31.103) and from every surfaced FCP carrying the `88` DO, including the spec's "SFI = the 5 LSBs of the FID" rule when the DO is absent; an SFI used before its relation is learned stays provisional and merges into the resolved entry. A file's DF context @@ -229,6 +230,17 @@ image freezes past 64 KB of observed bytes per file (`image_truncated`) and the change log keeps the last 1000 events per file (`changes_total` keeps counting). Counters are returned by the API but not shown in this view yet. +File names are **DF-scoped**: `simtrace_analyser/ef_tables.py` holds the complete +per-DF file tables and SFI lists extracted from the pinned specs (TS 102 221, +TS 31.102 including its H.1–H.12 SFI annexes, TS 31.103, TS 51.011), so a FID +reused across DFs resolves to the file its DF context names — `4F09` is +EF_SUPI_NAI under DF_5GS but EF_PROSE_GC under DF_ProSe, `6F07` is EF_IMSI in +the USIM/SIM but EF_IST in the ISIM, `6F4E` is EF_EXT4 in DF_TELECOM but +EF_EXT5 in the USIM ADF — with the flat table as the fallback when no DF is +known. `tests/extract_spec_tables.py` regenerates the pinned fixture +(`tests/data/spec_files.json`) and `--check` re-verifies the tables against the +PDFs. + Server options: ``` diff --git a/README_RUS.md b/README_RUS.md index 9cc8a87..ac2326d 100644 --- a/README_RUS.md +++ b/README_RUS.md @@ -224,7 +224,8 @@ Files — это таймлайн файловых операций по все известных образу (никогда не виденные байты просто заполняют образ — это наблюдение, а не изменение). Файлы, доступные по SFI, попадают в ту же запись, что и доступ по пути/FID: соответствия SFI ↔ FID берутся из -нормативных таблиц (TS 102 221 Annex H, TS 31.102 Annex H) и из каждого +нормативных таблиц (анекс H.1 TS 102 221 и анексы H.1–H.12 TS 31.102; +строки SFI ISIM — из TS 31.103) и из каждого показанного FCP с DO `88`, включая правило спецификации «SFI = 5 младших бит FID» при отсутствии DO; SFI, использованный до того, как связь выучена, остаётся предварительным и сливается с разрешённой записью. Контекст DF @@ -236,6 +237,16 @@ DF, включая выбор по пути); приложения, выбран последние 1000 событий на файл (`changes_total` продолжает считать). Счётчики API отдаёт, но в этом виде пока не показываются. +Имена файлов **привязаны к DF**: `simtrace_analyser/ef_tables.py` содержит +полные таблицы файлов и SFI по каждому DF, извлечённые из закреплённых +спецификаций (TS 102 221, TS 31.102 включая анексы SFI H.1–H.12, TS 31.103, +TS 51.011), поэтому FID, повторно используемый в разных DF, разрешается в имя +своего контекста — `4F09` это EF_SUPI_NAI в DF_5GS, но EF_PROSE_GC в DF_ProSe; +`6F07` — EF_IMSI в USIM/SIM, но EF_IST в ISIM; `6F4E` — EF_EXT4 в DF_TELECOM, +но EF_EXT5 в ADF USIM, — а плоская таблица служит запасным вариантом, когда DF +неизвестен. `tests/extract_spec_tables.py` пересобирает закреплённый фикстур +(`tests/data/spec_files.json`), `--check` повторно сверяет таблицы с PDF. + Параметры сервера: ``` diff --git a/frontend/index.html b/frontend/index.html index a3adea9..7575fac 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -3168,7 +3168,7 @@ function onFilterChange() { } // ──────────────────── Versioning ──────────────────── -const PWA_VERSION = '2.4.2'; +const PWA_VERSION = '2.5.0'; const MIN_SERVER_VERSION = '2.3.0'; function parseVersion(v) { diff --git a/frontend/sw.js b/frontend/sw.js index d44d407..6e54c08 100644 --- a/frontend/sw.js +++ b/frontend/sw.js @@ -1,4 +1,4 @@ -const CACHE = 'simtrace-analyser-v70'; +const CACHE = 'simtrace-analyser-v71'; const PRECACHE = [ 'index.html', diff --git a/pyproject.toml b/pyproject.toml index 4fca784..eaf75dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.backends._legacy:_Backend" [project] name = "simtrace-analyser" -version = "2.4.2" +version = "2.5.0" description = "SIMtrace Analyser — APDU trace analyser for GSMTAP/PCAP captures (bundles the simtrace2-pysniff utility)" readme = {file = "README.md", content-type = "text/markdown"} license = {text = "GPL-2.0-or-later"} diff --git a/simtrace_analyser/decode.py b/simtrace_analyser/decode.py index ba66666..c56055a 100644 --- a/simtrace_analyser/decode.py +++ b/simtrace_analyser/decode.py @@ -6,6 +6,8 @@ the SIMtrace2 sniffer into structured dicts for the PWA to display. from simtrace2_pysniff.gsmtap import GSMTAP_FLAG_BAD_FCS +from .ef_tables import KNOWN_FIDS, DF_CHILDREN, SFI_BY_DF as _SFI_BY_DF + # ──────────────────── Status Word names ──────────────────── SW_NAMES = { @@ -149,276 +151,8 @@ def decode_cla(cla): # ──────────────────── Known FIDs ──────────────────── -# Sourced from pySim (TS 102 221 common, TS 51.011 SIM, TS 31.102 USIM, -# TS 31.103 ISIM). Note: some FIDs are reused across DFs/ADFs; the -# USIM/ISIM interpretation is preferred here because the '7FFF' (current -# ADF) SELECT-path resolution is the primary use case. - -KNOWN_FIDS = { - # Master File and common files (TS 102 221 / TS 51.011) - '3f00': 'MF', - '2f00': 'EF_DIR', - '2f05': 'EF_PL', - '2f06': 'EF_ARR', - '2f07': 'EF_ENV-CLASSES', - '2f08': 'EF_UMPC', - '2fe2': 'EF_ICCID', - - # Dedicated files (MF/ADF level) - '7f10': 'DF_TELECOM', - '7f20': 'DF_GSM', - '5f3b': 'DF_GSM_ACCESS', - '5f40': 'DF_WLAN', - '5f50': 'DF_HNB', - '5f90': 'DF_ProSe', - '5ff0': 'DF_5G_ProSe', - '5fc0': 'DF_5GS', - '5fe0': 'DF_SNPN', - '5ff1': 'DF_5MBSUECONFIG', - - # SIM (TS 51.011) — DF_TELECOM / DF_GSM - '6f05': 'EF_LI', - '6f07': 'EF_IMSI', - '6f20': 'EF_Kc', - '6f2c': 'EF_DCK', - '6f30': 'EF_PLMNsel', - '6f31': 'EF_HPPLMN', - '6f32': 'EF_CNL', - '6f37': 'EF_ACMmax', - '6f38': 'EF_UST', - '6f39': 'EF_ACM', - '6f3a': 'EF_ADN', - '6f3b': 'EF_FDN', - '6f3c': 'EF_SMS', - '6f3d': 'EF_CCP', - '6f3e': 'EF_GID1', - '6f3f': 'EF_GID2', - '6f40': 'EF_MSISDN', - '6f41': 'EF_PUCT', - '6f42': 'EF_SMSP', - '6f43': 'EF_SMSS', - '6f44': 'EF_LND', - '6f45': 'EF_CBMI', - '6f46': 'EF_SPN', - '6f47': 'EF_SMSR', - '6f48': 'EF_CBMID', - '6f49': 'EF_SDN', - '6f4a': 'EF_EXT1', - '6f4b': 'EF_EXT2', - '6f4c': 'EF_EXT3', - '6f4d': 'EF_BDN', - '6f4e': 'EF_EXT4', - '6f4f': 'EF_ECCP', - '6f50': 'EF_CBMIR', - '6f51': 'EF_NIA', - '6f52': 'EF_KcGPRS', - '6f53': 'EF_LOCI_GPRS', - '6f58': 'EF_CMI', - '6f63': 'EF_CPBCCH', - '6f64': 'EF_InvScan', - '6f74': 'EF_BCCH', - '6f78': 'EF_ACC', - '6f7b': 'EF_FPLMN', - '6f7e': 'EF_LOCI', - '6fad': 'EF_AD', - '6fae': 'EF_Phase', - '6fb1': 'EF_VGCS', - '6fb2': 'EF_VGCSS', - '6fb3': 'EF_VBS', - '6fb4': 'EF_VBSS', - '6fb5': 'EF_eMLPP', - '6fb6': 'EF_AAeM', - '6fb7': 'EF_ECC', - '6fc5': 'EF_PNN', - '6fc6': 'EF_OPL', - '6fc9': 'EF_MBI', - '6fca': 'EF_MWIS', - '6fcb': 'EF_CFIS', - '6fcd': 'EF_SPDI', - '6fce': 'EF_MMSN', - '6fd0': 'EF_MMSICP', - '6fd1': 'EF_MMSUP', - '6fd2': 'EF_MMSUCP', - - # USIM (TS 31.102) — ADF_USIM - '6f01': 'EF_eAKA', - '6f02': 'EF_IMPI', # ISIM EF_IMPI (USIM reuses 6F02 for EF_OCST) - '6f06': 'EF_ARR', - '6f08': 'EF_Keys', - '6f09': 'EF_KeysPS', # ISIM EF_P-CSCF also reuses 6F09 - '6f55': 'EF_EXT4', - '6f56': 'EF_EST', - '6f57': 'EF_ACL', - '6f5b': 'EF_START-HFN', - '6f5c': 'EF_THRESHOLD', - '6f60': 'EF_PLMNwAcT', - '6f61': 'EF_OPLMNwAcT', - '6f62': 'EF_HPLMNwAcT', - '6f65': 'EF_RPLMNAcT', - '6f73': 'EF_PSLOCI', - '6f80': 'EF_ICI', - '6f81': 'EF_OCI', - '6f82': 'EF_ICT', - '6f83': 'EF_OCT', - '6fc4': 'EF_NETPAR', - '6fc7': 'EF_MBDN', - '6fc8': 'EF_EXT6', - '6fcc': 'EF_EXT7', - '6fcf': 'EF_EXT8', - '6fd3': 'EF_NIA', - '6fd4': 'EF_VGCSCA', - '6fd5': 'EF_VBSCA', - '6fd6': 'EF_GBABP', - '6fd7': 'EF_MSK', - '6fd8': 'EF_MUK', - '6fd9': 'EF_EHPLMN', - '6fda': 'EF_GBANL', - '6fdb': 'EF_EHPLMNPI', - '6fdd': 'EF_NAFKCA', - '6fde': 'EF_SPNI', - '6fdf': 'EF_PNNI', - '6fe2': 'EF_NCP-IP', - '6fe3': 'EF_EPSLOCI', - '6fe4': 'EF_EPSNSC', - '6fe6': 'EF_UFC', - '6fe8': 'EF_NASCONFIG', - '6fec': 'EF_PWS', - '6fed': 'EF_FDNURI', - '6fee': 'EF_BDNURI', - '6fef': 'EF_SDNURI', - '6ff1': 'EF_IPS', - '6ff3': 'EF_ePDGId', - '6ff4': 'EF_ePDGSelection', - '6ff5': 'EF_ePDGIdEm', - '6ff6': 'EF_ePDGSelectionEm', - '6ff7': 'EF_FromPreferred', - '6ff8': 'EF_IMSConfigData', - '6ffa': 'EF_WebRTCURI', - '6ffc': 'EF_XCAPConfigData', - '6ffd': 'EF_EARFCNList', - '6ffe': 'EF_MuDMiDConfigData', - - # ISIM (TS 31.103) — ADF_ISIM - '6f03': 'EF_DOMAIN', - '6f04': 'EF_IMPU', - '6f0a': 'EF_GBAUAPI', - '6f0b': 'EF_IMSDCI', - '6fe7': 'EF_UICCIARI', -} - - -# DF/ADF children (child FID → name), for context-aware SELECT-path -# resolution. The '4Fxx' files are only meaningful within their parent -# DF, and many FIDs collide across DFs (e.g. '4F01' differs between -# DF_ProSe, DF_5GS, DF_MCS, DF_V2X, DF_SNPN), so they cannot live in the -# flat KNOWN_FIDS table. Sourced from pySim (TS 31.102 / TS 31.102 telecom). -DF_CHILDREN = { - '7f10': { # DF_TELECOM - '5f3a': 'DF_PHONEBOOK', - '5f3b': 'DF_MULTIMEDIA', - '5f3d': 'DF_MCS', - '5f3e': 'DF_V2X', - }, - '5f3a': { # DF_PHONEBOOK - '4f22': 'EF_PSC', - '4f23': 'EF_CC', - '4f24': 'EF_PUID', - '4f30': 'EF_PBR', - }, - '5f3b': { # DF_GSM_ACCESS (ADF) + DF_MULTIMEDIA (DF_TELECOM) share 5F3B - '4f20': 'EF_Kc', - '4f52': 'EF_KcGPRS', - '4f63': 'EF_CPBCCH', - '4f64': 'EF_InvScan', - '4f47': 'EF_MML', - '4f48': 'EF_MMDF', - }, - '5f3d': { # DF_MCS - '4f01': 'EF_MST', - '4f02': 'EF_MCS_CONFIG', - }, - '5f3e': { # DF_V2X - '4f01': 'EF_VST', - '4f02': 'EF_V2X_CONFIG', - }, - '5f40': { # DF_WLAN - '4f41': 'EF_Pseudo', - '4f42': 'EF_UPLMNWLAN', - '4f43': 'EF_OPLMNWLAN', - '4f44': 'EF_UWSIDL', - '4f45': 'EF_OWSIDL', - '4f46': 'EF_WRI', - '4f47': 'EF_HWSIDL', - '4f48': 'EF_WEHPLMNPI', - '4f49': 'EF_WHPI', - '4f4a': 'EF_WLRPLMN', - '4f4b': 'EF_HPLMNDAI', - }, - '5f50': { # DF_HNB - '4f81': 'EF_ACSGL', - '4f82': 'EF_CSGT', - '4f83': 'EF_HNBN', - '4f84': 'EF_OCSGL', - '4f85': 'EF_OCSGT', - '4f86': 'EF_OHNBN', - }, - '5f90': { # DF_ProSe - '4f01': 'EF_PROSE_MON', - '4f02': 'EF_PROSE_ANN', - '4f03': 'EF_PROSEFUNC', - '4f04': 'EF_PROSE_RADIO_COM', - '4f05': 'EF_PROSE_RADIO_MON', - '4f06': 'EF_PROSE_RADIO_ANN', - '4f07': 'EF_PROSE_POLICY', - '4f08': 'EF_PROSE_PLMN', - '4f09': 'EF_PROSE_GC', - '4f10': 'EF_PST', - '4f11': 'EF_UIRC', - '4f12': 'EF_PROSE_GM_DISCOVERY', - '4f13': 'EF_PROSE_RELAY', - '4f14': 'EF_PROSE_RELAY_DISCOVERY', - }, - '5fc0': { # DF_5GS - '4f01': 'EF_5GS3GPPLOCI', - '4f02': 'EF_5GSN3GPPLOCI', - '4f03': 'EF_5GS3GPPNSC', - '4f04': 'EF_5GSN3GPPNSC', - '4f05': 'EF_5GAUTHKEYS', - '4f06': 'EF_UAC_AIC', - '4f07': 'EF_SUCI_Calc_Info', - '4f08': 'EF_OPL5G', - '4f09': 'EF_SUPI_NAI', - '4f0a': 'EF_Routing_Indicator', - '4f0b': 'EF_URSP', - '4f0c': 'EF_TN3GPPSNN', - '4f0d': 'EF_CAG', - '4f0e': 'EF_SOR-CMCI', - '4f0f': 'EF_DRI', - '4f10': 'EF_5GSEDRX', - '4f11': 'EF_5GNSWO_CONF', - '4f15': 'EF_MCHPPLMN', - '4f16': 'EF_KAUSF_DERIVATION', - }, - '5fd0': { # DF_SAIP - '4f01': 'EF_SUCI_Calc_Info', - }, - '5fe0': { # DF_SNPN - '4f01': 'EF_PWS_SNPN', - '4f02': 'EF_NID', - }, - '5ff0': { # DF_5G_ProSe - '4f02': 'EF_5G_PROSE_DD', - '4f03': 'EF_5G_PROSE_DC', - '4f04': 'EF_5G_PROSE_U2NRU', - '4f05': 'EF_5G_PROSE_RU', - '4f06': 'EF_5G_PROSE_UIR', - '4f07': 'EF_5G_PROSE_U2URU', - '4f08': 'EF_5G_PROSE_EU', - }, - '5ff1': { # DF_5MBSUECONFIG - '4f01': 'EF_5MBSUECONFIG', - }, -} +# The spec-sourced file/DF/SFI tables live in ef_tables.py (see its +# header for the sources); _file_name() resolves DF-first. def _decode_select_path(body_hex): @@ -427,15 +161,21 @@ def _decode_select_path(body_hex): Splits the body into 2-byte FIDs and names each component, tracking the parent DF so DF-specific children (e.g. '4Fxx') resolve against the correct DF. '7fff' is the implicit FID for the current - application's ADF, whose children are the flat KNOWN_FIDS files. + application's ADF; its children resolve against DF_CHILDREN['7fff'] + (the USIM ADF table — a message-level path has no AID context, so an + AID-selected application resolves exactly only in file operations, + where the tracker's DF identity carries the AID). """ pairs = [body_hex[i:i + 4] for i in range(0, len(body_hex), 4)] parts = [] parent = None for p in pairs: if p == '7fff': + # The running application's ADF; its children resolve against the + # USIM ADF table (an AID-selected application is tracked by its AID + # in file operations, where the DF is known exactly). parts.append('current ADF') - parent = None + parent = '7fff' continue name = None if parent and parent in DF_CHILDREN: @@ -465,58 +205,6 @@ def select_target_fid(d): return None -# Short File Identifier (SFI) → FID, scoped to the containing DF. -# SFI is only unique within a DF (TS 102 221 §8.3). Sourced from the -# normative SFI lists: TS 31.102 Annex H (USIM ADF, DF GSM-ACCESS) and -# TS 102 221 Annex H (MF level). -_SFI_BY_DF = { - '3f00': { # MF (TS 102 221 Annex H) - 0x02: '2fe2', # EF_ICCID - 0x05: '2f05', # EF_PL - 0x06: '2f06', # EF_ARR - 0x07: '2f07', # EF_ENV-CLASSES (TS 102 671) - 0x08: '2f08', # EF_UMPC - 0x1e: '2f00', # EF_DIR - }, - '7fff': { # ADF_USIM (TS 31.102 Annex H H.1) - 0x01: '6fb7', # EF_ECC - 0x02: '6f05', # EF_LI - 0x03: '6fad', # EF_AD - 0x04: '6f38', # EF_UST - 0x05: '6f56', # EF_EST - 0x06: '6f78', # EF_ACC - 0x07: '6f07', # EF_IMSI - 0x08: '6f08', # EF_Keys - 0x09: '6f09', # EF_KeysPS - 0x0a: '6f60', # EF_PLMNwAcT - 0x0b: '6f7e', # EF_LOCI - 0x0c: '6f73', # EF_PSLOCI - 0x0d: '6f7b', # EF_FPLMN - 0x0e: '6f48', # EF_CBMID - 0x0f: '6f5b', # EF_START-HFN - 0x10: '6f5c', # EF_THRESHOLD - 0x11: '6f61', # EF_OPLMNwAcT - 0x12: '6f31', # EF_HPPLMN - 0x13: '6f62', # EF_HPLMNwAcT - 0x14: '6f80', # EF_ICI - 0x15: '6f81', # EF_OCI - 0x16: '6f4f', # EF_CCP2 - 0x17: '6f06', # EF_ARR - 0x18: '6fe4', # EF_EPSNSC - 0x19: '6fc5', # EF_PNN - 0x1a: '6fc6', # EF_OPL - 0x1b: '6fcd', # EF_SPDI - 0x1c: '6f39', # EF_ACM - 0x1d: '6fd9', # EF_EHPLMN - 0x1e: '6fe3', # EF_EPSLOCI - }, - '5f3b': { # DF_GSM_ACCESS (TS 31.102 Annex H H.2) - 0x01: '4f20', # EF_Kc - 0x02: '4f52', # EF_KcGPRS - }, -} - - def _is_df_fid(fid): """True if *fid* (2-hex-byte string) denotes a DF/ADF (MF, 7Fxx, 5Fxx).""" return fid == '3f00' or fid.startswith('7f') or fid.startswith('5f') @@ -3744,7 +3432,15 @@ FILE_DECODERS = { } -def _decode_file_data(fid, raw, p1=None): +def _file_name(fid, df=None): + """The display name of a file: DF-scoped names first (DF_CHILDREN, so a + FID reused across DFs resolves to the right file), then the flat table, then + the FID itself.""" + name = (DF_CHILDREN.get(df or '', {}) or {}).get(fid) + return name or KNOWN_FIDS.get(fid) or fid.upper() + + +def _decode_file_data(fid, raw, p1=None, df=None): """Decode file data using the FID's registered decoder.""" if not raw: return None @@ -3758,7 +3454,7 @@ def _decode_file_data(fid, raw, p1=None): if not out: return None out['fid'] = fid - out['ef'] = KNOWN_FIDS.get(fid, fid.upper()) + out['ef'] = _file_name(fid, df) return out @@ -3782,7 +3478,7 @@ def _is_record_file(sel): def _stale_file_note(sel): """Mark a decode as skipped because the selection is inconsistent.""" fid = sel.get('fid') or '' - return {'fid': fid, 'ef': KNOWN_FIDS.get(fid, fid.upper()), 'stale': True} + return {'fid': fid, 'ef': _file_name(fid), 'stale': True} def _file_summary(f): @@ -4219,11 +3915,11 @@ def decode_message(raw_data, prev=None): sfi = p1p2['sfi'] fid = (prev.get('sfi_map') or {}).get(sfi) if fid: - file_dec = _decode_file_data(fid, body, p1=p1) + file_dec = _decode_file_data(fid, body, p1=p1, df=prev.get('df')) if file_dec: result['file'] = file_dec else: - result['file'] = {'fid': fid, 'ef': KNOWN_FIDS.get(fid, fid.upper()), + result['file'] = {'fid': fid, 'ef': _file_name(fid, prev.get('df')), 'raw': body.hex().upper()} else: result['file'] = {'sfi': sfi, 'unknown': True, 'ef': f'SFI {sfi}'} @@ -4231,11 +3927,11 @@ def decode_message(raw_data, prev=None): # Record command with SFI (P2 b8-b4) → resolve the target EF. fid = (prev.get('sfi_map') or {}).get(rec_sfi) if fid: - file_dec = _decode_file_data(fid, body, p1=p1) + file_dec = _decode_file_data(fid, body, p1=p1, df=prev.get('df')) if file_dec: result['file'] = file_dec else: - result['file'] = {'fid': fid, 'ef': KNOWN_FIDS.get(fid, fid.upper()), + result['file'] = {'fid': fid, 'ef': _file_name(fid, prev.get('df')), 'raw': body.hex().upper()} else: result['file'] = {'sfi': rec_sfi, 'unknown': True, 'ef': f'SFI {rec_sfi}'} @@ -4244,7 +3940,7 @@ def decode_message(raw_data, prev=None): # is a search pattern, not file data). fid = (prev.get('sfi_map') or {}).get(rec_sfi) if fid: - result['file'] = {'fid': fid, 'ef': KNOWN_FIDS.get(fid, fid.upper())} + result['file'] = {'fid': fid, 'ef': _file_name(fid, prev.get('df'))} else: result['file'] = {'sfi': rec_sfi, 'unknown': True, 'ef': f'SFI {rec_sfi}'} elif ins in (0xB0, 0xB2, 0xD6, 0xDC) and prev and prev.get('sel'): @@ -4254,7 +3950,8 @@ def decode_message(raw_data, prev=None): else: file_dec = None if ins in (0xB2, 0xDC) or offset == 0: - file_dec = _decode_file_data(prev['sel']['fid'], body, p1=p1) + file_dec = _decode_file_data(prev['sel']['fid'], body, p1=p1, + df=prev.get('df')) if file_dec: result['file'] = file_dec else: @@ -4262,8 +3959,7 @@ def decode_message(raw_data, prev=None): # still attribute the operation to the selected file. result['file'] = { 'fid': prev['sel']['fid'], - 'ef': KNOWN_FIDS.get(prev['sel']['fid'], - prev['sel']['fid'].upper()), + 'ef': _file_name(prev['sel']['fid'], prev.get('df')), } # SW @@ -4281,7 +3977,9 @@ def decode_message(raw_data, prev=None): if prev_ins == 0xB2 and not _is_record_file(prev['sel']): result['file'] = _stale_file_note(prev['sel']) else: - file_dec = _decode_file_data(prev['sel']['fid'], remaining[:cmd_body_len]) + file_dec = _decode_file_data(prev['sel']['fid'], + remaining[:cmd_body_len], + df=prev.get('df')) if file_dec: result['file'] = file_dec else: @@ -4289,7 +3987,7 @@ def decode_message(raw_data, prev=None): fid = prev['sel']['fid'] result['file'] = { 'fid': fid, - 'ef': KNOWN_FIDS.get(fid, fid.upper()), + 'ef': _file_name(fid, prev.get('df')), } elif prev_ins == 0xA2 and prev.get('sel') and prev.get('file_ok'): if not _is_record_file(prev['sel']): @@ -4299,7 +3997,7 @@ def decode_message(raw_data, prev=None): fid = prev['sel']['fid'] result['file'] = { 'fid': fid, - 'ef': KNOWN_FIDS.get(fid, fid.upper()), + 'ef': _file_name(fid, prev.get('df')), 'record_numbers': list(remaining[:cmd_body_len]), } elif prev_ins == 0xC2: # ENVELOPE → Response Packet (PoR) diff --git a/simtrace_analyser/ef_tables.py b/simtrace_analyser/ef_tables.py new file mode 100644 index 0000000..80c5690 --- /dev/null +++ b/simtrace_analyser/ef_tables.py @@ -0,0 +1,751 @@ +"""EF file-identifier tables — extracted from the pinned specs. + +``KNOWN_FIDS`` is the flat FID → name table used when no DF context is +known; ``DF_CHILDREN`` maps a DF (or the ADF of an AID-selected +application, keyed ``aid:``) to its child files/DFs; ``SFI_BY_DF`` +maps an SFI to its FID within a DF (TS 102 221 §8.3: SFIs are unique only +within a DF). + +Sources (PDFs in the workspace ``docs/uicc/``): + * ETSI TS 102 221 V18.4.0 — MF files, Annex H.1 SFI list + * 3GPP TS 31.102 V18.10.0 — ADF USIM files (4.2), the child DFs + (4.3/4.4/4.6), the SFI annexes H.1–H.12, the Relay Node and + certificate files (Annex L.6); Annex A/E were used as the ADF/DF file + completeness cross-check + * 3GPP TS 31.103 V18.3.0 — ADF ISIM files (4.2) and their per-file SFI + rows (the spec has no SFI annex) + * 3GPP TS 51.011 V4.15.0 — MF/DF_GSM/DF_TELECOM files (10.1–10.6) + +A FID reused across DFs keeps its DF-specific name here (``4F09`` is +EF_SUPI_NAI under DF_5GS but EF_PROSE_GC under DF_ProSe; ``6F07`` is +EF_IMSI in the USIM/SIM but EF_IST in the ISIM; ``6F4E`` is EF_EXT4 in +DF_TELECOM but EF_EXT5 in the USIM ADF); only names that are identical in +every DF they occur in live in the flat table. + +The tables are pinned by ``tests/data/spec_files.json`` (the extracted +tables) and ``tests/test_decode.py``; regenerate the fixture with +``tests/extract_spec_tables.py`` (`--check` re-verifies the tables +against the PDFs) when a spec revision is pinned. +""" + +KNOWN_FIDS = { + # ── Master File and common files (TS 102 221 Annex H; TS 51.011 10.1) + '3f00': 'MF', + '2f00': 'EF_DIR', + '2f05': 'EF_PL', + '2f06': 'EF_ARR', + '2f07': 'EF_ENV-CLASSES', + '2f08': 'EF_UMPC', + '2fe2': 'EF_ICCID', + + # ── DF/ADF identifiers (TS 31.102 4.3/4.6; TS 51.011 10.2/10.4/10.6) + # Children of the USIM ADF + '5f3a': 'DF_PHONEBOOK', + '5f3b': 'DF_GSM_ACCESS', # TS 51.011 10.6: DF_MULTIMEDIA under DF_TELECOM + '5f3c': 'DF_MexE', # TS 51.011 10.6: DF_MMSS under DF_TELECOM + '5f3d': 'DF_MCS', + '5f3e': 'DF_V2X', + '5f3f': 'DF_A2X', + '5f40': 'DF_WLAN', + '5f50': 'DF_HNB', # TS 51.011 10.6: DF_GRAPHICS under DF_TELECOM + '5f70': 'DF_SoLSA', + '5f80': 'DF_BCAST', # reserved for OMA BCAST (TS 31.102 4.3 note) + '5f90': 'DF_ProSe', + '5fa0': 'DF_ACDC', + '5fb0': 'DF_TV', + '5fc0': 'DF_5GS', + '5fd0': 'DF_SAIP', + '5fe0': 'DF_SNPN', + '5ff0': 'DF_5G_ProSe', + '5ff1': 'DF_5MBSUECONFIG', + # Children of DF_GSM (SIM) + '5f30': 'DF_IRIDIUM', + '5f31': 'DF_GLOBALSTAR', + '5f32': 'DF_ICO', + '5f33': 'DF_ACES', + '5f60': 'DF_CTS', + # MF-level DFs + '7f10': 'DF_TELECOM', + '7f20': 'DF_GSM', + + # ── SIM DF_GSM / DF_TELECOM files (TS 51.011 10.3/10.5) + '6f05': 'EF_LI', # DF_GSM: EF_LP (TS 51.011 10.3.1) + '6f07': 'EF_IMSI', # ADF ISIM: EF_IST (TS 31.103 4.2.6) + '6f20': 'EF_Kc', + '6f2c': 'EF_DCK', + '6f30': 'EF_PLMNsel', + '6f31': 'EF_HPPLMN', + '6f32': 'EF_CNL', + '6f37': 'EF_ACMmax', + '6f38': 'EF_UST', # DF_GSM: EF_SST (TS 51.011 10.3.x) + '6f39': 'EF_ACM', + '6f3a': 'EF_ADN', + '6f3b': 'EF_FDN', + '6f3c': 'EF_SMS', + '6f3d': 'EF_CCP', + '6f3e': 'EF_GID1', + '6f3f': 'EF_GID2', + '6f40': 'EF_MSISDN', + '6f41': 'EF_PUCT', + '6f42': 'EF_SMSP', + '6f43': 'EF_SMSS', + '6f44': 'EF_LND', + '6f45': 'EF_CBMI', + '6f46': 'EF_SPN', + '6f47': 'EF_SMSR', + '6f48': 'EF_CBMID', + '6f49': 'EF_SDN', + '6f4a': 'EF_EXT1', + '6f4b': 'EF_EXT2', + '6f4c': 'EF_EXT3', + '6f4d': 'EF_BDN', + '6f4e': 'EF_EXT4', # ADF USIM: EF_EXT5 (TS 31.102 4.2.37) + '6f4f': 'EF_ECCP', # ADF USIM: EF_CCP2 (TS 31.102 4.2.38) + '6f50': 'EF_CBMIR', + '6f51': 'EF_NIA', + '6f52': 'EF_KcGPRS', + '6f53': 'EF_LOCIGPRS', # DF_TELECOM: EF_RMA (TS 31.102 4.5.8) + '6f54': 'EF_SUME', + '6f55': 'EF_EXT4', # ADF USIM only (TS 31.102 4.2.45) + '6f56': 'EF_EST', + '6f57': 'EF_ACL', + '6f58': 'EF_CMI', + '6f5b': 'EF_START-HFN', + '6f5c': 'EF_THRESHOLD', + '6f60': 'EF_PLMNwAcT', + '6f61': 'EF_OPLMNwAcT', + '6f62': 'EF_HPLMNwAcT', + '6f63': 'EF_CPBCCH', + '6f64': 'EF_InvScan', + '6f65': 'EF_RPLMNAcT', # legacy (TS 31.102 4.2.45 NOTE 5) + '6f73': 'EF_PSLOCI', + '6f74': 'EF_BCCH', + '6f78': 'EF_ACC', + '6f7b': 'EF_FPLMN', + '6f7e': 'EF_LOCI', + '6fad': 'EF_AD', + '6fae': 'EF_Phase', + '6fb1': 'EF_VGCS', + '6fb2': 'EF_VGCSS', + '6fb3': 'EF_VBS', + '6fb4': 'EF_VBSS', + '6fb5': 'EF_eMLPP', + '6fb6': 'EF_AAeM', # TS 31.102 4.2.40: EFAaeM + '6fb7': 'EF_ECC', + '6fc5': 'EF_PNN', + '6fc6': 'EF_OPL', + '6fc7': 'EF_MBDN', + '6fc9': 'EF_MBI', + '6fca': 'EF_MWIS', + '6fcb': 'EF_CFIS', + '6fcd': 'EF_SPDI', + '6fce': 'EF_MMSN', + '6fd0': 'EF_MMSICP', + '6fd1': 'EF_MMSUP', + '6fd2': 'EF_MMSUCP', + + # ── ADF USIM files (TS 31.102 4.2; see DF_CHILDREN['7fff']) + '6f01': 'EF_eAKA', + '6f02': 'EF_IMPI', # ADF USIM: EF_OCST (TS 31.102 4.2.81) + '6f06': 'EF_ARR', + '6f08': 'EF_Keys', + '6f09': 'EF_KeysPS', # ADF ISIM: EF_P-CSCF (TS 31.103 4.2.13) + '6f0a': 'EF_AC_GBAUAPI', # shared with ADF ISIM (TS 31.103 4.2.16) + '6fc3': 'EF_Hiddenkey', + '6fc4': 'EF_NETPAR', + '6fc8': 'EF_EXT6', + '6fcc': 'EF_EXT7', + '6fcf': 'EF_EXT8', + '6fd3': 'EF_NIA', + '6fd4': 'EF_VGCSCA', + '6fd5': 'EF_VBSCA', # ADF ISIM: EF_GBABP (TS 31.103 4.2.21) + '6fd6': 'EF_GBABP', + '6fd7': 'EF_MSK', # ADF ISIM: EF_GBANL (TS 31.103 4.2.22) + '6fd8': 'EF_MUK', + '6fd9': 'EF_EHPLMN', + '6fda': 'EF_GBANL', + '6fdb': 'EF_EHPLMNPI', + '6fdc': 'EF_LRPLMNSI', + '6fdd': 'EF_NAFKCA', + '6fde': 'EF_SPNI', + '6fdf': 'EF_PNNI', + '6fe0': 'EF_ICE_DN', + '6fe1': 'EF_ICE_FF', + '6fe2': 'EF_NCP-IP', + '6fe3': 'EF_EPSLOCI', + '6fe4': 'EF_EPSNSC', + '6fe5': 'EF_PSISMSC', + '6fe6': 'EF_UFC', + '6fe7': 'EF_UICCIARI', + '6fe8': 'EF_NASCONFIG', + '6fe9': 'EF_CERT', # TS 31.102 Annex L.6.1.1 + '6fea': 'EF_eFRNid', # TS 31.102 Annex L.6.2.1 + '6feb': 'EF_SCCmax', # TS 31.102 Annex L.6.2.2 + '6fec': 'EF_PWS', + '6fed': 'EF_FDNURI', + '6fee': 'EF_BDNURI', + '6fef': 'EF_SDNURI', + '6ff0': 'EF_IAL', + '6ff1': 'EF_IPS', + '6ff2': 'EF_IPD', + '6ff3': 'EF_ePDGId', + '6ff4': 'EF_ePDGSelection', + '6ff5': 'EF_ePDGIdEm', + '6ff6': 'EF_ePDGSelectionEm', + '6ff7': 'EF_FromPreferred', + '6ff8': 'EF_IMSConfigData', + '6ff9': 'EF_3GPPPSDATAOFF', + '6ffa': 'EF_WebRTCURI', # ADF USIM: EF_3GPPPSDATAOFFservicelist + '6ffb': 'EF_TVCONFIG', + '6ffc': 'EF_XCAPConfigData', + '6ffd': 'EF_EARFCNList', + '6ffe': 'EF_MuDMiDConfigData', + + # ── ADF ISIM files (TS 31.103 4.2; see DF_CHILDREN['aid:...']) + '6f03': 'EF_DOMAIN', + '6f04': 'EF_IMPU', + '6f0b': 'EF_IMSDCI', + '6fe7': 'EF_UICCIARI', +} + +# DF/ADF children (child FID -> name), for context-aware SELECT-path +# resolution and DF-scoped file naming: the complete per-DF file tables +# of the pinned specs. A FID reused across DFs (e.g. 4F01, 6F07) +# resolves against its parent; the flat KNOWN_FIDS above is the +# fallback for messages with no known DF context. +DF_CHILDREN = { + '3f00': { # MF (TS 102 221 Annex H; TS 51.011 10.1) + '2f00': 'EF_DIR', + '2f05': 'EF_PL', + '2f06': 'EF_ARR', + '2f07': 'EF_ENV-CLASSES', + '2f08': 'EF_UMPC', + '2fe2': 'EF_ICCID', + }, + '7fff': { # ADF USIM (TS 31.102 4.2; Annex L.6 for the RN/certificate files) + '5f3a': 'DF_PHONEBOOK', + '5f3b': 'DF_GSM_ACCESS', + '5f3c': 'DF_MexE', + '5f40': 'DF_WLAN', + '5f50': 'DF_HNB', + '5f70': 'DF_SoLSA', + '5f80': 'DF_BCAST', + '5f90': 'DF_ProSe', + '5fa0': 'DF_ACDC', + '5fb0': 'DF_TV', + '5fc0': 'DF_5GS', + '5fd0': 'DF_SAIP', + '5fe0': 'DF_SNPN', + '5ff0': 'DF_5G_ProSe', + '5ff1': 'DF_5MBSUECONFIG', + '6f01': 'EF_eAKA', + '6f02': 'EF_OCST', + '6f05': 'EF_LI', + '6f06': 'EF_ARR', + '6f07': 'EF_IMSI', + '6f08': 'EF_Keys', + '6f09': 'EF_KeysPS', + '6f0a': 'EF_AC_GBAUAPI', + '6f0b': 'EF_IMSDCI', + '6f2c': 'EF_DCK', + '6f31': 'EF_HPPLMN', + '6f32': 'EF_CNL', + '6f37': 'EF_ACMmax', + '6f38': 'EF_UST', + '6f39': 'EF_ACM', + '6f3b': 'EF_FDN', + '6f3c': 'EF_SMS', + '6f3e': 'EF_GID1', + '6f3f': 'EF_GID2', + '6f40': 'EF_MSISDN', + '6f41': 'EF_PUCT', + '6f42': 'EF_SMSP', + '6f43': 'EF_SMSS', + '6f45': 'EF_CBMI', + '6f46': 'EF_SPN', + '6f47': 'EF_SMSR', + '6f48': 'EF_CBMID', + '6f49': 'EF_SDN', + '6f4b': 'EF_EXT2', + '6f4c': 'EF_EXT3', + '6f4d': 'EF_BDN', + '6f4e': 'EF_EXT5', + '6f4f': 'EF_CCP2', + '6f50': 'EF_CBMIR', + '6f55': 'EF_EXT4', + '6f56': 'EF_EST', + '6f57': 'EF_ACL', + '6f58': 'EF_CMI', + '6f5b': 'EF_START-HFN', + '6f5c': 'EF_THRESHOLD', + '6f60': 'EF_PLMNwAcT', + '6f61': 'EF_OPLMNwAcT', + '6f62': 'EF_HPLMNwAcT', + '6f73': 'EF_PSLOCI', + '6f78': 'EF_ACC', + '6f7b': 'EF_FPLMN', + '6f7e': 'EF_LOCI', + '6f80': 'EF_ICI', + '6f81': 'EF_OCI', + '6f82': 'EF_ICT', + '6f83': 'EF_OCT', + '6fad': 'EF_AD', + '6fb1': 'EF_VGCS', + '6fb2': 'EF_VGCSS', + '6fb3': 'EF_VBS', + '6fb4': 'EF_VBSS', + '6fb5': 'EF_eMLPP', + '6fb6': 'EF_AAeM', + '6fb7': 'EF_ECC', + '6fc3': 'EF_Hiddenkey', + '6fc4': 'EF_NETPAR', + '6fc5': 'EF_PNN', + '6fc6': 'EF_OPL', + '6fc7': 'EF_MBDN', + '6fc8': 'EF_EXT6', + '6fc9': 'EF_MBI', + '6fca': 'EF_MWIS', + '6fcb': 'EF_CFIS', + '6fcc': 'EF_EXT7', + '6fcd': 'EF_SPDI', + '6fce': 'EF_MMSN', + '6fcf': 'EF_EXT8', + '6fd0': 'EF_MMSICP', + '6fd1': 'EF_MMSUP', + '6fd2': 'EF_MMSUCP', + '6fd3': 'EF_NIA', + '6fd4': 'EF_VGCSCA', + '6fd5': 'EF_VBSCA', + '6fd6': 'EF_GBABP', + '6fd7': 'EF_MSK', + '6fd8': 'EF_MUK', + '6fd9': 'EF_EHPLMN', + '6fda': 'EF_GBANL', + '6fdb': 'EF_EHPLMNPI', + '6fdc': 'EF_LRPLMNSI', + '6fdd': 'EF_NAFKCA', + '6fde': 'EF_SPNI', + '6fdf': 'EF_PNNI', + '6fe2': 'EF_NCP-IP', + '6fe3': 'EF_EPSLOCI', + '6fe4': 'EF_EPSNSC', + '6fe6': 'EF_UFC', + '6fe7': 'EF_UICCIARI', + '6fe8': 'EF_NASCONFIG', + '6fe9': 'EF_CERT', + '6fea': 'EF_eFRNid', + '6feb': 'EF_SCCmax', + '6fec': 'EF_PWS', + '6fed': 'EF_FDNURI', + '6fee': 'EF_BDNURI', + '6fef': 'EF_SDNURI', + '6ff0': 'EF_IAL', + '6ff1': 'EF_IPS', + '6ff2': 'EF_IPD', + '6ff3': 'EF_ePDGId', + '6ff4': 'EF_ePDGSelection', + '6ff5': 'EF_ePDGIdEm', + '6ff6': 'EF_ePDGSelectionEm', + '6ff7': 'EF_FromPreferred', + '6ff8': 'EF_IMSConfigData', + '6ff9': 'EF_3GPPPSDATAOFF', + '6ffa': 'EF_3GPPPSDATAOFFservicelist', + '6ffb': 'EF_TVCONFIG', + '6ffc': 'EF_XCAPConfigData', + '6ffd': 'EF_EARFCNList', + '6ffe': 'EF_MuDMiDConfigData', + }, + '7f10': { # DF_TELECOM (TS 51.011 10.5; TS 31.102 4.5) + '5f3a': 'DF_PHONEBOOK', + '5f3b': 'DF_MULTIMEDIA', + '5f3c': 'DF_MMSS', + '5f3d': 'DF_MCS', + '5f3e': 'DF_V2X', + '5f3f': 'DF_A2X', + '5f50': 'DF_GRAPHICS', + '6f06': 'EF_ARR', + '6f3a': 'EF_ADN', + '6f3b': 'EF_FDN', + '6f3c': 'EF_SMS', + '6f3d': 'EF_CCP', + '6f40': 'EF_MSISDN', + '6f42': 'EF_SMSP', + '6f43': 'EF_SMSS', + '6f44': 'EF_LND', + '6f47': 'EF_SMSR', + '6f49': 'EF_SDN', + '6f4a': 'EF_EXT1', + '6f4b': 'EF_EXT2', + '6f4c': 'EF_EXT3', + '6f4d': 'EF_BDN', + '6f4e': 'EF_EXT4', + '6f4f': 'EF_ECCP', + '6f53': 'EF_RMA', + '6f54': 'EF_SUME', + '6f58': 'EF_CMI', + '6fe0': 'EF_ICE_DN', + '6fe1': 'EF_ICE_FF', + '6fe5': 'EF_PSISMSC', + }, + '7f20': { # DF_GSM (TS 51.011 10.3) + '5f30': 'DF_IRIDIUM', + '5f31': 'DF_GLOBALSTAR', + '5f32': 'DF_ICO', + '5f33': 'DF_ACES', + '5f3c': 'DF_MexE', + '5f60': 'DF_CTS', + '5f70': 'DF_SoLSA', + '6f05': 'EF_LP', + '6f07': 'EF_IMSI', + '6f20': 'EF_Kc', + '6f2c': 'EF_DCKDe-personalizationControlKeys', + '6f30': 'EF_PLMNsel', + '6f31': 'EF_HPPLMN', + '6f32': 'EF_CNL', + '6f37': 'EF_ACMmax', + '6f38': 'EF_SST', + '6f39': 'EF_ACM', + '6f3e': 'EF_GID1', + '6f3f': 'EF_GID2', + '6f41': 'EF_PUCT', + '6f45': 'EF_CBMI', + '6f46': 'EF_SPN', + '6f48': 'EF_CBMID', + '6f50': 'EF_CBMIR', + '6f51': 'EF_NIA', + '6f52': 'EF_KcGPRS', + '6f53': 'EF_LOCIGPRS', + '6f54': 'EF_SUME', + '6f60': 'EF_PLMNwAcT', + '6f61': 'EF_OPLMNwAcT', + '6f62': 'EF_HPLMNwAcT', + '6f63': 'EF_CPBCCH', + '6f64': 'EF_InvScan', + '6f74': 'EF_BCCH', + '6f78': 'EF_ACC', + '6f7b': 'EF_FPLMN', + '6f7e': 'EF_LOCI', + '6fad': 'EF_AD', + '6fae': 'EF_Phase', + '6fb1': 'EF_VGCS', + '6fb2': 'EF_VGCSS', + '6fb3': 'EF_VBS', + '6fb4': 'EF_VBSS', + '6fb5': 'EF_eMLPP', + '6fb6': 'EF_AAeM', + '6fb7': 'EF_ECC', + '6fc5': 'EF_PNN', + '6fc6': 'EF_OPL', + '6fc7': 'EF_MBDN', + '6fc8': 'EF_EXT6', + '6fc9': 'EF_MBI', + '6fca': 'EF_MWIS', + '6fcb': 'EF_CFIS', + '6fcc': 'EF_EXT7', + '6fcd': 'EF_SPDI', + '6fce': 'EF_MMSN', + '6fcf': 'EF_EXT8', + '6fd0': 'EF_MMSICP', + '6fd1': 'EF_MMSUP', + '6fd2': 'EF_MMSUCP', + }, + '5f3a': { # DF_PHONEBOOK (TS 31.102 4.4.2/4.6.2; EF_PBR assigns its files) + '4f22': 'EF_PSC', + '4f23': 'EF_CC', + '4f24': 'EF_PUID', + '4f30': 'EF_PBR', + }, + '5f3b': { # DF_GSM_ACCESS (TS 31.102 4.4.3) + DF_MULTIMEDIA (4.6.3) + '4f20': 'EF_Kc', + '4f47': 'EF_MML', + '4f48': 'EF_MMDF', + '4f52': 'EF_KcGPRS', + '4f63': 'EF_CPBCCH', + '4f64': 'EF_InvScan', + }, + '5f3c': { # DF_MexE (TS 31.102 4.4.4; TS 51.011 10.4.2) + '4f40': 'EF_MexE-ST', + '4f41': 'EF_ORPK', + '4f42': 'EF_ARPK', + '4f43': 'EF_TPRPK', + }, + '5f3d': { # DF_MCS (TS 31.102 4.6.4) + '4f01': 'EF_MST', + '4f02': 'EF_MCS_CONFIG', + }, + '5f3e': { # DF_V2X (TS 31.102 4.6.5) + '4f01': 'EF_VST', + '4f02': 'EF_V2X_CONFIG', + '4f03': 'EF_V2XP_PC5', + '4f04': 'EF_V2XP_Uu', + }, + '5f3f': { # DF_A2X (TS 31.102 4.6.6) + '4f01': 'EF_AST', + '4f02': 'EF_A2X_CONFIG', + '4f03': 'EF_A2XP_PC5', + '4f04': 'EF_A2X_DDAAP_PC5', + '4f05': 'EF_A2X_DC2P_PC5', + '4f06': 'EF_A2XP_Uu', + }, + '5f40': { # DF_WLAN (TS 31.102 4.4.5) + '4f41': 'EF_Pseudo', + '4f42': 'EF_UPLMNWLAN', + '4f43': 'EF_OPLMNWLAN', + '4f44': 'EF_UWSIDL', + '4f45': 'EF_OWSIDL', + '4f46': 'EF_WRI', + '4f47': 'EF_HWSIDL', + '4f48': 'EF_WEHPLMNPI', + '4f49': 'EF_WHPI', + '4f4a': 'EF_WLRPLMN', + '4f4b': 'EF_HPLMNDAI', + }, + '5f50': { # DF_HNB (TS 31.102 4.4.6) + DF_GRAPHICS (4.6.1/TS 51.011 10.6.1) + '4f20': 'EF_IMG', + '4f21': 'EF_ICE_graphics', + '4f81': 'EF_ACSGL', + '4f82': 'EF_CSGT', + '4f83': 'EF_HNBN', + '4f84': 'EF_OCSGL', + '4f85': 'EF_OCSGT', + '4f86': 'EF_OHNBN', + }, + '5f70': { # DF_SoLSA (TS 31.102 4.4.1; TS 51.011 10.4.1) + '4f30': 'EF_SAI', + '4f31': 'EF_SLL', + }, + '5f90': { # DF_ProSe (TS 31.102 4.4.8) + '4f01': 'EF_PROSE_MON', + '4f02': 'EF_PROSE_ANN', + '4f03': 'EF_PROSEFUNC', + '4f04': 'EF_PROSE_RADIO_COM', + '4f05': 'EF_PROSE_RADIO_MON', + '4f06': 'EF_PROSE_RADIO_ANN', + '4f07': 'EF_PROSE_POLICY', + '4f08': 'EF_PROSE_PLMN', + '4f09': 'EF_PROSE_GC', + '4f10': 'EF_PST', + '4f11': 'EF_PROSE_UIRC', + '4f12': 'EF_PROSE_GM_DISCOVERY', + '4f13': 'EF_PROSE_RELAY', + '4f14': 'EF_PROSE_RELAY_DISCOVERY', + }, + '5fa0': { # DF_ACDC (TS 31.102 4.4.9) + '4f01': 'EF_ACDC_LIST', + }, + '5fc0': { # DF_5GS (TS 31.102 4.4.11) + '4f01': 'EF_5GS3GPPLOCI', + '4f02': 'EF_5GSN3GPPLOCI', + '4f03': 'EF_5GS3GPPNSC', + '4f04': 'EF_5GSN3GPPNSC', + '4f05': 'EF_5GAUTHKEYS', + '4f06': 'EF_UAC_AIC', + '4f07': 'EF_SUCI_Calc_Info', + '4f08': 'EF_OPL5G', + '4f09': 'EF_SUPI_NAI', + '4f0a': 'EF_Routing_Indicator', + '4f0b': 'EF_URSP', + '4f0c': 'EF_TN3GPPSNN', + '4f0d': 'EF_CAG', + '4f0e': 'EF_SOR-CMCI', + '4f0f': 'EF_DRI', + '4f10': 'EF_5GSEDRX', + '4f11': 'EF_5GNSWO_CONF', + '4f15': 'EF_MCHPPLMN', + '4f16': 'EF_KAUSF_DERIVATION', + }, + '5fe0': { # DF_SNPN (TS 31.102 4.4.12) + '4f01': 'EF_PWS_SNPN', + '4f02': 'EF_NID', + }, + '5ff0': { # DF_5G_ProSe (TS 31.102 4.4.13) + '4f01': 'EF_5G_PROSE_ST', + '4f02': 'EF_5G_PROSE_DD', + '4f03': 'EF_5G_PROSE_DC', + '4f04': 'EF_5G_PROSE_U2NRU', + '4f05': 'EF_5G_PROSE_RU', + '4f06': 'EF_5G_PROSE_UIR', + '4f07': 'EF_5G_PROSE_U2URU', + '4f08': 'EF_5G_PROSE_EU', + }, + '5ff1': { # DF_5MBSUECONFIG (TS 31.102 4.4.14) + '4f01': 'EF_5MBSUECONFIG', + }, + 'aid:a0000000871004': { # ADF ISIM, selected by AID (TS 31.103 4.2) + '6f02': 'EF_IMPI', + '6f03': 'EF_DOMAIN', + '6f04': 'EF_IMPU', + '6f06': 'EF_ARR', + '6f07': 'EF_IST', + '6f09': 'EF_P-CSCF', + '6f0a': 'EF_AC_GBAUAPI', + '6f0b': 'EF_IMSDCI', + '6f3c': 'EF_SMS', + '6f42': 'EF_SMSP', + '6f43': 'EF_SMSS', + '6f47': 'EF_SMSR', + '6fad': 'EF_AD', + '6fd5': 'EF_GBABP', + '6fd7': 'EF_GBANL', + '6fdd': 'EF_NAFKCA', + '6fe5': 'EF_PSISMSC', + '6fe7': 'EF_UICCIARI', + '6ff7': 'EF_FromPreferred', + '6ff8': 'EF_IMSConfigData', + '6ffa': 'EF_WebRTCURI', + '6ffc': 'EF_XCAPConfigData', + '6ffe': 'EF_MuDMiDConfigData', + }, +} + +# Short File Identifier (SFI) -> FID, scoped to the containing DF +# (TS 102 221 8.3: an SFI is only unique within a DF). Sourced from +# the normative SFI annexes listed per DF; the ISIM annex does not +# exist, so its table is built from the SFI row of each file clause. +SFI_BY_DF = { + '3f00': { # MF (TS 102 221 Annex H.1) + 0x02: '2fe2', # EF_ICCID + 0x05: '2f05', # EF_PL + 0x06: '2f06', # EF_ARR + 0x07: '2f07', # EF_ENV-CLASSES + 0x08: '2f08', # EF_UMPC + 0x1e: '2f00', # EF_DIR + }, + '7fff': { # ADF USIM (TS 31.102 Annex H.1) + 0x01: '6fb7', # EF_ECC + 0x02: '6f05', # EF_LI + 0x03: '6fad', # EF_AD + 0x04: '6f38', # EF_UST + 0x05: '6f56', # EF_EST + 0x06: '6f78', # EF_ACC + 0x07: '6f07', # EF_IMSI + 0x08: '6f08', # EF_Keys + 0x09: '6f09', # EF_KeysPS + 0x0a: '6f60', # EF_PLMNwAcT + 0x0b: '6f7e', # EF_LOCI + 0x0c: '6f73', # EF_PSLOCI + 0x0d: '6f7b', # EF_FPLMN + 0x0e: '6f48', # EF_CBMID + 0x0f: '6f5b', # EF_START-HFN + 0x10: '6f5c', # EF_THRESHOLD + 0x11: '6f61', # EF_OPLMNwAcT + 0x12: '6f31', # EF_HPPLMN + 0x13: '6f62', # EF_HPLMNwAcT + 0x14: '6f80', # EF_ICI + 0x15: '6f81', # EF_OCI + 0x16: '6f4f', # EF_CCP2 + 0x17: '6f06', # EF_ARR + 0x18: '6fe4', # EF_EPSNSC + 0x19: '6fc5', # EF_PNN + 0x1a: '6fc6', # EF_OPL + 0x1b: '6fcd', # EF_SPDI + 0x1c: '6f39', # EF_ACM + 0x1d: '6fd9', # EF_EHPLMN + 0x1e: '6fe3', # EF_EPSLOCI + }, + '5f3b': { # DF_GSM_ACCESS (TS 31.102 Annex H.2) + 0x01: '4f20', # EF_Kc + 0x02: '4f52', # EF_KcGPRS + }, + '5f3d': { # DF_MCS (TS 31.102 Annex H.7) + 0x01: '4f01', # EF_MST + 0x02: '4f02', # EF_MCS_CONFIG + }, + '5f3e': { # DF_V2X (TS 31.102 Annex H.8) + 0x01: '4f01', # EF_VST + 0x02: '4f02', # EF_V2X_CONFIG + 0x06: '4f03', # EF_V2XP_PC5 + 0x07: '4f04', # EF_V2XP_Uu + }, + '5f3f': { # DF_A2X (TS 31.102 Annex H.12) + 0x01: '4f01', # EF_AST + 0x02: '4f02', # EF_A2X_CONFIG + 0x03: '4f03', # EF_A2XP_PC5 + 0x04: '4f04', # EF_A2X_DDAAP_PC5 + 0x05: '4f05', # EF_A2X_DC2P_PC5 + 0x06: '4f06', # EF_A2XP_Uu + }, + '5f40': { # DF_WLAN (TS 31.102 Annex H.3) + 0x01: '4f41', # EF_Pseudo + 0x02: '4f42', # EF_UPLMNWLAN + 0x03: '4f43', # EF_OPLMNWLAN + 0x04: '4f44', # EF_UWSIDL + 0x05: '4f45', # EF_OWSIDL + 0x06: '4f46', # EF_WRI + 0x07: '4f47', # EF_HWSIDL + 0x08: '4f48', # EF_WEHPLMNPI + 0x09: '4f49', # EF_WHPI + 0x0a: '4f4a', # EF_WLRPLMN + 0x0b: '4f4b', # EF_HPLMNDAI + }, + '5f50': { # DF_HNB (TS 31.102 Annex H.4) + 0x01: '4f81', # EF_ACSGL + 0x02: '4f82', # EF_CSGT + 0x03: '4f83', # EF_HNBN + 0x04: '4f84', # EF_OCSGL + 0x05: '4f85', # EF_OCSGT + 0x06: '4f86', # EF_OHNBN + }, + '5f90': { # DF_ProSe (TS 31.102 Annex H.5) + 0x01: '4f01', # EF_PROSE_MON + 0x02: '4f02', # EF_PROSE_ANN + 0x03: '4f03', # EF_PROSEFUNC + 0x04: '4f04', # EF_PROSE_RADIO_COM + 0x05: '4f05', # EF_PROSE_RADIO_MON + 0x06: '4f06', # EF_PROSE_RADIO_ANN + 0x07: '4f07', # EF_PROSE_POLICY + 0x08: '4f08', # EF_PROSE_PLMN + 0x09: '4f09', # EF_PROSE_GC + 0x10: '4f10', # EF_PST + 0x11: '4f11', # EF_PROSE_UIRC + 0x12: '4f12', # EF_PROSE_GM_DISCOVERY + 0x13: '4f13', # EF_PROSE_RELAY + 0x14: '4f14', # EF_PROSE_RELAY_DISCOVERY + }, + '5fa0': { # DF_ACDC (TS 31.102 Annex H.6) + 0x01: '4f01', # EF_ACDC_LIST + }, + '5fc0': { # DF_5GS (TS 31.102 Annex H.9) + 0x01: '4f01', # EF_5GS3GPPLOCI + 0x02: '4f02', # EF_5GSN3GPPLOCI + 0x03: '4f03', # EF_5GS3GPPNSC + 0x04: '4f04', # EF_5GSN3GPPNSC + 0x05: '4f05', # EF_5GAUTHKEYS + 0x06: '4f06', # EF_UAC_AIC + 0x07: '4f07', # EF_SUCI_Calc_Info + 0x08: '4f08', # EF_OPL5G + 0x09: '4f09', # EF_SUPI_NAI + 0x0a: '4f0a', # EF_Routing_Indicator + 0x0b: '4f0b', # EF_URSP + 0x0c: '4f0c', # EF_TN3GPPSNN + 0x0d: '4f0d', # EF_CAG + 0x0e: '4f0e', # EF_SOR-CMCI + 0x0f: '4f0f', # EF_DRI + 0x10: '4f10', # EF_5GSEDRX + 0x11: '4f11', # EF_5GNSWO_CONF + 0x15: '4f15', # EF_MCHPPLMN + 0x16: '4f16', # EF_KAUSF_DERIVATION + }, + '5fe0': { # DF_SNPN (TS 31.102 Annex H.11) + 0x01: '4f01', # EF_PWS_SNPN + 0x02: '4f02', # EF_NID + }, + '5ff0': { # DF_5G_ProSe (TS 31.102 Annex H.10) + 0x01: '4f01', # EF_5G_PROSE_ST + 0x02: '4f02', # EF_5G_PROSE_DD + 0x03: '4f03', # EF_5G_PROSE_DC + 0x04: '4f04', # EF_5G_PROSE_U2NRU + 0x05: '4f05', # EF_5G_PROSE_RU + 0x06: '4f06', # EF_5G_PROSE_UIR + 0x07: '4f07', # EF_5G_PROSE_U2URU + 0x08: '4f08', # EF_5G_PROSE_EU + }, + 'aid:a0000000871004': { # ADF ISIM (TS 31.103: the SFI row of each file) + 0x02: '6f02', # EF_IMPI + 0x03: '6fad', # EF_AD + 0x04: '6f04', # EF_IMPU + 0x05: '6f03', # EF_DOMAIN + 0x06: '6f06', # EF_ARR + 0x07: '6f07', # EF_IST + }, +} diff --git a/simtrace_analyser/filetrack.py b/simtrace_analyser/filetrack.py index 0c811f4..11fb385 100644 --- a/simtrace_analyser/filetrack.py +++ b/simtrace_analyser/filetrack.py @@ -39,8 +39,8 @@ keeps counting). """ from .database import _SUCCESS_SW1 -from .decode import (DF_CHILDREN, KNOWN_FIDS, _is_df_fid, select_target_fid, - selected_df_fid, sfi_table) +from .decode import (DF_CHILDREN, KNOWN_FIDS, _file_name, _is_df_fid, + select_target_fid, selected_df_fid, sfi_table) IMAGE_CAP = 64 * 1024 # observed bytes kept per file EVENTS_CAP = 1000 # change events kept per file @@ -266,8 +266,10 @@ class FileTracker: key = (df, fid) entry = self.files.get(key) if entry is None: + # A file first seen via its FCP (or an operation whose decode had + # no name) still gets its DF-scoped dictionary name. entry = _FileEntry(key, df, fid, - name or (fid.upper() if fid else '?'), + (name or _file_name(fid, df)) if fid else '?', image_cap=self._image_cap) self.files[key] = entry elif name: @@ -321,6 +323,8 @@ class FileTracker: provisional.key = (df, fid) provisional.fid = fid provisional.sfi = sfi + if provisional.name == f'SFI:{sfi}': + provisional.name = _file_name(fid, df) self.files[(df, fid)] = provisional target = provisional else: diff --git a/simtrace_analyser/version.py b/simtrace_analyser/version.py index d9645f0..8b1e131 100644 --- a/simtrace_analyser/version.py +++ b/simtrace_analyser/version.py @@ -1,3 +1,3 @@ """Single source of truth for the SIMtrace Analyser version.""" -__version__ = '2.4.2' +__version__ = '2.5.0' diff --git a/tests/data/spec_files.json b/tests/data/spec_files.json new file mode 100644 index 0000000..7deadb3 --- /dev/null +++ b/tests/data/spec_files.json @@ -0,0 +1,571 @@ +{ + "comment": "File/SFI tables extracted from the pinned specs by tools/extract_spec_tables.py; asserted by tests/test_decode.py. Regenerate when a spec revision is pinned.", + "df_fids": { + "5f30": "DF_IRIDIUM", + "5f31": "DF_GLOBALSTAR", + "5f32": "DF_ICO", + "5f33": "DF_ACES", + "5f3a": "DF_PHONEBOOK", + "5f3b": "DF_GSM_ACCESS", + "5f3c": "DF_MexE", + "5f3d": "DF_MCS", + "5f3e": "DF_V2X", + "5f3f": "DF_A2X", + "5f40": "DF_WLAN", + "5f50": "DF_HNB", + "5f60": "DF_CTS", + "5f70": "DF_SoLSA", + "5f80": "DF_BCAST", + "5f90": "DF_ProSe", + "5fa0": "DF_ACDC", + "5fb0": "DF_TV", + "5fc0": "DF_5GS", + "5fd0": "DF_SAIP", + "5fe0": "DF_SNPN", + "5ff0": "DF_5G_ProSe", + "5ff1": "DF_5MBSUECONFIG", + "7f10": "DF_TELECOM", + "7f20": "DF_GSM" + }, + "files": { + "3f00": { + "2f00": "EF_DIR", + "2f05": "EF_PL", + "2f06": "EF_ARR", + "2f07": "EF_ENV-CLASSES", + "2f08": "EF_UMPC", + "2fe2": "EF_ICCID" + }, + "5f3a": { + "4f22": "EF_PSC", + "4f23": "EF_CC", + "4f24": "EF_PUID", + "4f30": "EF_PBR" + }, + "5f3b": { + "4f20": "EF_Kc", + "4f47": "EF_MML", + "4f48": "EF_MMDF", + "4f52": "EF_KcGPRS", + "4f63": "EF_CPBCCH", + "4f64": "EF_InvScan" + }, + "5f3c": { + "4f40": "EF_MexE-ST", + "4f41": "EF_ORPK", + "4f42": "EF_ARPK", + "4f43": "EF_TPRPK" + }, + "5f3d": { + "4f01": "EF_MST", + "4f02": "EF_MCS_CONFIG" + }, + "5f3e": { + "4f01": "EF_VST", + "4f02": "EF_V2X_CONFIG", + "4f03": "EF_V2XP_PC5", + "4f04": "EF_V2XP_Uu" + }, + "5f3f": { + "4f01": "EF_AST", + "4f02": "EF_A2X_CONFIG", + "4f03": "EF_A2XP_PC5", + "4f04": "EF_A2X_DDAAP_PC5", + "4f05": "EF_A2X_DC2P_PC5", + "4f06": "EF_A2XP_Uu" + }, + "5f40": { + "4f41": "EF_Pseudo", + "4f42": "EF_UPLMNWLAN", + "4f43": "EF_OPLMNWLAN", + "4f44": "EF_UWSIDL", + "4f45": "EF_OWSIDL", + "4f46": "EF_WRI", + "4f47": "EF_HWSIDL", + "4f48": "EF_WEHPLMNPI", + "4f49": "EF_WHPI", + "4f4a": "EF_WLRPLMN", + "4f4b": "EF_HPLMNDAI" + }, + "5f50": { + "4f20": "EF_IMG", + "4f21": "EF_ICE_graphics", + "4f81": "EF_ACSGL", + "4f82": "EF_CSGT", + "4f83": "EF_HNBN", + "4f84": "EF_OCSGL", + "4f85": "EF_OCSGT", + "4f86": "EF_OHNBN" + }, + "5f70": { + "4f30": "EF_SAI", + "4f31": "EF_SLL" + }, + "5f90": { + "4f01": "EF_PROSE_MON", + "4f02": "EF_PROSE_ANN", + "4f03": "EF_PROSEFUNC", + "4f04": "EF_PROSE_RADIO_COM", + "4f05": "EF_PROSE_RADIO_MON", + "4f06": "EF_PROSE_RADIO_ANN", + "4f07": "EF_PROSE_POLICY", + "4f08": "EF_PROSE_PLMN", + "4f09": "EF_PROSE_GC", + "4f10": "EF_PST", + "4f11": "EF_PROSE_UIRC", + "4f12": "EF_PROSE_GM_DISCOVERY", + "4f13": "EF_PROSE_RELAY", + "4f14": "EF_PROSE_RELAY_DISCOVERY" + }, + "5fa0": { + "4f01": "EF_ACDC_LIST" + }, + "5fc0": { + "4f01": "EF_5GS3GPPLOCI", + "4f02": "EF_5GSN3GPPLOCI", + "4f03": "EF_5GS3GPPNSC", + "4f04": "EF_5GSN3GPPNSC", + "4f05": "EF_5GAUTHKEYS", + "4f06": "EF_UAC_AIC", + "4f07": "EF_SUCI_Calc_Info", + "4f08": "EF_OPL5G", + "4f09": "EF_SUPI_NAI", + "4f0a": "EF_Routing_Indicator", + "4f0b": "EF_URSP", + "4f0c": "EF_TN3GPPSNN", + "4f0d": "EF_CAG", + "4f0e": "EF_SOR-CMCI", + "4f0f": "EF_DRI", + "4f10": "EF_5GSEDRX", + "4f11": "EF_5GNSWO_CONF", + "4f15": "EF_MCHPPLMN", + "4f16": "EF_KAUSF_DERIVATION" + }, + "5fe0": { + "4f01": "EF_PWS_SNPN", + "4f02": "EF_NID" + }, + "5ff0": { + "4f01": "EF_5G_PROSE_ST", + "4f02": "EF_5G_PROSE_DD", + "4f03": "EF_5G_PROSE_DC", + "4f04": "EF_5G_PROSE_U2NRU", + "4f05": "EF_5G_PROSE_RU", + "4f06": "EF_5G_PROSE_UIR", + "4f07": "EF_5G_PROSE_U2URU", + "4f08": "EF_5G_PROSE_EU" + }, + "5ff1": { + "4f01": "EF_5MBSUECONFIG" + }, + "7f10": { + "5f3a": "DF_PHONEBOOK", + "5f3b": "DF_MULTIMEDIA", + "5f3c": "DF_MMSS", + "5f3d": "DF_MCS", + "5f3e": "DF_V2X", + "5f3f": "DF_A2X", + "5f50": "DF_GRAPHICS", + "6f06": "EF_ARR", + "6f3a": "EF_ADN", + "6f3b": "EF_FDN", + "6f3c": "EF_SMS", + "6f3d": "EF_CCP", + "6f40": "EF_MSISDN", + "6f42": "EF_SMSP", + "6f43": "EF_SMSS", + "6f44": "EF_LND", + "6f47": "EF_SMSR", + "6f49": "EF_SDN", + "6f4a": "EF_EXT1", + "6f4b": "EF_EXT2", + "6f4c": "EF_EXT3", + "6f4d": "EF_BDN", + "6f4e": "EF_EXT4", + "6f4f": "EF_ECCP", + "6f53": "EF_RMA", + "6f54": "EF_SUME", + "6f58": "EF_CMI", + "6fe0": "EF_ICE_DN", + "6fe1": "EF_ICE_FF", + "6fe5": "EF_PSISMSC" + }, + "7f20": { + "5f30": "DF_IRIDIUM", + "5f31": "DF_GLOBALSTAR", + "5f32": "DF_ICO", + "5f33": "DF_ACES", + "5f3c": "DF_MexE", + "5f60": "DF_CTS", + "5f70": "DF_SoLSA", + "6f05": "EF_LP", + "6f07": "EF_IMSI", + "6f20": "EF_Kc", + "6f2c": "EF_DCKDe-personalizationControlKeys", + "6f30": "EF_PLMNsel", + "6f31": "EF_HPPLMN", + "6f32": "EF_CNL", + "6f37": "EF_ACMmax", + "6f38": "EF_SST", + "6f39": "EF_ACM", + "6f3e": "EF_GID1", + "6f3f": "EF_GID2", + "6f41": "EF_PUCT", + "6f45": "EF_CBMI", + "6f46": "EF_SPN", + "6f48": "EF_CBMID", + "6f50": "EF_CBMIR", + "6f51": "EF_NIA", + "6f52": "EF_KcGPRS", + "6f53": "EF_LOCIGPRS", + "6f54": "EF_SUME", + "6f60": "EF_PLMNwAcT", + "6f61": "EF_OPLMNwAcT", + "6f62": "EF_HPLMNwAcT", + "6f63": "EF_CPBCCH", + "6f64": "EF_InvScan", + "6f74": "EF_BCCH", + "6f78": "EF_ACC", + "6f7b": "EF_FPLMN", + "6f7e": "EF_LOCI", + "6fad": "EF_AD", + "6fae": "EF_Phase", + "6fb1": "EF_VGCS", + "6fb2": "EF_VGCSS", + "6fb3": "EF_VBS", + "6fb4": "EF_VBSS", + "6fb5": "EF_eMLPP", + "6fb6": "EF_AAeM", + "6fb7": "EF_ECC", + "6fc5": "EF_PNN", + "6fc6": "EF_OPL", + "6fc7": "EF_MBDN", + "6fc8": "EF_EXT6", + "6fc9": "EF_MBI", + "6fca": "EF_MWIS", + "6fcb": "EF_CFIS", + "6fcc": "EF_EXT7", + "6fcd": "EF_SPDI", + "6fce": "EF_MMSN", + "6fcf": "EF_EXT8", + "6fd0": "EF_MMSICP", + "6fd1": "EF_MMSUP", + "6fd2": "EF_MMSUCP" + }, + "7fff": { + "5f3a": "DF_PHONEBOOK", + "5f3b": "DF_GSM_ACCESS", + "5f3c": "DF_MexE", + "5f40": "DF_WLAN", + "5f50": "DF_HNB", + "5f70": "DF_SoLSA", + "5f80": "DF_BCAST", + "5f90": "DF_ProSe", + "5fa0": "DF_ACDC", + "5fb0": "DF_TV", + "5fc0": "DF_5GS", + "5fd0": "DF_SAIP", + "5fe0": "DF_SNPN", + "5ff0": "DF_5G_ProSe", + "5ff1": "DF_5MBSUECONFIG", + "6f01": "EF_eAKA", + "6f02": "EF_OCST", + "6f05": "EF_LI", + "6f06": "EF_ARR", + "6f07": "EF_IMSI", + "6f08": "EF_Keys", + "6f09": "EF_KeysPS", + "6f0a": "EF_AC_GBAUAPI", + "6f0b": "EF_IMSDCI", + "6f2c": "EF_DCK", + "6f31": "EF_HPPLMN", + "6f32": "EF_CNL", + "6f37": "EF_ACMmax", + "6f38": "EF_UST", + "6f39": "EF_ACM", + "6f3b": "EF_FDN", + "6f3c": "EF_SMS", + "6f3e": "EF_GID1", + "6f3f": "EF_GID2", + "6f40": "EF_MSISDN", + "6f41": "EF_PUCT", + "6f42": "EF_SMSP", + "6f43": "EF_SMSS", + "6f45": "EF_CBMI", + "6f46": "EF_SPN", + "6f47": "EF_SMSR", + "6f48": "EF_CBMID", + "6f49": "EF_SDN", + "6f4b": "EF_EXT2", + "6f4c": "EF_EXT3", + "6f4d": "EF_BDN", + "6f4e": "EF_EXT5", + "6f4f": "EF_CCP2", + "6f50": "EF_CBMIR", + "6f55": "EF_EXT4", + "6f56": "EF_EST", + "6f57": "EF_ACL", + "6f58": "EF_CMI", + "6f5b": "EF_START-HFN", + "6f5c": "EF_THRESHOLD", + "6f60": "EF_PLMNwAcT", + "6f61": "EF_OPLMNwAcT", + "6f62": "EF_HPLMNwAcT", + "6f73": "EF_PSLOCI", + "6f78": "EF_ACC", + "6f7b": "EF_FPLMN", + "6f7e": "EF_LOCI", + "6f80": "EF_ICI", + "6f81": "EF_OCI", + "6f82": "EF_ICT", + "6f83": "EF_OCT", + "6fad": "EF_AD", + "6fb1": "EF_VGCS", + "6fb2": "EF_VGCSS", + "6fb3": "EF_VBS", + "6fb4": "EF_VBSS", + "6fb5": "EF_eMLPP", + "6fb6": "EF_AAeM", + "6fb7": "EF_ECC", + "6fc3": "EF_Hiddenkey", + "6fc4": "EF_NETPAR", + "6fc5": "EF_PNN", + "6fc6": "EF_OPL", + "6fc7": "EF_MBDN", + "6fc8": "EF_EXT6", + "6fc9": "EF_MBI", + "6fca": "EF_MWIS", + "6fcb": "EF_CFIS", + "6fcc": "EF_EXT7", + "6fcd": "EF_SPDI", + "6fce": "EF_MMSN", + "6fcf": "EF_EXT8", + "6fd0": "EF_MMSICP", + "6fd1": "EF_MMSUP", + "6fd2": "EF_MMSUCP", + "6fd3": "EF_NIA", + "6fd4": "EF_VGCSCA", + "6fd5": "EF_VBSCA", + "6fd6": "EF_GBABP", + "6fd7": "EF_MSK", + "6fd8": "EF_MUK", + "6fd9": "EF_EHPLMN", + "6fda": "EF_GBANL", + "6fdb": "EF_EHPLMNPI", + "6fdc": "EF_LRPLMNSI", + "6fdd": "EF_NAFKCA", + "6fde": "EF_SPNI", + "6fdf": "EF_PNNI", + "6fe2": "EF_NCP-IP", + "6fe3": "EF_EPSLOCI", + "6fe4": "EF_EPSNSC", + "6fe6": "EF_UFC", + "6fe7": "EF_UICCIARI", + "6fe8": "EF_NASCONFIG", + "6fe9": "EF_CERT", + "6fea": "EF_eFRNid", + "6feb": "EF_SCCmax", + "6fec": "EF_PWS", + "6fed": "EF_FDNURI", + "6fee": "EF_BDNURI", + "6fef": "EF_SDNURI", + "6ff0": "EF_IAL", + "6ff1": "EF_IPS", + "6ff2": "EF_IPD", + "6ff3": "EF_ePDGId", + "6ff4": "EF_ePDGSelection", + "6ff5": "EF_ePDGIdEm", + "6ff6": "EF_ePDGSelectionEm", + "6ff7": "EF_FromPreferred", + "6ff8": "EF_IMSConfigData", + "6ff9": "EF_3GPPPSDATAOFF", + "6ffa": "EF_3GPPPSDATAOFFservicelist", + "6ffb": "EF_TVCONFIG", + "6ffc": "EF_XCAPConfigData", + "6ffd": "EF_EARFCNList", + "6ffe": "EF_MuDMiDConfigData" + }, + "aid:a0000000871004": { + "6f02": "EF_IMPI", + "6f03": "EF_DOMAIN", + "6f04": "EF_IMPU", + "6f06": "EF_ARR", + "6f07": "EF_IST", + "6f09": "EF_P-CSCF", + "6f0a": "EF_AC_GBAUAPI", + "6f0b": "EF_IMSDCI", + "6f3c": "EF_SMS", + "6f42": "EF_SMSP", + "6f43": "EF_SMSS", + "6f47": "EF_SMSR", + "6fad": "EF_AD", + "6fd5": "EF_GBABP", + "6fd7": "EF_GBANL", + "6fdd": "EF_NAFKCA", + "6fe5": "EF_PSISMSC", + "6fe7": "EF_UICCIARI", + "6ff7": "EF_FromPreferred", + "6ff8": "EF_IMSConfigData", + "6ffa": "EF_WebRTCURI", + "6ffc": "EF_XCAPConfigData", + "6ffe": "EF_MuDMiDConfigData" + } + }, + "flat_special": { + "3f00": "MF", + "6f65": "EF_RPLMNAcT" + }, + "sfi": { + "3f00": { + "02": "2fe2", + "05": "2f05", + "06": "2f06", + "07": "2f07", + "08": "2f08", + "1e": "2f00" + }, + "5f3b": { + "01": "4f20", + "02": "4f52" + }, + "5f3d": { + "01": "4f01", + "02": "4f02" + }, + "5f3e": { + "01": "4f01", + "02": "4f02", + "06": "4f03", + "07": "4f04" + }, + "5f3f": { + "01": "4f01", + "02": "4f02", + "03": "4f03", + "04": "4f04", + "05": "4f05", + "06": "4f06" + }, + "5f40": { + "01": "4f41", + "02": "4f42", + "03": "4f43", + "04": "4f44", + "05": "4f45", + "06": "4f46", + "07": "4f47", + "08": "4f48", + "09": "4f49", + "0a": "4f4a", + "0b": "4f4b" + }, + "5f50": { + "01": "4f81", + "02": "4f82", + "03": "4f83", + "04": "4f84", + "05": "4f85", + "06": "4f86" + }, + "5f90": { + "01": "4f01", + "02": "4f02", + "03": "4f03", + "04": "4f04", + "05": "4f05", + "06": "4f06", + "07": "4f07", + "08": "4f08", + "09": "4f09", + "10": "4f10", + "11": "4f11", + "12": "4f12", + "13": "4f13", + "14": "4f14" + }, + "5fa0": { + "01": "4f01" + }, + "5fc0": { + "01": "4f01", + "02": "4f02", + "03": "4f03", + "04": "4f04", + "05": "4f05", + "06": "4f06", + "07": "4f07", + "08": "4f08", + "09": "4f09", + "0a": "4f0a", + "0b": "4f0b", + "0c": "4f0c", + "0d": "4f0d", + "0e": "4f0e", + "0f": "4f0f", + "10": "4f10", + "11": "4f11", + "15": "4f15", + "16": "4f16" + }, + "5fe0": { + "01": "4f01", + "02": "4f02" + }, + "5ff0": { + "01": "4f01", + "02": "4f02", + "03": "4f03", + "04": "4f04", + "05": "4f05", + "06": "4f06", + "07": "4f07", + "08": "4f08" + }, + "7fff": { + "01": "6fb7", + "02": "6f05", + "03": "6fad", + "04": "6f38", + "05": "6f56", + "06": "6f78", + "07": "6f07", + "08": "6f08", + "09": "6f09", + "0a": "6f60", + "0b": "6f7e", + "0c": "6f73", + "0d": "6f7b", + "0e": "6f48", + "0f": "6f5b", + "10": "6f5c", + "11": "6f61", + "12": "6f31", + "13": "6f62", + "14": "6f80", + "15": "6f81", + "16": "6f4f", + "17": "6f06", + "18": "6fe4", + "19": "6fc5", + "1a": "6fc6", + "1b": "6fcd", + "1c": "6f39", + "1d": "6fd9", + "1e": "6fe3" + }, + "aid:a0000000871004": { + "02": "6f02", + "03": "6fad", + "04": "6f04", + "05": "6f03", + "06": "6f06", + "07": "6f07" + } + }, + "sources": { + "ts_102_221": "ETSI TS 102 221 V18.4.0 (Annex H)", + "ts_31_102": "3GPP TS 31.102 V18.10.0 (4.2-4.6, Annex H, Annex L.6)", + "ts_31_103": "3GPP TS 31.103 V18.3.0 (4.2)", + "ts_51_011": "3GPP TS 51.011 V4.15.0 (10.1-10.6)" + } +} \ No newline at end of file diff --git a/tests/extract_spec_tables.py b/tests/extract_spec_tables.py new file mode 100644 index 0000000..8b1883b --- /dev/null +++ b/tests/extract_spec_tables.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Regenerate tests/data/spec_files.json (the pinned spec tables). + +The APDU decoder's file/SFI dictionaries (simtrace_analyser/ef_tables.py) are +extracted from the pinned specs in ``projects/docs/uicc``. This tool parses +the pdftotext output of those PDFs and writes the fixture that +tests/test_decode.py asserts against, so a table edit or a spec revision can +be cross-checked mechanically. + +Usage:: + + python tests/extract_spec_tables.py # regenerate the fixture + python tests/extract_spec_tables.py --check # diff the fixture vs the tables + +The parsing rules follow the specs' own layout: file clauses carry an +``Identifier: 'XXXX'`` row (possibly with a space inside the quotes, single or +double quotes) and an optional ``SFI: 'NN'`` row; a clause that only points at +another spec's definition is resolved by its file name (e.g. "For the +structure... see EFIMSDCI in TS 31.103"); the normative SFI annexes are +parsed as tables. Everything the parser cannot attribute to the specs is +reported, never invented. +""" +import argparse +import collections +import json +import os +import re +import subprocess +import sys +import tempfile + +SPECS = { + 'ts_102_221': ('ts_102221v180400p.pdf', 'ETSI TS 102 221 V18.4.0 (Annex H)'), + 'ts_31_102': ('ts_131102v181000p.pdf', '3GPP TS 31.102 V18.10.0 (4.2-4.6, Annex H, Annex L.6)'), + 'ts_31_103': ('ts_131103v180300p.pdf', '3GPP TS 31.103 V18.3.0 (4.2)'), + 'ts_51_011': ('ts_151011v041500p.pdf', '3GPP TS 51.011 V4.15.0 (10.1-10.6)'), +} +ISIM_AID = 'aid:a0000000871004' + +DF_BY_NAME = { + 'PHONEBOOK': '5f3a', 'GSM-ACCESS': '5f3b', 'MEXE': '5f3c', + 'MULTIMEDIA': '5f3b', 'MMSS': '5f3c', 'MCS': '5f3d', 'V2X': '5f3e', + 'A2X': '5f3f', 'WLAN': '5f40', 'HNB': '5f50', 'GRAPHICS': '5f50', + 'SOLSA': '5f70', 'BCAST': '5f80', 'PROSE': '5f90', 'ACDC': '5fa0', + 'TV': '5fb0', '5GS': '5fc0', 'SAIP': '5fd0', 'SNPN': '5fe0', + '5G-PROSE': '5ff0', '5GPROSE': '5ff0', + '5MBS-UE-PRE-CONFIGURATION': '5ff1', '5MBSUECONFIG': '5ff1', + 'IRIDIUM': '5f30', 'GLOBALSTAR': '5f31', 'ICO': '5f32', 'ACES': '5f33', + 'CTS': '5f60', 'IS-41': '7f22', 'FP-CTS': '7f23', +} + +SEC = re.compile(r'^\d+(?:\.\d+)*[a-z]?\s+(EF[A-Za-z0-9_ \-]+?)(?:\s*\(.*)?$') +FID = re.compile(r"""Identifier:\s*['"]([^'"]*)['"]""") +FIDS = re.compile(r"Identifier:\s*([0-9A-Fa-f]{4})\b") +SFI = re.compile(r"SFI:\s*'?([0-9A-Fa-f]{2})'?") +FID_SENT = re.compile(r"file identifier\s*'?([0-9A-Fa-f]{4})'?", re.I) +DFSEC = re.compile(r'^\d+(?:\.\d+)*\s+Contents of [Ff]iles? at the .+? level', re.I) +SFISEC = re.compile(r'^H\.(\d+)\s+List of SFI Values at the (.+?) Level', re.I) +SFIROW = re.compile(r"^\s*'([0-9A-Fa-f]{4})\s*'?\s+'([0-9A-Fa-f]{2})'") +REF = re.compile(r'see (?:clause [0-9.]+ )?\(?(EF[A-Za-z0-9_\-]+)') + +# Curated: clauses that borrowed a FID from another spec's same-named file but +# whose file is actually issuer-assigned (TS 31.102 4.4.2: the phonebook files +# are declared in EF_PBR's records), and the case-only spelling variants. +DROP = {('ts_31_102', '5f3a', '6f3a'), ('ts_31_102', '5f3a', '6f4a')} +CANON = {'EF_AaeM': 'EF_AAeM', 'EF_MExE-ST': 'EF_MexE-ST', + 'EF_OPLMNwACT': 'EF_OPLMNwAcT'} +# TS 31.102 Annex L.6 clauses state their identifiers in the clause body. +ANNEX_L = {'6fe9': 'EF_CERT', '6fea': 'EF_eFRNid', '6feb': 'EF_SCCmax'} +MF_FILES = {'2f00': 'EF_DIR', '2f05': 'EF_PL', '2f06': 'EF_ARR', + '2f07': 'EF_ENV-CLASSES', '2f08': 'EF_UMPC', '2fe2': 'EF_ICCID'} +# The DFs listed as children of their parents by the specs themselves +# (TS 31.102 4.3/4.6, TS 51.011 10.2/10.4/10.6). +CHILD_DFS = { + '7fff': {'5f3a': 'DF_PHONEBOOK', '5f3b': 'DF_GSM_ACCESS', + '5f3c': 'DF_MexE', '5f40': 'DF_WLAN', '5f50': 'DF_HNB', + '5f70': 'DF_SoLSA', '5f80': 'DF_BCAST', '5f90': 'DF_ProSe', + '5fa0': 'DF_ACDC', '5fb0': 'DF_TV', '5fc0': 'DF_5GS', + '5fd0': 'DF_SAIP', '5fe0': 'DF_SNPN', '5ff0': 'DF_5G_ProSe', + '5ff1': 'DF_5MBSUECONFIG'}, + '7f10': {'5f3a': 'DF_PHONEBOOK', '5f3b': 'DF_MULTIMEDIA', + '5f3c': 'DF_MMSS', '5f3d': 'DF_MCS', '5f3e': 'DF_V2X', + '5f3f': 'DF_A2X', '5f50': 'DF_GRAPHICS'}, + '7f20': {'5f30': 'DF_IRIDIUM', '5f31': 'DF_GLOBALSTAR', '5f32': 'DF_ICO', + '5f33': 'DF_ACES', '5f3c': 'DF_MexE', '5f60': 'DF_CTS', + '5f70': 'DF_SoLSA'}, +} +DF_FIDS = { + '5f30': 'DF_IRIDIUM', '5f31': 'DF_GLOBALSTAR', '5f32': 'DF_ICO', + '5f33': 'DF_ACES', '5f3a': 'DF_PHONEBOOK', '5f3b': 'DF_GSM_ACCESS', + '5f3c': 'DF_MexE', '5f3d': 'DF_MCS', '5f3e': 'DF_V2X', '5f3f': 'DF_A2X', + '5f40': 'DF_WLAN', '5f50': 'DF_HNB', '5f60': 'DF_CTS', '5f70': 'DF_SoLSA', + '5f80': 'DF_BCAST', '5f90': 'DF_ProSe', '5fa0': 'DF_ACDC', + '5fb0': 'DF_TV', '5fc0': 'DF_5GS', '5fd0': 'DF_SAIP', '5fe0': 'DF_SNPN', + '5ff0': 'DF_5G_ProSe', '5ff1': 'DF_5MBSUECONFIG', '7f10': 'DF_TELECOM', + '7f20': 'DF_GSM', +} +ORDER = ['3f00', '7fff', '7f10', '7f20', '5f3a', '5f3b', '5f3c', '5f3d', + '5f3e', '5f3f', '5f40', '5f50', '5f70', '5f90', '5fa0', '5fc0', + '5fe0', '5ff0', '5ff1', 'aid:a0000000871004'] +SFI_ORDER = ['3f00', '7fff', '5f3b', '5f3d', '5f3e', '5f3f', '5f40', '5f50', + '5f90', '5fa0', '5fc0', '5fe0', '5ff0', 'aid:a0000000871004'] + + +def norm_df_name(s): + s = re.sub(r'^(DF|THE|GSM|TELECOM)[\s-]*', '', s.strip().upper(), count=1) + return s.replace(' ', '-').replace('_', '-') + + +def parse_spec(path, spec): + lines = open(path, encoding='utf-8', errors='replace').read().splitlines() + files = collections.defaultdict(dict) + sfis = collections.defaultdict(dict) + per_file_sfi = collections.defaultdict(dict) + unresolved = {} + + def is_heading(line): + line = line.strip() + if '....' in line: + return False + return bool(SEC.match(line) or DFSEC.match(line) or SFISEC.match(line)) + + cur_df = cur_sfi_df = None + for i, raw in enumerate(lines): + # pdftotext emits typographic quotes in some tables; normalize them. + ln = (raw.strip().replace('\u2018', "'").replace('\u2019', "'") + .replace('\u201c', '"').replace('\u201d', '"')) + if '....' in ln: + continue + if DFSEC.match(ln): + mm = re.search(r'at the (.+?) level', ln, re.I) + name = norm_df_name(mm.group(1)) if mm else '' + cur_df = DF_BY_NAME.get(name) + if spec in ('ts_31_102', 'ts_51_011') and ln.startswith(('4.1 ', '10.1 ')) \ + and 'MF level' in ln: + cur_df = '3f00' + mm = SFISEC.match(ln) + if mm: + name = norm_df_name(mm.group(2)) + if spec == 'ts_31_103': + cur_sfi_df = ISIM_AID + elif name in ('USIM-ADF', 'USIM-ADF-APPLICATION-DF'): + cur_sfi_df = '7fff' + else: + cur_sfi_df = DF_BY_NAME.get(name) + if cur_sfi_df is None and spec == 'ts_102_221': + cur_sfi_df = '3f00' if 'MF' in mm.group(2).upper() else None + if cur_sfi_df: + row = SFIROW.match(ln) + if row: + fid = row.group(1).lower() + prev = sfis[cur_sfi_df].get(int(row.group(2), 16)) + if prev and prev != fid: + print(f'WARN {spec}: SFI clash {cur_sfi_df} {fid} vs {prev}', + file=sys.stderr) + sfis[cur_sfi_df][int(row.group(2), 16)] = fid + m = SEC.match(ln) + if not (m and m.group(1).startswith('EF')): + continue + sec_no = ln.split()[0] + df = cur_df + if spec == 'ts_31_103': + df = ISIM_AID + elif spec == 'ts_31_102': + if sec_no.startswith('4.2'): + df = '7fff' + elif sec_no.startswith('4.1'): + df = '3f00' + elif sec_no.startswith('4.5'): + df = '7f10' + elif spec == 'ts_51_011': + if sec_no.startswith('10.3'): + df = '7f20' + elif sec_no.startswith('10.5'): + df = '7f10' + elif sec_no.startswith('10.1'): + df = '3f00' + fid = sfi = None + body = [] + for line in lines[i + 1:]: + if is_heading(line): + break + body.append(line) + if fid is None: + fm = FID.search(line) or FIDS.search(line) or FID_SENT.search(line) + if fm: + cand = re.sub(r'\s', '', fm.group(1)) + if re.fullmatch(r'[0-9A-Fa-f]{4}', cand): + fid = cand.lower() + if sfi is None: + sm = SFI.search(line) + if sm: + sfi = int(sm.group(1), 16) + name = 'EF_' + m.group(1).replace(' ', '')[2:] + if not df or 'This EF is not used' in '\n'.join(body): + continue + if fid: + prev = files[df].get(fid) + if prev and prev != name: + print(f'WARN {spec}: FID clash {df} {fid}: {prev} vs {name}', + file=sys.stderr) + files[df][fid] = name + if sfi is not None: + per_file_sfi[df][fid] = sfi + else: + unresolved[(spec, df, name)] = REF.findall('\n'.join(body)) + return files, sfis, per_file_sfi, unresolved + + +def build(docs_dir): + texts = {} + for spec, (pdf, _) in SPECS.items(): + src = os.path.join(docs_dir, pdf) + with tempfile.NamedTemporaryFile(suffix='.txt', delete=False) as tmp: + subprocess.run(['pdftotext', '-layout', src, tmp.name], check=True) + texts[spec] = tmp.name + files = collections.defaultdict(dict) + sfi = collections.defaultdict(dict) + unresolved = {} + for spec, path in texts.items(): + fl, sf, pfs, unres = parse_spec(path, spec) + unresolved.update(unres) + for df, entries in fl.items(): + for fid, name in entries.items(): + if (spec, df, fid) in DROP: + continue + name = CANON.get(name, name) + prev = files[df].get(fid) + if prev and prev != name: + raise SystemExit(f'curation needed: {df} {fid}: ' + f'{prev} vs {name} ({spec})') + files[df][fid] = name + for df, table in sf.items(): + sfi[df].update(table) + for df, table in pfs.items(): + for fid, val in table.items(): + if df.startswith('aid:'): + sfi[df].setdefault(val, fid) + # clauses that only reference another spec's file by name + by_name = collections.defaultdict(set) + for entries in files.values(): + for fid, name in entries.items(): + by_name[name].add(fid) + for (spec, df, name), refs in sorted(unresolved.items()): + cands = by_name.get(name) or set() + if not cands: + for ref in refs: + cands |= by_name.get(ref, set()) + if len(cands) == 1 and df and (spec, df, next(iter(cands))) not in DROP: + files[df][next(iter(cands))] = name + files['3f00'].update(MF_FILES) + files['7fff'].update(ANNEX_L) + for df, children in CHILD_DFS.items(): + files[df].update(children) + return files, sfi + + +def fixture(files, sfi): + return { + 'comment': ('File/SFI tables extracted from the pinned specs by ' + 'tools/extract_spec_tables.py; asserted by ' + 'tests/test_decode.py. Regenerate when a spec revision ' + 'is pinned.'), + 'sources': {spec: meta[1] for spec, meta in SPECS.items()}, + 'files': {df: dict(files[df]) for df in ORDER}, + 'sfi': {df: {f'{s:02x}': fid for s, fid in sfi[df].items()} + for df in SFI_ORDER}, + 'df_fids': DF_FIDS, + # flat-table entries that are not spec file/DF names: the MF itself + # and the legacy 6F65 (its value was withdrawn, TS 31.102 4.2.45 NOTE 5) + 'flat_special': {'3f00': 'MF', '6f65': 'EF_RPLMNAcT'}, + } + + +def check(fixture_data, repo_root): + sys.path.insert(0, repo_root) + from simtrace_analyser import ef_tables + + bad = 0 + for df, entries in fixture_data['files'].items(): + table = ef_tables.DF_CHILDREN.get(df, {}) + for fid, name in entries.items(): + if table.get(fid) != name: + print(f'files mismatch {df}/{fid}: fixture {name} table {table.get(fid)}') + bad += 1 + for df, table in fixture_data['sfi'].items(): + want = {int(s, 16): fid for s, fid in table.items()} + got = ef_tables.SFI_BY_DF.get(df, {}) + if want != got: + print(f'sfi mismatch {df}: fixture {want} table {got}') + bad += 1 + names = collections.defaultdict(set) + for entries in fixture_data['files'].values(): + for fid, name in entries.items(): + names[fid].add(name) + for fid, name in fixture_data['df_fids'].items(): + names[fid].add(name) + for fid, name in fixture_data['flat_special'].items(): + names[fid].add(name) + for fid, name in ef_tables.KNOWN_FIDS.items(): + if name not in names.get(fid, set()): + print(f'flat mismatch {fid}: {name}') + bad += 1 + return bad + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + ap.add_argument('--docs-dir', default=os.path.join(root, '..', 'docs', 'uicc'), + help='directory holding the pinned spec PDFs') + ap.add_argument('--out', default=os.path.join(root, 'tests', 'data', + 'spec_files.json')) + ap.add_argument('--check', action='store_true', + help='only compare the current tables with the specs') + args = ap.parse_args() + + files, sfi = build(args.docs_dir) + data = fixture(files, sfi) + if args.check: + sys.exit(1 if check(data, root) else 0) + with open(args.out, 'w') as fh: + json.dump(data, fh, indent=1, sort_keys=True) + print(f'wrote {args.out}: ' + f'{sum(len(v) for v in files.values())} files, ' + f'{sum(len(v) for v in sfi.values())} SFIs') + + +if __name__ == '__main__': + main() diff --git a/tests/test_decode.py b/tests/test_decode.py index 9f9dab4..a583c9c 100644 --- a/tests/test_decode.py +++ b/tests/test_decode.py @@ -1699,12 +1699,16 @@ class TestSfiResolution(unittest.TestCase): 'aid:a00000015141434c00') self.assertIsNone(selected_df_fid({'ins_hex': 'a4', 'p1': {'raw': '04'}, 'body': {'hex': ''}})) - # An AID DF has no normative SFI table. + # An unknown AID has no SFI table; the ISIM AID is keyed by its AID + # (TS 31.103 has no SFI annex, so its table comes from the file rows). self.assertEqual(sfi_table('aid:a00000015141434c00'), {}) + self.assertEqual(sfi_table('aid:a0000000871004')[0x07], '6f07') # EF_IST self.assertEqual(sfi_table('7fff')[0x1e], '6fe3') self.assertEqual(sfi_table('3f00')[0x1e], '2f00') self.assertEqual(sfi_table('7fff')[0x17], '6f06') self.assertEqual(sfi_table('7fff')[0x1a], '6fc6') + self.assertEqual(sfi_table('5f40')[0x07], '4f47') # EF_HWSIDL (H.3) + self.assertEqual(sfi_table('5fc0')[0x10], '4f10') # EF_5GSEDRX (H.9) self.assertEqual(sfi_table('6f7e'), {}) # TS 102 221 §8.4.1: selecting an EF leaves its parent DF current. # 'path from MF', single component → directly under MF (Table 8.2). @@ -2863,5 +2867,68 @@ class TestLineEvents(unittest.TestCase): self.assertEqual(msgs[0]['decoded']['data_rate'], 10753) +class TestSpecTables(unittest.TestCase): + """The file/SFI dictionaries are spec-sourced: this pins them against + tests/data/spec_files.json (extracted from the PDFs in docs/uicc/).""" + + @classmethod + def setUpClass(cls): + import json + import pathlib + cls.fixture = json.loads( + (pathlib.Path(__file__).parent / 'data' / 'spec_files.json') + .read_text()) + + def test_file_name_is_df_aware(self): + from simtrace_analyser.decode import _file_name + # 4F09: SUPI-as-NAI under DF_5GS, group communication under DF_ProSe. + self.assertEqual(_file_name('4f09', '5fc0'), 'EF_SUPI_NAI') + self.assertEqual(_file_name('4f09', '5f90'), 'EF_PROSE_GC') + # 6F07: IMSI in the USIM/SIM, the ISIM service table in the ISIM. + self.assertEqual(_file_name('6f07', '7fff'), 'EF_IMSI') + self.assertEqual(_file_name('6f07', '7f20'), 'EF_IMSI') + self.assertEqual(_file_name('6f07', 'aid:a0000000871004'), 'EF_IST') + # 6F4E: EF_EXT4 under DF_TELECOM, EF_EXT5 in the USIM ADF. + self.assertEqual(_file_name('6f4e', '7f10'), 'EF_EXT4') + self.assertEqual(_file_name('6f4e', '7fff'), 'EF_EXT5') + # Without a DF the unambiguous flat table is the fallback; a FID + # that is ambiguous (or unknown) stays numeric. + self.assertEqual(_file_name('2fe2'), 'EF_ICCID') + self.assertEqual(_file_name('2fe2', '7fff'), 'EF_ICCID') + self.assertEqual(_file_name('4f09'), '4F09') + self.assertEqual(_file_name('1234', '7fff'), '1234') + + def test_file_tables_cover_the_pinned_specs(self): + from simtrace_analyser.decode import _file_name + for df, files in self.fixture['files'].items(): + for fid, name in files.items(): + self.assertEqual(_file_name(fid, df), name, f'{df}/{fid}') + + def test_sfi_tables_match_the_pinned_specs(self): + from simtrace_analyser.decode import sfi_table + for df, table in self.fixture['sfi'].items(): + self.assertEqual(sfi_table(df), + {int(s, 16): fid for s, fid in table.items()}, + df) + + def test_flat_names_are_spec_verified(self): + import collections + from simtrace_analyser.decode import KNOWN_FIDS + # Every flat name must be the spec name of that FID somewhere (the DF + # tables, the DF identifiers, or the two intentional specials): a typo + # or an invented name fails, and a DF-specific name can no longer + # sneak into the flat table under a conflicting FID. + names = collections.defaultdict(set) + for files in self.fixture['files'].values(): + for fid, name in files.items(): + names[fid].add(name) + for fid, name in self.fixture['df_fids'].items(): + names[fid].add(name) + names['3f00'].add('MF') + names['6f65'].add('EF_RPLMNAcT') # legacy value, not re-assigned + for fid, name in KNOWN_FIDS.items(): + self.assertIn(name, names.get(fid, set()), f'{fid}={name}') + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_filetrack.py b/tests/test_filetrack.py index d891b71..2149886 100644 --- a/tests/test_filetrack.py +++ b/tests/test_filetrack.py @@ -499,6 +499,29 @@ class TestChangeImages(TrackerCase): {'offset': 6, 'hex': 'cc00'}]) +class TestNaming(TrackerCase): + def test_select_only_file_gets_its_dictionary_name(self): + # A file whose FCP was surfaced but which was never read still carries + # its DF-scoped dictionary name: the entry is created by the select. + data = self.track( + sel_fid('7fff'), + sel_fid('6f3c'), gr(fcp('6f3c', structure=0b010, rec_len=176, n_rec=10)), + ) + entry = data['files']['7fff/6f3c'] + self.assertEqual(entry['name'], 'EF_SMS') + self.assertEqual(entry['path'], 'ADF/EF_SMS (6F3C)') + self.assertEqual(entry['counters']['select'], 1) + + def test_sim_df_names(self): + # The same FID resolves against the SIM DF_GSM table. + data = self.track( + sel_fid('7f20'), + sel_fid('6f38'), gr(fcp('6f38', size=1)), + ) + self.assertEqual(data['files']['7f20/6f38']['path'], + 'DF_GSM/EF_SST (6F38)') + + class TestFilePath(unittest.TestCase): def test_paths(self): from simtrace_analyser.filetrack import _file_path @@ -511,6 +534,9 @@ class TestFilePath(unittest.TestCase): # A DF-qualified name from DF_CHILDREN wins over the flat FID. self.assertEqual(_file_path('5f3b', '4f52', '4F52'), 'DF_GSM_ACCESS/EF_KcGPRS (4F52)') + # The DF_TELECOM files are named from TS 51.011 10.5 / TS 31.102 4.5. + self.assertEqual(_file_path('7f10', '6f3a', '6F3A'), + 'DF_TELECOM/EF_ADN (6F3A)') # An application selected by an unknown AID: the AID is the DF name, # with the EF.DIR label when the trace surfaced it. self.assertEqual(_file_path('aid:a00000015141434c00', '4f09', '4F09'), @@ -520,7 +546,7 @@ class TestFilePath(unittest.TestCase): {'A0000000871002FF49FF0589': 'USIM'}), 'AID A0000000871002FF49FF0589 (USIM)/EF_IMSI (6F07)') # An unnamed file: the FID alone, no duplicated parentheses. - self.assertEqual(_file_path('7f10', '6f3a', '6F3A'), 'DF_TELECOM/6F3A') + self.assertEqual(_file_path('3f00', '2f99', '2F99'), 'MF/2F99') class TestEndpoint(unittest.TestCase): diff --git a/tests/test_pcap_import.py b/tests/test_pcap_import.py index 7ab0251..c39b37a 100644 --- a/tests/test_pcap_import.py +++ b/tests/test_pcap_import.py @@ -224,7 +224,7 @@ class TestImportEndpoint(unittest.TestCase): class TestVersion(unittest.TestCase): def test_package_version(self): from simtrace2_pysniff import __version__ - self.assertEqual(__version__, '2.4.2') + self.assertEqual(__version__, '2.5.0') def test_status_includes_version(self): import os