catarrh/simtrace-analyser
1
0 Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

File names: the DF-scoped spec dictionary + refreshed EF/SFI tables (v2.5.0)

Browse Source 
- the file/DF/SFI dictionaries move to simtrace_analyser/ef_tables.py and are
  extracted from the pinned specs: TS 102 221 (MF, Annex H.1), TS 31.102
  (ADF USIM 4.2 and the child DFs 4.3/4.4/4.6, the H.1-H.12 SFI annexes, the
  Annex L.6 Relay Node/certificate files; Annex A/E as the ADF/DF file
  completeness cross-check), TS 31.103 and TS 51.011 - 20 DF tables (~370
  entries) and 14 SFI tables, up from ~180 flat names and 3 SFI tables
- names are DF-scoped: a FID reused across DFs resolves to the file its DF
  context names - 4F09 is EF_SUPI_NAI under DF_5GS but EF_PROSE_GC under
  DF_ProSe, 6F07 is EF_IMSI in the USIM/SIM but EF_IST in the ISIM, 6F4E is
  EF_EXT4 in DF_TELECOM but EF_EXT5 in the USIM ADF; the flat table keeps
  only names that are identical everywhere and is the fallback when no DF is
  known (the two names the old table had wrong there are fixed: 6F0A is
  EF_AC_GBAUAPI, 6F53 EF_LOCIGPRS)
- the SELECT-path decoder resolves the 7FFF children against the USIM ADF
  table, and the Files view names files DF-first - including files first seen
  only via their FCP (a select-only file used to keep the bare FID, e.g.
  ADF/6F3C is now ADF/EF_SMS (6F3C))
- tests/data/spec_files.json pins the extracted tables; tests/test_decode.py
  asserts the DF/ADF tables cover them, the SFI tables match them exactly and
  every flat name is a spec name for that FID; tests/extract_spec_tables.py
  regenerates the fixture from the PDFs and --check re-verifies the tables
- README/README_RUS document the dictionary and its sources; PWA_VERSION and
  pyproject/version.py 2.5.0, SW cache simtrace-analyser-v71
This commit is contained in:
catarrh committed 2026-10-17 22:11:47 +03:00
1 parent bd0c9a2c4a
commit cbeeed3608
14 files changed
+1820 -350
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/data/spec_files.json
+571
View File
@@ -0,0 +1,571 @@
{
"comment": "File/SFI tables extracted from the pinned specs by tools/extract_spec_tables.py; asserted by tests/test_decode.py. Regenerate when a spec revision is pinned.",
"df_fids": {
"5f30": "DF_IRIDIUM",
"5f31": "DF_GLOBALSTAR",
"5f32": "DF_ICO",
"5f33": "DF_ACES",
"5f3a": "DF_PHONEBOOK",
"5f3b": "DF_GSM_ACCESS",
"5f3c": "DF_MexE",
"5f3d": "DF_MCS",
"5f3e": "DF_V2X",
"5f3f": "DF_A2X",
"5f40": "DF_WLAN",
"5f50": "DF_HNB",
"5f60": "DF_CTS",
"5f70": "DF_SoLSA",
"5f80": "DF_BCAST",
"5f90": "DF_ProSe",
"5fa0": "DF_ACDC",
"5fb0": "DF_TV",
"5fc0": "DF_5GS",
"5fd0": "DF_SAIP",
"5fe0": "DF_SNPN",
"5ff0": "DF_5G_ProSe",
"5ff1": "DF_5MBSUECONFIG",
"7f10": "DF_TELECOM",
"7f20": "DF_GSM"
},
"files": {
"3f00": {
"2f00": "EF_DIR",
"2f05": "EF_PL",
"2f06": "EF_ARR",
"2f07": "EF_ENV-CLASSES",
"2f08": "EF_UMPC",
"2fe2": "EF_ICCID"
},
"5f3a": {
"4f22": "EF_PSC",
"4f23": "EF_CC",
"4f24": "EF_PUID",
"4f30": "EF_PBR"
},
"5f3b": {
"4f20": "EF_Kc",
"4f47": "EF_MML",
"4f48": "EF_MMDF",
"4f52": "EF_KcGPRS",
"4f63": "EF_CPBCCH",
"4f64": "EF_InvScan"
},
"5f3c": {
"4f40": "EF_MexE-ST",
"4f41": "EF_ORPK",
"4f42": "EF_ARPK",
"4f43": "EF_TPRPK"
},
"5f3d": {
"4f01": "EF_MST",
"4f02": "EF_MCS_CONFIG"
},
"5f3e": {
"4f01": "EF_VST",
"4f02": "EF_V2X_CONFIG",
"4f03": "EF_V2XP_PC5",
"4f04": "EF_V2XP_Uu"
},
"5f3f": {
"4f01": "EF_AST",
"4f02": "EF_A2X_CONFIG",
"4f03": "EF_A2XP_PC5",
"4f04": "EF_A2X_DDAAP_PC5",
"4f05": "EF_A2X_DC2P_PC5",
"4f06": "EF_A2XP_Uu"
},
"5f40": {
"4f41": "EF_Pseudo",
"4f42": "EF_UPLMNWLAN",
"4f43": "EF_OPLMNWLAN",
"4f44": "EF_UWSIDL",
"4f45": "EF_OWSIDL",
"4f46": "EF_WRI",
"4f47": "EF_HWSIDL",
"4f48": "EF_WEHPLMNPI",
"4f49": "EF_WHPI",
"4f4a": "EF_WLRPLMN",
"4f4b": "EF_HPLMNDAI"
},
"5f50": {
"4f20": "EF_IMG",
"4f21": "EF_ICE_graphics",
"4f81": "EF_ACSGL",
"4f82": "EF_CSGT",
"4f83": "EF_HNBN",
"4f84": "EF_OCSGL",
"4f85": "EF_OCSGT",
"4f86": "EF_OHNBN"
},
"5f70": {
"4f30": "EF_SAI",
"4f31": "EF_SLL"
},
"5f90": {
"4f01": "EF_PROSE_MON",
"4f02": "EF_PROSE_ANN",
"4f03": "EF_PROSEFUNC",
"4f04": "EF_PROSE_RADIO_COM",
"4f05": "EF_PROSE_RADIO_MON",
"4f06": "EF_PROSE_RADIO_ANN",
"4f07": "EF_PROSE_POLICY",
"4f08": "EF_PROSE_PLMN",
"4f09": "EF_PROSE_GC",
"4f10": "EF_PST",
"4f11": "EF_PROSE_UIRC",
"4f12": "EF_PROSE_GM_DISCOVERY",
"4f13": "EF_PROSE_RELAY",
"4f14": "EF_PROSE_RELAY_DISCOVERY"
},
"5fa0": {
"4f01": "EF_ACDC_LIST"
},
"5fc0": {
"4f01": "EF_5GS3GPPLOCI",
"4f02": "EF_5GSN3GPPLOCI",
"4f03": "EF_5GS3GPPNSC",
"4f04": "EF_5GSN3GPPNSC",
"4f05": "EF_5GAUTHKEYS",
"4f06": "EF_UAC_AIC",
"4f07": "EF_SUCI_Calc_Info",
"4f08": "EF_OPL5G",
"4f09": "EF_SUPI_NAI",
"4f0a": "EF_Routing_Indicator",
"4f0b": "EF_URSP",
"4f0c": "EF_TN3GPPSNN",
"4f0d": "EF_CAG",
"4f0e": "EF_SOR-CMCI",
"4f0f": "EF_DRI",
"4f10": "EF_5GSEDRX",
"4f11": "EF_5GNSWO_CONF",
"4f15": "EF_MCHPPLMN",
"4f16": "EF_KAUSF_DERIVATION"
},
"5fe0": {
"4f01": "EF_PWS_SNPN",
"4f02": "EF_NID"
},
"5ff0": {
"4f01": "EF_5G_PROSE_ST",
"4f02": "EF_5G_PROSE_DD",
"4f03": "EF_5G_PROSE_DC",
"4f04": "EF_5G_PROSE_U2NRU",
"4f05": "EF_5G_PROSE_RU",
"4f06": "EF_5G_PROSE_UIR",
"4f07": "EF_5G_PROSE_U2URU",
"4f08": "EF_5G_PROSE_EU"
},
"5ff1": {
"4f01": "EF_5MBSUECONFIG"
},
"7f10": {
"5f3a": "DF_PHONEBOOK",
"5f3b": "DF_MULTIMEDIA",
"5f3c": "DF_MMSS",
"5f3d": "DF_MCS",
"5f3e": "DF_V2X",
"5f3f": "DF_A2X",
"5f50": "DF_GRAPHICS",
"6f06": "EF_ARR",
"6f3a": "EF_ADN",
"6f3b": "EF_FDN",
"6f3c": "EF_SMS",
"6f3d": "EF_CCP",
"6f40": "EF_MSISDN",
"6f42": "EF_SMSP",
"6f43": "EF_SMSS",
"6f44": "EF_LND",
"6f47": "EF_SMSR",
"6f49": "EF_SDN",
"6f4a": "EF_EXT1",
"6f4b": "EF_EXT2",
"6f4c": "EF_EXT3",
"6f4d": "EF_BDN",
"6f4e": "EF_EXT4",
"6f4f": "EF_ECCP",
"6f53": "EF_RMA",
"6f54": "EF_SUME",
"6f58": "EF_CMI",
"6fe0": "EF_ICE_DN",
"6fe1": "EF_ICE_FF",
"6fe5": "EF_PSISMSC"
},
"7f20": {
"5f30": "DF_IRIDIUM",
"5f31": "DF_GLOBALSTAR",
"5f32": "DF_ICO",
"5f33": "DF_ACES",
"5f3c": "DF_MexE",
"5f60": "DF_CTS",
"5f70": "DF_SoLSA",
"6f05": "EF_LP",
"6f07": "EF_IMSI",
"6f20": "EF_Kc",
"6f2c": "EF_DCKDe-personalizationControlKeys",
"6f30": "EF_PLMNsel",
"6f31": "EF_HPPLMN",
"6f32": "EF_CNL",
"6f37": "EF_ACMmax",
"6f38": "EF_SST",
"6f39": "EF_ACM",
"6f3e": "EF_GID1",
"6f3f": "EF_GID2",
"6f41": "EF_PUCT",
"6f45": "EF_CBMI",
"6f46": "EF_SPN",
"6f48": "EF_CBMID",
"6f50": "EF_CBMIR",
"6f51": "EF_NIA",
"6f52": "EF_KcGPRS",
"6f53": "EF_LOCIGPRS",
"6f54": "EF_SUME",
"6f60": "EF_PLMNwAcT",
"6f61": "EF_OPLMNwAcT",
"6f62": "EF_HPLMNwAcT",
"6f63": "EF_CPBCCH",
"6f64": "EF_InvScan",
"6f74": "EF_BCCH",
"6f78": "EF_ACC",
"6f7b": "EF_FPLMN",
"6f7e": "EF_LOCI",
"6fad": "EF_AD",
"6fae": "EF_Phase",
"6fb1": "EF_VGCS",
"6fb2": "EF_VGCSS",
"6fb3": "EF_VBS",
"6fb4": "EF_VBSS",
"6fb5": "EF_eMLPP",
"6fb6": "EF_AAeM",
"6fb7": "EF_ECC",
"6fc5": "EF_PNN",
"6fc6": "EF_OPL",
"6fc7": "EF_MBDN",
"6fc8": "EF_EXT6",
"6fc9": "EF_MBI",
"6fca": "EF_MWIS",
"6fcb": "EF_CFIS",
"6fcc": "EF_EXT7",
"6fcd": "EF_SPDI",
"6fce": "EF_MMSN",
"6fcf": "EF_EXT8",
"6fd0": "EF_MMSICP",
"6fd1": "EF_MMSUP",
"6fd2": "EF_MMSUCP"
},
"7fff": {
"5f3a": "DF_PHONEBOOK",
"5f3b": "DF_GSM_ACCESS",
"5f3c": "DF_MexE",
"5f40": "DF_WLAN",
"5f50": "DF_HNB",
"5f70": "DF_SoLSA",
"5f80": "DF_BCAST",
"5f90": "DF_ProSe",
"5fa0": "DF_ACDC",
"5fb0": "DF_TV",
"5fc0": "DF_5GS",
"5fd0": "DF_SAIP",
"5fe0": "DF_SNPN",
"5ff0": "DF_5G_ProSe",
"5ff1": "DF_5MBSUECONFIG",
"6f01": "EF_eAKA",
"6f02": "EF_OCST",
"6f05": "EF_LI",
"6f06": "EF_ARR",
"6f07": "EF_IMSI",
"6f08": "EF_Keys",
"6f09": "EF_KeysPS",
"6f0a": "EF_AC_GBAUAPI",
"6f0b": "EF_IMSDCI",
"6f2c": "EF_DCK",
"6f31": "EF_HPPLMN",
"6f32": "EF_CNL",
"6f37": "EF_ACMmax",
"6f38": "EF_UST",
"6f39": "EF_ACM",
"6f3b": "EF_FDN",
"6f3c": "EF_SMS",
"6f3e": "EF_GID1",
"6f3f": "EF_GID2",
"6f40": "EF_MSISDN",
"6f41": "EF_PUCT",
"6f42": "EF_SMSP",
"6f43": "EF_SMSS",
"6f45": "EF_CBMI",
"6f46": "EF_SPN",
"6f47": "EF_SMSR",
"6f48": "EF_CBMID",
"6f49": "EF_SDN",
"6f4b": "EF_EXT2",
"6f4c": "EF_EXT3",
"6f4d": "EF_BDN",
"6f4e": "EF_EXT5",
"6f4f": "EF_CCP2",
"6f50": "EF_CBMIR",
"6f55": "EF_EXT4",
"6f56": "EF_EST",
"6f57": "EF_ACL",
"6f58": "EF_CMI",
"6f5b": "EF_START-HFN",
"6f5c": "EF_THRESHOLD",
"6f60": "EF_PLMNwAcT",
"6f61": "EF_OPLMNwAcT",
"6f62": "EF_HPLMNwAcT",
"6f73": "EF_PSLOCI",
"6f78": "EF_ACC",
"6f7b": "EF_FPLMN",
"6f7e": "EF_LOCI",
"6f80": "EF_ICI",
"6f81": "EF_OCI",
"6f82": "EF_ICT",
"6f83": "EF_OCT",
"6fad": "EF_AD",
"6fb1": "EF_VGCS",
"6fb2": "EF_VGCSS",
"6fb3": "EF_VBS",
"6fb4": "EF_VBSS",
"6fb5": "EF_eMLPP",
"6fb6": "EF_AAeM",
"6fb7": "EF_ECC",
"6fc3": "EF_Hiddenkey",
"6fc4": "EF_NETPAR",
"6fc5": "EF_PNN",
"6fc6": "EF_OPL",
"6fc7": "EF_MBDN",
"6fc8": "EF_EXT6",
"6fc9": "EF_MBI",
"6fca": "EF_MWIS",
"6fcb": "EF_CFIS",
"6fcc": "EF_EXT7",
"6fcd": "EF_SPDI",
"6fce": "EF_MMSN",
"6fcf": "EF_EXT8",
"6fd0": "EF_MMSICP",
"6fd1": "EF_MMSUP",
"6fd2": "EF_MMSUCP",
"6fd3": "EF_NIA",
"6fd4": "EF_VGCSCA",
"6fd5": "EF_VBSCA",
"6fd6": "EF_GBABP",
"6fd7": "EF_MSK",
"6fd8": "EF_MUK",
"6fd9": "EF_EHPLMN",
"6fda": "EF_GBANL",
"6fdb": "EF_EHPLMNPI",
"6fdc": "EF_LRPLMNSI",
"6fdd": "EF_NAFKCA",
"6fde": "EF_SPNI",
"6fdf": "EF_PNNI",
"6fe2": "EF_NCP-IP",
"6fe3": "EF_EPSLOCI",
"6fe4": "EF_EPSNSC",
"6fe6": "EF_UFC",
"6fe7": "EF_UICCIARI",
"6fe8": "EF_NASCONFIG",
"6fe9": "EF_CERT",
"6fea": "EF_eFRNid",
"6feb": "EF_SCCmax",
"6fec": "EF_PWS",
"6fed": "EF_FDNURI",
"6fee": "EF_BDNURI",
"6fef": "EF_SDNURI",
"6ff0": "EF_IAL",
"6ff1": "EF_IPS",
"6ff2": "EF_IPD",
"6ff3": "EF_ePDGId",
"6ff4": "EF_ePDGSelection",
"6ff5": "EF_ePDGIdEm",
"6ff6": "EF_ePDGSelectionEm",
"6ff7": "EF_FromPreferred",
"6ff8": "EF_IMSConfigData",
"6ff9": "EF_3GPPPSDATAOFF",
"6ffa": "EF_3GPPPSDATAOFFservicelist",
"6ffb": "EF_TVCONFIG",
"6ffc": "EF_XCAPConfigData",
"6ffd": "EF_EARFCNList",
"6ffe": "EF_MuDMiDConfigData"
},
"aid:a0000000871004": {
"6f02": "EF_IMPI",
"6f03": "EF_DOMAIN",
"6f04": "EF_IMPU",
"6f06": "EF_ARR",
"6f07": "EF_IST",
"6f09": "EF_P-CSCF",
"6f0a": "EF_AC_GBAUAPI",
"6f0b": "EF_IMSDCI",
"6f3c": "EF_SMS",
"6f42": "EF_SMSP",
"6f43": "EF_SMSS",
"6f47": "EF_SMSR",
"6fad": "EF_AD",
"6fd5": "EF_GBABP",
"6fd7": "EF_GBANL",
"6fdd": "EF_NAFKCA",
"6fe5": "EF_PSISMSC",
"6fe7": "EF_UICCIARI",
"6ff7": "EF_FromPreferred",
"6ff8": "EF_IMSConfigData",
"6ffa": "EF_WebRTCURI",
"6ffc": "EF_XCAPConfigData",
"6ffe": "EF_MuDMiDConfigData"
}
},
"flat_special": {
"3f00": "MF",
"6f65": "EF_RPLMNAcT"
},
"sfi": {
"3f00": {
"02": "2fe2",
"05": "2f05",
"06": "2f06",
"07": "2f07",
"08": "2f08",
"1e": "2f00"
},
"5f3b": {
"01": "4f20",
"02": "4f52"
},
"5f3d": {
"01": "4f01",
"02": "4f02"
},
"5f3e": {
"01": "4f01",
"02": "4f02",
"06": "4f03",
"07": "4f04"
},
"5f3f": {
"01": "4f01",
"02": "4f02",
"03": "4f03",
"04": "4f04",
"05": "4f05",
"06": "4f06"
},
"5f40": {
"01": "4f41",
"02": "4f42",
"03": "4f43",
"04": "4f44",
"05": "4f45",
"06": "4f46",
"07": "4f47",
"08": "4f48",
"09": "4f49",
"0a": "4f4a",
"0b": "4f4b"
},
"5f50": {
"01": "4f81",
"02": "4f82",
"03": "4f83",
"04": "4f84",
"05": "4f85",
"06": "4f86"
},
"5f90": {
"01": "4f01",
"02": "4f02",
"03": "4f03",
"04": "4f04",
"05": "4f05",
"06": "4f06",
"07": "4f07",
"08": "4f08",
"09": "4f09",
"10": "4f10",
"11": "4f11",
"12": "4f12",
"13": "4f13",
"14": "4f14"
},
"5fa0": {
"01": "4f01"
},
"5fc0": {
"01": "4f01",
"02": "4f02",
"03": "4f03",
"04": "4f04",
"05": "4f05",
"06": "4f06",
"07": "4f07",
"08": "4f08",
"09": "4f09",
"0a": "4f0a",
"0b": "4f0b",
"0c": "4f0c",
"0d": "4f0d",
"0e": "4f0e",
"0f": "4f0f",
"10": "4f10",
"11": "4f11",
"15": "4f15",
"16": "4f16"
},
"5fe0": {
"01": "4f01",
"02": "4f02"
},
"5ff0": {
"01": "4f01",
"02": "4f02",
"03": "4f03",
"04": "4f04",
"05": "4f05",
"06": "4f06",
"07": "4f07",
"08": "4f08"
},
"7fff": {
"01": "6fb7",
"02": "6f05",
"03": "6fad",
"04": "6f38",
"05": "6f56",
"06": "6f78",
"07": "6f07",
"08": "6f08",
"09": "6f09",
"0a": "6f60",
"0b": "6f7e",
"0c": "6f73",
"0d": "6f7b",
"0e": "6f48",
"0f": "6f5b",
"10": "6f5c",
"11": "6f61",
"12": "6f31",
"13": "6f62",
"14": "6f80",
"15": "6f81",
"16": "6f4f",
"17": "6f06",
"18": "6fe4",
"19": "6fc5",
"1a": "6fc6",
"1b": "6fcd",
"1c": "6f39",
"1d": "6fd9",
"1e": "6fe3"
},
"aid:a0000000871004": {
"02": "6f02",
"03": "6fad",
"04": "6f04",
"05": "6f03",
"06": "6f06",
"07": "6f07"
}
},
"sources": {
"ts_102_221": "ETSI TS 102 221 V18.4.0 (Annex H)",
"ts_31_102": "3GPP TS 31.102 V18.10.0 (4.2-4.6, Annex H, Annex L.6)",
"ts_31_103": "3GPP TS 31.103 V18.3.0 (4.2)",
"ts_51_011": "3GPP TS 51.011 V4.15.0 (10.1-10.6)"
}
}
tests/extract_spec_tables.py
+330
View File
@@ -0,0 +1,330 @@
#!/usr/bin/env python3
"""Regenerate tests/data/spec_files.json (the pinned spec tables).
The APDU decoder's file/SFI dictionaries (simtrace_analyser/ef_tables.py) are
extracted from the pinned specs in ``projects/docs/uicc``. This tool parses
the pdftotext output of those PDFs and writes the fixture that
tests/test_decode.py asserts against, so a table edit or a spec revision can
be cross-checked mechanically.
Usage::
python tests/extract_spec_tables.py # regenerate the fixture
python tests/extract_spec_tables.py --check # diff the fixture vs the tables
The parsing rules follow the specs' own layout: file clauses carry an
``Identifier: 'XXXX'`` row (possibly with a space inside the quotes, single or
double quotes) and an optional ``SFI: 'NN'`` row; a clause that only points at
another spec's definition is resolved by its file name (e.g. "For the
structure... see EFIMSDCI in TS 31.103"); the normative SFI annexes are
parsed as tables. Everything the parser cannot attribute to the specs is
reported, never invented.
"""
import argparse
import collections
import json
import os
import re
import subprocess
import sys
import tempfile
SPECS = {
'ts_102_221': ('ts_102221v180400p.pdf', 'ETSI TS 102 221 V18.4.0 (Annex H)'),
'ts_31_102': ('ts_131102v181000p.pdf', '3GPP TS 31.102 V18.10.0 (4.2-4.6, Annex H, Annex L.6)'),
'ts_31_103': ('ts_131103v180300p.pdf', '3GPP TS 31.103 V18.3.0 (4.2)'),
'ts_51_011': ('ts_151011v041500p.pdf', '3GPP TS 51.011 V4.15.0 (10.1-10.6)'),
}
ISIM_AID = 'aid:a0000000871004'
DF_BY_NAME = {
'PHONEBOOK': '5f3a', 'GSM-ACCESS': '5f3b', 'MEXE': '5f3c',
'MULTIMEDIA': '5f3b', 'MMSS': '5f3c', 'MCS': '5f3d', 'V2X': '5f3e',
'A2X': '5f3f', 'WLAN': '5f40', 'HNB': '5f50', 'GRAPHICS': '5f50',
'SOLSA': '5f70', 'BCAST': '5f80', 'PROSE': '5f90', 'ACDC': '5fa0',
'TV': '5fb0', '5GS': '5fc0', 'SAIP': '5fd0', 'SNPN': '5fe0',
'5G-PROSE': '5ff0', '5GPROSE': '5ff0',
'5MBS-UE-PRE-CONFIGURATION': '5ff1', '5MBSUECONFIG': '5ff1',
'IRIDIUM': '5f30', 'GLOBALSTAR': '5f31', 'ICO': '5f32', 'ACES': '5f33',
'CTS': '5f60', 'IS-41': '7f22', 'FP-CTS': '7f23',
}
SEC = re.compile(r'^\d+(?:\.\d+)*[a-z]?\s+(EF[A-Za-z0-9_ \-]+?)(?:\s*\(.*)?$')
FID = re.compile(r"""Identifier:\s*['"]([^'"]*)['"]""")
FIDS = re.compile(r"Identifier:\s*([0-9A-Fa-f]{4})\b")
SFI = re.compile(r"SFI:\s*'?([0-9A-Fa-f]{2})'?")
FID_SENT = re.compile(r"file identifier\s*'?([0-9A-Fa-f]{4})'?", re.I)
DFSEC = re.compile(r'^\d+(?:\.\d+)*\s+Contents of [Ff]iles? at the .+? level', re.I)
SFISEC = re.compile(r'^H\.(\d+)\s+List of SFI Values at the (.+?) Level', re.I)
SFIROW = re.compile(r"^\s*'([0-9A-Fa-f]{4})\s*'?\s+'([0-9A-Fa-f]{2})'")
REF = re.compile(r'see (?:clause [0-9.]+ )?\(?(EF[A-Za-z0-9_\-]+)')
# Curated: clauses that borrowed a FID from another spec's same-named file but
# whose file is actually issuer-assigned (TS 31.102 4.4.2: the phonebook files
# are declared in EF_PBR's records), and the case-only spelling variants.
DROP = {('ts_31_102', '5f3a', '6f3a'), ('ts_31_102', '5f3a', '6f4a')}
CANON = {'EF_AaeM': 'EF_AAeM', 'EF_MExE-ST': 'EF_MexE-ST',
'EF_OPLMNwACT': 'EF_OPLMNwAcT'}
# TS 31.102 Annex L.6 clauses state their identifiers in the clause body.
ANNEX_L = {'6fe9': 'EF_CERT', '6fea': 'EF_eFRNid', '6feb': 'EF_SCCmax'}
MF_FILES = {'2f00': 'EF_DIR', '2f05': 'EF_PL', '2f06': 'EF_ARR',
'2f07': 'EF_ENV-CLASSES', '2f08': 'EF_UMPC', '2fe2': 'EF_ICCID'}
# The DFs listed as children of their parents by the specs themselves
# (TS 31.102 4.3/4.6, TS 51.011 10.2/10.4/10.6).
CHILD_DFS = {
'7fff': {'5f3a': 'DF_PHONEBOOK', '5f3b': 'DF_GSM_ACCESS',
'5f3c': 'DF_MexE', '5f40': 'DF_WLAN', '5f50': 'DF_HNB',
'5f70': 'DF_SoLSA', '5f80': 'DF_BCAST', '5f90': 'DF_ProSe',
'5fa0': 'DF_ACDC', '5fb0': 'DF_TV', '5fc0': 'DF_5GS',
'5fd0': 'DF_SAIP', '5fe0': 'DF_SNPN', '5ff0': 'DF_5G_ProSe',
'5ff1': 'DF_5MBSUECONFIG'},
'7f10': {'5f3a': 'DF_PHONEBOOK', '5f3b': 'DF_MULTIMEDIA',
'5f3c': 'DF_MMSS', '5f3d': 'DF_MCS', '5f3e': 'DF_V2X',
'5f3f': 'DF_A2X', '5f50': 'DF_GRAPHICS'},
'7f20': {'5f30': 'DF_IRIDIUM', '5f31': 'DF_GLOBALSTAR', '5f32': 'DF_ICO',
'5f33': 'DF_ACES', '5f3c': 'DF_MexE', '5f60': 'DF_CTS',
'5f70': 'DF_SoLSA'},
}
DF_FIDS = {
'5f30': 'DF_IRIDIUM', '5f31': 'DF_GLOBALSTAR', '5f32': 'DF_ICO',
'5f33': 'DF_ACES', '5f3a': 'DF_PHONEBOOK', '5f3b': 'DF_GSM_ACCESS',
'5f3c': 'DF_MexE', '5f3d': 'DF_MCS', '5f3e': 'DF_V2X', '5f3f': 'DF_A2X',
'5f40': 'DF_WLAN', '5f50': 'DF_HNB', '5f60': 'DF_CTS', '5f70': 'DF_SoLSA',
'5f80': 'DF_BCAST', '5f90': 'DF_ProSe', '5fa0': 'DF_ACDC',
'5fb0': 'DF_TV', '5fc0': 'DF_5GS', '5fd0': 'DF_SAIP', '5fe0': 'DF_SNPN',
'5ff0': 'DF_5G_ProSe', '5ff1': 'DF_5MBSUECONFIG', '7f10': 'DF_TELECOM',
'7f20': 'DF_GSM',
}
ORDER = ['3f00', '7fff', '7f10', '7f20', '5f3a', '5f3b', '5f3c', '5f3d',
'5f3e', '5f3f', '5f40', '5f50', '5f70', '5f90', '5fa0', '5fc0',
'5fe0', '5ff0', '5ff1', 'aid:a0000000871004']
SFI_ORDER = ['3f00', '7fff', '5f3b', '5f3d', '5f3e', '5f3f', '5f40', '5f50',
'5f90', '5fa0', '5fc0', '5fe0', '5ff0', 'aid:a0000000871004']
def norm_df_name(s):
s = re.sub(r'^(DF|THE|GSM|TELECOM)[\s-]*', '', s.strip().upper(), count=1)
return s.replace(' ', '-').replace('_', '-')
def parse_spec(path, spec):
lines = open(path, encoding='utf-8', errors='replace').read().splitlines()
files = collections.defaultdict(dict)
sfis = collections.defaultdict(dict)
per_file_sfi = collections.defaultdict(dict)
unresolved = {}
def is_heading(line):
line = line.strip()
if '....' in line:
return False
return bool(SEC.match(line) or DFSEC.match(line) or SFISEC.match(line))
cur_df = cur_sfi_df = None
for i, raw in enumerate(lines):
# pdftotext emits typographic quotes in some tables; normalize them.
ln = (raw.strip().replace('\u2018', "'").replace('\u2019', "'")
.replace('\u201c', '"').replace('\u201d', '"'))
if '....' in ln:
continue
if DFSEC.match(ln):
mm = re.search(r'at the (.+?) level', ln, re.I)
name = norm_df_name(mm.group(1)) if mm else ''
cur_df = DF_BY_NAME.get(name)
if spec in ('ts_31_102', 'ts_51_011') and ln.startswith(('4.1 ', '10.1 ')) \
and 'MF level' in ln:
cur_df = '3f00'
mm = SFISEC.match(ln)
if mm:
name = norm_df_name(mm.group(2))
if spec == 'ts_31_103':
cur_sfi_df = ISIM_AID
elif name in ('USIM-ADF', 'USIM-ADF-APPLICATION-DF'):
cur_sfi_df = '7fff'
else:
cur_sfi_df = DF_BY_NAME.get(name)
if cur_sfi_df is None and spec == 'ts_102_221':
cur_sfi_df = '3f00' if 'MF' in mm.group(2).upper() else None
if cur_sfi_df:
row = SFIROW.match(ln)
if row:
fid = row.group(1).lower()
prev = sfis[cur_sfi_df].get(int(row.group(2), 16))
if prev and prev != fid:
print(f'WARN {spec}: SFI clash {cur_sfi_df} {fid} vs {prev}',
file=sys.stderr)
sfis[cur_sfi_df][int(row.group(2), 16)] = fid
m = SEC.match(ln)
if not (m and m.group(1).startswith('EF')):
continue
sec_no = ln.split()[0]
df = cur_df
if spec == 'ts_31_103':
df = ISIM_AID
elif spec == 'ts_31_102':
if sec_no.startswith('4.2'):
df = '7fff'
elif sec_no.startswith('4.1'):
df = '3f00'
elif sec_no.startswith('4.5'):
df = '7f10'
elif spec == 'ts_51_011':
if sec_no.startswith('10.3'):
df = '7f20'
elif sec_no.startswith('10.5'):
df = '7f10'
elif sec_no.startswith('10.1'):
df = '3f00'
fid = sfi = None
body = []
for line in lines[i + 1:]:
if is_heading(line):
break
body.append(line)
if fid is None:
fm = FID.search(line) or FIDS.search(line) or FID_SENT.search(line)
if fm:
cand = re.sub(r'\s', '', fm.group(1))
if re.fullmatch(r'[0-9A-Fa-f]{4}', cand):
fid = cand.lower()
if sfi is None:
sm = SFI.search(line)
if sm:
sfi = int(sm.group(1), 16)
name = 'EF_' + m.group(1).replace(' ', '')[2:]
if not df or 'This EF is not used' in '\n'.join(body):
continue
if fid:
prev = files[df].get(fid)
if prev and prev != name:
print(f'WARN {spec}: FID clash {df} {fid}: {prev} vs {name}',
file=sys.stderr)
files[df][fid] = name
if sfi is not None:
per_file_sfi[df][fid] = sfi
else:
unresolved[(spec, df, name)] = REF.findall('\n'.join(body))
return files, sfis, per_file_sfi, unresolved
def build(docs_dir):
texts = {}
for spec, (pdf, _) in SPECS.items():
src = os.path.join(docs_dir, pdf)
with tempfile.NamedTemporaryFile(suffix='.txt', delete=False) as tmp:
subprocess.run(['pdftotext', '-layout', src, tmp.name], check=True)
texts[spec] = tmp.name
files = collections.defaultdict(dict)
sfi = collections.defaultdict(dict)
unresolved = {}
for spec, path in texts.items():
fl, sf, pfs, unres = parse_spec(path, spec)
unresolved.update(unres)
for df, entries in fl.items():
for fid, name in entries.items():
if (spec, df, fid) in DROP:
continue
name = CANON.get(name, name)
prev = files[df].get(fid)
if prev and prev != name:
raise SystemExit(f'curation needed: {df} {fid}: '
f'{prev} vs {name} ({spec})')
files[df][fid] = name
for df, table in sf.items():
sfi[df].update(table)
for df, table in pfs.items():
for fid, val in table.items():
if df.startswith('aid:'):
sfi[df].setdefault(val, fid)
# clauses that only reference another spec's file by name
by_name = collections.defaultdict(set)
for entries in files.values():
for fid, name in entries.items():
by_name[name].add(fid)
for (spec, df, name), refs in sorted(unresolved.items()):
cands = by_name.get(name) or set()
if not cands:
for ref in refs:
cands |= by_name.get(ref, set())
if len(cands) == 1 and df and (spec, df, next(iter(cands))) not in DROP:
files[df][next(iter(cands))] = name
files['3f00'].update(MF_FILES)
files['7fff'].update(ANNEX_L)
for df, children in CHILD_DFS.items():
files[df].update(children)
return files, sfi
def fixture(files, sfi):
return {
'comment': ('File/SFI tables extracted from the pinned specs by '
'tools/extract_spec_tables.py; asserted by '
'tests/test_decode.py. Regenerate when a spec revision '
'is pinned.'),
'sources': {spec: meta[1] for spec, meta in SPECS.items()},
'files': {df: dict(files[df]) for df in ORDER},
'sfi': {df: {f'{s:02x}': fid for s, fid in sfi[df].items()}
for df in SFI_ORDER},
'df_fids': DF_FIDS,
# flat-table entries that are not spec file/DF names: the MF itself
# and the legacy 6F65 (its value was withdrawn, TS 31.102 4.2.45 NOTE 5)
'flat_special': {'3f00': 'MF', '6f65': 'EF_RPLMNAcT'},
}
def check(fixture_data, repo_root):
sys.path.insert(0, repo_root)
from simtrace_analyser import ef_tables
bad = 0
for df, entries in fixture_data['files'].items():
table = ef_tables.DF_CHILDREN.get(df, {})
for fid, name in entries.items():
if table.get(fid) != name:
print(f'files mismatch {df}/{fid}: fixture {name} table {table.get(fid)}')
bad += 1
for df, table in fixture_data['sfi'].items():
want = {int(s, 16): fid for s, fid in table.items()}
got = ef_tables.SFI_BY_DF.get(df, {})
if want != got:
print(f'sfi mismatch {df}: fixture {want} table {got}')
bad += 1
names = collections.defaultdict(set)
for entries in fixture_data['files'].values():
for fid, name in entries.items():
names[fid].add(name)
for fid, name in fixture_data['df_fids'].items():
names[fid].add(name)
for fid, name in fixture_data['flat_special'].items():
names[fid].add(name)
for fid, name in ef_tables.KNOWN_FIDS.items():
if name not in names.get(fid, set()):
print(f'flat mismatch {fid}: {name}')
bad += 1
return bad
def main():
ap = argparse.ArgumentParser(description=__doc__)
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
ap.add_argument('--docs-dir', default=os.path.join(root, '..', 'docs', 'uicc'),
help='directory holding the pinned spec PDFs')
ap.add_argument('--out', default=os.path.join(root, 'tests', 'data',
'spec_files.json'))
ap.add_argument('--check', action='store_true',
help='only compare the current tables with the specs')
args = ap.parse_args()
files, sfi = build(args.docs_dir)
data = fixture(files, sfi)
if args.check:
sys.exit(1 if check(data, root) else 0)
with open(args.out, 'w') as fh:
json.dump(data, fh, indent=1, sort_keys=True)
print(f'wrote {args.out}: '
f'{sum(len(v) for v in files.values())} files, '
f'{sum(len(v) for v in sfi.values())} SFIs')
if __name__ == '__main__':
main()
tests/test_decode.py
+68 -1
View File
@@ -1699,12 +1699,16 @@ class TestSfiResolution(unittest.TestCase):
'aid:a00000015141434c00')
self.assertIsNone(selected_df_fid({'ins_hex': 'a4', 'p1': {'raw': '04'},
'body': {'hex': ''}}))
# An AID DF has no normative SFI table.
# An unknown AID has no SFI table; the ISIM AID is keyed by its AID
# (TS 31.103 has no SFI annex, so its table comes from the file rows).
self.assertEqual(sfi_table('aid:a00000015141434c00'), {})
self.assertEqual(sfi_table('aid:a0000000871004')[0x07], '6f07') # EF_IST
self.assertEqual(sfi_table('7fff')[0x1e], '6fe3')
self.assertEqual(sfi_table('3f00')[0x1e], '2f00')
self.assertEqual(sfi_table('7fff')[0x17], '6f06')
self.assertEqual(sfi_table('7fff')[0x1a], '6fc6')
self.assertEqual(sfi_table('5f40')[0x07], '4f47') # EF_HWSIDL (H.3)
self.assertEqual(sfi_table('5fc0')[0x10], '4f10') # EF_5GSEDRX (H.9)
self.assertEqual(sfi_table('6f7e'), {})
# TS 102 221 §8.4.1: selecting an EF leaves its parent DF current.
# 'path from MF', single component → directly under MF (Table 8.2).
@@ -2863,5 +2867,68 @@ class TestLineEvents(unittest.TestCase):
self.assertEqual(msgs[0]['decoded']['data_rate'], 10753)
class TestSpecTables(unittest.TestCase):
"""The file/SFI dictionaries are spec-sourced: this pins them against
tests/data/spec_files.json (extracted from the PDFs in docs/uicc/)."""
@classmethod
def setUpClass(cls):
import json
import pathlib
cls.fixture = json.loads(
(pathlib.Path(__file__).parent / 'data' / 'spec_files.json')
.read_text())
def test_file_name_is_df_aware(self):
from simtrace_analyser.decode import _file_name
# 4F09: SUPI-as-NAI under DF_5GS, group communication under DF_ProSe.
self.assertEqual(_file_name('4f09', '5fc0'), 'EF_SUPI_NAI')
self.assertEqual(_file_name('4f09', '5f90'), 'EF_PROSE_GC')
# 6F07: IMSI in the USIM/SIM, the ISIM service table in the ISIM.
self.assertEqual(_file_name('6f07', '7fff'), 'EF_IMSI')
self.assertEqual(_file_name('6f07', '7f20'), 'EF_IMSI')
self.assertEqual(_file_name('6f07', 'aid:a0000000871004'), 'EF_IST')
# 6F4E: EF_EXT4 under DF_TELECOM, EF_EXT5 in the USIM ADF.
self.assertEqual(_file_name('6f4e', '7f10'), 'EF_EXT4')
self.assertEqual(_file_name('6f4e', '7fff'), 'EF_EXT5')
# Without a DF the unambiguous flat table is the fallback; a FID
# that is ambiguous (or unknown) stays numeric.
self.assertEqual(_file_name('2fe2'), 'EF_ICCID')
self.assertEqual(_file_name('2fe2', '7fff'), 'EF_ICCID')
self.assertEqual(_file_name('4f09'), '4F09')
self.assertEqual(_file_name('1234', '7fff'), '1234')
def test_file_tables_cover_the_pinned_specs(self):
from simtrace_analyser.decode import _file_name
for df, files in self.fixture['files'].items():
for fid, name in files.items():
self.assertEqual(_file_name(fid, df), name, f'{df}/{fid}')
def test_sfi_tables_match_the_pinned_specs(self):
from simtrace_analyser.decode import sfi_table
for df, table in self.fixture['sfi'].items():
self.assertEqual(sfi_table(df),
{int(s, 16): fid for s, fid in table.items()},
df)
def test_flat_names_are_spec_verified(self):
import collections
from simtrace_analyser.decode import KNOWN_FIDS
# Every flat name must be the spec name of that FID somewhere (the DF
# tables, the DF identifiers, or the two intentional specials): a typo
# or an invented name fails, and a DF-specific name can no longer
# sneak into the flat table under a conflicting FID.
names = collections.defaultdict(set)
for files in self.fixture['files'].values():
for fid, name in files.items():
names[fid].add(name)
for fid, name in self.fixture['df_fids'].items():
names[fid].add(name)
names['3f00'].add('MF')
names['6f65'].add('EF_RPLMNAcT') # legacy value, not re-assigned
for fid, name in KNOWN_FIDS.items():
self.assertIn(name, names.get(fid, set()), f'{fid}={name}')
if __name__ == '__main__':
unittest.main()
tests/test_filetrack.py
+27 -1
View File
@@ -499,6 +499,29 @@ class TestChangeImages(TrackerCase):
{'offset': 6, 'hex': 'cc00'}])
class TestNaming(TrackerCase):
def test_select_only_file_gets_its_dictionary_name(self):
# A file whose FCP was surfaced but which was never read still carries
# its DF-scoped dictionary name: the entry is created by the select.
data = self.track(
sel_fid('7fff'),
sel_fid('6f3c'), gr(fcp('6f3c', structure=0b010, rec_len=176, n_rec=10)),
)
entry = data['files']['7fff/6f3c']
self.assertEqual(entry['name'], 'EF_SMS')
self.assertEqual(entry['path'], 'ADF/EF_SMS (6F3C)')
self.assertEqual(entry['counters']['select'], 1)
def test_sim_df_names(self):
# The same FID resolves against the SIM DF_GSM table.
data = self.track(
sel_fid('7f20'),
sel_fid('6f38'), gr(fcp('6f38', size=1)),
)
self.assertEqual(data['files']['7f20/6f38']['path'],
'DF_GSM/EF_SST (6F38)')
class TestFilePath(unittest.TestCase):
def test_paths(self):
from simtrace_analyser.filetrack import _file_path
@@ -511,6 +534,9 @@ class TestFilePath(unittest.TestCase):
# A DF-qualified name from DF_CHILDREN wins over the flat FID.
self.assertEqual(_file_path('5f3b', '4f52', '4F52'),
'DF_GSM_ACCESS/EF_KcGPRS (4F52)')
# The DF_TELECOM files are named from TS 51.011 10.5 / TS 31.102 4.5.
self.assertEqual(_file_path('7f10', '6f3a', '6F3A'),
'DF_TELECOM/EF_ADN (6F3A)')
# An application selected by an unknown AID: the AID is the DF name,
# with the EF.DIR label when the trace surfaced it.
self.assertEqual(_file_path('aid:a00000015141434c00', '4f09', '4F09'),
@@ -520,7 +546,7 @@ class TestFilePath(unittest.TestCase):
{'A0000000871002FF49FF0589': 'USIM'}),
'AID A0000000871002FF49FF0589 (USIM)/EF_IMSI (6F07)')
# An unnamed file: the FID alone, no duplicated parentheses.
self.assertEqual(_file_path('7f10', '6f3a', '6F3A'), 'DF_TELECOM/6F3A')
self.assertEqual(_file_path('3f00', '2f99', '2F99'), 'MF/2F99')
class TestEndpoint(unittest.TestCase):
tests/test_pcap_import.py
+1 -1
View File
@@ -224,7 +224,7 @@ class TestImportEndpoint(unittest.TestCase):
class TestVersion(unittest.TestCase):
def test_package_version(self):
from simtrace2_pysniff import __version__
self.assertEqual(__version__, '2.4.2')
self.assertEqual(__version__, '2.5.0')
def test_status_includes_version(self):
import os