personalization: SMSP: set alpha_id = ''

Change-Id: Ief6e02517f3e96158a2509d763b88aec4bd5a296

README.md

@@ -97,7 +97,7 @@ Please install the following dependencies:
  - pyscard
  - pyserial
  - pytlv
- - pyyaml >= 5.4
+ - pyyaml >= 5.1
  - smpp.pdu (from `github.com/hologram-io/smpp.pdu`)
  - termcolor
 

contrib/jenkins.sh

@@ -10,11 +10,6 @@
 
 export PYTHONUNBUFFERED=1
 
-setup_venv() {
-	virtualenv -p python3 venv --system-site-packages
-	. venv/bin/activate
-}
-
 if [ ! -d "./tests/" ] ; then
 	echo "###############################################"
 	echo "Please call from pySim-prog top directory"
@@ -28,7 +23,8 @@ fi
 
 case "$JOB_TYPE" in
 "test")
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install -r requirements.txt
 	pip install pyshark
@@ -36,27 +32,23 @@ case "$JOB_TYPE" in
 	# Execute automatically discovered unit tests first
 	python -m unittest discover -v -s tests/unittests
 
-	# Run pySim-trace test
+	# Run pySim-prog integration tests (requires physical cards)
-	tests/pySim-trace_test/pySim-trace_test.sh
-	;;
-"card-test") # tests requiring physical cards
-	setup_venv
-
-	pip install -r requirements.txt
-
-	# Run pySim-prog integration tests
 	cd tests/pySim-prog_test/
-	./pySim-prog_test.sh
+        ./pySim-prog_test.sh
 	cd ../../
 
-	# Run pySim-shell integration tests
+	# Run pySim-trace test
+	tests/pySim-trace_test/pySim-trace_test.sh
+
+	# Run pySim-shell integration tests (requires physical cards)
 	python3 -m unittest discover -v -s ./tests/pySim-shell_test/
 
 	# Run pySim-smpp2sim test
 	tests/pySim-smpp2sim_test/pySim-smpp2sim_test.sh
 	;;
 "distcheck")
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install .
 	pip install pyshark
@@ -69,7 +61,8 @@ case "$JOB_TYPE" in
 	# Print pylint version
 	pip3 freeze | grep pylint
 
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install .
 
@@ -87,7 +80,8 @@ case "$JOB_TYPE" in
 		contrib/*.py
 	;;
 "docs")
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install -r requirements.txt
 

pySim-prog.py

@@ -27,6 +27,7 @@
 import hashlib
 import argparse
 import os
+import random
 import re
 import sys
 import traceback
@@ -435,7 +436,7 @@ def gen_parameters(opts):
         if not re.match('^[0-9a-fA-F]{32}$', ki):
             raise ValueError('Ki needs to be 128 bits, in hex format')
     else:
-        ki = os.urandom(16).hex()
+        ki = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
 
     # OPC (random)
     if opts.opc is not None:
@@ -446,7 +447,7 @@ def gen_parameters(opts):
     elif opts.op is not None:
         opc = derive_milenage_opc(ki, opts.op)
     else:
-        opc = os.urandom(16).hex()
+        opc = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
 
     pin_adm = sanitize_pin_adm(opts.pin_adm, opts.pin_adm_hex)
 

pySim/esim/saip/__init__.py

@@ -34,7 +34,7 @@ from pySim import ts_102_222
 from pySim.utils import dec_imsi
 from pySim.ts_102_221 import FileDescriptor
 from pySim.filesystem import CardADF, Path
-from pySim.ts_31_102 import ADF_USIM, EF_UST, EF_SUCI_Calc_Info
+from pySim.ts_31_102 import ADF_USIM
 from pySim.ts_31_103 import ADF_ISIM
 from pySim.esim import compile_asn1_subdir
 from pySim.esim.saip import templates
@@ -1517,11 +1517,8 @@ class ProfileElementHeader(ProfileElement):
     def mandatory_service_add(self, service_name):
         self.decoded['eUICC-Mandatory-services'][service_name] = None
 
-    def mandatory_service_present(self, service_name):
-        return service_name in self.decoded['eUICC-Mandatory-services'].keys()
-
     def mandatory_service_remove(self, service_name):
-        if self.mandatory_service_present(service_name):
+        if service_name in self.decoded['eUICC-Mandatory-services'].keys():
             del self.decoded['eUICC-Mandatory-services'][service_name]
         else:
             raise ValueError("service not in eUICC-Mandatory-services list, cannot remove")
@@ -1729,64 +1726,12 @@ class ProfileElementSequence:
         if 'BT' in ftype_list:
             svc_set.add('ber-tlv')
         # FIXME:dfLinked files (scan all files, check for non-empty Fcp.linkPath presence of DFs)
-
+        # TODO: 5G related bits (derive from EF.UST or file presence?)
-        # 5G:
-        # - When SUCI is:
-        #   - enabled (EF.UST 124 = true)
-        #   AND
-        #   - calculated in the USIM (EF.UST 125 = true),
-        #   then eUICC-Mandatory-services needs 'get-identity'.
-        # - 'get-identity' implies that the eUICC must support ONE OF profile-A OR profile-B.
-        #   So, when SUCI-CalcInfo for USIM in DF.SAIP contains both key types,
-        #   then no profile-A or B services need to be requested explicitly.
-        # - When the SUCI-CalcInfo for USIM (DF.SAIP) contains ONLY a key of profile-A ("identifier": 1),
-        #   then eUICC-Mandatory-services needs 'profile-a-x25519'.
-        # - Same: ONLY profile-B ("identifier": 2) needs 'profile-b-p256'.
-        # - (When SUCI is calculated in the UE, then the eUICC does not need to provide any of these services.)
-        suci_in_usim_enabled = False
-        try:
-            f_ust = self.get_pe_for_type("usim").files["ef-ust"]
-            ust = EF_UST().decode_bin(f_ust.body)
-            suci_in_usim_enabled = ust[124]['activated'] and ust[125]['activated']
-        except (KeyError, AttributeError):
-            pass
-        if suci_in_usim_enabled:
-            svc_set.add('get-identity')
-            # now check for profile-a and profile-b
-            suci_calcinfo_has_profile_a = False
-            suci_calcinfo_has_profile_b = False
-            try:
-                f_sucici = self.get_pe_for_type("df-saip").files["ef-suci-calc-info-usim"]
-                sucici = EF_SUCI_Calc_Info().decode_bin(f_sucici.body) or {}
-                for prot_scheme in sucici['prot_scheme_id_list']:
-                    if not isinstance(prot_scheme, dict):
-                        continue
-                    ps_id = prot_scheme["identifier"]
-                    if ps_id == 1:
-                        suci_calcinfo_has_profile_a = True
-                    elif ps_id == 2:
-                        suci_calcinfo_has_profile_b = True
-            except (KeyError, AttributeError):
-                pass
-            if suci_calcinfo_has_profile_a and suci_calcinfo_has_profile_b:
-                # 'get-identity' implies that the eUICC supports one of the above. Do not require a specific one.
-                pass
-            elif suci_calcinfo_has_profile_a:
-                # The profile has only a profile-A key, so require that
-                svc_set.add('profile-a-x25519')
-            elif suci_calcinfo_has_profile_b:
-                # The profile has only a profile-B key, so require that
-                svc_set.add('profile-b-p256')
-
         hdr_pe = self.get_pe_for_type('header')
         # patch in the 'manual' services from the existing list:
-        old_svc_set = set()
         for old_svc in hdr_pe.decoded['eUICC-Mandatory-services'].keys():
             if old_svc in manual_services:
-                old_svc_set.add(old_svc)
+                svc_set.add(old_svc)
-        logger.debug(f"{svc_set=}  +  {old_svc_set=}")
-        svc_set = svc_set.union(old_svc_set)
-        logger.debug(f"{svc_set=}")
         hdr_pe.decoded['eUICC-Mandatory-services'] = {x: None for x in svc_set}
 
     def rebuild_mandatory_gfstelist(self):

pySim/esim/saip/batch.py

@@ -126,8 +126,6 @@ class BatchPersonalization:
                     logger.error('during %s: %r', _func_(), e)
                     raise ValueError(f'{p.param.name} fed by {p.src.name}: {e!r}') from e
 
-            pes.rebuild_mandatory_services()
-
             yield pes
 
 
@@ -327,15 +325,12 @@ class BatchAudit(list):
 
         return batch_audit
 
-    def to_csv_rows(self, headers=True, sort_key=None, column_blacklist=None):
+    def to_csv_rows(self, headers=True, sort_key=None):
         '''generator that yields all audits' values as rows, useful feed to a csv.writer.'''
         columns = set()
         for audit in self:
             columns.update(audit.keys())
 
-        if column_blacklist:
-            columns.difference_update(set(column_blacklist))
-
         columns = tuple(sorted(columns, key=sort_key))
 
         if headers:

pySim/esim/saip/personalization.py

@@ -20,26 +20,19 @@ import io
 import os
 import re
 import pprint
-import json
 from typing import List, Tuple, Generator, Optional
 
 from construct.core import StreamError
 from osmocom.tlv import camel_to_snake
 from osmocom.utils import hexstr
 from pySim.utils import enc_iccid, dec_iccid, enc_imsi, dec_imsi, h2b, b2h, rpad, sanitize_iccid
-from pySim.ts_31_102 import EF_AD, EF_UST, EF_Routing_Indicator, EF_SUCI_Calc_Info, DF_USIM_5GS
+from pySim.ts_31_102 import EF_AD
 from pySim.ts_51_011 import EF_SMSP
 from pySim.esim.saip import param_source
 from pySim.esim.saip import ProfileElement, ProfileElementSD, ProfileElementSequence
-from pySim.esim.saip import ProfileElementHeader
 from pySim.esim.saip import SecurityDomainKey, SecurityDomainKeyComponent
 from pySim.global_platform import KeyUsageQualifier, KeyType
 
-# optimization: instantiate class instance to get the fid only once.
-file_path_df_5gs = bytes.fromhex(DF_USIM_5GS().fid)
-fid_ri = bytes.fromhex(EF_Routing_Indicator().fid)
-fid_sucici = bytes.fromhex(EF_SUCI_Calc_Info().fid)
-
 def unrpad(s: hexstr, c='f') -> hexstr:
     return hexstr(s.rstrip(c))
 
@@ -298,9 +291,7 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
            May be overridden by subclasses.
            This default implementation returns the maximum allowed value length -- a good fit for most subclasses.
            '''
-        l = cls.get_len_range()[1] or 16
+        return cls.get_len_range()[1] or 16
-        l = min(10*80, l)
-        return l
 
     @classmethod
     def is_super_of(cls, other_class):
@@ -627,28 +618,10 @@ class SmspTpScAddr(ConfigurableParameter):
             ef_smsp_dec['tp_sc_addr']['ton_npi']['type_of_number'] = 'international' if international else 'unknown'
             # ensure the parameter_indicators.tp_sc_addr is True
             ef_smsp_dec['parameter_indicators']['tp_sc_addr'] = True
-
-            # alpha_id padding: to make room for a human readable SMSC name that can be provisioned to the profile later
-            # on, alpha_id needs to be empty but padded 0xff to some length.
-            # - alpha_id is optional, setting alpha_id = '' ensures the IE is present.
-            # - the length of the file is 28+Y where Y is the length of the alpha_id -- here the intended length of our padding
-            #   (see 3GPP TS 31.102 4.2.27 EF.SMSP). So if we want a maximum length of alpha_id = 14, we set the total
-            #   file size to 28+14 = 42.
-            # - this file size has to go in two places: encode_record_bin() needs to know the length to encode the right
-            #   length of fillFileContent.
-            # - the f_smsp needs to show the right file size in the PES, as in
-            #      'ef-smsp': [('fileDescriptor', {'efFileSize': '2a', ...
-            #   (where 2a == 42)
-            # - To generate the right amount of fillFileContent, pass total_len=42 to encode_record_bin().
-            # - To show the right size in the PES, set f_smsp.rec_len = 42
             ef_smsp_dec['alpha_id'] = ''
-
+            # re-encode into the File body
-            # we can set this to choose a fixed length:
+            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1)
-            #f_smsp.rec_len = 42
+            #print("SMSP  (new): %s" % f_smsp.body)
-            # but leave rec_len unchanged to keep the same length as was found in the eSIM template.
-
-            # re-encode into the File body.
-            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1, total_len=f_smsp.rec_len)
             # re-generate the pe.decoded member from the File instance
             pe.file2pe(f_smsp)
 
@@ -676,12 +649,24 @@ class SmspTpScAddr(ConfigurableParameter):
             yield { cls.name: cls.tuple_to_str((international, digits)) }
 
 
-class MncLen(EnumParam):
+class MncLen(ConfigurableParameter):
     """MNC length. Must be either 2 or 3. Sets only the MNC length field in EF-AD (Administrative Data)."""
     name = 'MNC-LEN'
-    value_map = { '2': 2, '3': 3 }
+    allow_chars = '23'
-    default_source = param_source.ConstantSource
+    strip_chars = ' \t\r\n'
+    numeric_base = 10
+    max_len = 1
+    min_len = 1
     example_input = '2'
+    default_source = param_source.ConstantSource
+
+    @classmethod
+    def validate_val(cls, val):
+        val = super().validate_val(val)
+        val = int(val)
+        if val not in (2, 3):
+            raise ValueError(f"MNC-LEN must be either 2 or 3, not {val!r}")
+        return val
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -727,7 +712,7 @@ class MncLen(EnumParam):
             if mnc_len is None:
                 continue
 
-            yield { cls.name: cls.map_val_to_name(int(mnc_len)) }
+            yield { cls.name: str(mnc_len) }
 
 
 class SdKey(BinaryParam):
@@ -1163,7 +1148,7 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):
 
 class MilenageXoringConstants(BinaryParam, AlgoConfig):
     """XOR-ing constants c1,c2,c3,c4,c5 of Milenage, 128bit each. See 3GPP TS 35.206 Sections 2.3 + 5.3.
-    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concatenation
+    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concetenation
     of::
 
      00000000000000000000000000000000
@@ -1191,411 +1176,3 @@ class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
     max_val = 255
     example_input = '1'
     default_source = param_source.ConstantSource
-    numeric_base = None  # indicate that this won't need random number sources
-
-
-class EfUstServiceParam(EnumParam):
-    """superclass for EF-UST service flag parameters"""
-    service_idx = 0
-    value_map = { 'enabled': True, 'disabled': False }
-    default_source = param_source.ConstantSource
-    example_input = sorted(value_map.keys())[0]
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ust = pe.files['ef-ust']
-            ef_ust = EF_UST()
-            ust = ef_ust.decode_bin(f_ust.body)
-
-            ust[cls.service_idx]['activated'] = val
-
-            f_ust.body = ef_ust.encode_bin(ust)
-            pe.file2pe(f_ust)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ust = pe.files.get('ef-ust', None)
-            if not f_ust:
-                continue
-            ef_ust = EF_UST()
-            try:
-                ust = ef_ust.decode_bin(f_ust.body)
-
-                service_flag = ust[cls.service_idx]['activated']
-                yield { cls.name: cls.map_val_to_name(service_flag) }
-            except:
-                pass
-
-class SuciActive(EfUstServiceParam):
-    """EF-UST service nr 124: enable or disable the SUCI service."""
-    service_idx = 124
-    name = '5G-SUCI-active'
-    value_map = { 'SUCI-off': False, 'SUCI-on': True }
-    example_input = 'SUCI-on'
-
-class SuciInUsim(EfUstServiceParam):
-    """EF-UST service nr 125: calculate SUCI in UE or in USIM"""
-    service_idx = 125
-    name = '5G-SUCI-in-USIM'
-    value_map = { 'SUCI-in-UE': False, 'SUCI-in-USIM': True }
-    example_input = 'SUCI-in-USIM'
-
-class SuciRi(ConfigurableParameter):
-    """SUCI Routing Indicator as in section 4.4.11.11 of 3GPP TS 31.102"""
-    name = '5G-SUCI-RI'
-    allow_chars = '0123456789'
-    min_len = 1
-    max_len = 4
-    allow_types = (str,)
-    example_input = '0'
-    default_source = param_source.ConstantSource
-
-    KEY_RI = "routing_indicator"
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        for pe in pes.get_pes_for_type('df-5gs'):
-            f_ri = pe.files.get('ef-routing-indicator', None)
-            if f_ri is None:
-                continue
-            ef_ri = EF_Routing_Indicator()
-            ri = ef_ri.decode_bin(f_ri.body)
-
-            ri[cls.KEY_RI] = str(val)
-
-            f_ri.body = ef_ri.encode_bin(ri)
-            pe.file2pe(f_ri)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('df-5gs'):
-            f_ri = pe.files.get('ef-routing-indicator', None)
-            if f_ri is None:
-                continue
-            ef_ri = EF_Routing_Indicator()
-            try:
-                ri = ef_ri.decode_bin(f_ri.body)
-                yield { cls.name: ri.get(cls.KEY_RI) }
-            except:
-                pass
-
-class SuciCalcInfoParameter(ConfigurableParameter):
-    """SUCI Calculation Information as in section 4.4.11.8 of 3GPP TS 31.102"""
-    name = '5G-SUCI-CalcInfo'
-    default_source = param_source.ConstantSource
-    allow_types = (str,)
-    max_len = 4096  # to indicate a large input field to UI renderers
-    example_input = '{"prot_scheme_id_list": [{"priority": 0, "identifier": 0, "key_index": 0}], "hnet_pubkey_list": []}'
-
-    PE_IN_UE = ("df-5gs", "ef-suci-calc-info")
-    PE_IN_USIM = ("df-saip", "ef-suci-calc-info-usim")
-    suci_calc_info_pe = None
-
-    @classmethod
-    def validate_val(cls, val):
-        val = super().validate_val(val)
-
-        if not val:
-            val = "{}"
-
-        # check that it is a dict something like
-        # {
-        #     "prot_scheme_id_list": [
-        #         {"priority": 0, "identifier": 2, "key_index": 1},
-        #         {"priority": 1, "identifier": 1, "key_index": 2},
-        #         ],
-        #     "hnet_pubkey_list": [
-        #         {"hnet_pubkey_identifier": 27,
-        #          "hnet_pubkey": "0472DA71976234CE833A6907425867B82E074D44EF907DFB4B3E21C1C2256EBCD15A7DED52FCBB097A4ED250E036C7B9C8C7004C4EEDC4F068CD7BF8D3F900E3B4"},
-        #         {"hnet_pubkey_identifier": 30,
-        #          "hnet_pubkey": "5A8D38864820197C3394B92613B20B91633CBD897119273BF8E4A6F4EEC0A650"},
-        #          ],
-        # }
-
-        try:
-            d = json.loads(val)
-        except json.decoder.JSONDecodeError as e:
-            raise ValueError(f"Cannot parse SUCI Calc Info: {e}") from e
-
-        KEY_PSI_LIST = 'prot_scheme_id_list'
-        KEY_HPK_LIST = 'hnet_pubkey_list'
-        KEYS_D = set((KEY_HPK_LIST, KEY_PSI_LIST))
-        KEYS_PSI = set(('identifier', 'key_index', 'priority'))
-        KEYS_HPK = set(('hnet_pubkey_identifier', 'hnet_pubkey'))
-
-        if not d:
-            d = { KEY_PSI_LIST: [], KEY_HPK_LIST: [] }
-
-        if not (isinstance(d, dict)
-                and set(d.keys()) == KEYS_D):
-            raise ValueError(f"Unexpected structure in SUCI Calc Info: expected dict with entries {KEYS_D}")
-
-        psi = d.get(KEY_PSI_LIST, None)
-        if not all((set(e.keys()) == KEYS_PSI) for e in psi):
-            raise ValueError("Unexpected structure in SUCI Calc Info:"
-                             f" in {KEY_PSI_LIST}, expected dict with entries {KEYS_PSI}")
-
-        hpk = d.get(KEY_HPK_LIST, None)
-        if not all((set(e.keys()) == KEYS_HPK) for e in hpk):
-            raise ValueError("Unexpected structure in SUCI Calc Info:"
-                             f" in {KEY_HPK_LIST}, expected dict with entries {KEYS_HPK}")
-        return d
-
-    @classmethod
-    def _apply_suci(cls, pes: ProfileElementSequence, val, pe_type="df-5gs", pe_file="ef-suci-calc-info"):
-        for pe in pes.get_pes_for_type(pe_type):
-            f_sucici = pe.files.get(pe_file, None)
-            if not f_sucici:
-                continue
-            ef_sucici = EF_SUCI_Calc_Info()
-            body = ef_sucici.encode_bin(val)
-
-            # 0xff pad up to the existing file size, so that the underlying template doesn't come through
-            is_size = f_sucici.file_size
-            pad_n = is_size - len(body)
-            if pad_n > 0:
-                body = body + b'\xff' * pad_n
-
-            f_sucici.body = body
-            pe.file2pe(f_sucici)
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        cls._apply_suci(pes, val, *cls.suci_calc_info_pe)
-
-    @staticmethod
-    def normalize_sucici(sucici:dict):
-        """Normalize the CalcInfo dict so it can be json encoded:
-           convert bytes to hex strings."""
-        if not sucici:
-            sucici = {}
-
-        for hnet_pubkey in sucici.get('hnet_pubkey_list', ()):
-            val = hnet_pubkey['hnet_pubkey']
-            if isinstance(val, bytes):
-                val = b2h(val)
-                hnet_pubkey['hnet_pubkey'] = val
-
-        return sucici
-
-    @classmethod
-    def _get_suci(cls, pes: ProfileElementSequence, pe_type="df-5gs", pe_file="ef-suci-calc-info"):
-        for pe in pes.get_pes_for_type(pe_type):
-            f_sucici = pe.files.get(pe_file, None)
-            if not f_sucici:
-                continue
-            ef_sucici = EF_SUCI_Calc_Info()
-            sucici = ef_sucici.decode_bin(f_sucici.body)
-
-            # normalize to string (bytes cannot go into json)
-            sucici = cls.normalize_sucici(sucici)
-
-            yield { cls.name: json.dumps(sucici) }
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        yield from cls._get_suci(pes, *cls.suci_calc_info_pe)
-
-class SuciCalcInfoUe(SuciCalcInfoParameter):
-    """SUCI Calculation Information as in section 4.4.11.8 of 3GPP TS 31.102, readable by UE (DF-5GS)"""
-    name = '5G-SUCI-CalcInfo-UE'
-    suci_calc_info_pe = SuciCalcInfoParameter.PE_IN_UE
-
-class SuciCalcInfoUsim(SuciCalcInfoParameter):
-    """SUCI Calculation Information as in section 4.4.11.8 of 3GPP TS 31.102, readable only by USIM (DF-SAIP)"""
-    name = '5G-SUCI-CalcInfo-USIM'
-    suci_calc_info_pe = SuciCalcInfoParameter.PE_IN_USIM
-
-def gfm_find(pes: ProfileElementSequence, file_path:bytes, ef_fid:bytes):
-    """look through genericFileManagement PE and return the fmc list with start and end indexes as
-       (fmc_list, first_idx, after_last_idx)
-       so that fmc_list[first_idx:after_last_idx] is the slice of file management commands relevant to the given
-       file_path/ef_fid.
-    """
-    for pe in pes.get_pes_for_type('genericFileManagement'):
-        path_match = False
-        creating_fid = False
-
-        for fmc in pe.decoded['fileManagementCMD']:
-            first = None
-            last = None
-
-            for idx in range(len(fmc)):
-                cmd, arg = fmc[idx]
-
-                if cmd == 'filePath':
-                    path_match = (arg == file_path)
-                    if not path_match:
-                        creating_fid = False
-                elif path_match and cmd == 'createFCP':
-                    creating_fid = (arg.get('fileID') == ef_fid)
-                if creating_fid:
-                    if first is None:
-                        first = idx
-                        last = idx
-                    first = min(first, idx)
-                    last = max(last, idx)
-
-            if first is not None:
-                yield fmc, first, last + 1
-
-# genericFileManagement 5G params
-
-def pes_get_adf_fid(pes:ProfileElementSequence, naa_name="usim", adf_name="adf-usim"):
-    adf = pes.get_pe_for_type(naa_name)
-    return adf.decoded[adf_name][0][1]['fileID']
-
-def mk_adf_df_path(pes, naa:str, adf:str, file_path:bytes) -> bytes:
-    adf_file_id = pes_get_adf_fid(pes, naa, adf)
-    return b''.join((adf_file_id, file_path))
-
-def gfm_get_file_content(pes: ProfileElementSequence, naa:str, adf:str, file_path:bytes, ef_fid:bytes) -> bytes:
-    '''find a given file in the genericFileManagement section, and return the bytes from the first fillFileContent
-       item.
-       TODO: implement File.from_gfm() and return the full resulting bytes?
-    '''
-    adf_df_path = mk_adf_df_path(pes, naa, adf, file_path)
-
-    data = []
-    for fmc, first_idx, after_last_idx in gfm_find(pes, adf_df_path, ef_fid):
-        assert fmc[first_idx][0] == 'createFCP'
-        assert after_last_idx > first_idx
-
-        idx = first_idx + 1
-        while idx < after_last_idx:
-            if fmc[idx][0] == 'fillFileContent':
-                data.append(fmc[idx][1])
-            idx += 1
-
-    return data
-
-def gfm_set_file_content(pes: ProfileElementSequence, naa:str, adf:str, file_path:bytes, ef_fid:bytes, file_content:bytes) -> int:
-    adf_df_path = mk_adf_df_path(pes, naa, adf, file_path)
-
-    found = 0
-    for fmc, first_idx, after_last_idx in gfm_find(pes, adf_df_path, ef_fid):
-        assert fmc[first_idx][0] == 'createFCP'
-        assert after_last_idx > first_idx
-
-        new_fmc = [
-                fmc[first_idx],
-                ('fillFileContent', file_content),
-            ]
-        new_fmc[0][1]['efFileSize'] = bytes((len(file_content), ))
-
-        fmc[first_idx:after_last_idx] = new_fmc
-
-        found += 1
-    return found
-
-class GfmSuciRi(SuciRi):
-    """SUCI Routing Indicator as in section 4.4.11.11 of 3GPP TS 31.102,
-       applied via General File Management. Intended for SAIP 2.1 profiles."""
-    name = 'GFM-5G-SUCI-RI'
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        ri = {
-            "routing_indicator": str(val),
-            "rfu": "ffff"
-        }
-        ef_ri = EF_Routing_Indicator()
-        found = gfm_set_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_ri,
-                                     ef_ri.encode_bin(ri))
-        if not found:
-            raise ValueError(f"No target file found, Cannot apply {cls.name} = {ri}")
-
-        data = gfm_get_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_ri)
-        val = ef_ri.decode_bin(b''.join(data))
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        data = gfm_get_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_ri)
-        if not data:
-            return
-
-        data = b''.join(data)
-        if not data:
-            return
-
-        ef_ri = EF_Routing_Indicator()
-        ri = ef_ri.decode_bin(data)
-        yield { cls.name: ri.get(cls.KEY_RI) }
-
-class GfmSuciCalcInfoUe(SuciCalcInfoUe):
-    """SUCI Calculation Information as in section 4.4.11.8 of 3GPP TS 31.102, readable by UE (DF-5GS),
-       applied via General File Management. Intended for SAIP 2.1 profiles."""
-    name = 'GFM-5G-SUCI-CalcInfo-UE'
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        if not isinstance(val, dict):
-            raise ValueError("val should be a dict, after 'val = SuciCalcInfoParameter.validate_val(val)'")
-
-        ef_sucici = EF_SUCI_Calc_Info()
-        body = ef_sucici.encode_bin(val)
-        gfm_set_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_sucici,
-                             body)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        data = gfm_get_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_sucici)
-        if not data:
-            return
-
-        data = b''.join(data)
-        if not data:
-            return
-
-        ef_sucici = EF_SUCI_Calc_Info()
-        sucici = ef_sucici.decode_bin(data)
-        sucici = cls.normalize_sucici(sucici)
-        yield { cls.name: json.dumps(sucici) }
-
-
-class EuiccMandatoryServiceParam(EnumParam):
-    """superclass for managing items of the ProfileHeader / eUICC-Mandatory-services ServicesList"""
-    service_name = None
-    value_map = { 'mandatory': True, 'optional': False }
-    default_source = param_source.ConstantSource
-    example_input = sorted(value_map.keys())[0]
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        for pe in pes.get_pes_for_type('header'):
-            assert isinstance(pe, ProfileElementHeader)
-            if val:
-                pe.mandatory_service_add(cls.service_name)
-            else:
-                # explicitly check to avoid exception when then service is already not present
-                if pe.mandatory_service_present(cls.service_name):
-                    pe.mandatory_service_remove(cls.service_name)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('header'):
-            assert isinstance(pe, ProfileElementHeader)
-            val = bool(pe.mandatory_service_present(cls.service_name))
-            yield { cls.name: cls.map_val_to_name(val) }
-
-class EuiccMandatoryServiceGetIdentity(EuiccMandatoryServiceParam):
-    """eUICC Mandatory Services: get-identity. The eUICC must be capable of providing a 5G identity using SUCI-CalcInfo
-    located in the USIM's DF-SAIP, see parameter 5G-SUCI-CalcInfo-USIM."""
-    name = '5G-eUICC-get-identity'
-    service_name = 'get-identity'
-
-class EuiccMandatoryServiceProfileA(EuiccMandatoryServiceParam):
-    """eUICC Mandatory Services: profile-a-x25519. The eUICC must be able to estblish a 5G identity using an X25519 key,
-    as provided in a profile-A ("identifier": 1) key in SUCI-CalcInfo located in the USIM's DF-SAIP, see parameter
-    5G-SUCI-CalcInfo-USIM."""
-    name = '5G-eUICC-profile-a-x25519'
-    service_name = 'profile-a-x25519'
-
-class EuiccMandatoryServiceProfileB(EuiccMandatoryServiceParam):
-    """eUICC Mandatory Services: profile-b-p256. The eUICC must be able to estblish a 5G identity using a P256 key, as
-    provided in a profile-B ("identifier": 2) key in SUCI-CalcInfo located in the USIM's DF-SAIP, see parameter
-    5G-SUCI-CalcInfo-USIM."""
-    name = '5G-eUICC-profile-b-p256'
-    service_name = 'profile-b-p256'

pySim/ts_31_102.py

@@ -327,7 +327,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         """conversion method to generate list of {hnet_pubkey_identifier, hnet_pubkey} dicts
         from flat [{hnet_pubkey_identifier: }, {net_pubkey: }, ...] list"""
         out = []
-        while l:
+        while len(l):
             a = l.pop(0)
             b = l.pop(0)
             z = {**a, **b}

pySim/ts_51_011.py

@@ -251,16 +251,6 @@ class EF_SMSP(LinFixedEF):
                                                       "numbering_plan_id": "isdn_e164" },
                             "call_number": "4915790109999" },
             "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
-        ( 'e1ffffffffffffffffffffffff0891945197109099f9ffffff0000a9',
-          { "alpha_id": "", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
-                                                          "tp_pid": True, "tp_dcs": True, "tp_vp": True },
-            "tp_dest_addr": { "length": 255, "ton_npi": { "ext": True, "type_of_number": "reserved_for_extension",
-                                                          "numbering_plan_id": "reserved_for_extension" },
-                              "call_number": "" },
-            "tp_sc_addr": { "length": 8, "ton_npi": { "ext": True, "type_of_number": "international",
-                                                      "numbering_plan_id": "isdn_e164" },
-                            "call_number": "4915790109999" },
-            "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
         ( '454e6574776f726b73fffffffffffffff1ffffffffffffffffffffffffffffffffffffffffffffffff0000a7',
           { "alpha_id": "ENetworks", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
                                                                "tp_pid": True, "tp_dcs": True, "tp_vp": False },
@@ -341,8 +331,7 @@ class EF_SMSP(LinFixedEF):
                         'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
         DestAddr = Struct('length'/Rebuild(Int8ub, lambda ctx: EF_SMSP.dest_addr_len(ctx)),
                         'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
-        # (see comment below)
+        self._construct = Struct('alpha_id'/COptional(GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28)))),
-        self._construct = Struct('alpha_id'/GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28))),
                                  'parameter_indicators'/InvertAdapter(BitStruct(
                                                                         Const(7, BitsInteger(3)),
                                                                         'tp_vp'/Flag,
@@ -356,25 +345,6 @@ class EF_SMSP(LinFixedEF):
                                  'tp_dcs'/Bytes(1),
                                  'tp_vp_minutes'/EF_SMSP.ValidityPeriodAdapter(Byte))
 
-    # Ensure 'alpha_id' is always present
-    def encode_record_hex(self, abstract_data: dict, record_nr: int, total_len: int = None) -> str:
-        # Problem: TS 51.011 Section 10.5.6 describes the 'alpha_id' field as optional. However, this is only true
-        # at the time when the record length of the file is set up in the file system. A card manufacturer may decide
-        # to remove the field by setting the record length to 28. Likewise, the card manaufacturer may also decide to
-        # set the field to a distinct length by setting the record length to a value greater than 28 (e.g. 14 bytes
-        # 'alpha_id' + 28 bytes). Due to the fixed nature of the record length, this eventually means that in practice
-        # 'alpha_id' is a mandatory field with a fixed length.
-        #
-        # Due to the problematic specification of 'alpha_id' as a pseudo-optional field at the beginning of a
-        # fixed-size memory, the construct definition in self._construct has been incorrectly implemented and the field
-        # has been marked as COptional. We may correct the problem by removing COptional. But to maintain compatibility,
-        # we then have to ensure that in case the field is not provided (None), it is set to an empty string ('').
-        #
-        # See also ts_31_102.py, class EF_OCI for a correct example.
-        if abstract_data['alpha_id'] is None:
-            abstract_data['alpha_id'] = ''
-        return super().encode_record_hex(abstract_data, record_nr, total_len)
-
 # TS 51.011 Section 10.5.7
 class EF_SMSS(TransparentEF):
     class MemCapAdapter(Adapter):

requirements.txt

@@ -6,7 +6,7 @@ jsonpath-ng
 construct>=2.10.70
 bidict
 pyosmocom>=0.0.12
-pyyaml>=5.4
+pyyaml>=5.1
 termcolor
 colorlog
 pycryptodomex

setup.py

@@ -26,7 +26,7 @@ setup(
         "construct >= 2.10.70",
         "bidict",
         "pyosmocom >= 0.0.12",
-        "pyyaml >= 5.4",
+        "pyyaml >= 5.1",
         "termcolor",
         "colorlog",
         "pycryptodomex",

tests/pySim-smpp2sim_test/pySim-smpp2sim_test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Utility to verify the functionality of pySim-smpp2sim.py
+# Utility to verify the functionality of pySim-trace.py
 #
 # (C) 2026 by sysmocom - s.f.m.c. GmbH
 # All Rights Reserved

tests/unittests/test_configurable_parameters.py

@@ -21,7 +21,6 @@ import io
 import sys
 import unittest
 import io
-import json
 from importlib import resources
 from osmocom.utils import hexstr
 from pySim.esim.saip import ProfileElementSequence
@@ -54,21 +53,18 @@ class ConfigurableParameterTest(unittest.TestCase):
     def test_parameters(self):
 
         upp_fnames = (
-                'SAIP2.1_gfmsuci.der',
+                'TS48v5_SAIP2.1A_NoBERTLV.der',
+                'TS48v5_SAIP2.3_BERTLV_SUCI.der',
                 'TS48v5_SAIP2.1B_NoBERTLV.der',
                 'TS48v5_SAIP2.3_NoBERTLV.der',
                )
 
         class Paramtest:
-            iff_present_default = False
+            def __init__(self, param_cls, val, expect_val, expect_clean_val=None):
-            def __init__(self, param_cls, val, expect_val, expect_clean_val=None, iff_present=None):
                 self.param_cls = param_cls
                 self.val = val
                 self.expect_clean_val = expect_clean_val
                 self.expect_val = expect_val
-                if iff_present is None:
-                    iff_present = Paramtest.iff_present_default
-                self.iff_present = iff_present
 
         param_tests = [
             Paramtest(param_cls=p13n.Imsi, val='123456',
@@ -281,102 +277,7 @@ class ConfigurableParameterTest(unittest.TestCase):
                       expect_clean_val=3,
                       expect_val='3'),
 
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceGetIdentity,
+            ]
-                      val='mandatory',
-                      expect_clean_val=True,
-                      expect_val='mandatory'),
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceGetIdentity,
-                      val='optional',
-                      expect_clean_val=False,
-                      expect_val='optional'),
-
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileA,
-                      val='mandatory',
-                      expect_clean_val=True,
-                      expect_val='mandatory'),
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileA,
-                      val='optional',
-                      expect_clean_val=False,
-                      expect_val='optional'),
-
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileB,
-                      val='mandatory',
-                      expect_clean_val=True,
-                      expect_val='mandatory'),
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileB,
-                      val='optional',
-                      expect_clean_val=False,
-                      expect_val='optional'),
-        ]
-
-        Paramtest.iff_present_default = True
-
-        sucici = {
-             "prot_scheme_id_list": [
-                 {"priority": 0, "identifier": 2, "key_index": 1},
-                 {"priority": 1, "identifier": 1, "key_index": 2},
-                 ],
-             "hnet_pubkey_list": [
-                 {"hnet_pubkey_identifier": 27,
-                  "hnet_pubkey": "0472da71976234ce833a6907425867b82e074d44ef907dfb4b3e21c1c2256ebcd15a7ded52fcbb097a4ed250e036c7b9c8c7004c4eedc4f068cd7bf8d3f900e3b4"},
-                 {"hnet_pubkey_identifier": 30,
-                  "hnet_pubkey": "5a8d38864820197c3394b92613b20b91633cbd897119273bf8e4a6f4eec0a650"},
-                  ],
-        }
-
-        param_tests.extend([
-            Paramtest(param_cls=p13n.SuciActive, val='SUCI-on',
-                      expect_clean_val=True,
-                      expect_val={'5G-SUCI-active': 'SUCI-on'}),
-            Paramtest(param_cls=p13n.SuciActive, val='SUCI-off',
-                      expect_clean_val=False,
-                      expect_val={'5G-SUCI-active': 'SUCI-off'}),
-
-            Paramtest(param_cls=p13n.SuciInUsim, val='SUCI-in-UE',
-                      expect_clean_val=False,
-                      expect_val={'5G-SUCI-in-USIM': 'SUCI-in-UE'}),
-            Paramtest(param_cls=p13n.SuciInUsim, val='SUCI-in-USIM',
-                      expect_clean_val=True,
-                      expect_val={'5G-SUCI-in-USIM': 'SUCI-in-USIM'}),
-
-            Paramtest(param_cls=p13n.SuciRi, val='123',
-                      expect_clean_val='123',
-                      expect_val={'5G-SUCI-RI': '123'}),
-            Paramtest(param_cls=p13n.SuciRi, val='0',
-                      expect_clean_val='0',
-                      expect_val={'5G-SUCI-RI': '0'}),
-            Paramtest(param_cls=p13n.SuciRi, val='9999',
-                      expect_clean_val='9999',
-                      expect_val={'5G-SUCI-RI': '9999'}),
-
-            Paramtest(param_cls=p13n.SuciCalcInfoUe,
-                      val=json.dumps(sucici),
-                      expect_clean_val=sucici,
-                      expect_val={'5G-SUCI-CalcInfo-UE': json.dumps(sucici)}),
-
-            Paramtest(param_cls=p13n.SuciCalcInfoUsim,
-                      val=json.dumps(sucici),
-                      expect_clean_val=sucici,
-                      expect_val={'5G-SUCI-CalcInfo-USIM': json.dumps(sucici)}),
-
-            Paramtest(param_cls=p13n.GfmSuciRi, val='123',
-                      expect_clean_val='123',
-                      expect_val={'GFM-5G-SUCI-RI': '123'}),
-            Paramtest(param_cls=p13n.GfmSuciRi, val='0',
-                      expect_clean_val='0',
-                      expect_val={'GFM-5G-SUCI-RI': '0'}),
-            Paramtest(param_cls=p13n.GfmSuciRi, val='9999',
-                      expect_clean_val='9999',
-                      expect_val={'GFM-5G-SUCI-RI': '9999'}),
-
-            Paramtest(param_cls=p13n.GfmSuciCalcInfoUe,
-                      val=json.dumps(sucici),
-                      expect_clean_val=sucici,
-                      expect_val={'GFM-5G-SUCI-CalcInfo-UE': json.dumps(sucici)}),
-
-        ])
-
-        Paramtest.iff_present_default = False
 
         for sdkey_cls in (
             # thin out the number of tests, as a compromise between completeness and test runtime
@@ -472,8 +373,7 @@ class ConfigurableParameterTest(unittest.TestCase):
 
                 for t in param_tests:
                     test_idx += 1
-                    testlog = []
+                    logloc = f'{upp_fname} {t.param_cls.__name__}(val={valtypestr(t.val)})'
-                    testlog.append(f'{upp_fname} {t.param_cls.__name__}(val={valtypestr(t.val)})')
 
                     param = None
                     try:
@@ -481,32 +381,21 @@ class ConfigurableParameterTest(unittest.TestCase):
                         param.input_value = t.val
                         param.validate()
                     except ValueError as e:
-                        raise ValueError(f'{" ".join(testlog)}: {e}') from e
+                        raise ValueError(f'{logloc}: {e}') from e
 
                     clean_val = param.value
-                    testlog.append(f'clean_val={valtypestr(clean_val)}')
+                    logloc = f'{logloc} clean_val={valtypestr(clean_val)}'
                     if t.expect_clean_val is not None and t.expect_clean_val != clean_val:
-                        raise ValueError(f'{" ".join(testlog)}: expected'
+                        raise ValueError(f'{logloc}: expected'
                                          f' expect_clean_val={valtypestr(t.expect_clean_val)}')
 
                     # on my laptop, deepcopy is about 30% slower than decoding the DER from scratch:
                     # pes = copy.deepcopy(orig_pes)
                     pes = ProfileElementSequence.from_der(der)
-
-                    found = list((t.param_cls.get_value_from_pes(pes) or {}).values())
-                    testlog.append(f"previous value: {found}")
-
-                    if t.iff_present and not found:
-                        testlog.append("skipping, param not in template.")
-                        output = "\nskip: " + "\n  ".join(testlog)
-                        outputs.append(output)
-                        print(output)
-                        continue
-
                     try:
                         param.apply(pes)
                     except ValueError as e:
-                        raise ValueError(f'{" ".join(testlog)} apply_val(clean_val): {e}') from e
+                        raise ValueError(f'{logloc} apply_val(clean_val): {e}') from e
 
                     changed_der = pes.to_der()
 
@@ -524,18 +413,22 @@ class ConfigurableParameterTest(unittest.TestCase):
                     else:
                         read_back_val_type = f'{type(read_back_val).__name__}'
 
-                    testlog.append(f'read_back_val={valtypestr(read_back_val)}')
+                    logloc = (f'{logloc} read_back_val={valtypestr(read_back_val)}')
 
                     if isinstance(read_back_val, dict) and not t.param_cls.get_name() in read_back_val.keys():
-                        raise ValueError(f'{" ".join(testlog)}: expected to find name {t.param_cls.get_name()!r} in read_back_val')
+                        raise ValueError(f'{logloc}: expected to find name {t.param_cls.get_name()!r} in read_back_val')
 
                     expect_val = t.expect_val
                     if not isinstance(expect_val, dict):
                         expect_val = { t.param_cls.get_name(): expect_val }
                     if read_back_val != expect_val:
-                        raise ValueError(f'{" ".join(testlog)}: expected {expect_val=!r}:{type(t.expect_val).__name__}')
+                        raise ValueError(f'{logloc}: expected {expect_val=!r}:{type(t.expect_val).__name__}')
 
-                    output = "\nok: " + "\n  ".join(testlog)
+                    ok = logloc.replace(' clean_val', '\n\tclean_val'
+                              ).replace(' read_back_val', '\n\tread_back_val'
+                              ).replace('=', '=\t'
+                              )
+                    output = f'\nok: {ok}'
                     outputs.append(output)
                     print(output)