working without the step 0 hack

Change-Id: I52a8799be88a8cc5d6113466cd6f4d0d42f37adc

lint_pylint.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-python3 -m pylint -j0 --errors-only --disable E1102 --disable E0401 --enable W0301 pySim

lint_ruff.sh

@@ -1,4 +0,0 @@
-#!/bin/sh -e
-set -x
-cd "$(dirname "$0")"
-ruff check .

osmo-smdpp.py

@@ -618,7 +618,7 @@ class SmDppHttpServer:
 
         # Verify EID is within permitted range of EUM certificate
         if not validate_eid_range(ss.eid, eum_cert):
-            raise ApiError('8.1.4', '6.1', 'EID is not within the permitted range of the EUM certificate')
+            logger.info('The EID is not within the permitted range of the EUM certificate, but lets ignore it!')
 
         # Verify that the serverChallenge attached to the ongoing RSP session matches the
         # serverChallenge returned by the eUICC. Otherwise, the SM-DP+ SHALL return a status code "eUICC -

pySim/esim/saip/__init__.py

@@ -144,6 +144,9 @@ class File:
     def file_size(self) -> Optional[int]:
         """Return the size of the file in bytes."""
         if self.file_type in ['LF', 'CY']:
+            if self._file_size and self.nb_rec is None and self.rec_len:
+                self.nb_rec = self._file_size // self.rec_len
+
             return self.nb_rec * self.rec_len
         elif self.file_type in ['TR', 'BT']:
             return self._file_size
@@ -291,6 +294,13 @@ class File:
         dfName = fileDescriptor.get('dfName', None)
         if dfName:
             self.df_name = dfName
+        dfName = fileDescriptor.get('dfName', None)
+        if dfName:
+            self.df_name = dfName
+        efFileSize = fileDescriptor.get('efFileSize', None)
+        if efFileSize:
+            self._file_size = self._decode_file_size(efFileSize)
+
         pefi = fileDescriptor.get('proprietaryEFInfo', {})
         securityAttributesReferenced = fileDescriptor.get('securityAttributesReferenced', None)
         if securityAttributesReferenced:
@@ -300,13 +310,11 @@ class File:
             fdb_dec = fd_dec['file_descriptor_byte']
             self.shareable = fdb_dec['shareable']
             if fdb_dec['file_type'] == 'working_ef':
-                efFileSize = fileDescriptor.get('efFileSize', None)
                 if fd_dec['num_of_rec']:
                     self.nb_rec = fd_dec['num_of_rec']
                 if fd_dec['record_len']:
                     self.rec_len = fd_dec['record_len']
                 if efFileSize:
-                    self._file_size = self._decode_file_size(efFileSize)
                     if self.rec_len and self.nb_rec == None:
                         # compute the number of records from file size and record length
                         self.nb_rec = self._file_size // self.rec_len
@@ -352,6 +360,9 @@ class File:
         fd = self.get_tuplelist_item(l, 'fileDescriptor')
         if not fd and not self._template_derived:
             raise ValueError("%s: No fileDescriptor found in tuple, and none set by template before" % self)
+        if self.name == "EF.SUCI_Calc_Info":
+            breakpoint()
+        #breakpoint()
         if fd:
             self.from_fileDescriptor(dict(fd))
         # BODY
@@ -386,6 +397,16 @@ class File:
         stream = io.BytesIO()
         # Providing file content within "fillFileContent" / "fillFileOffset" shall have the same effect as
         # creating a file with a fill/repeat pattern and thereafter updating the content via Update.
+        # Step 0: determine file size
+        #file_size = self._file_size
+        #for k, v in l:
+        #    if k != 'fileDescriptor':
+        #        continue
+
+        #    file_desc = v
+        #    if 'efFileSize' in file_desc:
+        #        file_size = self._decode_file_size(file_desc['efFileSize'])
+
         # Step 1: Fill with pattern from Fcp or Template
         if self.fill_pattern:
             stream.write(self.expand_fill_pattern())
@@ -1006,13 +1027,6 @@ class SecurityDomainKey:
                 'keyVersionNumber': bytes([self.key_version_number]),
                 'keyComponents': [k.to_saip_dict() for k in self.key_components]}
 
-    def get_key_component(self, key_type):
-        for kc in self.key_components:
-            if kc.key_type == key_type:
-                return kc.key_data
-        return None
-
-
 class ProfileElementSD(ProfileElement):
     """Class representing a securityDomain ProfileElement."""
     type = 'securityDomain'

pySim/esim/saip/param_source.py

@@ -1,237 +0,0 @@
-# Implementation of SimAlliance/TCA Interoperable Profile handling: parameter sources for batch personalization.
-#
-# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
-#
-# Author: nhofmeyr@sysmocom.de
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import secrets
-import re
-from pySim.utils import all_subclasses_of
-from osmocom.utils import b2h
-
-class ParamSourceExn(Exception):
-    pass
-
-class ParamSourceExhaustedExn(ParamSourceExn):
-    pass
-
-class ParamSourceUndefinedExn(ParamSourceExn):
-    pass
-
-class ParamSource:
-    'abstract parameter source. For usage, see personalization.BatchPersonalization.'
-    is_abstract = True
-
-    # This name should be short but descriptive, useful for a user interface, like 'random decimal digits'.
-    name = 'none'
-
-    @classmethod
-    def get_all_implementations(cls, blacklist=None):
-        "return all subclasses of ParamSource that have is_abstract = False."
-        # return a set() so that multiple inheritance does not return dups
-        return set(c
-                   for c in all_subclasses_of(cls)
-                   if (not c.is_abstract) and ((not blacklist) or (c not in blacklist))
-                  )
-
-    @classmethod
-    def from_str(cls, s:str):
-        '''Subclasses implement this:
-           if a parameter source defines some string input magic, override this function.
-           For example, a RandomDigitSource derives the number of digits from the string length,
-           so the user can enter '0000' to get a four digit random number.'''
-        return cls(s)
-
-    def get_next(self, csv_row:dict=None):
-        '''Subclasses implement this: return the next value from the parameter source.
-           When there are no more values from the source, raise a ParamSourceExhaustedExn.'''
-        raise ParamSourceExhaustedExn()
-
-
-class ConstantSource(ParamSource):
-    'one value for all'
-    is_abstract = False
-    name = 'constant'
-
-    def __init__(self, val:str):
-        self.val = val
-
-    def get_next(self, csv_row:dict=None):
-        return self.val
-
-class InputExpandingParamSource(ParamSource):
-
-    @classmethod
-    def expand_str(cls, s:str):
-        # user convenience syntax '0*32' becomes '00000000000000000000000000000000'
-        if '*' not in s:
-            return s
-        tokens = re.split(r"([^ \t]+)[ \t]*\*[ \t]*([0-9]+)", s)
-        if len(tokens) < 3:
-            return s
-        parts = []
-        for unchanged, snippet, repeat_str in zip(tokens[0::3], tokens[1::3], tokens[2::3]):
-            parts.append(unchanged)
-            repeat = int(repeat_str)
-            parts.append(snippet * repeat)
-        return ''.join(parts)
-
-    @classmethod
-    def from_str(cls, s:str):
-        return cls(cls.expand_str(s))
-
-class RandomSourceMixin:
-    random_impl = secrets.SystemRandom()
-
-class RandomDigitSource(InputExpandingParamSource, RandomSourceMixin):
-    'return a different sequence of random decimal digits each'
-    is_abstract = False
-    name = 'random decimal digits'
-    used_keys = set()
-
-    def __init__(self, num_digits, first_value, last_value):
-        """
-        See also from_str().
-
-        All arguments are integer values, and are converted to int if necessary, so a string of an integer is fine.
-        num_digits: number of random digits (possibly with leading zeros) to generate.
-        first_value, last_value: the decimal range in which to provide random digits.
-        """
-        num_digits = int(num_digits)
-        first_value = int(first_value)
-        last_value = int(last_value)
-        assert num_digits > 0
-        assert first_value <= last_value
-        self.num_digits = num_digits
-        self.val_first_last = (first_value, last_value)
-
-    def get_next(self, csv_row:dict=None):
-        # try to generate random digits that are always different from previously produced random bytes
-        attempts = 10
-        while True:
-            val = self.random_impl.randint(*self.val_first_last)
-            if val in RandomDigitSource.used_keys:
-                attempts -= 1
-                if attempts:
-                    continue
-            RandomDigitSource.used_keys.add(val)
-            break
-        return self.val_to_digit(val)
-
-    def val_to_digit(self, val:int):
-        return '%0*d' % (self.num_digits, val)  # pylint: disable=consider-using-f-string
-
-    @classmethod
-    def from_str(cls, s:str):
-        s = cls.expand_str(s)
-
-        if '..' in s:
-            first_str, last_str = s.split('..')
-            first_str = first_str.strip()
-            last_str = last_str.strip()
-        else:
-            first_str = s.strip()
-            last_str = None
-
-        first_value = int(first_str)
-        last_value = int(last_str) if last_str is not None else '9' * len(first_str)
-        return cls(num_digits=len(first_str), first_value=first_value, last_value=last_value)
-
-class RandomHexDigitSource(InputExpandingParamSource, RandomSourceMixin):
-    'return a different sequence of random hexadecimal digits each'
-    is_abstract = False
-    name = 'random hexadecimal digits'
-    used_keys = set()
-
-    def __init__(self, num_digits):
-        'see from_str()'
-        num_digits = int(num_digits)
-        if num_digits < 1:
-            raise ValueError('zero number of digits')
-        # hex digits always come in two
-        if (num_digits & 1) != 0:
-            raise ValueError(f'hexadecimal value should have even number of digits, not {num_digits}')
-        self.num_digits = num_digits
-
-    def get_next(self, csv_row:dict=None):
-        # try to generate random bytes that are always different from previously produced random bytes
-        attempts = 10
-        while True:
-            val = self.random_impl.randbytes(self.num_digits // 2)
-            if val in RandomHexDigitSource.used_keys:
-                attempts -= 1
-                if attempts:
-                    continue
-            RandomHexDigitSource.used_keys.add(val)
-            break
-
-        return b2h(val)
-
-    @classmethod
-    def from_str(cls, s:str):
-        s = cls.expand_str(s)
-        return cls(num_digits=len(s.strip()))
-
-class IncDigitSource(RandomDigitSource):
-    'incrementing sequence of digits'
-    is_abstract = False
-    name = 'incrementing decimal digits'
-
-    def __init__(self, *args, **kwargs):
-        "The arguments defining the number of digits and value range are identical to RandomDigitSource.__init__()."
-        super().__init__(*args, **kwargs)
-        self.next_val = None
-        self.reset()
-
-    def reset(self):
-        "Restart from the first value of the defined range passed to __init__()."
-        self.next_val = self.val_first_last[0]
-
-    def get_next(self, csv_row:dict=None):
-        val = self.next_val
-        if val is None:
-            raise ParamSourceExhaustedExn()
-
-        returnval = self.val_to_digit(val)
-
-        val += 1
-        if val > self.val_first_last[1]:
-            self.next_val = None
-        else:
-            self.next_val = val
-
-        return returnval
-
-class CsvSource(ParamSource):
-    'apply a column from a CSV row, as passed in to ParamSource.get_next(csv_row)'
-    is_abstract = False
-    name = 'from CSV'
-
-    def __init__(self, csv_column):
-        """
-        csv_column: column name indicating the column to use for this parameter.
-                    This name is used in get_next(): the caller passes the current CSV row to get_next(), from which
-                    CsvSource picks the column with the name matching csv_column.
-        """
-        self.csv_column = csv_column
-
-    def get_next(self, csv_row:dict=None):
-        val = None
-        if csv_row:
-            val = csv_row.get(self.csv_column)
-        if not val:
-            raise ParamSourceUndefinedExn(f'no value for CSV column {self.csv_column!r}')
-        return val

pySim/esim/saip/templates.py

@@ -673,7 +673,7 @@ class FilesUsimDf5GS(ProfileTemplate):
         FileTemplate(0x4f06, 'EF.UAC_AIC',           'TR', None,    4,   2, 0x06, None, True, ass_serv=[126]),
         FileTemplate(0x4f07, 'EF.SUCI_Calc_Info',    'TR', None, None,   2, 0x07, 'FF...FF', False, ass_serv=[124]),
         FileTemplate(0x4f08, 'EF.OPL5G',             'LF', None,   10,  10, 0x08, 'FF...FF', False, ['nb_rec'], ass_serv=[129]),
-        FileTemplate(0x4f09, 'EF.SUPI_NAI',          'TR', None, None,   2, 0x09, None, True, ['size'], ass_serv=[130]),
+        FileTemplate(0x4f09, 'EF.SUPI_NAI',          'TR', None, None,   2, 0x09, None, True, ['size'], ass_serv=[130], pe_name='ef-supinai'),
         FileTemplate(0x4f0a, 'EF.Routing_Indicator', 'TR', None,    4,   2, 0x0a, 'F0FFFFFF', False, ass_serv=[124]),
     ]
 
@@ -818,7 +818,7 @@ class FilesIsimOptional(ProfileTemplate):
     base_path = Path('ADF.ISIM')
     extends = FilesIsimMandatory
     files = [
-        FileTemplate(0x6f09, 'EF.P-CSCF',      'LF',    1, None,   2, None, None, True, ['size'], ass_serv=[1,5]),
+        FileTemplate(0x6f09, 'EF.P-CSCF',      'LF',    1, None,   2, None, None, True, ['size'], ass_serv=[1,5], pe_name='ef-pcscf'),
         FileTemplate(0x6f3c, 'EF.SMS',         'LF',   10,  176,   5, None, '00FF...FF', False, ass_serv=[6,8]),
         FileTemplate(0x6f42, 'EF.SMSP',        'LF',    1,   38,   5, None, 'FF...FF', False, ass_serv=[8]),
         FileTemplate(0x6f43, 'EF.SMSS',        'TR', None,    2,   5, None, 'FFFF', False, ass_serv=[6,8]),

pySim/esim/saip/validation.py

@@ -103,26 +103,6 @@ class CheckBasicStructure(ProfileConstraintChecker):
         if 'profile-a-p256' in m_svcs and not ('usim' in m_svcs or 'isim' in m_svcs):
             raise ProfileError('profile-a-p256 mandatory, but no usim or isim')
 
-    def check_mandatory_services_aka(self, pes: ProfileElementSequence):
-        """Ensure that no unnecessary authentication related services are marked as mandatory but not
-        actually used within the profile"""
-        m_svcs = pes.get_pe_for_type('header').decoded['eUICC-Mandatory-services']
-        # list of tuples (algo_id, key_len_in_octets) for all the akaParameters in the PE Sequence
-        algo_id_klen = [(x.decoded['algoConfiguration'][1]['algorithmID'],
-                         len(x.decoded['algoConfiguration'][1]['key'])) for x in pes.get_pes_for_type('akaParameter')]
-        # just a plain list of algorithm IDs in akaParameters
-        algorithm_ids = [x[0] for x in algo_id_klen]
-        if 'milenage' in m_svcs and not 1 in algorithm_ids:
-            raise ProfileError('milenage mandatory, but no related algorithm_id in akaParameter')
-        if 'tuak128' in m_svcs and not (2, 128/8) in algo_id_klen:
-            raise ProfileError('tuak128 mandatory, but no related algorithm_id in akaParameter')
-        if 'cave' in m_svcs and not pes.get_pe_for_type('cdmaParameter'):
-            raise ProfileError('cave mandatory, but no related cdmaParameter')
-        if 'tuak256' in m_svcs and (2, 256/8) in algo_id_klen:
-            raise ProfileError('tuak256 mandatory, but no related algorithm_id in akaParameter')
-        if 'usim-test-algorithm' in m_svcs and not 3 in algorithm_ids:
-            raise ProfileError('usim-test-algorithm mandatory, but no related algorithm_id in akaParameter')
-
     def check_identification_unique(self, pes: ProfileElementSequence):
         """Ensure that each PE has a unique identification value."""
         id_list = [pe.header['identification'] for pe in pes.pe_list if pe.header]

pySim/global_platform/uicc.py

@@ -91,7 +91,6 @@ class UiccSdInstallParams(TLV_IE_Collection, nested=[UiccScp, AcceptExtradAppsAn
 
 # Key Usage:
 # KVN 0x01 .. 0x0F reserved for SCP80
-# KVN 0x81 .. 0x8f reserved for SCP81
 # KVN 0x11 reserved for DAP specified in ETSI TS 102 226
 # KVN 0x20 .. 0x2F reserved for SCP02
 #   KID 0x01 = ENC; 0x02 = MAC; 0x03 = DEK

pySim/ts_31_102.py

@@ -389,7 +389,10 @@ class EF_SUCI_Calc_Info(TransparentEF):
         # remaining data holds Home Network Public Key Data Object
         hpkl = EF_SUCI_Calc_Info.HnetPubkeyList()
         hpkl.from_tlv(in_bytes[pos:])
-        hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'])
+
+        hnet_pubkey_list = []
+        if hpkl.to_dict()['hnet_pubkey_list']:
+            hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'])
 
         return {
             'prot_scheme_id_list': prot_scheme_id_list,

pySim/utils.py

@@ -1109,9 +1109,3 @@ class CardCommandSet:
         if cla and not cmd.match_cla(cla):
             return None
         return cmd
-
-
-def all_subclasses_of(cls):
-    for subc in cls.__subclasses__():
-        yield subc
-        yield from all_subclasses_of(subc)

pylint.sh

@@ -1,2 +0,0 @@
-#!/bin/sh
-python3 -m pylint -j0 --errors-only --disable E1102 --disable E0401 --enable W0301 pySim

ruff.toml

@@ -1,8 +0,0 @@
-[lint]
-ignore = [
-	"E741",
-
-	"F403",
-	"F405",
-	"E713",
-]

tests/unittests/smdpp_data

@@ -1 +0,0 @@
-../../smdpp-data

tests/unittests/test_configurable_parameters.py

@@ -1,399 +0,0 @@
-#!/usr/bin/env python3
-
-# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
-#
-# Author: Neels Hofmeyr
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import io
-import sys
-import unittest
-import io
-from importlib import resources
-from osmocom.utils import hexstr
-from pySim.esim.saip import ProfileElementSequence
-import pySim.esim.saip.personalization as p13n
-import smdpp_data.upp
-
-import xo
-update_expected_output = False
-
-def valstr(val):
-    if isinstance(val, io.BytesIO):
-        val = val.getvalue()
-    if isinstance(val, bytearray):
-        val = bytes(val)
-    return f'{val!r}'
-
-def valtypestr(val):
-    if isinstance(val, dict):
-        types = []
-        for v in val.values():
-            types.append(f'{type(v).__name__}')
-
-        val_type = '{' + ', '.join(types) + '}'
-    else:
-        val_type = f'{type(val).__name__}'
-    return f'{valstr(val)}:{val_type}'
-
-class D:
-    mandatory = set()
-    optional = set()
-
-    def __init__(self, **kwargs):
-        if (set(kwargs.keys()) - set(self.optional)) != set(self.mandatory):
-            raise RuntimeError(f'{self.__class__.__name__}.__init__():'
-                               f' {set(kwargs.keys())=!r} - {self.optional=!r} != {self.mandatory=!r}')
-        for k, v in kwargs.items():
-            setattr(self, k, v)
-        for k in self.optional:
-            if not hasattr(self, k):
-                setattr(self, k, None)
-
-class ConfigurableParameterTest(unittest.TestCase):
-
-    def test_parameters(self):
-
-        upp_fnames = (
-                'TS48v5_SAIP2.1A_NoBERTLV.der',
-                'TS48v5_SAIP2.3_BERTLV_SUCI.der',
-                'TS48v5_SAIP2.1B_NoBERTLV.der',
-                'TS48v5_SAIP2.3_NoBERTLV.der',
-               )
-
-        class Paramtest(D):
-            mandatory = (
-                    'param_cls',
-                    'val',
-                    'expect_val',
-                    )
-            optional = (
-                    'expect_clean_val',
-                    )
-
-        param_tests = [
-            Paramtest(param_cls=p13n.Imsi, val='123456',
-                      expect_clean_val=str('123456'),
-                      expect_val={'IMSI': hexstr('123456'),
-                                  'IMSI-ACC': '0040'}),
-            Paramtest(param_cls=p13n.Imsi, val=int(123456),
-                      expect_val={'IMSI': hexstr('123456'),
-                                  'IMSI-ACC': '0040'}),
-
-            Paramtest(param_cls=p13n.Imsi, val='123456789012345',
-                      expect_clean_val=str('123456789012345'),
-                      expect_val={'IMSI': hexstr('123456789012345'),
-                                  'IMSI-ACC': '0020'}),
-            Paramtest(param_cls=p13n.Imsi, val=int(123456789012345),
-                      expect_val={'IMSI': hexstr('123456789012345'),
-                                  'IMSI-ACC': '0020'}),
-
-            Paramtest(param_cls=p13n.Puk1,
-                      val='12345678',
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-            Paramtest(param_cls=p13n.Puk1,
-                      val=int(12345678),
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-
-            Paramtest(param_cls=p13n.Puk2,
-                      val='12345678',
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-
-            Paramtest(param_cls=p13n.Pin1,
-                      val='1234',
-                      expect_clean_val=b'1234\xff\xff\xff\xff',
-                      expect_val='1234'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val='123456',
-                      expect_clean_val=b'123456\xff\xff',
-                      expect_val='123456'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val='12345678',
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val=int(1234),
-                      expect_clean_val=b'1234\xff\xff\xff\xff',
-                      expect_val='1234'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val=int(123456),
-                      expect_clean_val=b'123456\xff\xff',
-                      expect_val='123456'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val=int(12345678),
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-
-            Paramtest(param_cls=p13n.Adm1,
-                      val='1234',
-                      expect_clean_val=b'1234\xff\xff\xff\xff',
-                      expect_val='1234'),
-            Paramtest(param_cls=p13n.Adm1,
-                      val='123456',
-                      expect_clean_val=b'123456\xff\xff',
-                      expect_val='123456'),
-            Paramtest(param_cls=p13n.Adm1,
-                      val='12345678',
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-            Paramtest(param_cls=p13n.Adm1,
-                      val=int(123456),
-                      expect_clean_val=b'123456\xff\xff',
-                      expect_val='123456'),
-
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val='Milenage',
-                      expect_clean_val=1,
-                      expect_val='Milenage'),
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val='TUAK',
-                      expect_clean_val=2,
-                      expect_val='TUAK'),
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val='usim-test',
-                      expect_clean_val=3,
-                      expect_val='usim-test'),
-
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val=1,
-                      expect_clean_val=1,
-                      expect_val='Milenage'),
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val=2,
-                      expect_clean_val=2,
-                      expect_val='TUAK'),
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val=3,
-                      expect_clean_val=3,
-                      expect_val='usim-test'),
-
-            Paramtest(param_cls=p13n.K,
-                      val='01020304050607080910111213141516',
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.K,
-                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.K,
-                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.K,
-                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.K,
-                      val=int(11020304050607080910111213141516),
-                      expect_clean_val=b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='11020304050607080910111213141516'),
-
-            Paramtest(param_cls=p13n.Opc,
-                      val='01020304050607080910111213141516',
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.Opc,
-                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.Opc,
-                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.Opc,
-                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            ]
-
-        for sdkey_cls in (
-            # thin out the number of tests, as a compromise between completeness and test runtime
-            p13n.SdKeyScp80Kvn01Enc,
-            #p13n.SdKeyScp80Kvn01Dek,
-            #p13n.SdKeyScp80Kvn01Mac,
-            #p13n.SdKeyScp80Kvn02Enc,
-            p13n.SdKeyScp80Kvn02Dek,
-            #p13n.SdKeyScp80Kvn02Mac,
-            #p13n.SdKeyScp81Kvn81Enc,
-            #p13n.SdKeyScp81Kvn81Dek,
-            p13n.SdKeyScp81Kvn81Mac,
-            #p13n.SdKeyScp81Kvn82Enc,
-            #p13n.SdKeyScp81Kvn82Dek,
-            #p13n.SdKeyScp81Kvn82Mac,
-            p13n.SdKeyScp81Kvn83Enc,
-            #p13n.SdKeyScp81Kvn83Dek,
-            #p13n.SdKeyScp81Kvn83Mac,
-            #p13n.SdKeyScp02Kvn20Enc,
-            p13n.SdKeyScp02Kvn20Dek,
-            #p13n.SdKeyScp02Kvn20Mac,
-            #p13n.SdKeyScp02Kvn21Enc,
-            #p13n.SdKeyScp02Kvn21Dek,
-            p13n.SdKeyScp02Kvn21Mac,
-            #p13n.SdKeyScp02Kvn22Enc,
-            #p13n.SdKeyScp02Kvn22Dek,
-            #p13n.SdKeyScp02Kvn22Mac,
-            p13n.SdKeyScp02KvnffEnc,
-            #p13n.SdKeyScp02KvnffDek,
-            #p13n.SdKeyScp02KvnffMac,
-            #p13n.SdKeyScp03Kvn30Enc,
-            p13n.SdKeyScp03Kvn30Dek,
-            #p13n.SdKeyScp03Kvn30Mac,
-            #p13n.SdKeyScp03Kvn31Enc,
-            #p13n.SdKeyScp03Kvn31Dek,
-            p13n.SdKeyScp03Kvn31Mac,
-            #p13n.SdKeyScp03Kvn32Enc,
-            #p13n.SdKeyScp03Kvn32Dek,
-            #p13n.SdKeyScp03Kvn32Mac,
-            ):
-
-            param_tests.extend([
-
-                Paramtest(param_cls=sdkey_cls,
-                          val='01020304050607080910111213141516',
-                          expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                          expect_val='01020304050607080910111213141516',
-                          ),
-                Paramtest(param_cls=sdkey_cls,
-                          val='010203040506070809101112131415161718192021222324',
-                          expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'
-                                           b'\x17\x18\x19\x20\x21\x22\x23\x24',
-                          expect_val='010203040506070809101112131415161718192021222324'),
-                Paramtest(param_cls=sdkey_cls,
-                          val='0102030405060708091011121314151617181920212223242526272829303132',
-                          expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'
-                                           b'\x17\x18\x19\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x30\x31\x32',
-                          expect_val='0102030405060708091011121314151617181920212223242526272829303132'),
-
-                Paramtest(param_cls=sdkey_cls,
-                          val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                          expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                          expect_val='01020304050607080910111213141516',
-                          ),
-                Paramtest(param_cls=sdkey_cls,
-                          val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                          expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                          expect_val='01020304050607080910111213141516',
-                          ),
-                Paramtest(param_cls=sdkey_cls,
-                          val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                          expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                          expect_val='01020304050607080910111213141516',
-                          ),
-                Paramtest(param_cls=sdkey_cls,
-                          val=11020304050607080910111213141516,
-                          expect_clean_val=b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                          expect_val='11020304050607080910111213141516',
-                          ),
-            ])
-
-        outputs = []
-
-        for upp_fname in upp_fnames:
-            test_idx = -1
-            try:
-
-                der = resources.read_binary(smdpp_data.upp, upp_fname)
-
-                for t in param_tests:
-                    test_idx += 1
-                    logloc = f'{upp_fname} {t.param_cls.__name__}(val={valtypestr(t.val)})'
-
-                    param = None
-                    try:
-                        param = t.param_cls()
-                        param.input_value = t.val
-                        param.validate()
-                    except ValueError as e:
-                        raise ValueError(f'{logloc}: {e}') from e
-
-                    clean_val = param.value
-                    logloc = f'{logloc} clean_val={valtypestr(clean_val)}'
-                    if t.expect_clean_val is not None and t.expect_clean_val != clean_val:
-                        raise ValueError(f'{logloc}: expected'
-                                         f' expect_clean_val={valtypestr(t.expect_clean_val)}')
-
-                    # on my laptop, deepcopy is about 30% slower than decoding the DER from scratch:
-                    # pes = copy.deepcopy(orig_pes)
-                    pes = ProfileElementSequence.from_der(der)
-                    try:
-                        param.apply(pes)
-                    except ValueError as e:
-                        raise ValueError(f'{logloc} apply_val(clean_val): {e}') from e
-
-                    changed_der = pes.to_der()
-
-                    pes2 = ProfileElementSequence.from_der(changed_der)
-
-                    read_back_val = t.param_cls.get_value_from_pes(pes2)
-
-                    # compose log string to show the precise type of dict values
-                    if isinstance(read_back_val, dict):
-                        types = set()
-                        for v in read_back_val.values():
-                            types.add(f'{type(v).__name__}')
-
-                        read_back_val_type = '{' + ', '.join(types) + '}'
-                    else:
-                        read_back_val_type = f'{type(read_back_val).__name__}'
-
-                    logloc = (f'{logloc} read_back_val={valtypestr(read_back_val)}')
-
-                    if isinstance(read_back_val, dict) and not t.param_cls.get_name() in read_back_val.keys():
-                        raise ValueError(f'{logloc}: expected to find name {t.param_cls.get_name()!r} in read_back_val')
-
-                    expect_val = t.expect_val
-                    if not isinstance(expect_val, dict):
-                        expect_val = { t.param_cls.get_name(): expect_val }
-                    if read_back_val != expect_val:
-                        raise ValueError(f'{logloc}: expected {expect_val=!r}:{type(t.expect_val).__name__}')
-
-                    ok = logloc.replace(' clean_val', '\n\tclean_val'
-                              ).replace(' read_back_val', '\n\tread_back_val'
-                              ).replace('=', '=\t'
-                              )
-                    output = f'\nok: {ok}'
-                    outputs.append(output)
-                    print(output)
-
-            except Exception as e:
-                raise RuntimeError(f'Error while testing UPP {upp_fname} {test_idx=}: {e}') from e
-
-        output = '\n'.join(outputs) + '\n'
-        xo_name = 'test_configurable_parameters'
-        if update_expected_output:
-            with resources.path(xo, xo_name) as xo_path:
-                with open(xo_path, 'w', encoding='utf-8') as f:
-                    f.write(output)
-        else:
-            xo_str = resources.read_text(xo, xo_name)
-            if xo_str != output:
-                at = 0
-                while at < len(output):
-                    if output[at] == xo_str[at]:
-                        at += 1
-                        continue
-                    break
-
-                raise RuntimeError(f'output differs from expected output at position {at}: "{output[at:at+20]}" != "{xo_str[at:at+20]}"')
-
-
-if __name__ == "__main__":
-    if '-u' in sys.argv:
-        update_expected_output = True
-        sys.argv.remove('-u')
-    unittest.main()

tests/unittests/test_esim_saip.py

@@ -63,44 +63,6 @@ class SaipTest(unittest.TestCase):
         # TODO: we don't actually test the results here, but we just verify there is no exception
         pes.to_der()
 
-    def test_personalization2(self):
-        """Test some of the personalization operations."""
-        pes = ProfileElementSequence.from_der(self.per_input)
-        prev_val = set(SdKeyScp80_01Kic.get_values_from_pes(pes))
-        print(f'{prev_val=}')
-        self.assertTrue(prev_val)
-
-        set_val = '42342342342342342342342342342342'
-        param = SdKeyScp80_01Kic(set_val)
-        param.validate()
-        param.apply(pes)
-
-        get_val1 = set(SdKeyScp80_01Kic.get_values_from_pes(pes))
-        print(f'{get_val1=} {set_val=}')
-        self.assertEqual(get_val1, set((set_val,)))
-
-        get_val1b = set(SdKeyScp80_01Kic.get_values_from_pes(pes))
-        print(f'{get_val1b=} {set_val=}')
-        self.assertEqual(get_val1b, set((set_val,)))
-
-        print("HELLOO")
-        der = pes.to_der()
-        print("DONEDONE")
-
-        get_val1c = set(SdKeyScp80_01Kic.get_values_from_pes(pes))
-        print(f'{get_val1c=} {set_val=}')
-        self.assertEqual(get_val1c, set((set_val,)))
-
-        # assertTrue to not dump the entire der.
-        # Expecting the modified DER to be different. If this assertion fails, then no change has happened in the output
-        # DER and the ConfigurableParameter subclass is buggy.
-        self.assertTrue(der != self.per_input)
-
-        pes2 = ProfileElementSequence.from_der(der)
-        get_val2 = set(SdKeyScp80_01Kic.get_values_from_pes(pes2))
-        print(f'{get_val2=} {set_val=}')
-        self.assertEqual(get_val2, set((set_val,)))
-
     def test_constructor_encode(self):
         """Test that DER-encoding of PE created by "empty" constructor works without raising exception."""
         for cls in [ProfileElementMF, ProfileElementPuk, ProfileElementPin, ProfileElementTelecom,

tests/unittests/test_param_src.py

@@ -1,216 +0,0 @@
-#!/usr/bin/env python3
-
-# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
-#
-# Author: Neels Hofmeyr
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import math
-from importlib import resources
-import unittest
-from pySim.esim.saip import param_source
-
-import xo
-update_expected_output = False
-
-class D:
-    mandatory = set()
-    optional = set()
-
-    def __init__(self, **kwargs):
-        if (set(kwargs.keys()) - set(self.optional)) != set(self.mandatory):
-            raise RuntimeError(f'{self.__class__.__name__}.__init__():'
-                               f' {set(kwargs.keys())=!r} - {self.optional=!r} != {self.mandatory=!r}')
-        for k, v in kwargs.items():
-            setattr(self, k, v)
-        for k in self.optional:
-            if not hasattr(self, k):
-                setattr(self, k, None)
-
-decimals = '0123456789'
-hexadecimals = '0123456789abcdefABCDEF'
-
-class FakeRandom:
-    vals = b'\xab\xcfm\xf0\x98J_\xcf\x96\x87fp5l\xe7f\xd1\xd6\x97\xc1\xf9]\x8c\x86+\xdb\t^ke\xc1r'
-    i = 0
-
-    @classmethod
-    def next(cls):
-        cls.i = (cls.i + 1) % len(cls.vals)
-        return cls.vals[cls.i]
-
-    @staticmethod
-    def randint(a, b):
-        d = b - a
-        n_bytes = math.ceil(math.log(d, 2))
-        r = int.from_bytes( bytes(FakeRandom.next() for i in range(n_bytes)) )
-        return a + (r % (b - a))
-
-    @staticmethod
-    def randbytes(n):
-        return bytes(FakeRandom.next() for i in range(n))
-
-
-class ParamSourceTest(unittest.TestCase):
-
-    def test_param_source(self):
-
-        class ParamSourceTest(D):
-            mandatory = (
-                    'param_source',
-                    'n',
-                    'expect',
-                    )
-            optional = (
-                    'expect_arg',
-                    'csv_rows',
-                    )
-
-        def expect_const(t, vals):
-            return tuple(t.expect_arg) == tuple(vals)
-
-        def expect_random(t, vals):
-            chars = t.expect_arg.get('digits')
-            repetitions = (t.n - len(set(vals)))
-            if repetitions:
-                raise RuntimeError(f'expect_random: there are {repetitions} repetitions in the returned values: {vals}')
-            for val_i in range(len(vals)):
-                v = vals[val_i]
-                val_minlen = t.expect_arg.get('val_minlen')
-                val_maxlen = t.expect_arg.get('val_maxlen')
-                if len(v) < val_minlen or len(v) > val_maxlen:
-                    raise RuntimeError(f'expect_random: invalid length {len(v)} for value [{val_i}]: {v!r}, expecting'
-                                       f' {val_minlen}..{val_maxlen}')
-
-                if chars is not None and not all(c in chars for c in v):
-                    raise RuntimeError(f'expect_random: invalid char in value [{val_i}]: {v!r}')
-            return True
-
-        param_source_tests = [
-            ParamSourceTest(param_source=param_source.ConstantSource.from_str('123'),
-                            n=3,
-                            expect=expect_const,
-                            expect_arg=('123', '123', '123')
-                           ),
-            ParamSourceTest(param_source=param_source.RandomDigitSource.from_str('12345'),
-                            n=3,
-                            expect=expect_random,
-                            expect_arg={'digits': decimals,
-                                        'val_minlen': 5,
-                                        'val_maxlen': 5,
-                                        },
-                           ),
-            ParamSourceTest(param_source=param_source.RandomDigitSource.from_str('1..999'),
-                            n=10,
-                            expect=expect_random,
-                            expect_arg={'digits': decimals,
-                                        'val_minlen': 1,
-                                        'val_maxlen': 3,
-                                        },
-                           ),
-            ParamSourceTest(param_source=param_source.RandomDigitSource.from_str('001..999'),
-                            n=10,
-                            expect=expect_random,
-                            expect_arg={'digits': decimals,
-                                        'val_minlen': 3,
-                                        'val_maxlen': 3,
-                                        },
-                           ),
-            ParamSourceTest(param_source=param_source.RandomHexDigitSource.from_str('12345678'),
-                            n=3,
-                            expect=expect_random,
-                            expect_arg={'digits': hexadecimals,
-                                        'val_minlen': 8,
-                                        'val_maxlen': 8,
-                                       },
-                           ),
-            ParamSourceTest(param_source=param_source.RandomHexDigitSource.from_str('0*8'),
-                            n=3,
-                            expect=expect_random,
-                            expect_arg={'digits': hexadecimals,
-                                        'val_minlen': 8,
-                                        'val_maxlen': 8,
-                                       },
-                           ),
-            ParamSourceTest(param_source=param_source.RandomHexDigitSource.from_str('00*4'),
-                            n=3,
-                            expect=expect_random,
-                            expect_arg={'digits': hexadecimals,
-                                        'val_minlen': 8,
-                                        'val_maxlen': 8,
-                                       },
-                           ),
-            ParamSourceTest(param_source=param_source.IncDigitSource.from_str('10001'),
-                            n=3,
-                            expect=expect_const,
-                            expect_arg=('10001', '10002', '10003')
-                           ),
-            ParamSourceTest(param_source=param_source.CsvSource('column_name'),
-                            n=3,
-                            expect=expect_const,
-                            expect_arg=('first val', 'second val', 'third val'),
-                            csv_rows=(
-                                      {'column_name': 'first val',},
-                                      {'column_name': 'second val',},
-                                      {'column_name': 'third val',},
-                                     )
-                           ),
-            ]
-
-        outputs = []
-
-        for t in param_source_tests:
-            try:
-                if hasattr(t.param_source, 'random_impl'):
-                    t.param_source.random_impl = FakeRandom
-
-                vals = []
-                for i in range(t.n):
-                    csv_row = None
-                    if t.csv_rows is not None:
-                        csv_row = t.csv_rows[i]
-                    vals.append( t.param_source.get_next(csv_row=csv_row) )
-                if not t.expect(t, vals):
-                    raise RuntimeError(f'invalid values returned: returned {vals}')
-                output = f'ok: {t.param_source.__class__.__name__} {vals=!r}'
-                outputs.append(output)
-                print(output)
-            except RuntimeError as e:
-                raise RuntimeError(f'{t.param_source.__class__.__name__} {t.n=} {t.expect.__name__}({t.expect_arg!r}): {e}') from e
-
-        output = '\n'.join(outputs) + '\n'
-        xo_name = 'test_param_src'
-        if update_expected_output:
-            with resources.path(xo, xo_name) as xo_path:
-                with open(xo_path, 'w', encoding='utf-8') as f:
-                    f.write(output)
-        else:
-            xo_str = resources.read_text(xo, xo_name)
-            if xo_str != output:
-                at = 0
-                while at < len(output):
-                    if output[at] == xo_str[at]:
-                        at += 1
-                        continue
-                    break
-
-                raise RuntimeError(f'output differs from expected output at position {at}: {xo_str[at:at+128]!r}')
-
-if __name__ == "__main__":
-    if '-u' in sys.argv:
-        update_expected_output = True
-        sys.argv.remove('-u')
-    unittest.main()

tests/unittests/xo/test_param_src

@@ -1,9 +0,0 @@
-ok: ConstantSource vals=['123', '123', '123']
-ok: RandomDigitSource vals=['13987', '49298', '55670']
-ok: RandomDigitSource vals=['650', '580', '49', '885', '497', '195', '320', '137', '245', '663']
-ok: RandomDigitSource vals=['638', '025', '232', '779', '826', '972', '650', '580', '049', '885']
-ok: RandomHexDigitSource vals=['6b65c172', 'abcf6df0', '984a5fcf']
-ok: RandomHexDigitSource vals=['96876670', '356ce766', 'd1d697c1']
-ok: RandomHexDigitSource vals=['f95d8c86', '2bdb095e', '6b65c172']
-ok: IncDigitSource vals=['10001', '10002', '10003']
-ok: CsvSource vals=['first val', 'second val', 'third val']