fix

Change-Id: I1af2fd04ba935fc70a2306342eeb247d941c67af

README.md

@@ -97,7 +97,7 @@ Please install the following dependencies:
  - pyscard
  - pyserial
  - pytlv
- - pyyaml >= 5.4
+ - pyyaml >= 5.1
  - smpp.pdu (from `github.com/hologram-io/smpp.pdu`)
  - termcolor
 

contrib/jenkins.sh

@@ -10,11 +10,6 @@
 
 export PYTHONUNBUFFERED=1
 
-setup_venv() {
-	virtualenv -p python3 venv --system-site-packages
-	. venv/bin/activate
-}
-
 if [ ! -d "./tests/" ] ; then
 	echo "###############################################"
 	echo "Please call from pySim-prog top directory"
@@ -28,7 +23,8 @@ fi
 
 case "$JOB_TYPE" in
 "test")
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install -r requirements.txt
 	pip install pyshark
@@ -36,27 +32,23 @@ case "$JOB_TYPE" in
 	# Execute automatically discovered unit tests first
 	python -m unittest discover -v -s tests/unittests
 
-	# Run pySim-trace test
-	tests/pySim-trace_test/pySim-trace_test.sh
-	;;
-"card-test") # tests requiring physical cards
-	setup_venv
-
-	pip install -r requirements.txt
-
-	# Run pySim-prog integration tests
+	# Run pySim-prog integration tests (requires physical cards)
 	cd tests/pySim-prog_test/
-	./pySim-prog_test.sh
+        ./pySim-prog_test.sh
 	cd ../../
 
-	# Run pySim-shell integration tests
+	# Run pySim-trace test
+	tests/pySim-trace_test/pySim-trace_test.sh
+
+	# Run pySim-shell integration tests (requires physical cards)
 	python3 -m unittest discover -v -s ./tests/pySim-shell_test/
 
 	# Run pySim-smpp2sim test
 	tests/pySim-smpp2sim_test/pySim-smpp2sim_test.sh
 	;;
 "distcheck")
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install .
 	pip install pyshark
@@ -69,7 +61,8 @@ case "$JOB_TYPE" in
 	# Print pylint version
 	pip3 freeze | grep pylint
 
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install .
 
@@ -87,7 +80,8 @@ case "$JOB_TYPE" in
 		contrib/*.py
 	;;
 "docs")
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install -r requirements.txt
 

osmo-smdpp.py

@@ -640,7 +640,7 @@ class SmDppHttpServer:
                 # look up profile based on matchingID.  We simply check if a given file exists for now..
                 path = os.path.join(self.upp_dir, matchingId) + '.der'
                 # prevent directory traversal attack
-                if os.path.commonpath((os.path.realpath(path),self.upp_dir)) != self.upp_dir:
+                if os.path.commonprefix((os.path.realpath(path),self.upp_dir)) != self.upp_dir:
                     raise ApiError('8.2.6', '3.8', 'Refused')
                 if not os.path.isfile(path) or not os.access(path, os.R_OK):
                     raise ApiError('8.2.6', '3.8', 'Refused')

pySim-prog.py

@@ -27,6 +27,7 @@
 import hashlib
 import argparse
 import os
+import random
 import re
 import sys
 import traceback
@@ -435,7 +436,7 @@ def gen_parameters(opts):
         if not re.match('^[0-9a-fA-F]{32}$', ki):
             raise ValueError('Ki needs to be 128 bits, in hex format')
     else:
-        ki = os.urandom(16).hex()
+        ki = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
 
     # OPC (random)
     if opts.opc is not None:
@@ -446,7 +447,7 @@ def gen_parameters(opts):
     elif opts.op is not None:
         opc = derive_milenage_opc(ki, opts.op)
     else:
-        opc = os.urandom(16).hex()
+        opc = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
 
     pin_adm = sanitize_pin_adm(opts.pin_adm, opts.pin_adm_hex)
 

pySim-shell.py

@@ -24,21 +24,21 @@ import traceback
 import re
 import cmd2
 from packaging import version
+from cmd2 import style
+
 import logging
 from pySim.log import PySimLogger
 from osmocom.utils import auto_uint8
 
-# cmd2 >= 3.0 replaced Fg + style() with Color + stylize()
-if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
-    from cmd2 import Color, stylize # pylint: disable=no-name-in-module
-    RED = Color.RED
-    YELLOW = Color.YELLOW
-    LIGHT_RED = Color.BRIGHT_RED
-    LIGHT_GREEN = Color.BRIGHT_GREEN
-    def style(text, fg=None, bg=None, bold=False): # pylint: disable=function-redefined
-        return stylize(text, fg) if fg else text
+# cmd2 >= 2.3.0 has deprecated the bg/fg in favor of Bg/Fg :(
+if version.parse(cmd2.__version__) < version.parse("2.3.0"):
+    from cmd2 import fg, bg # pylint: disable=no-name-in-module
+    RED = fg.red
+    YELLOW = fg.yellow
+    LIGHT_RED = fg.bright_red
+    LIGHT_GREEN = fg.bright_green
 else:
-    from cmd2 import style, Fg # pylint: disable=no-name-in-module
+    from cmd2 import Fg, Bg # pylint: disable=no-name-in-module
     RED = Fg.RED
     YELLOW = Fg.YELLOW
     LIGHT_RED = Fg.LIGHT_RED
@@ -69,26 +69,50 @@ from pySim.ts_102_222 import Ts102222Commands
 from pySim.gsm_r import DF_EIRENE
 from pySim.cat import ProactiveCommand
 
-from pySim.card_key_provider import card_key_provider_argparse_add_args, card_key_provider_init
-from pySim.card_key_provider import card_key_provider_get_field, card_key_provider_get
+from pySim.card_key_provider import CardKeyProviderCsv, CardKeyProviderPgsql
+from pySim.card_key_provider import card_key_provider_register, card_key_provider_get_field, card_key_provider_get
 
 from pySim.app import init_card
 
 log = PySimLogger.get(Path(__file__).stem)
 
-class PysimApp(cmd2.Cmd):
+class Cmd2Compat(cmd2.Cmd):
+    """Backwards-compatibility wrapper around cmd2.Cmd to support older and newer
+    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
+    def run_editor(self, file_path: Optional[str] = None) -> None:
+        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
+            return self._run_editor(file_path) # pylint: disable=no-member
+        else:
+            return super().run_editor(file_path) # pylint: disable=no-member
+
+class Settable2Compat(cmd2.Settable):
+    """Backwards-compatibility wrapper around cmd2.Settable to support older and newer
+    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
+    def __init__(self, name, val_type, description, settable_object, **kwargs):
+        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
+            super().__init__(name, val_type, description, **kwargs) # pylint: disable=no-value-for-parameter
+        else:
+            super().__init__(name, val_type, description, settable_object, **kwargs) # pylint: disable=too-many-function-args
+
+class PysimApp(Cmd2Compat):
     CUSTOM_CATEGORY = 'pySim Commands'
     BANNER = """Welcome to pySim-shell!
 (C) 2021-2023 by Harald Welte, sysmocom - s.f.m.c. GmbH and contributors
 Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/shell.html """
 
     def __init__(self, verbose, card, rs, sl, ch, script=None):
+        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
+            kwargs = {'use_ipython': True}
+        else:
+            kwargs = {'include_ipy': True}
+
         self.verbose = verbose
         PySimLogger.setup(self.poutput, {logging.WARN: YELLOW})
         self._onchange_verbose('verbose', False, self.verbose)
 
+        # pylint: disable=unexpected-keyword-arg
         super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
-                         auto_load_commands=False, startup_script=script, include_ipy=True)
+                         auto_load_commands=False, startup_script=script, **kwargs)
         self.intro = style(self.BANNER, fg=RED)
         self.default_category = 'pySim-shell built-in commands'
         self.card = None
@@ -104,24 +128,18 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         self.apdu_trace = False
         self.apdu_strict = False
 
-        self.add_settable(cmd2.Settable('numeric_path', bool,
-                                        'Print File IDs instead of names',
-                                        self, onchange_cb=self._onchange_numeric_path))
-        self.add_settable(cmd2.Settable('conserve_write', bool,
-                                        'Read and compare before write',
-                                        self, onchange_cb=self._onchange_conserve_write))
-        self.add_settable(cmd2.Settable('json_pretty_print', bool,
-                                        'Pretty-Print JSON output',
-                                        self))
-        self.add_settable(cmd2.Settable('apdu_trace', bool,
-                                        'Trace and display APDUs exchanged with card',
-                                        self, onchange_cb=self._onchange_apdu_trace))
-        self.add_settable(cmd2.Settable('apdu_strict', bool,
-                                        'Strictly apply APDU format according to ISO/IEC 7816-3, table 12',
-                                        self))
-        self.add_settable(cmd2.Settable('verbose', bool,
-                                        'Enable/disable verbose logging',
-                                        self, onchange_cb=self._onchange_verbose))
+        self.add_settable(Settable2Compat('numeric_path', bool, 'Print File IDs instead of names', self,
+                                          onchange_cb=self._onchange_numeric_path))
+        self.add_settable(Settable2Compat('conserve_write', bool, 'Read and compare before write', self,
+                                          onchange_cb=self._onchange_conserve_write))
+        self.add_settable(Settable2Compat('json_pretty_print', bool, 'Pretty-Print JSON output', self))
+        self.add_settable(Settable2Compat('apdu_trace', bool, 'Trace and display APDUs exchanged with card', self,
+                                          onchange_cb=self._onchange_apdu_trace))
+        self.add_settable(Settable2Compat('apdu_strict', bool,
+                                          'Strictly apply APDU format according to ISO/IEC 7816-3, table 12', self))
+        self.add_settable(Settable2Compat('verbose', bool,
+                                          'Enable/disable verbose logging', self,
+                                          onchange_cb=self._onchange_verbose))
         self.equip(card, rs)
 
     def equip(self, card, rs):
@@ -1128,6 +1146,18 @@ global_group.add_argument("--skip-card-init", help="Skip all card/profile initia
 global_group.add_argument("--verbose", help="Enable verbose logging",
                           action='store_true', default=False)
 
+card_key_group = option_parser.add_argument_group('Card Key Provider Options')
+card_key_group.add_argument('--csv', metavar='FILE',
+                            default="~/.osmocom/pysim/card_data.csv",
+                            help='Read card data from CSV file')
+card_key_group.add_argument('--pgsql', metavar='FILE',
+                            default="~/.osmocom/pysim/card_data_pgsql.cfg",
+                            help='Read card data from PostgreSQL database (config file)')
+card_key_group.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                            help=argparse.SUPPRESS, dest='column_key')
+card_key_group.add_argument('--column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                            help='per-column AES transport key', dest='column_key')
+
 adm_group = global_group.add_mutually_exclusive_group()
 adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM1', dest='pin_adm', default=None,
                        help='ADM PIN used for provisioning (overwrites default)')
@@ -1140,7 +1170,6 @@ option_parser.add_argument("command", nargs='?',
                            help="A pySim-shell command that would optionally be executed at startup")
 option_parser.add_argument('command_args', nargs=argparse.REMAINDER,
                            help="Optional Arguments for command")
-card_key_provider_argparse_add_args(option_parser)
 
 if __name__ == '__main__':
     startup_errors = False
@@ -1149,8 +1178,16 @@ if __name__ == '__main__':
     # Ensure that we are able to print formatted warnings from the beginning.
     PySimLogger.setup(print, {logging.WARN: YELLOW}, opts.verbose)
 
-    # Init card key provider for automatic card key retrieval
-    card_key_provider_init(opts)
+    # Register csv-file as card data provider, either from specified CSV
+    # or from CSV file in home directory
+    column_keys = {}
+    for par in opts.column_key:
+        name, key = par.split(':')
+        column_keys[name] = key
+    if os.path.isfile(os.path.expanduser(opts.csv)):
+        card_key_provider_register(CardKeyProviderCsv(os.path.expanduser(opts.csv), column_keys))
+    if os.path.isfile(os.path.expanduser(opts.pgsql)):
+        card_key_provider_register(CardKeyProviderPgsql(os.path.expanduser(opts.pgsql), column_keys))
 
     # Init card reader driver
     sl = init_reader(opts, proactive_handler = Proact())

pySim-trace.py

@@ -117,7 +117,7 @@ class Tracer:
             try:
                 apdu = self.source.read()
                 apdu_counter = apdu_counter + 1
-            except (StopIteration, KeyboardInterrupt):
+            except StopIteration:
                 print("%i APDUs parsed, stop iteration." % apdu_counter)
                 return 0
 

pySim/app.py

@@ -26,9 +26,6 @@ from pySim.cdma_ruim import CardProfileRUIM
 from pySim.ts_102_221 import CardProfileUICC
 from pySim.utils import all_subclasses
 from pySim.exceptions import SwMatchError
-from pySim.log import PySimLogger
-
-log = PySimLogger.get(__name__)
 
 # we need to import this module so that the SysmocomSJA2 sub-class of
 # CardModel is created, which will add the ATR-based matching and
@@ -57,7 +54,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
 
     # Wait up to three seconds for a card in reader and try to detect
     # the card type.
-    log.info("Waiting for card...")
+    print("Waiting for card...")
     sl.wait_for_card(3)
 
     # The user may opt to skip all card initialization. In this case only the
@@ -69,7 +66,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
     generic_card = False
     card = card_detect(scc)
     if card is None:
-        log.warning("Could not detect card type - assuming a generic card type...")
+        print("Warning: Could not detect card type - assuming a generic card type...")
         card = SimCardBase(scc)
         generic_card = True
 
@@ -79,7 +76,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
         # just means that pySim was unable to recognize the card profile. This
         # may happen in particular with unprovisioned cards that do not have
         # any files on them yet.
-        log.warning("Unsupported card type!")
+        print("Unsupported card type!")
         return None, card
 
     # ETSI TS 102 221, Table 9.3 specifies a default for the PIN key
@@ -90,7 +87,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
     if generic_card and isinstance(profile, CardProfileUICC):
         card._adm_chv_num = 0x0A
 
-    log.info("Card is of type: %s", str(profile))
+    print("Info: Card is of type: %s" % str(profile))
 
     # FIXME: this shouldn't really be here but somewhere else/more generic.
     # We cannot do it within pySim/profile.py as that would create circular

pySim/ara_m.py

@@ -300,51 +300,6 @@ class ADF_ARAM(CardADF):
                                'major': v_major, 'minor': v_minor, 'patch': v_patch}}])
         return ADF_ARAM.xceive_apdu_tlv(scc, '80cadf21', cmd_do, ResponseAramConfigDO)
 
-    @staticmethod
-    def store_ref_ar_do(scc, aid:Hexstr, aid_empty:bool, device_app_id:Hexstr, pkg_ref:str,
-                        apdu_filter:Hexstr, apdu_never:bool, apdu_always:bool,
-                        nfc_always:bool, nfc_never:bool, android_permissions:Hexstr):
-        # REF
-        ref_do_content = []
-        if aid is not None:
-            ref_do_content += [{'aid_ref_do': aid}]
-        elif aid_empty:
-            ref_do_content += [{'aid_ref_empty_do': None}]
-        ref_do_content += [{'dev_app_id_ref_do': device_app_id}]
-        if pkg_ref:
-            ref_do_content += [{'pkg_ref_do': {'package_name_string': pkg_ref}}]
-        # AR
-        ar_do_content = []
-        if apdu_never:
-            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
-        elif apdu_always:
-            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
-        elif apdu_filter:
-            if len(apdu_filter) % 16:
-                raise ValueError(f'Invalid non-modulo-16 length of APDU filter: {len(apdu_filter)}')
-            offset = 0
-            apdu_filter_list = []
-            while offset < len(apdu_filter):
-                apdu_filter_list += [{'header': apdu_filter[offset:offset+8],
-                                      'mask': apdu_filter[offset+8:offset+16]}]
-                offset += 16 # Move offset to the beginning of the next apdu_filter object
-            ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter_list}}]
-        if nfc_never:
-            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
-        elif nfc_always:
-            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
-        if android_permissions:
-            ar_do_content += [{'perm_ar_do': {'permissions': android_permissions}}]
-        d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
-        csrado = CommandStoreRefArDO()
-        csrado.from_val_dict(d)
-        return ADF_ARAM.store_data(scc, csrado)
-
-    @staticmethod
-    def aram_delete_all(scc):
-        deldo = CommandDelete()
-        return ADF_ARAM.store_data(scc, deldo)
-
     @with_default_category('Application-Specific Commands')
     class AddlShellCommands(CommandSet):
         def do_aram_get_all(self, _opts):
@@ -379,25 +334,58 @@ class ADF_ARAM(CardADF):
         apdu_grp.add_argument(
             '--apdu-filter', help='APDU filter: multiple groups of 8 hex bytes (4 byte CLA/INS/P1/P2 followed by 4 byte mask)')
         nfc_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
-        nfc_grp.add_argument('--nfc-never', action='store_true',
-                             help='NFC event access is not allowed')
         nfc_grp.add_argument('--nfc-always', action='store_true',
                              help='NFC event access is allowed')
+        nfc_grp.add_argument('--nfc-never', action='store_true',
+                             help='NFC event access is not allowed')
         store_ref_ar_do_parse.add_argument(
             '--android-permissions', help='Android UICC Carrier Privilege Permissions (8 hex bytes)')
 
         @cmd2.with_argparser(store_ref_ar_do_parse)
         def do_aram_store_ref_ar_do(self, opts):
             """Perform STORE DATA [Command-Store-REF-AR-DO] to store a (new) access rule."""
-            res_do = ADF_ARAM.store_ref_ar_do(self._cmd.lchan.scc, opts.aid, opts.aid_empty, opts.device_app_id,
-                                              opts.pkg_ref, opts.apdu_filter, opts.apdu_never, opts.apdu_always,
-                                              opts.nfc_always, opts.nfc_never, opts.android_permissions)
+            # REF
+            ref_do_content = []
+            if opts.aid is not None:
+                ref_do_content += [{'aid_ref_do': opts.aid}]
+            elif opts.aid_empty:
+                ref_do_content += [{'aid_ref_empty_do': None}]
+            ref_do_content += [{'dev_app_id_ref_do': opts.device_app_id}]
+            if opts.pkg_ref:
+                ref_do_content += [{'pkg_ref_do': {'package_name_string': opts.pkg_ref}}]
+            # AR
+            ar_do_content = []
+            if opts.apdu_never:
+                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
+            elif opts.apdu_always:
+                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
+            elif opts.apdu_filter:
+                if len(opts.apdu_filter) % 16:
+                    raise ValueError(f'Invalid non-modulo-16 length of APDU filter: {len(opts.apdu_filter)}')
+                offset = 0
+                apdu_filter = []
+                while offset < len(opts.apdu_filter):
+                    apdu_filter += [{'header': opts.apdu_filter[offset:offset+8],
+                                     'mask': opts.apdu_filter[offset+8:offset+16]}]
+                    offset += 16 # Move offset to the beginning of the next apdu_filter object
+                ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter}}]
+            if opts.nfc_always:
+                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
+            elif opts.nfc_never:
+                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
+            if opts.android_permissions:
+                ar_do_content += [{'perm_ar_do': {'permissions': opts.android_permissions}}]
+            d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
+            csrado = CommandStoreRefArDO()
+            csrado.from_val_dict(d)
+            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, csrado)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
         def do_aram_delete_all(self, _opts):
             """Perform STORE DATA [Command-Delete[all]] to delete all access rules."""
-            res_do = ADF_ARAM.aram_delete_all(self._cmd.lchan.scc)
+            deldo = CommandDelete()
+            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, deldo)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
@@ -406,6 +394,7 @@ class ADF_ARAM(CardADF):
             (Proprietary feature that is specific to sysmocom's fork of Bertrand Martel’s ARA-M implementation.)"""
             self._cmd.lchan.scc.send_apdu_checksw('80e2900001A1', '9000')
 
+
 # SEAC v1.1 Section 4.1.2.2 + 5.1.2.2
 sw_aram = {
     'ARA-M': {

pySim/card_key_provider.py

@@ -33,12 +33,10 @@ from Cryptodome.Cipher import AES
 from osmocom.utils import h2b, b2h
 from pySim.log import PySimLogger
 
-import os
 import abc
 import csv
 import logging
 import yaml
-import argparse
 
 log = PySimLogger.get(__name__)
 
@@ -132,31 +130,6 @@ class CardKeyFieldCryptor:
         cipher = AES.new(h2b(self.transport_keys[field_name.upper()]), AES.MODE_CBC, self.__IV)
         return b2h(cipher.encrypt(h2b(plaintext_val)))
 
-    @staticmethod
-    def argparse_add_args(arg_parser: argparse.ArgumentParser):
-        arg_parser.add_argument('--column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
-                                help='per-column AES transport key', dest='column_key')
-        # Depprecated argument, replaced by --column-key (see above)
-        arg_parser.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
-                                help=argparse.SUPPRESS, dest='column_key')
-
-    @staticmethod
-    def transport_keys_from_opts(opts: argparse.Namespace) -> dict:
-        """
-        Transport keys are passed via the commandline using the '--column-key' option. Each column requires a
-        dedicated transport key. This method can be used to extract the column keys parameters from the commandline
-        options into a dict that can be directly passed to the construtor with the transport_keys argument.
-
-        Args:
-                opts: parsed commandline options (Namespace)
-        """
-
-        transport_keys = {}
-        for par in opts.column_key:
-            name, key = par.split(':')
-            transport_keys[name] = key
-        return transport_keys
-
 class CardKeyProvider(abc.ABC):
     """Base class, not containing any concrete implementation."""
 
@@ -175,33 +148,24 @@ class CardKeyProvider(abc.ABC):
                 fond None shall be returned.
         """
 
-    @staticmethod
-    def argparse_add_args(arg_parser: argparse.ArgumentParser):
-        """
-        Add the commandline arguments relevant for this card key provider.
-
-        Args:
-                arg_parser : argument parser group
-        """
-
     def __str__(self):
         return type(self).__name__
 
 class CardKeyProviderCsv(CardKeyProvider):
     """Card key provider implementation that allows to query against a specified CSV file."""
 
-    def __init__(self, csv_filename: str, field_cryptor: CardKeyFieldCryptor):
+    def __init__(self, csv_filename: str, transport_keys: dict):
         """
         Args:
                 csv_filename : file name (path) of CSV file containing card-individual key/data
-                field_cryptor : (see class CardKeyFieldCryptor)
+                transport_keys : (see class CardKeyFieldCryptor)
         """
         log.info("Using CSV file as card key data source: %s" % csv_filename)
         self.csv_file = open(csv_filename, 'r')
         if not self.csv_file:
             raise RuntimeError("Could not open CSV file '%s'" % csv_filename)
         self.csv_filename = csv_filename
-        self.crypt = field_cryptor
+        self.crypt = CardKeyFieldCryptor(transport_keys)
 
     def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         self.csv_file.seek(0)
@@ -224,20 +188,14 @@ class CardKeyProviderCsv(CardKeyProvider):
             return None
         return return_dict
 
-    @staticmethod
-    def argparse_add_args(arg_parser: argparse.ArgumentParser):
-        arg_parser.add_argument('--csv', metavar='FILE',
-                                default="~/.osmocom/pysim/card_data.csv",
-                                help='Read card data from CSV file')
-
 class CardKeyProviderPgsql(CardKeyProvider):
     """Card key provider implementation that allows to query against a specified PostgreSQL database table."""
 
-    def __init__(self, config_filename: str, field_cryptor: CardKeyFieldCryptor):
+    def __init__(self, config_filename: str, transport_keys: dict):
         """
         Args:
                 config_filename : file name (path) of CSV file containing card-individual key/data
-                field_cryptor : (see class CardKeyFieldCryptor)
+                transport_keys : (see class CardKeyFieldCryptor)
         """
         import psycopg2
         log.info("Using SQL database as card key data source: %s" % config_filename)
@@ -254,7 +212,7 @@ class CardKeyProviderPgsql(CardKeyProvider):
                                          host=config.get('host'))
             self.tables = config.get('table_names')
             log.info("Card key database tables: %s" % str(self.tables))
-            self.crypt = field_cryptor
+            self.crypt = CardKeyFieldCryptor(transport_keys)
 
     def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         import psycopg2
@@ -294,11 +252,6 @@ class CardKeyProviderPgsql(CardKeyProvider):
             result[k] = self.crypt.decrypt_field(k, result.get(k))
         return result
 
-    @staticmethod
-    def argparse_add_args(arg_parser: argparse.ArgumentParser):
-        arg_parser.add_argument('--pgsql', metavar='FILE',
-                                default="~/.osmocom/pysim/card_data_pgsql.cfg",
-                                help='Read card data from PostgreSQL database (config file)')
 
 def card_key_provider_register(provider: CardKeyProvider, provider_list=card_key_providers):
     """Register a new card key provider.
@@ -352,19 +305,3 @@ def card_key_provider_get_field(field: str, key: str, value: str, provider_list=
     fields = [field]
     result = card_key_provider_get(fields, key, value, card_key_providers)
     return result.get(field.upper())
-
-def card_key_provider_argparse_add_args(arg_parser: argparse.ArgumentParser):
-    """Add card key provider commandline options to the given argument parser"""
-    card_key_group = arg_parser.add_argument_group('Card Key Provider Options')
-    CardKeyProviderCsv.argparse_add_args(card_key_group)
-    CardKeyProviderPgsql.argparse_add_args(card_key_group)
-    CardKeyFieldCryptor.argparse_add_args(card_key_group)
-
-def card_key_provider_init(opts: argparse.Namespace):
-    """Initialize card key provider depending on the user provided commandline options"""
-    transport_keys = CardKeyFieldCryptor.transport_keys_from_opts(opts)
-    card_key_field_cryptor = CardKeyFieldCryptor(transport_keys)
-    if os.path.isfile(os.path.expanduser(opts.csv)):
-        card_key_provider_register(CardKeyProviderCsv(os.path.expanduser(opts.csv), card_key_field_cryptor))
-    if os.path.isfile(os.path.expanduser(opts.pgsql)):
-        card_key_provider_register(CardKeyProviderPgsql(os.path.expanduser(opts.pgsql), card_key_field_cryptor))

pySim/esim/saip/__init__.py

@@ -34,7 +34,7 @@ from pySim import ts_102_222
 from pySim.utils import dec_imsi
 from pySim.ts_102_221 import FileDescriptor
 from pySim.filesystem import CardADF, Path
-from pySim.ts_31_102 import ADF_USIM, EF_UST, EF_SUCI_Calc_Info
+from pySim.ts_31_102 import ADF_USIM
 from pySim.ts_31_103 import ADF_ISIM
 from pySim.esim import compile_asn1_subdir
 from pySim.esim.saip import templates
@@ -1079,13 +1079,6 @@ class SecurityDomainKey:
                 'keyVersionNumber': bytes([self.key_version_number]),
                 'keyComponents': [k.to_saip_dict() for k in self.key_components]}
 
-    def get_key_component(self, key_type):
-        for kc in self.key_components:
-            if kc.key_type == key_type:
-                return kc.key_data
-        return None
-
-
 class ProfileElementSD(ProfileElement):
     """Class representing a securityDomain ProfileElement."""
     type = 'securityDomain'
@@ -1726,52 +1719,7 @@ class ProfileElementSequence:
         if 'BT' in ftype_list:
             svc_set.add('ber-tlv')
         # FIXME:dfLinked files (scan all files, check for non-empty Fcp.linkPath presence of DFs)
-
-        # 5G:
-        # - When SUCI is:
-        #   - enabled (EF.UST 124 = true)
-        #   AND
-        #   - calculated in the USIM (EF.UST 125 = true),
-        #   then eUICC-Mandatory-services needs 'get-identity'.
-        # - 'get-identity' implies that the eUICC must support ONE OF profile-A OR profile-B.
-        #   (One might assume from this that, when SUCI-CalcInfo for USIM in DF.SAIP contains both key types, then no
-        #   profile-A or B services need to be requested explicitly. However, the correct logic is:)
-        # - Iff the SUCI-CalcInfo for USIM (DF.SAIP) contains a key of profile-A ("identifier": 1),
-        #   then eUICC-Mandatory-services needs 'profile-a-x25519'.
-        # - Same: profile-B ("identifier": 2) needs 'profile-b-p256'.
-        # - (When SUCI is calculated in the UE, then the eUICC does not need to provide any of these services.)
-        suci_in_usim_enabled = False
-        try:
-            f_ust = self.get_pe_for_type("usim").files["ef-ust"]
-            ust = EF_UST().decode_bin(f_ust.body)
-            suci_in_usim_enabled = ust[124]['activated'] and ust[125]['activated']
-        except (KeyError, AttributeError):
-            pass
-        if suci_in_usim_enabled:
-            svc_set.add('get-identity')
-            # now check for profile-a and profile-b presence
-            suci_calcinfo_has_profile_a = False
-            suci_calcinfo_has_profile_b = False
-            try:
-                f_sucici = self.get_pe_for_type("df-saip").files["ef-suci-calc-info-usim"]
-                sucici = EF_SUCI_Calc_Info().decode_bin(f_sucici.body) or {}
-                for prot_scheme in sucici['prot_scheme_id_list']:
-                    if not isinstance(prot_scheme, dict):
-                        continue
-                    ps_id = prot_scheme["identifier"]
-                    if ps_id == 1:
-                        suci_calcinfo_has_profile_a = True
-                    elif ps_id == 2:
-                        suci_calcinfo_has_profile_b = True
-            except (KeyError, AttributeError):
-                pass
-            if suci_calcinfo_has_profile_a:
-                # The profile has a profile-A key, so require that
-                svc_set.add('profile-a-x25519')
-            if suci_calcinfo_has_profile_b:
-                # The profile has a profile-B key, so require that
-                svc_set.add('profile-b-p256')
-
+        # TODO: 5G related bits (derive from EF.UST or file presence?)
         hdr_pe = self.get_pe_for_type('header')
         # patch in the 'manual' services from the existing list:
         for old_svc in hdr_pe.decoded['eUICC-Mandatory-services'].keys():

pySim/esim/saip/batch.py

@@ -1,362 +0,0 @@
-"""Implementation of Personalization of eSIM profiles in SimAlliance/TCA Interoperable Profile:
-   Run a batch of N personalizations"""
-
-# (C) 2025-2026 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
-#
-# Author: nhofmeyr@sysmocom.de
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import copy
-import pprint
-from typing import Generator, Union
-from pySim.esim.saip.personalization import ConfigurableParameter
-from pySim.esim.saip import param_source
-from pySim.esim.saip import ProfileElementSequence, ProfileElementSD
-from pySim.global_platform import KeyUsageQualifier
-from osmocom.utils import b2h
-
-# a list of ConfigurableParameter classes and/or ConfigurableParameter class instances
-ParamList = list[Union[type[ConfigurableParameter], ConfigurableParameter]]
-
-class BatchPersonalization:
-    """Produce a series of eSIM profiles from predefined parameters.
-    Personalization parameters are derived from pysim.esim.saip.param_source.ParamSource.
-
-    Usage example:
-
-    der_input = open('some_file', 'rb').read()
-    pes = ProfileElementSequence.from_der(der_input)
-    p = BatchPersonalization(
-        n=10,
-        src_pes=pes,
-        csv_rows=get_csv_reader())
-
-    p.add_param_and_src(
-        personalization.Iccid(),
-        param_source.IncDigitSource(
-            num_digits=18,
-            first_value=123456789012340001,
-            last_value=123456789012340010))
-
-    # add more parameters here, using ConfigurableParameter and ParamSource subclass instances to define the profile
-    # ...
-
-    # generate all 10 profiles (from n=10 above)
-    for result_pes in p.generate_profiles():
-        upp = result_pes.to_der()
-        store_upp(upp)
-    """
-
-    class ParamAndSrc:
-        """tie a ConfigurableParameter to a source of actual values"""
-        def __init__(self, param: ConfigurableParameter, src: param_source.ParamSource):
-            if isinstance(param, type):
-                self.param_cls = param
-            else:
-                self.param_cls = param.__class__
-            self.src = src
-
-    def __init__(self,
-                 n: int,
-                 src_pes: ProfileElementSequence,
-                 params: list[ParamAndSrc]=None,
-                 csv_rows: Generator=None,
-                ):
-        """
-        n: number of eSIM profiles to generate.
-        src_pes: a decoded eSIM profile as ProfileElementSequence, to serve as template. This is not modified, only
-                 copied.
-        params: list of ParamAndSrc instances, defining a ConfigurableParameter and corresponding ParamSource to fill in
-                profile values.
-        csv_rows: A generator (e.g. iter(list_of_rows)) producing all CSV rows one at a time, starting with a row
-                  containing the column headers. This is compatible with the python csv.reader. Each row gets passed to
-                  ParamSource.get_next(), such that ParamSource implementations can access the row items. See
-                  param_source.CsvSource.
-        """
-        self.n = n
-        self.params = params or []
-        self.src_pes = src_pes
-        self.csv_rows = csv_rows
-
-    def add_param_and_src(self, param:ConfigurableParameter, src:param_source.ParamSource):
-        self.params.append(BatchPersonalization.ParamAndSrc(param, src))
-
-    def generate_profiles(self):
-        # get first row of CSV: column names
-        csv_columns = None
-        if self.csv_rows:
-            try:
-                csv_columns = next(self.csv_rows)
-            except StopIteration as e:
-                raise ValueError('the input CSV file appears to be empty') from e
-
-        for i in range(self.n):
-            csv_row = None
-            if self.csv_rows and csv_columns:
-                try:
-                    csv_row_list = next(self.csv_rows)
-                except StopIteration as e:
-                    raise ValueError(f'not enough rows in the input CSV for eSIM nr {i+1} of {self.n}') from e
-
-                csv_row = dict(zip(csv_columns, csv_row_list))
-
-            pes = copy.deepcopy(self.src_pes)
-
-            for p in self.params:
-                try:
-                    input_value = p.src.get_next(csv_row=csv_row)
-                    assert input_value is not None
-                    value = p.param_cls.validate_val(input_value)
-                    p.param_cls.apply_val(pes, value)
-                except Exception as e:
-                    raise ValueError(f'{p.param_cls.get_name()} fed by {p.src.name}: {e}') from e
-
-            pes.rebuild_mandatory_services()
-
-            yield pes
-
-
-class UppAudit(dict):
-    """
-    Key-value pairs collected from a single UPP DER or PES.
-
-    UppAudit itself is a dict, callers may use the standard python dict API to access key-value pairs read from the UPP.
-    """
-
-    @classmethod
-    def from_der(cls, der: bytes, params: ParamList, der_size=False, additional_sd_keys=False):
-        """return a dict of parameter name and set of selected parameter values found in a DER encoded profile. Note:
-        some ConfigurableParameter implementations return more than one key-value pair, for example, Imsi returns
-        both 'IMSI' and 'IMSI-ACC' parameters.
-
-        e.g.
-            UppAudit.from_der(my_der, [Imsi, ])
-            --> {'IMSI': {'001010000000023'}, 'IMSI-ACC': {'5'}}
-
-        (where 'IMSI' == Imsi.name)
-
-        Read all parameters listed in params. params is a list of either ConfigurableParameter classes or
-        ConfigurableParameter class instances. This calls only classmethods, so each entry in params can either be the
-        class itself, or a class-instance of, a (non-abstract) ConfigurableParameter subclass.
-        For example, params = [Imsi, ] is equivalent to params = [Imsi(), ].
-
-        For der_size=True, also include a {'der_size':12345} entry.
-
-        For additional_sd_keys=True, output also all Security Domain KVN that there are *no* ConfigurableParameter
-        subclasses for. For example, SCP80 has reserved kvn 0x01..0x0f, but we offer only Scp80Kvn01, Scp80Kvn02,
-        Scp80Kvn03. So we would not show kvn 0x04..0x0f in an audit. additional_sd_keys=True includes audits of all SD
-        key KVN there may be in the UPP. This helps to spot SD keys that may already be present in a UPP template, with
-        unexpected / unusual kvn.
-        """
-
-        # make an instance of this class
-        upp_audit = cls()
-
-        if der_size:
-            upp_audit['der_size'] = set((len(der), ))
-
-        pes = ProfileElementSequence.from_der(der)
-        for param in params:
-            try:
-                for valdict in param.get_values_from_pes(pes):
-                    upp_audit.add_values(valdict)
-            except Exception as e:
-                raise ValueError(f'Error during audit for parameter {param}: {e}') from e
-
-        if not additional_sd_keys:
-            return upp_audit
-
-        # additional_sd_keys
-        for pe in pes.pe_list:
-            if pe.type != 'securityDomain':
-                continue
-            assert isinstance(pe, ProfileElementSD)
-
-            for key in pe.keys:
-                audit_key = f'SdKey_KVN{key.key_version_number:02x}_ID{key.key_identifier:02x}'
-                kuq_bin = KeyUsageQualifier.build(key.key_usage_qualifier).hex()
-                audit_val = f'{key.key_components=!r} key_usage_qualifier=0x{kuq_bin}={key.key_usage_qualifier!r}'
-                upp_audit.add_values({audit_key: audit_val})
-
-        return upp_audit
-
-    def get_single_val(self, key, allow_absent=False, absent_val=None):
-        """
-        Return the audit's value for the given audit key (like 'IMSI' or 'IMSI-ACC').
-        Any kind of value may occur multiple times in a profile. When all of these agree to the same unambiguous value,
-        return that value. When they do not agree, raise a ValueError.
-        """
-        # key should be a string, but if someone passes a ConfigurableParameter, just use its default name
-        if ConfigurableParameter.is_super_of(key):
-            key = key.get_name()
-
-        assert isinstance(key, str)
-        v = self.get(key)
-        if v is None and allow_absent:
-            return absent_val
-        if not isinstance(v, set):
-            raise ValueError(f'audit value should be a set(), got {v!r}')
-        if len(v) != 1:
-            raise ValueError(f'expected a single value for {key}, got {v!r}')
-        v = tuple(v)[0]
-        return v
-
-    @staticmethod
-    def audit_val_to_str(v):
-        """
-        Usually, we want to see a single value in an audit. Still, to be able to collect multiple ambiguous values,
-        audit values are always python sets. Turn it into a nice string representation: only the value when it is
-        unambiguous, otherwise a list of the ambiguous values.
-        A value may also be completely absent, then return 'not present'.
-        """
-        def try_single_val(w):
-            'change single-entry sets to just the single value'
-            if isinstance(w, set):
-                if len(w) == 1:
-                    return tuple(w)[0]
-                if len(w) == 0:
-                    return None
-            return w
-
-        v = try_single_val(v)
-        if isinstance(v, bytes):
-            v = b2h(v)
-        if v is None:
-            return 'not present'
-        return str(v)
-
-    def get_val_str(self, key):
-        """Return a string of the value stored for the given key"""
-        return UppAudit.audit_val_to_str(self.get(key))
-
-    def add_values(self, src:dict):
-        """Merge a plain dict of values into self, which is a dict of sets.
-        For example from
-            self == { 'a': {123} }
-        and
-            src == { 'a': 456, 'b': 789 }
-        then after this function call:
-            self == { 'a': {123, 456}, 'b': {789} }
-        """
-        assert isinstance(src, dict)
-        for key, srcval in src.items():
-            dstvalset = self.get(key)
-            if dstvalset is None:
-                dstvalset = set()
-                self[key] = dstvalset
-            dstvalset.add(srcval)
-
-    def __str__(self):
-        return '\n'.join(f'{key}: {self.get_val_str(key)}' for key in sorted(self.keys()))
-
-class BatchAudit(list):
-    """
-    Collect UppAudit instances for a batch of UPP, for example from a personalization.BatchPersonalization.
-    Produce an output CSV.
-
-    Usage example:
-
-        ba = BatchAudit(params=(personalization.Iccid, ))
-        for upp_der in upps:
-            ba.add_audit(upp_der)
-        print(ba.summarize())
-
-        with open('output.csv', 'wb') as csv_data:
-            csv_str = io.TextIOWrapper(csv_data, 'utf-8', newline='')
-            csv.writer(csv_str).writerows( ba.to_csv_rows() )
-            csv_str.flush()
-
-    BatchAudit itself is a list, callers may use the standard python list API to access the UppAudit instances.
-    """
-
-    def __init__(self, params: ParamList):
-        assert params
-        self.params = params
-
-    def add_audit(self, upp_der:bytes):
-        audit = UppAudit.from_der(upp_der, self.params)
-        self.append(audit)
-        return audit
-
-    def summarize(self):
-        batch_audit = UppAudit()
-
-        audits = self
-
-        if len(audits) > 2:
-            val_sep = ', ..., '
-        else:
-            val_sep = ', '
-
-        first_audit = None
-        last_audit = None
-        if len(audits) >= 1:
-            first_audit = audits[0]
-        if len(audits) >= 2:
-            last_audit = audits[-1]
-
-        if first_audit:
-            if last_audit:
-                for key in first_audit.keys():
-                    first_val = first_audit.get_val_str(key)
-                    last_val = last_audit.get_val_str(key)
-
-                    if first_val == last_val:
-                        val = first_val
-                    else:
-                        val_sep_with_newline = f"{val_sep.rstrip()}\n{' ' * (len(key) + 2)}"
-                        val = val_sep_with_newline.join((first_val, last_val))
-                    batch_audit[key] = val
-            else:
-                batch_audit.update(first_audit)
-
-        return batch_audit
-
-    def to_csv_rows(self, headers=True, sort_key=None):
-        """generator that yields all audits' values as rows, useful feed to a csv.writer."""
-        columns = set()
-        for audit in self:
-            columns.update(audit.keys())
-
-        columns = tuple(sorted(columns, key=sort_key))
-
-        if headers:
-            yield columns
-
-        for audit in self:
-            yield (audit.get_single_val(col, allow_absent=True, absent_val="") for col in columns)
-
-def esim_profile_introspect(upp):
-    pes = ProfileElementSequence.from_der(upp.read())
-    d = {}
-    d['upp'] = repr(pes)
-
-    def show_bytes_as_hexdump(item):
-        if isinstance(item, bytes):
-            return b2h(item)
-        if isinstance(item, list):
-            return list(show_bytes_as_hexdump(i) for i in item)
-        if isinstance(item, tuple):
-            return tuple(show_bytes_as_hexdump(i) for i in item)
-        if isinstance(item, dict):
-            d = {}
-            for k, v in item.items():
-                d[k] = show_bytes_as_hexdump(v)
-            return d
-        return item
-
-    l = list((pe.type, show_bytes_as_hexdump(pe.decoded)) for pe in pes)
-    d['pp'] = pprint.pformat(l, width=120)
-    return d

pySim/esim/saip/param_source.py

@@ -1,221 +0,0 @@
-# Implementation of SimAlliance/TCA Interoperable Profile handling: parameter sources for batch personalization.
-#
-# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
-#
-# Author: nhofmeyr@sysmocom.de
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import secrets
-import re
-from osmocom.utils import b2h
-
-class ParamSourceExn(Exception):
-    pass
-
-class ParamSourceExhaustedExn(ParamSourceExn):
-    pass
-
-class ParamSourceUndefinedExn(ParamSourceExn):
-    pass
-
-class ParamSource:
-    """abstract parameter source. For usage, see personalization.BatchPersonalization."""
-
-    # This name should be short but descriptive, useful for a user interface, like 'random decimal digits'.
-    name = "none"
-    numeric_base = None # or 10 or 16
-
-    def __init__(self, input_str:str):
-        """Subclasses should call super().__init__(input_str) before evaluating self.input_str. Each subclass __init__()
-        may in turn manipulate self.input_str to apply expansions or decodings."""
-        self.input_str = input_str
-
-    def get_next(self, csv_row:dict=None):
-        """Subclasses implement this: return the next value from the parameter source.
-           When there are no more values from the source, raise a ParamSourceExhaustedExn.
-           This default implementation is an empty source."""
-        raise ParamSourceExhaustedExn()
-
-    @classmethod
-    def from_str(cls, input_str:str):
-        """compatibility with earlier version of ParamSource. Just use the constructor."""
-        return cls(input_str)
-
-class ConstantSource(ParamSource):
-    """one value for all"""
-    name = "constant"
-
-    def get_next(self, csv_row:dict=None):
-        return self.input_str
-
-class InputExpandingParamSource(ParamSource):
-
-    def __init__(self, input_str:str):
-        super().__init__(input_str)
-        self.input_str = self.expand_input_str(self.input_str)
-
-    @classmethod
-    def expand_input_str(cls, input_str:str):
-        # user convenience syntax '0*32' becomes '00000000000000000000000000000000'
-        if "*" not in input_str:
-            return input_str
-        # re:                "XX            *        123" with optional spaces
-        tokens = re.split(r"([^ \t]+)[ \t]*\*[ \t]*([0-9]+)", input_str)
-        if len(tokens) < 3:
-            return input_str
-        parts = []
-        for unchanged, snippet, repeat_str in zip(tokens[0::3], tokens[1::3], tokens[2::3]):
-            parts.append(unchanged)
-            repeat = int(repeat_str)
-            parts.append(snippet * repeat)
-
-        return "".join(parts)
-
-class DecimalRangeSource(InputExpandingParamSource):
-    """abstract: decimal numbers with a value range"""
-
-    numeric_base = 10
-
-    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
-        """Constructor to set up values from a (user entered) string: DecimalRangeSource(input_str).
-        Constructor to set up values directly: DecimalRangeSource(num_digits=3, first_value=123, last_value=456)
-
-        num_digits produces leading zeros when first_value..last_value are shorter.
-        """
-        assert ((input_str is not None and (num_digits, first_value, last_value) == (None, None, None))
-                or (input_str is None and None not in (num_digits, first_value, last_value)))
-
-        if input_str is not None:
-            super().__init__(input_str)
-
-            input_str = self.input_str
-
-            if ".." in input_str:
-                first_str, last_str = input_str.split('..')
-                first_str = first_str.strip()
-                last_str = last_str.strip()
-            else:
-                first_str = input_str.strip()
-                last_str = None
-
-            num_digits = len(first_str)
-            first_value = int(first_str)
-            last_value = int(last_str if last_str is not None else "9" * num_digits)
-
-        assert num_digits > 0
-        assert first_value <= last_value
-        self.num_digits = num_digits
-        self.first_value = first_value
-        self.last_value = last_value
-
-    def val_to_digit(self, val:int):
-        return "%0*d" % (self.num_digits, val)  # pylint: disable=consider-using-f-string
-
-class RandomSourceMixin:
-    random_impl = secrets.SystemRandom()
-
-class RandomDigitSource(DecimalRangeSource, RandomSourceMixin):
-    """return a different sequence of random decimal digits each"""
-    name = "random decimal digits"
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.used_keys = set()
-
-    def get_next(self, csv_row:dict=None):
-        # try to generate random digits that are always different from previously produced random digits
-        for _ in range(10):
-            val = self.random_impl.randint(self.first_value, self.last_value)
-            if val not in self.used_keys:
-                break
-        self.used_keys.add(val)
-        return self.val_to_digit(val)
-
-class RandomHexDigitSource(InputExpandingParamSource, RandomSourceMixin):
-    """return a different sequence of random hexadecimal digits each"""
-    name = "random hexadecimal digits"
-    numeric_base = 16
-    def __init__(self, input_str:str):
-        super().__init__(input_str)
-        input_str = self.input_str
-
-        num_digits = len(input_str.strip())
-        if num_digits < 1:
-            raise ValueError("zero number of digits")
-        # hex digits always come in two
-        if (num_digits & 1) != 0:
-            raise ValueError(f"hexadecimal value should have even number of digits, not {num_digits}")
-        self.num_digits = num_digits
-        self.used_keys = set()
-
-    def get_next(self, csv_row:dict=None):
-        # try to generate random bytes that are always different from previously produced random bytes
-        for _ in range(10):
-            val = self.random_impl.randbytes(self.num_digits // 2)
-            if val not in self.used_keys:
-                break
-        self.used_keys.add(val)
-
-        return b2h(val)
-
-class IncDigitSource(DecimalRangeSource):
-    """incrementing sequence of digits"""
-    name = "incrementing decimal digits"
-
-    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
-        """input_str: the range of values to iterate. Format: 'FIRST..LAST' (e.g. '0001..9999') or
-        just 'FIRST' (iterates to the maximum value for the given digit width). Leading zeros in
-        FIRST determine the digit width and are preserved in returned values."""
-        super().__init__(input_str, num_digits, first_value, last_value)
-        self.next_val = None
-        self.reset()
-
-    def reset(self):
-        """Restart from the first value of the defined range passed to __init__()."""
-        self.next_val = self.first_value
-
-    def get_next(self, csv_row:dict=None):
-        val = self.next_val
-        if val is None:
-            raise ParamSourceExhaustedExn()
-
-        returnval = self.val_to_digit(val)
-
-        val += 1
-        if val > self.last_value:
-            self.next_val = None
-        else:
-            self.next_val = val
-
-        return returnval
-
-class CsvSource(ParamSource):
-    """apply a column from a CSV row, as passed in to ParamSource.get_next(csv_row)"""
-    name = "from CSV"
-
-    def __init__(self, input_str:str):
-        """input_str: the CSV column name to read values from.
-        The caller passes the current CSV row to get_next(), from which CsvSource picks the column matching
-        this name."""
-        super().__init__(input_str)
-        self.csv_column = self.input_str
-
-    def get_next(self, csv_row:dict=None):
-        val = None
-        if csv_row:
-            val = csv_row.get(self.csv_column)
-        if val is None:
-            raise ParamSourceUndefinedExn(f"no value for CSV column {self.csv_column!r}")
-        return val

pySim/esim/saip/personalization.py

@@ -16,24 +16,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import abc
-import enum
 import io
-import re
-from typing import List, Tuple, Generator, Optional
+from typing import List, Tuple
 
-from construct.core import StreamError
 from osmocom.tlv import camel_to_snake
-from osmocom.utils import hexstr
-from pySim.utils import enc_iccid, dec_iccid, enc_imsi, dec_imsi, h2b, b2h, rpad, sanitize_iccid
-from pySim.ts_31_102 import EF_AD
+from pySim.utils import enc_iccid, enc_imsi, h2b, rpad, sanitize_iccid
+from pySim.esim.saip import ProfileElement, ProfileElementSequence
 from pySim.ts_51_011 import EF_SMSP
-from pySim.esim.saip import param_source
-from pySim.esim.saip import ProfileElement, ProfileElementSD, ProfileElementSequence
-from pySim.esim.saip import SecurityDomainKey, SecurityDomainKeyComponent
-from pySim.global_platform import KeyUsageQualifier, KeyType
-
-def unrpad(s: hexstr, c='f') -> hexstr:
-    return hexstr(s.rstrip(c))
 
 def remove_unwanted_tuples_from_list(l: List[Tuple], unwanted_keys: List[str]) -> List[Tuple]:
     """In a list of tuples, remove all tuples whose first part equals 'unwanted_key'."""
@@ -54,6 +43,7 @@ class ClassVarMeta(abc.ABCMeta):
         x = super().__new__(metacls, name, bases, namespace)
         for k, v in kwargs.items():
             setattr(x, k, v)
+        setattr(x, 'name', camel_to_snake(name))
         return x
 
 class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
@@ -73,7 +63,6 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
     min_len: minimum length of an input str; min_len = 4
     max_len: maximum length of an input str; max_len = 8
     allow_len: permit only specific lengths; allow_len = (8, 16, 32)
-    numeric_base: indicate hex / decimal, if any; numeric_base = None; numeric_base = 10; numeric_base = 16
 
     Subclasses may change the meaning of these by overriding validate_val(), for example that the length counts
     resulting bytes instead of a hexstring length. Most subclasses will be covered by the default validate_val().
@@ -128,8 +117,6 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
     max_len = None
     allow_len = None # a list of specific lengths
     example_input = None
-    default_source = None # a param_source.ParamSource subclass
-    numeric_base = None # or 10 or 16
 
     def __init__(self, input_value=None):
         self.input_value = input_value # the raw input value as given by caller
@@ -191,28 +178,19 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
             if cls.allow_chars is not None:
                 if any(c not in cls.allow_chars for c in val):
                     raise ValueError(f"invalid characters in input value {val!r}, valid chars are {cls.allow_chars}")
-        elif isinstance(val, io.BytesIO):
-            val = val.getvalue()
-
-        if hasattr(val, '__len__'):
-            val_len = len(val)
-        else:
-            # e.g. int length
-            val_len = len(str(val))
-
         if cls.allow_len is not None:
             l = cls.allow_len
             # cls.allow_len could be one int, or a tuple of ints. Wrap a single int also in a tuple.
             if not isinstance(l, (tuple, list)):
                 l = (l,)
-            if val_len not in l:
-                raise ValueError(f'length must be one of {cls.allow_len}, not {val_len}: {val!r}')
+            if len(val) not in l:
+                raise ValueError(f'length must be one of {cls.allow_len}, not {len(val)}: {val!r}')
         if cls.min_len is not None:
-            if val_len < cls.min_len:
-                raise ValueError(f'length must be at least {cls.min_len}, not {val_len}: {val!r}')
+            if len(val) < cls.min_len:
+                raise ValueError(f'length must be at least {cls.min_len}, not {len(val)}: {val!r}')
         if cls.max_len is not None:
-            if val_len > cls.max_len:
-                raise ValueError(f'length must be at most {cls.max_len}, not {val_len}: {val!r}')
+            if len(val) > cls.max_len:
+                raise ValueError(f'length must be at most {cls.max_len}, not {len(val)}: {val!r}')
         return val
 
     @classmethod
@@ -221,49 +199,6 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
         Write the given val in the right format in all the right places in pes."""
         pass
 
-    @classmethod
-    def get_value_from_pes(cls, pes: ProfileElementSequence):
-        """Same as get_values_from_pes() but expecting a single value.
-           get_values_from_pes() may return values like this:
-             [{ 'AlgorithmID': 'Milenage' }, { 'AlgorithmID': 'Milenage' }]
-           This ensures that all these entries are identical and would return only
-              { 'AlgorithmID': 'Milenage' }.
-
-           This is relevant for any profile element that may appear multiple times in the same PES (only a few),
-           where each occurrence should reflect the same value (all currently known parameters).
-        """
-
-        val = None
-        for v in cls.get_values_from_pes(pes):
-            if val is None:
-                val = v
-            elif val != v:
-                raise ValueError(f'get_value_from_pes(): got distinct values: {val!r} != {v!r}')
-        return val
-
-    @classmethod
-    @abc.abstractmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence) -> Generator:
-        """This is what subclasses implement: yield all values from a decoded profile package.
-           Find all values in the pes, and yield them decoded to a valid cls.input_value format.
-           Should be a generator function, i.e. use 'yield' instead of 'return'.
-
-           Yielded value must be a dict(). Usually, an implementation will return only one key, like
-
-              { "ICCID": "1234567890123456789" }
-
-           Some implementations have more than one value to return, like
-
-              { "IMSI": "00101012345678", "IMSI-ACC" : "5" }
-
-           Implementation example:
-
-             for pe in pes:
-                if my_condition(pe):
-                    yield { cls.name: b2h(my_bin_value_from(pe)) }
-           """
-        pass
-
     @classmethod
     def get_len_range(cls):
         """considering all of min_len, max_len and allow_len, get a tuple of the resulting (min, max) of permitted
@@ -284,20 +219,6 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
             return (None, None)
         return (min(vals), max(vals))
 
-    @classmethod
-    def get_typical_input_len(cls):
-        '''return a good length to use as the visible width of a user interface input field.
-           May be overridden by subclasses.
-           This default implementation returns the maximum allowed value length -- a good fit for most subclasses.
-           '''
-        return cls.get_len_range()[1] or 16
-
-    @classmethod
-    def is_super_of(cls, other_class):
-        try:
-            return issubclass(other_class, cls)
-        except TypeError:
-            return False
 
 class DecimalParam(ConfigurableParameter):
     """Decimal digits. The input value may be a string of decimal digits like '012345', or an int. The output of
@@ -305,7 +226,6 @@ class DecimalParam(ConfigurableParameter):
     """
     allow_types = (str, int)
     allow_chars = '0123456789'
-    numeric_base = 10
 
     @classmethod
     def validate_val(cls, val):
@@ -329,7 +249,6 @@ class DecimalHexParam(DecimalParam):
     @classmethod
     def validate_val(cls, val):
         val = super().validate_val(val)
-        assert isinstance(val, str)
         val = ''.join('%02x' % ord(x) for x in val)
         if cls.rpad is not None:
             c = cls.rpad_char
@@ -337,21 +256,9 @@ class DecimalHexParam(DecimalParam):
         # a DecimalHexParam subclass expects the apply_val() input to be a bytes instance ready for the pes
         return h2b(val)
 
-    @classmethod
-    def decimal_hex_to_str(cls, val):
-        """useful for get_values_from_pes() implementations of subclasses"""
-        if isinstance(val, bytes):
-            val = b2h(val)
-        assert isinstance(val, hexstr)
-        if cls.rpad is not None:
-            c = cls.rpad_char or 'f'
-            val = unrpad(val, c)
-        return val.to_bytes().decode('ascii')
-
 class IntegerParam(ConfigurableParameter):
     allow_types = (str, int)
     allow_chars = '0123456789'
-    numeric_base = 10
 
     # two integers, if the resulting int should be range limited
     min_val = None
@@ -372,28 +279,14 @@ class IntegerParam(ConfigurableParameter):
             raise ValueError(f'Value {val} is out of range, must be [{cls.min_val}..{cls.max_val}]')
         return val
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for valdict in super().get_values_from_pes(pes):
-            for key, val in valdict.items():
-                if isinstance(val, int):
-                    valdict[key] = str(val)
-            yield valdict
-
 class BinaryParam(ConfigurableParameter):
-    allow_types = (str, io.BytesIO, bytes, bytearray, int)
+    allow_types = (str, io.BytesIO, bytes, bytearray)
     allow_chars = '0123456789abcdefABCDEF'
     strip_chars = ' \t\r\n'
-    numeric_base = 16
-    default_source = param_source.RandomHexDigitSource
 
     @classmethod
     def validate_val(cls, val):
         # take care that min_len and max_len are applied to the binary length by converting to bytes first
-        if isinstance(val, int):
-            min_len, _max_len = cls.get_len_range()
-            val = '%0*d' % (min_len, val)
-
         if isinstance(val, str):
             if cls.strip_chars is not None:
                 val = ''.join(c for c in val if c not in cls.strip_chars)
@@ -408,82 +301,6 @@ class BinaryParam(ConfigurableParameter):
         val = super().validate_val(val)
         return bytes(val)
 
-    @classmethod
-    def get_typical_input_len(cls):
-        # override to return twice the length, because of hex digits.
-        min_len, max_len = cls.get_len_range()
-        if max_len is None:
-            return None
-        # two hex characters per value octet.
-        # (maybe *3 to also allow for spaces?)
-        return max_len * 2
-
-
-class EnumParam(ConfigurableParameter):
-    """ConfigurableParameter for named integer enumeration values.
-
-    Subclasses must define a nested enum.IntEnum named 'Values' listing all valid names and their
-    integer codes.  apply_val() and get_values_from_pes() are not implemented here and this must
-    be inherited from another mixin."""
-
-    class Values(enum.IntEnum):
-        pass  # subclasses override this
-
-    @classmethod
-    def validate_val(cls, val) -> int:
-        if isinstance(val, int):
-            try:
-                return int(cls.Values(val))
-            except ValueError:
-                pass
-        elif isinstance(val, str):
-            member = cls.map_name_to_val(val, strict=False)
-            if member is not None:
-                return member
-
-        valid = ', '.join(m.name for m in cls.Values)
-        raise ValueError(f"{cls.get_name()}: invalid argument: {val!r}. Valid arguments are: {valid}")
-
-    @classmethod
-    def map_name_to_val(cls, name: str, strict=True) -> int:
-        """Return the integer value for a given enum member name. Performs an exact match first,
-        then falls back to fuzzy matching (case-insensitive, punctuation-insensitive)."""
-        try:
-            return int(cls.Values[name])
-        except KeyError:
-            pass
-
-        clean = cls.clean_name_str(name)
-        for member in cls.Values:
-            if cls.clean_name_str(member.name) == clean:
-                return int(member)
-
-        if strict:
-            valid = ', '.join(m.name for m in cls.Values)
-            raise ValueError(f"{cls.get_name()}: {name!r} is not a known value. Known values are: {valid}")
-        return None
-
-    @classmethod
-    def map_val_to_name(cls, val, strict=False) -> str:
-        """Return the enum member name for a given integer value."""
-        try:
-            return cls.Values(val).name
-        except ValueError:
-            if strict:
-                raise ValueError(f"{cls.get_name()}: {val!r} ({type(val).__name__}) is not a known value.")
-            return None
-
-    @classmethod
-    def name_normalize(cls, name: str) -> str:
-        """Map a (possibly fuzzy) name to its canonical enum member name."""
-        return cls.Values(cls.map_name_to_val(name)).name
-
-    @classmethod
-    def clean_name_str(cls, val: str) -> str:
-        """Strip punctuation and case for fuzzy name comparison.
-        Treats hyphens and underscores as equivalent (both removed)."""
-        return re.sub('[^0-9A-Za-z]', '', val).lower()
-
 
 class Iccid(DecimalParam):
     """ICCID Parameter. Input: string of decimal digits.
@@ -492,7 +309,6 @@ class Iccid(DecimalParam):
     min_len = 18
     max_len = 20
     example_input = '998877665544332211'
-    default_source = param_source.IncDigitSource
 
     @classmethod
     def validate_val(cls, val):
@@ -506,17 +322,6 @@ class Iccid(DecimalParam):
         # patch MF/EF.ICCID
         file_replace_content(pes.get_pe_for_type('mf').decoded['ef-iccid'], h2b(enc_iccid(val)))
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        padded = b2h(pes.get_pe_for_type('header').decoded['iccid'])
-        iccid = unrpad(padded)
-        yield { cls.name: iccid }
-
-        for pe in pes.get_pes_for_type('mf'):
-            iccid_f = pe.files.get('ef-iccid', None)
-            if iccid_f is not None:
-                yield { cls.name: dec_iccid(b2h(iccid_f.body)) }
-
 class Imsi(DecimalParam):
     """Configurable IMSI. Expects value to be a string of digits. Automatically sets the ACC to
     the last digit of the IMSI."""
@@ -525,7 +330,6 @@ class Imsi(DecimalParam):
     min_len = 6
     max_len = 15
     example_input = '00101' + ('0' * 10)
-    default_source = param_source.IncDigitSource
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -538,18 +342,6 @@ class Imsi(DecimalParam):
             file_replace_content(pe.decoded['ef-acc'], acc.to_bytes(2, 'big'))
         # TODO: DF.GSM_ACCESS if not linked?
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            imsi_f = pe.files.get('ef-imsi', None)
-            acc_f = pe.files.get('ef-acc', None)
-            y = {}
-            if imsi_f:
-                y[cls.name] = dec_imsi(b2h(imsi_f.body))
-            if acc_f:
-                y[cls.name + '-ACC'] = b2h(acc_f.body)
-            yield y
-
 class SmspTpScAddr(ConfigurableParameter):
     """Configurable SMSC (SMS Service Centre) TP-SC-ADDR. Expects to be a phone number in national or
     international format (designated by a leading +). Automatically sets the NPI to E.164 and the TON based on
@@ -558,45 +350,25 @@ class SmspTpScAddr(ConfigurableParameter):
     name = 'SMSP-TP-SC-ADDR'
     allow_chars = '+0123456789'
     strip_chars = ' \t\r\n'
-    numeric_base = 10
     max_len = 21 # '+' and 20 digits
     min_len = 1
     example_input = '+49301234567'
-    default_source = param_source.ConstantSource
 
-    @staticmethod
-    def str_to_tuple(addr_str):
+    @classmethod
+    def validate_val(cls, val):
+        val = super().validate_val(val)
+        addr_str = str(val)
         if addr_str[0] == '+':
             digits = addr_str[1:]
             international = True
         else:
             digits = addr_str
             international = False
-        return (international, digits)
-
-    @staticmethod
-    def tuple_to_str(addr_tuple):
-        international, digits = addr_tuple
-        if international:
-            ret = '+'
-        else:
-            ret = ''
-        ret += digits
-        return ret
-
-    @classmethod
-    def validate_val(cls, val):
-        val = super().validate_val(val)
-
-        addr_tuple = cls.str_to_tuple(str(val))
-
-        international, digits = addr_tuple
         if len(digits) > 20:
             raise ValueError(f'TP-SC-ADDR must not exceed 20 digits: {digits!r}')
         if not digits.isdecimal():
             raise ValueError(f'TP-SC-ADDR must only contain decimal digits: {digits!r}')
-
-        return addr_tuple
+        return (international, digits)
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -620,318 +392,98 @@ class SmspTpScAddr(ConfigurableParameter):
             ef_smsp_dec['tp_sc_addr']['ton_npi']['type_of_number'] = 'international' if international else 'unknown'
             # ensure the parameter_indicators.tp_sc_addr is True
             ef_smsp_dec['parameter_indicators']['tp_sc_addr'] = True
-
-            # alpha_id padding: to make room for a human readable SMSC name that can be provisioned to the profile later
-            # on, alpha_id needs to be empty but padded 0xff to some length.
-            # - alpha_id is optional, setting alpha_id = '' ensures the IE is present.
-            # - the length of the file is 28+Y where Y is the length of the alpha_id -- here the intended length of our padding
-            #   (see 3GPP TS 31.102 4.2.27 EF.SMSP). So if we want a maximum length of alpha_id = 14, we set the total
-            #   file size to 28+14 = 42.
-            # - this file size has to go in two places: encode_record_bin() needs to know the length to encode the right
-            #   length of fillFileContent.
-            # - the f_smsp needs to show the right file size in the PES, as in
-            #      'ef-smsp': [('fileDescriptor', {'efFileSize': '2a', ...
-            #   (where 2a == 42)
-            # - To generate the right amount of fillFileContent, pass total_len=42 to encode_record_bin().
-            # - To show the right size in the PES, set f_smsp.rec_len = 42
-            ef_smsp_dec['alpha_id'] = ''
-            f_smsp.rec_len = 42
-
-            # re-encode into the File body.
-            #
+            # re-encode into the File body
+            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1)
             #print("SMSP  (new): %s" % f_smsp.body)
             # re-generate the pe.decoded member from the File instance
-            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1, total_len=f_smsp.rec_len)
             pe.file2pe(f_smsp)
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            f_smsp = pe.files['ef-smsp']
-            ef_smsp = EF_SMSP()
-            ef_smsp_dec = ef_smsp.decode_record_bin(f_smsp.body, 1)
-
-            tp_sc_addr = ef_smsp_dec.get('tp_sc_addr', None)
-
-            digits = tp_sc_addr.get('call_number', None)
-
-            ton_npi = tp_sc_addr.get('ton_npi', None)
-            international = ton_npi.get('type_of_number', None)
-            international = (international == 'international')
-
-            yield { cls.name: cls.tuple_to_str((international, digits)) }
-
-
-class MncLen(EnumParam):
-    """MNC length.  Sets only the MNC length field in EF.AD (Administrative Data).
-    Accepted values: integer 2 or 3, digit strings '2' or '3', or enum names 'MNC2'/'MNC3'.
-    """
-    name = 'MNC-LEN'
-    example_input = '2'
-    default_source = param_source.ConstantSource
-
-    class Values(enum.IntEnum):
-        MNC2 = 2
-        MNC3 = 3
-
-    @classmethod
-    def validate_val(cls, val):
-        if isinstance(val, str) and val.isdigit():
-            val = int(val)
-        return super().validate_val(val)
-
-    @classmethod
-    def _get_f_ad(cls, pe: ProfileElement):
-        if not hasattr(pe, 'files'):
-            return None
-        f_ad = pe.files.get('ef-ad', None)
-        if f_ad and f_ad.body:
-            return f_ad
-        return None
-
-    @classmethod
-    def _decode_f_ad(cls, f_ad):
-        try:
-            ef_ad_dec = EF_AD().decode_bin(f_ad.body)
-        except StreamError:
-            return None
-        if 'mnc_len' not in ef_ad_dec:
-            return None
-        return ef_ad_dec
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val: int):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ad = cls._get_f_ad(pe)
-            if f_ad is None:
-                continue
-            # decode existing values
-            ef_ad_dec = cls._decode_f_ad(f_ad)
-            if ef_ad_dec is None:
-                continue
-            # change mnc_len
-            ef_ad_dec['mnc_len'] = val
-            # re-encode into the File body
-            f_ad.body = EF_AD().encode_bin(ef_ad_dec)
-            pe.file2pe(f_ad)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ad = cls._get_f_ad(pe)
-            if f_ad is None:
-                continue
-            ef_ad_dec = cls._decode_f_ad(f_ad)
-            if ef_ad_dec is None:
-                continue
-            mnc_len = ef_ad_dec.get('mnc_len')
-            yield { cls.name: str(mnc_len) }
-
-
-class SdKey(BinaryParam):
-    """Configurable Security Domain (SD) Key.  Value is presented as bytes.
-       Non-abstract implementations are generated in SdKey.generate_sd_key_classes"""
+class SdKey(BinaryParam, metaclass=ClassVarMeta):
+    """Configurable Security Domain (SD) Key.  Value is presented as bytes."""
     # these will be set by subclasses
     key_type = None
-    kvn = None
     key_id = None
+    kvn = None
     key_usage_qual = None
 
     @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        set_components = [ SecurityDomainKeyComponent(cls.key_type, val) ]
-
-        for pe in pes.pe_list:
-            if pe.type != 'securityDomain':
-                continue
-            assert isinstance(pe, ProfileElementSD)
-
-            key = pe.find_key(key_version_number=cls.kvn, key_id=cls.key_id)
-            if not key:
-                # Could not find matching key to patch, create a new one
-                key = SecurityDomainKey(
-                        key_version_number=cls.kvn,
-                        key_id=cls.key_id,
-                        key_usage_qualifier=cls.key_usage_qual,
-                        key_components=set_components,
-                        )
-                pe.add_key(key)
-            else:
-                # A key of this KVN and ID already exists in the profile.
-
-                # Keep the key_usage_qualifier as it was in the profile, so skip this here:
-                # key.key_usage_qualifier = cls.key_usage_qual
-
-                key.key_components = set_components
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.pe_list:
-            if pe.type != 'securityDomain':
-                continue
-            assert isinstance(pe, ProfileElementSD)
-
-            key = pe.find_key(key_version_number=cls.kvn, key_id=cls.key_id)
-            if not key:
-                continue
-            kc = key.get_key_component(cls.key_type)
-            if kc:
-                yield { cls.name: b2h(kc) }
-
-
-    LEN_128 = (16,)
-    LEN_128_192_256 = (16, 24, 32)
-    LEN_128_256 = (16, 32)
-
-    DES =    ('DES', dict(key_type=KeyType.des, allow_len=LEN_128) )
-    AES =    ('AES', dict(key_type=KeyType.aes, allow_len=LEN_128_192_256) )
-
-    ENC =    ('ENC', dict(key_id=0x01, key_usage_qual=0x18) )
-    MAC =    ('MAC', dict(key_id=0x02, key_usage_qual=0x14) )
-    DEK =    ('DEK', dict(key_id=0x03, key_usage_qual=0x48) )
-
-    TLSPSK_PSK = ('TLSPSK', dict(key_type=KeyType.tls_psk, key_id=0x01, key_usage_qual=0x3c, allow_len=LEN_128_192_256) )
-    TLSPSK_DEK = ('DEK',    dict(key_id=0x02, key_usage_qual=0x48) )
-
-    # THIS IS THE LIST that controls which SdKeyXxx subclasses exist:
-    SD_KEY_DEFS = (
-        # name    KVN                     x variants            x variants
-        ('SCP02', (0x20, 0x21, 0x22, 0xff), (AES, ),              (ENC, MAC, DEK) ),
-        ('SCP03', (0x30, 0x31, 0x32),       (AES, ),              (ENC, MAC, DEK) ),
-        ('SCP80', (0x01, 0x02, 0x03),       (DES, AES),           (ENC, MAC, DEK) ),
-
-        # key_id=1
-        ('SCP81', (0x40, 0x41, 0x42),       (TLSPSK_PSK, ),                       ),
-        # key_id=2
-        ('SCP81', (0x40, 0x41, 0x42),       (DES, AES),           (TLSPSK_DEK, )  ),
-    )
-
-    all_implementations = None
-
-    @classmethod
-    def generate_sd_key_classes(cls, sd_key_defs=SD_KEY_DEFS):
-        '''This generates python classes to be exported in this module, as subclasses of class SdKey.
-
-        We create SdKey subclasses dynamically from a list.
-        You can list all of them via:
-          from pySim.esim.saip.personalization import SdKey
-          SdKey.all_implementations
-        or
-          print('\n'.join(sorted(f'{x.__name__}\t{x.name}' for x in SdKey.all_implementations)))
-
-        at time of writing this comment, this prints:
-
-        SdKeyScp02Kvn20AesDek	SCP02-KVN20-AES-DEK
-        SdKeyScp02Kvn20AesEnc	SCP02-KVN20-AES-ENC
-        SdKeyScp02Kvn20AesMac	SCP02-KVN20-AES-MAC
-        SdKeyScp02Kvn21AesDek	SCP02-KVN21-AES-DEK
-        SdKeyScp02Kvn21AesEnc	SCP02-KVN21-AES-ENC
-        SdKeyScp02Kvn21AesMac	SCP02-KVN21-AES-MAC
-        SdKeyScp02Kvn22AesDek	SCP02-KVN22-AES-DEK
-        SdKeyScp02Kvn22AesEnc	SCP02-KVN22-AES-ENC
-        SdKeyScp02Kvn22AesMac	SCP02-KVN22-AES-MAC
-        SdKeyScp02KvnffAesDek	SCP02-KVNff-AES-DEK
-        SdKeyScp02KvnffAesEnc	SCP02-KVNff-AES-ENC
-        SdKeyScp02KvnffAesMac	SCP02-KVNff-AES-MAC
-        SdKeyScp03Kvn30AesDek	SCP03-KVN30-AES-DEK
-        SdKeyScp03Kvn30AesEnc	SCP03-KVN30-AES-ENC
-        SdKeyScp03Kvn30AesMac	SCP03-KVN30-AES-MAC
-        SdKeyScp03Kvn31AesDek	SCP03-KVN31-AES-DEK
-        SdKeyScp03Kvn31AesEnc	SCP03-KVN31-AES-ENC
-        SdKeyScp03Kvn31AesMac	SCP03-KVN31-AES-MAC
-        SdKeyScp03Kvn32AesDek	SCP03-KVN32-AES-DEK
-        SdKeyScp03Kvn32AesEnc	SCP03-KVN32-AES-ENC
-        SdKeyScp03Kvn32AesMac	SCP03-KVN32-AES-MAC
-        SdKeyScp80Kvn01AesDek	SCP80-KVN01-AES-DEK
-        SdKeyScp80Kvn01AesEnc	SCP80-KVN01-AES-ENC
-        SdKeyScp80Kvn01AesMac	SCP80-KVN01-AES-MAC
-        SdKeyScp80Kvn01DesDek	SCP80-KVN01-DES-DEK
-        SdKeyScp80Kvn01DesEnc	SCP80-KVN01-DES-ENC
-        SdKeyScp80Kvn01DesMac	SCP80-KVN01-DES-MAC
-        SdKeyScp80Kvn02AesDek	SCP80-KVN02-AES-DEK
-        SdKeyScp80Kvn02AesEnc	SCP80-KVN02-AES-ENC
-        SdKeyScp80Kvn02AesMac	SCP80-KVN02-AES-MAC
-        SdKeyScp80Kvn02DesDek	SCP80-KVN02-DES-DEK
-        SdKeyScp80Kvn02DesEnc	SCP80-KVN02-DES-ENC
-        SdKeyScp80Kvn02DesMac	SCP80-KVN02-DES-MAC
-        SdKeyScp80Kvn03AesDek	SCP80-KVN03-AES-DEK
-        SdKeyScp80Kvn03AesEnc	SCP80-KVN03-AES-ENC
-        SdKeyScp80Kvn03AesMac	SCP80-KVN03-AES-MAC
-        SdKeyScp80Kvn03DesDek	SCP80-KVN03-DES-DEK
-        SdKeyScp80Kvn03DesEnc	SCP80-KVN03-DES-ENC
-        SdKeyScp80Kvn03DesMac	SCP80-KVN03-DES-MAC
-        SdKeyScp81Kvn40AesDek	SCP81-KVN40-AES-DEK
-        SdKeyScp81Kvn40DesDek	SCP81-KVN40-DES-DEK
-        SdKeyScp81Kvn40Tlspsk	SCP81-KVN40-TLSPSK
-        SdKeyScp81Kvn41AesDek	SCP81-KVN41-AES-DEK
-        SdKeyScp81Kvn41DesDek	SCP81-KVN41-DES-DEK
-        SdKeyScp81Kvn41Tlspsk	SCP81-KVN41-TLSPSK
-        SdKeyScp81Kvn42AesDek	SCP81-KVN42-AES-DEK
-        SdKeyScp81Kvn42DesDek	SCP81-KVN42-DES-DEK
-        SdKeyScp81Kvn42Tlspsk	SCP81-KVN42-TLSPSK
-        '''
-
-        SdKey.all_implementations = []
-
-        def camel(s):
-            return s[:1].upper() + s[1:].lower()
-
-        def do_variants(name, kvn, remaining_variants, labels=[], attrs={}):
-            'recurse to unfold as many variants as there may be'
-            if remaining_variants:
-                # not a leaf node, collect more labels and attrs
-                variants = remaining_variants[0]
-                remaining_variants = remaining_variants[1:]
-
-                for label, valdict in variants:
-                    # pass copies to recursion
-                    inner_labels = list(labels)
-                    inner_attrs = dict(attrs)
-
-                    inner_labels.append(label)
-                    inner_attrs.update(valdict)
-                    do_variants(name, kvn, remaining_variants,
-                                labels=inner_labels,
-                                attrs=inner_attrs)
+    def _apply_sd(cls, pe: ProfileElement, value):
+        assert pe.type == 'securityDomain'
+        for key in pe.decoded['keyList']:
+            if key['keyIdentifier'][0] == cls.key_id and key['keyVersionNumber'][0] == cls.kvn:
+                assert len(key['keyComponents']) == 1
+                key['keyComponents'][0]['keyData'] = value
                 return
+        # Could not find matching key to patch, create a new one
+        key = {
+            'keyUsageQualifier': bytes([cls.key_usage_qual]),
+            'keyIdentifier': bytes([cls.key_id]),
+            'keyVersionNumber': bytes([cls.kvn]),
+            'keyComponents': [
+                { 'keyType': bytes([cls.key_type]), 'keyData': value },
+            ]
+        }
+        pe.decoded['keyList'].append(key)
 
-            # leaf node. create a new class with all the accumulated vals
-            parts = [name, f'KVN{kvn:02x}',] + labels
-            cls_label = '-'.join(p for p in parts if p)
+    @classmethod
+    def apply_val(cls, pes: ProfileElementSequence, value):
+        for pe in pes.get_pes_for_type('securityDomain'):
+            cls._apply_sd(pe, value)
 
-            parts = ['Sd', 'Key', name, f'Kvn{kvn:02x}'] + labels
-            clsname = ''.join(camel(p) for p in parts)
+class SdKeyScp80_01(SdKey, kvn=0x01, key_type=0x88, permitted_len=[16,24,32]): # AES key type
+    pass
+class SdKeyScp80_01Kic(SdKeyScp80_01, key_id=0x01, key_usage_qual=0x18): # FIXME: ordering?
+    pass
+class SdKeyScp80_01Kid(SdKeyScp80_01, key_id=0x02, key_usage_qual=0x14):
+    pass
+class SdKeyScp80_01Kik(SdKeyScp80_01, key_id=0x03, key_usage_qual=0x48):
+    pass
 
-            max_key_len = attrs.get('allow_len')[-1]
+class SdKeyScp81_01(SdKey, kvn=0x81): # FIXME
+    pass
+class SdKeyScp81_01Psk(SdKeyScp81_01, key_id=0x01, key_type=0x85, key_usage_qual=0x3C):
+    pass
+class SdKeyScp81_01Dek(SdKeyScp81_01, key_id=0x02, key_type=0x88, key_usage_qual=0x48):
+    pass
 
-            attrs.update({
-                'name' : cls_label,
-                'kvn': kvn,
-                'example_input': f'00*{max_key_len}',
-                })
+class SdKeyScp02_20(SdKey, kvn=0x20, key_type=0x88, permitted_len=[16,24,32]): # AES key type
+    pass
+class SdKeyScp02_20Enc(SdKeyScp02_20, key_id=0x01, key_usage_qual=0x18):
+    pass
+class SdKeyScp02_20Mac(SdKeyScp02_20, key_id=0x02, key_usage_qual=0x14):
+    pass
+class SdKeyScp02_20Dek(SdKeyScp02_20, key_id=0x03, key_usage_qual=0x48):
+    pass
 
-            # below line is like
-            # class SdKeyScpNNKvnXXYyyZzz(SdKey):
-            #     <set attrs>
-            cls_def = type(clsname, (cls,), attrs)
+class SdKeyScp03_30(SdKey, kvn=0x30, key_type=0x88, permitted_len=[16,24,32]): # AES key type
+    pass
+class SdKeyScp03_30Enc(SdKeyScp03_30, key_id=0x01, key_usage_qual=0x18):
+    pass
+class SdKeyScp03_30Mac(SdKeyScp03_30, key_id=0x02, key_usage_qual=0x14):
+    pass
+class SdKeyScp03_30Dek(SdKeyScp03_30, key_id=0x03, key_usage_qual=0x48):
+    pass
 
-            # for some unknown reason, subclassing from abc.ABC makes cls_def.__module__ == 'abc',
-            # but we don't want 'abc.SdKeyScp03Kvn32AesEnc'.
-            # Make sure it is 'pySim.esim.saip.personalization.SdKeyScp03Kvn32AesEnc'
-            cls_def.__module__ = __name__
+class SdKeyScp03_31(SdKey, kvn=0x31, key_type=0x88, permitted_len=[16,24,32]): # AES key type
+    pass
+class SdKeyScp03_31Enc(SdKeyScp03_31, key_id=0x01, key_usage_qual=0x18):
+    pass
+class SdKeyScp03_31Mac(SdKeyScp03_31, key_id=0x02, key_usage_qual=0x14):
+    pass
+class SdKeyScp03_31Dek(SdKeyScp03_31, key_id=0x03, key_usage_qual=0x48):
+    pass
 
-            globals()[clsname] = cls_def
-            SdKey.all_implementations.append(cls_def)
+class SdKeyScp03_32(SdKey, kvn=0x32, key_type=0x88, permitted_len=[16,24,32]): # AES key type
+    pass
+class SdKeyScp03_32Enc(SdKeyScp03_32, key_id=0x01, key_usage_qual=0x18):
+    pass
+class SdKeyScp03_32Mac(SdKeyScp03_32, key_id=0x02, key_usage_qual=0x14):
+    pass
+class SdKeyScp03_32Dek(SdKeyScp03_32, key_id=0x03, key_usage_qual=0x48):
+    pass
 
 
-        for items in sd_key_defs:
-            name, kvns = items[:2]
-            variants = items[2:]
-            for kvn in kvns:
-                do_variants(name, kvn, variants)
 
-# this creates all of the classes named like SdKeyScp02Kvn20AesDek to be published in this python module:
-SdKey.generate_sd_key_classes()
 
 def obtain_all_pe_from_pelist(l: List[ProfileElement], wanted_type: str) -> ProfileElement:
     return (pe for pe in l if pe.type == wanted_type)
@@ -950,8 +502,7 @@ class Puk(DecimalHexParam):
     allow_len = 8
     rpad = 16
     keyReference = None
-    example_input = f'0*{allow_len}'
-    default_source = param_source.RandomDigitSource
+    example_input = '0' * allow_len
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -965,14 +516,6 @@ class Puk(DecimalHexParam):
         raise ValueError("input template UPP has unexpected structure:"
                          f" cannot find pukCode with keyReference={cls.keyReference}")
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        mf_pes = pes.pes_by_naa['mf'][0]
-        for pukCodes in obtain_all_pe_from_pelist(mf_pes, 'pukCodes'):
-            for pukCode in pukCodes.decoded['pukCodes']:
-                if pukCode['keyReference'] == cls.keyReference:
-                    yield { cls.name: cls.decimal_hex_to_str(pukCode['pukValue']) }
-
 class Puk1(Puk):
     name = 'PUK1'
     keyReference = 0x01
@@ -986,8 +529,7 @@ class Pin(DecimalHexParam):
     rpad = 16
     min_len = 4
     max_len = 8
-    example_input = f'0*{max_len}'
-    default_source = param_source.RandomDigitSource
+    example_input = '0' * max_len
     keyReference = None
 
     @staticmethod
@@ -1009,24 +551,9 @@ class Pin(DecimalHexParam):
             raise ValueError('input template UPP has unexpected structure:'
                              + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference}')
 
-    @classmethod
-    def _read_all_pinvalues_from_pe(cls, pe: ProfileElement):
-        "This is a separate function because subclasses may feed different pe arguments."
-        for pinCodes in obtain_all_pe_from_pelist(pe, 'pinCodes'):
-            if pinCodes.decoded['pinCodes'][0] != 'pinconfig':
-                continue
-
-            for pinCode in pinCodes.decoded['pinCodes'][1]:
-                if pinCode['keyReference'] == cls.keyReference:
-                    yield { cls.name: cls.decimal_hex_to_str(pinCode['pinValue']) }
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        yield from cls._read_all_pinvalues_from_pe(pes.pes_by_naa['mf'][0])
-
 class Pin1(Pin):
     name = 'PIN1'
-    example_input = '0*4'  # PIN are usually 4 digits
+    example_input = '0' * 4  # PIN are usually 4 digits
     keyReference = 0x01
 
 class Pin2(Pin1):
@@ -1045,14 +572,6 @@ class Pin2(Pin1):
                     raise ValueError('input template UPP has unexpected structure:'
                             + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference} in {naa=}')
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for naa in pes.pes_by_naa:
-            if naa not in ['usim','isim','csim','telecom']:
-                continue
-            for pe in pes.pes_by_naa[naa]:
-                yield from cls._read_all_pinvalues_from_pe(pe)
-
 class Adm1(Pin):
     name = 'ADM1'
     keyReference = 0x0A
@@ -1077,59 +596,26 @@ class AlgoConfig(ConfigurableParameter):
             raise ValueError('input template UPP has unexpected structure:'
                              f' {cls.__name__} cannot find algoParameter with key={cls.algo_config_key}')
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('akaParameter'):
-            algoConfiguration = pe.decoded['algoConfiguration']
-            if len(algoConfiguration) < 2:
-                continue
-            if algoConfiguration[0] != 'algoParameter':
-                continue
-            if not algoConfiguration[1]:
-                continue
-            val = algoConfiguration[1].get(cls.algo_config_key, None)
-            if val is None:
-                continue
-            if isinstance(val, bytes):
-                val = b2h(val)
-            # if it is an int (algorithmID), just pass thru as int
-            yield { cls.name: val }
-
-class AlgorithmID(EnumParam, AlgoConfig):
-    """use validate_val() from EnumParam, and apply_val() from AlgoConfig.
-    In get_values_from_pes(), return enum value names, not raw values."""
-    name = "Algorithm"
+class AlgorithmID(DecimalParam, AlgoConfig):
     algo_config_key = 'algorithmID'
-    example_input = "Milenage"
-    default_source = param_source.ConstantSource
-
-    # as in pySim/esim/asn1/saip/PE_Definitions-3.3.1.asn
-    class Values(enum.IntEnum):
-        Milenage  = 1
-        TUAK      = 2
-        usim_test = 3  # input 'usim-test' also accepted via fuzzy matching
-
-    # EnumParam.validate_val() returns the int values from Values
+    allow_len = 1
+    example_input = 1  # Milenage
 
     @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        # return enum names, not raw values.
-        # use of super(): this intends to call AlgoConfig.get_values_from_pes() so that the cls argument is this cls
-        # here (AlgorithmID); i.e. AlgoConfig.get_values_from_pes(pes) doesn't work, because AlgoConfig needs to look up
-        # cls.algo_config_key.
-        for d in super(cls, cls).get_values_from_pes(pes):
-            if cls.name in d:
-                # convert int to value string
-                val = d[cls.name]
-                d[cls.name] = cls.map_val_to_name(val, strict=True)
-            yield d
+    def validate_val(cls, val):
+        val = super().validate_val(val)
+        val = int(val)
+        valid = (1, 2, 3)
+        if val not in valid:
+            raise ValueError(f'Invalid algorithmID {val!r}, must be one of {valid}')
+        return val
 
 class K(BinaryParam, AlgoConfig):
     """use validate_val() from BinaryParam, and apply_val() from AlgoConfig"""
     name = 'K'
     algo_config_key = 'key'
     allow_len = (128 // 8, 256 // 8) # length in bytes (from BinaryParam); TUAK also allows 256 bit
-    example_input = f'00*{allow_len[0]}'
+    example_input = '00' * allow_len[0]
 
 class Opc(K):
     name = 'OPc'
@@ -1143,7 +629,6 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):
     algo_config_key = 'rotationConstants'
     allow_len = 5 # length in bytes (from BinaryParam)
     example_input = '40 00 20 40 60'
-    default_source = param_source.ConstantSource
 
     @classmethod
     def validate_val(cls, val):
@@ -1156,7 +641,7 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):
 
 class MilenageXoringConstants(BinaryParam, AlgoConfig):
     """XOR-ing constants c1,c2,c3,c4,c5 of Milenage, 128bit each. See 3GPP TS 35.206 Sections 2.3 + 5.3.
-    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concatenation
+    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concetenation
     of::
 
      00000000000000000000000000000000
@@ -1174,7 +659,6 @@ class MilenageXoringConstants(BinaryParam, AlgoConfig):
                      ' 00000000000000000000000000000002'
                      ' 00000000000000000000000000000004'
                      ' 00000000000000000000000000000008')
-    default_source = param_source.ConstantSource
 
 class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
     """Number of iterations of Keccak-f[1600] permutation as recomended by Section 7.2 of 3GPP TS 35.231"""
@@ -1183,4 +667,3 @@ class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
     min_val = 1
     max_val = 255
     example_input = '1'
-    default_source = param_source.ConstantSource

pySim/euicc.py

@@ -226,28 +226,9 @@ class Icon(BER_TLV_IE, tag=0x94):
     _construct = GreedyBytes
 class ProfileClass(BER_TLV_IE, tag=0x95):
     _construct = Enum(Int8ub, test=0, provisioning=1, operational=2)
-class ProfilePolicyRules(BER_TLV_IE, tag=0x99):
-    _construct = GreedyBytes
-class NotificationConfigurationInfo(BER_TLV_IE, tag=0xb6):
-    _construct = GreedyBytes
-
-# ProfileOwner
-class ProfileOwnerPLMN(BER_TLV_IE, tag=0x80):
-    _construct = PlmnAdapter(Bytes(3))
-class ProfileOwnerGID1(BER_TLV_IE, tag=0x81):
-    _construct = GreedyBytes
-class ProfileOwnerGID2(BER_TLV_IE, tag=0x82):
-    _construct = GreedyBytes
-class ProfileOwner(BER_TLV_IE, tag=0xb7, nested=[ProfileOwnerPLMN, ProfileOwnerGID1, ProfileOwnerGID2]):
-    _construct = GreedyBytes
-
-class SMDPPProprietaryData(BER_TLV_IE, tag=0xb8):
-    _construct = GreedyBytes
-
 class ProfileInfo(BER_TLV_IE, tag=0xe3, nested=[Iccid, IsdpAid, ProfileState, ProfileNickname,
                                                 ServiceProviderName, ProfileName, IconType, Icon,
-                                                ProfileClass, ProfilePolicyRules, NotificationConfigurationInfo,
-                                                ProfileOwner, SMDPPProprietaryData]):
+                                                ProfileClass]): # FIXME: more IEs
     pass
 class ProfileInfoSeq(BER_TLV_IE, tag=0xa0, nested=[ProfileInfo]):
     pass
@@ -463,28 +444,9 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             d = rn.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['notification_sent_resp']))
 
-        get_profiles_info_parser = argparse.ArgumentParser()
-        get_profiles_info_parser.add_argument('--all', action='store_true', help='Retrieve all known tags of a profile')
-
-        @cmd2.with_argparser(get_profiles_info_parser)
-        def do_get_profiles_info(self, opts):
+        def do_get_profiles_info(self, _opts):
             """Perform an ES10c GetProfilesInfo function."""
-            if opts.all:
-                tags = [nest.tag for nest in ProfileInfo.nested_collection_cls().nested]
-                u8tags = []
-                # TODO: rework TagList to support 2 byte tags to not filter it into u8 tags
-                for tag in tags:
-                    if tag <= 255:
-                        u8tags.append(tag)
-                    elif tag <= 65535:
-                        u8tags.append(tag >> 8)
-                        u8tags.append(tag & 0xff)
-                    # Ignoring 3 byte tags
-                req = ProfileInfoListReq(children=[TagList(decoded=u8tags)])
-            else:
-                req = ProfileInfoListReq()
-
-            pi = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, req, ProfileInfoListResp)
+            pi = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, ProfileInfoListReq(), ProfileInfoListResp)
             d = pi.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['profile_info_list_resp']))
 

pySim/filesystem.py

@@ -44,7 +44,6 @@ from pySim.utils import sw_match, decomposeATR
 from pySim.jsonpath import js_path_modify
 from pySim.commands import SimCardCommands
 from pySim.exceptions import SwMatchError
-from pySim.log import PySimLogger
 
 # int: a single service is associated with this file
 # list: any of the listed services requires this file
@@ -53,8 +52,6 @@ CardFileService = Union[int, List[int], Tuple[int, ...]]
 
 Size = Tuple[int, Optional[int]]
 
-log = PySimLogger.get(__name__)
-
 class CardFile:
     """Base class for all objects in the smart card filesystem.
     Serve as a common ancestor to all other file types; rarely used directly.
@@ -1612,14 +1609,14 @@ class CardModel(abc.ABC):
         card_atr = scc.get_atr()
         for atr in cls._atrs:
             if atr == card_atr:
-                log.info("Detected CardModel: %s", cls.__name__)
+                print("Detected CardModel:", cls.__name__)
                 return True
         # if nothing found try to just compare the Historical Bytes of the ATR
         card_atr_hb = decomposeATR(card_atr)['hb']
         for atr in cls._atrs:
             atr_hb = decomposeATR(atr)['hb']
             if atr_hb == card_atr_hb:
-                log.info("Detected CardModel: %s", cls.__name__)
+                print("Detected CardModel:", cls.__name__)
                 return True
         return False
 

pySim/global_platform/__init__.py

@@ -18,12 +18,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
 import io
-import hashlib
 from copy import deepcopy
 from typing import Optional, List, Dict, Tuple
 from construct import Optional as COptional
 from construct import Struct, GreedyRange, FlagsEnum, Int16ub, Int24ub, Padding, Bit, Const
-from construct import Construct, stream_read, stream_write
 from Cryptodome.Random import get_random_bytes
 from Cryptodome.Cipher import DES, DES3, AES
 from osmocom.utils import *
@@ -150,24 +148,6 @@ sw_table = {
     },
 }
 
-class PutKeyLength(Construct):
-    """A length field of a PUT KEY data field, GP CardSpec v2.3.1 11.8.2.3.1
-    - all lengths ASN.1 BER-TLV (ITU-T X.690 Section 8.1.3)
-    - except that the length 128 may also be coded on one byte as '80' for backwards compatibility
-    80 does not introduce the indefinite form here which is unused in GP as far as i know.
-    That legacy form is accepted when parsing, but never generated, which agrees with the spec"""
-    def _parse(self, stream, context, path):
-        first = stream_read(stream, 1, path)[0]
-        if first <= 0x80:
-            return first
-        return int.from_bytes(stream_read(stream, first & 0x7f, path), 'big')
-
-    def _build(self, obj, stream, context, path):
-        data = bertlv_encode_len(obj)
-        stream_write(stream, data, len(data), path)
-        return obj
-
-
 # GlobalPlatform 2.1.1 Section 9.1.6
 KeyType = Enum(Byte,    des=0x80,
                         tls_psk=0x85,                           # v2.3.1 Section 11.1.8
@@ -622,8 +602,8 @@ class ADF_SD(CardADF):
             See GlobalPlatform CardSpecification v2.3 Section 11.8 for details.
 
             The KCV (Key Check Values) can either be explicitly specified using `--key-check`, or will
-            otherwise be automatically generated for DES, AES and TLS-PSK keys.  You can suppress the
-            latter using `--suppress-key-check`.
+            otherwise be automatically generated for DES and AES keys.  You can suppress the latter using
+            `--suppress-key-check`.
 
             Example (SCP80 KIC/KID/KIK):
                 put_key --key-version-nr 1 --key-id 0x01    --key-type aes --key-data 000102030405060708090a0b0c0d0e0f
@@ -640,81 +620,33 @@ class ADF_SD(CardADF):
             kdb = []
             for i in range(0, len(opts.key_type)):
                 if opts.key_check and len(opts.key_check) > i:
-                    kcv = h2b(opts.key_check[i])
+                    kcv = opts.key_check[i]
                 elif opts.suppress_key_check:
-                    kcv = b''
+                    kcv = ''
                 else:
-                    kcv = compute_kcv(opts.key_type[i], h2b(opts.key_data[i])) or b''
-                kdb.append({'key_type': opts.key_type[i], 'clear_key': h2b(opts.key_data[i]), 'kcv': kcv})
+                    kcv_bin = compute_kcv(opts.key_type[i], h2b(opts.key_data[i])) or b''
+                    kcv = b2h(kcv_bin)
+                if self._cmd.lchan.scc.scp:
+                    # encrypted key data with DEK of current SCP
+                    kcb = b2h(self._cmd.lchan.scc.scp.encrypt_key(h2b(opts.key_data[i])))
+                else:
+                    # (for example) during personalization, DEK might not be required)
+                    kcb = opts.key_data[i]
+                kdb.append({'key_type': opts.key_type[i], 'kcb': kcb, 'kcv': kcv})
             p2 = opts.key_id
             if len(opts.key_type) > 1:
                 p2 |= 0x80
             self.put_key(opts.old_key_version_nr, opts.key_version_nr, p2, kdb)
 
-        # Table 11-68: Key Data Field - Format 1 (Basic Format).  The key component block length is
-        # BER-TLV coded (Section 11.8.2.3.1), the key check value length is always '00' - '7F'.
-        KeyDataBasic = Struct('key_type'/KeyType,
-                              'kcb'/Prefixed(PutKeyLength(), GreedyBytes),
-                              'kcv'/Prefixed(Int8ub, GreedyBytes))
+        # Table 11-68: Key Data Field - Format 1 (Basic Format)
+        KeyDataBasic = GreedyRange(Struct('key_type'/KeyType,
+                                          'kcb'/Prefixed(Int8ub, GreedyBytes),
+                                          'kcv'/Prefixed(Int8ub, GreedyBytes)))
 
-        @classmethod
-        def encode_key_data_basic(cls, key_type: str, kcb: bytes, kcv: bytes) -> bytes:
-            """Generic Basic key data field, GP CardSpec v2.3 Table 11-68):
-                tag || L1 || <maybe L2> KCB || <1-byte length> KCV"""
-            return cls.KeyDataBasic.build({'key_type': key_type, 'kcb': kcb, 'kcv': kcv})
-
-        @classmethod
-        def encode_key_data_psk(cls, clear_key: bytes, ciphered_key: bytes, kcv: bytes) -> bytes:
-            """Single PSK TLS '85' key data field per GP Amendment B 1.2, 3.9.1 / Table 3-13:
-                85 | L1 | <L2>  <ciphered PSK key> | <KCV length> | <KCV>
-            - framing is like Basic Format, but the kcb is always GP CardSpec Table 11-70
-            so always with the length of the clear text key value, even without padding!
-            - 'ciphered_key' is DEK(block-padded clear key), no additional length prefix."""
-            kcb = bertlv_encode_len(len(clear_key)) + ciphered_key
-            return cls.encode_key_data_basic('tls_psk', kcb, kcv)
-
-        @classmethod
-        def build_put_key_data(cls, kvn: int, keys: List[dict], scp) -> bytes:
-            """Assemble the PUT KEY data field, mixed PSK + DES DEK is supported:
-            - new KVN followed by one key data field per key.
-            - tls_psk keys per GP Amendment B
-            - other key types generic Basic format
-            Param 'keys' is a dict:
-            - 'key_type' (str)
-            - 'clear_key' (bytes)
-            - 'kcv' (bytes / empty).
-            'scp' may be None (e.g. during personalization, when the DEK may not be required)."""
-            key_data = kvn.to_bytes(1, 'big')
-            for k in keys:
-                clear = k['clear_key']
-                if k['key_type'] == 'tls_psk':
-                    # len always part of the data see CardSpec Table 11-70 vs Table 11-71
-                    if scp:
-                        ciphered = scp.dek_encrypt(scp.pad_to_blocksize(clear))
-                    else:
-                        ciphered = clear
-                    key_data += cls.encode_key_data_psk(clear, ciphered, k['kcv'])
-                else:
-                    if scp:
-                        ciphered = scp.encrypt_key(clear)
-                    else:
-                        # (for example) during personalization, DEK might not be required
-                        ciphered = clear
-                    key_data += cls.encode_key_data_basic(k['key_type'], ciphered, k['kcv'])
-            return key_data
-
-        def put_key(self, old_kvn:int, kvn: int, kid: int, keys: List[dict]) -> bytes:
+        def put_key(self, old_kvn:int, kvn: int, kid: int, key_dict: dict) -> bytes:
             """Perform the GlobalPlatform PUT KEY command in order to store a new key on the card.
             See GlobalPlatform CardSpecification v2.3 Section 11.8 for details."""
-            key_data = self.build_put_key_data(kvn, keys, self._cmd.lchan.scc.scp)
-            # Lc of Table 11-64 is a single byte, while LOAD or STORE DATA splits we can't:
-            # 11.8.2.3.3 splits a key at component boundaries -> not helping here
-            max_cmd_len = self._cmd.lchan.scc.max_cmd_len
-            if len(key_data) > max_cmd_len:
-                raise ValueError('key data field of %u bytes exceeds the maximum command length of %u '
-                                 '(limited by the overhead of the current secure channel); use fewer '
-                                 'keys per command, a single key component that large needs STORE DATA' %
-                                 (len(key_data), max_cmd_len))
+            key_data = kvn.to_bytes(1, 'big') + build_construct(ADF_SD.AddlShellCommands.KeyDataBasic, key_dict)
             hdr = "80D8%02x%02x%02x" % (old_kvn, kid, len(key_data))
             data, _sw = self._cmd.lchan.scc.send_apdu_checksw(hdr + b2h(key_data) + "00")
             return data
@@ -894,32 +826,23 @@ class ADF_SD(CardADF):
         load_parser_from_grp.add_argument('--from-hex', type=is_hexstr, help='load from hex string')
         load_parser_from_grp.add_argument('--from-file', type=argparse.FileType('rb', 0), help='load from binary file')
         load_parser_from_grp.add_argument('--from-cap-file', type=argparse.FileType('rb', 0), help='load from JAVA-card CAP file')
-        load_parser.add_argument('--chunk-len', type=auto_uint8, default=None,
-                                 help='Block size for the LOAD command; default: as large as the current secure channel overhead permits, at most 240')
 
         @cmd2.with_argparser(load_parser)
         def do_load(self, opts):
             """Perform a GlobalPlatform LOAD command. (We currently only support loading without DAP and
             without ciphering.)"""
             if opts.from_hex is not None:
-                self.load(h2b(opts.from_hex), opts.chunk_len)
+                self.load(h2b(opts.from_hex))
             elif opts.from_file is not None:
-                self.load(opts.from_file.read(), opts.chunk_len)
+                self.load(opts.from_file.read())
             elif opts.from_cap_file is not None:
                 cap = CapFile(opts.from_cap_file)
-                self.load(cap.get_loadfile(), opts.chunk_len)
+                self.load(cap.get_loadfile())
             else:
                 raise ValueError('load source not specified!')
 
-        def load(self, contents:bytes, chunk_len:Optional[int] = None):
-            # scc.max_cmd_len knows the overhead the currently active SCP
-            # 240 is the old default, keep it for now.
-            max_chunk_len = self._cmd.lchan.scc.max_cmd_len
-            if chunk_len is None:
-                chunk_len = min(240, max_chunk_len)
-            elif not 1 <= chunk_len <= max_chunk_len:
-                raise ValueError('chunk_len must be in range 1..%u (limited by the overhead of the current secure channel)' %
-                                 max_chunk_len)
+        def load(self, contents:bytes, chunk_len:int = 240):
+            # TODO:tune chunk_len based on the overhead of the used SCP?
             # build TLV according to GPC_SPE_034 section 11.6.2.3 / Table 11-58 for unencrypted case
             remainder = b'\xC4' + bertlv_encode_len(len(contents)) + contents
             # transfer this in various chunks to the card
@@ -958,8 +881,6 @@ class ADF_SD(CardADF):
         install_cap_parser_inst_prm_grp.add_argument('--install-parameters-stk',
                                                      type=is_hexstr, default=None,
                                                      help='Load Parameters (ETSI TS 102 226, section 8.2.1.3.2.1)')
-        install_cap_parser.add_argument('--chunk-len', type=auto_uint8, default=None,
-                                        help='Block size for the LOAD command; default: as large as the current secure channel overhead permits, at most 240')
 
         @cmd2.with_argparser(install_cap_parser)
         def do_install_cap(self, opts):
@@ -998,7 +919,7 @@ class ADF_SD(CardADF):
             self._cmd.poutput("step #1: install for load...")
             self.do_install_for_load("--load-file-aid %s --security-domain-aid %s" % (load_file_aid, security_domain_aid))
             self._cmd.poutput("step #2: load...")
-            self.load(load_file, opts.chunk_len)
+            self.load(load_file)
             self._cmd.poutput("step #3: install_for_install (and make selectable)...")
             self.do_install_for_install("--load-file-aid %s --module-aid %s --application-aid %s --install-parameters %s --make-selectable" %
                                         (load_file_aid, module_aid, application_aid, install_parameters))
@@ -1144,16 +1065,10 @@ def compute_kcv_aes(key:bytes) -> bytes:
     cipher = AES.new(key, AES.MODE_ECB)
     return cipher.encrypt(plaintext)
 
-def compute_kcv_psk(key:bytes) -> bytes:
-    # GP Amendment B v1.2, 3.9.1 / Table 3-13
-    # KCV of a PSK TLS key is the 3 highest-order bytes of the SHA-1 digest of the clear key value.
-    return hashlib.sha1(key).digest()
-
 # dict is keyed by the string name of the KeyType enum above in this file
 KCV_CALCULATOR = {
         'aes': compute_kcv_aes,
         'des': compute_kcv_des,
-        'tls_psk': compute_kcv_psk,
     }
 
 def compute_kcv(key_type: str, key: bytes) -> Optional[bytes]:

pySim/global_platform/scp.py

@@ -27,9 +27,9 @@ from osmocom.utils import b2h
 from osmocom.tlv import bertlv_parse_len, bertlv_encode_len
 from pySim.utils import parse_command_apdu
 from pySim.secure_channel import SecureChannel
-from pySim.log import PySimLogger
 
-log = PySimLogger.get(__name__)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
 
 def scp02_key_derivation(constant: bytes, counter: int, base_key: bytes) -> bytes:
     assert len(constant) == 2
@@ -75,7 +75,7 @@ class Scp02SessionKeys:
             h = e.encrypt(strxor(h, bytes(padded_data[8*i:8*(i+1)])))
         h = d.decrypt(h)
         h = e.encrypt(h)
-        log.debug("mac_1des(%s,icv=%s) -> %s", b2h(data), b2h(icv), b2h(h))
+        logger.debug("mac_1des(%s,icv=%s) -> %s", b2h(data), b2h(icv), b2h(h))
         if self.des_icv_enc:
             self.icv = self.des_icv_enc.encrypt(h)
         else:
@@ -89,7 +89,7 @@ class Scp02SessionKeys:
         h = b'\x00' * 8
         for i in range(q):
             h = e.encrypt(strxor(h, bytes(padded_data[8*i:8*(i+1)])))
-        log.debug("mac_3des(%s) -> %s", b2h(data), b2h(h))
+        logger.debug("mac_3des(%s) -> %s", b2h(data), b2h(h))
         return h
 
     def __init__(self, counter: int, card_keys: 'GpCardKeyset', icv_encrypt=True):
@@ -182,29 +182,6 @@ class SCP(SecureChannel, abc.ABC):
         """Should we perform R-ENC?"""
         return self.security_level & 0x20
 
-    @property
-    @abc.abstractmethod
-    def mac_len(self) -> int:
-        """Length of the appended C-MAC, to be provided by derived class."""
-
-    @property
-    def overhead(self) -> int:
-        """Worst-case len that wrapping a command APDU adds to its data field at the
-        current sec level is (255 - overhead), C-MAC + C-DECRYPTION encryption padding."""
-        if not self.do_cmac:
-            return 0
-        if not self.do_cenc:
-            return self.mac_len
-        # see Secure Channel Protocol '03' Card Specification v2.3 - Amendment D v1.1.2
-        # which defers to GPCS v2.3 Section B.2 which then defers to
-        # NIST SP 800-38B for encryption and points out that
-        # the padding is, as expected, just the usual padding from NIST SP 800-38A
-        # C-DECRYPTION pads with ('80'+['00'...] at least 1 byte) up to
-        # the cipher block size + C-MAC on top -> largest usable data field
-        # is one byte less than the largest block-size multiple within 255 - mac_len.
-        bs = self.sk.blocksize
-        return 255 - ((255 - self.mac_len) // bs * bs - 1)
-
     def __str__(self) -> str:
         return "%s[%02x]" % (self.__class__.__name__, self.security_level)
 
@@ -238,20 +215,11 @@ class SCP(SecureChannel, abc.ABC):
     def gen_ext_auth_apdu(self, security_level: int = 0x01) -> bytes:
         pass
 
-    def pad_to_blocksize(self, data: bytes) -> bytes:
-        """Right pad the data with zero bytes to a multiple of the DEK cipher block size."""
-        if len(data) % self.sk.blocksize:
-            # not '+=' which would mutate the callers bytearray in place..
-            data = data + b'\x00' * (self.sk.blocksize - len(data) % self.sk.blocksize)
-        return data
-
     def encrypt_key(self, key: bytes) -> bytes:
         """Encrypt a key with the DEK."""
-        if len(key) % self.sk.blocksize:
-            # The kcv is right padded before encryption and the kcb
-            # is formatted as described in Table 11-70: preceded by the actual length of the
-            # clear text kcv.
-            return bertlv_encode_len(len(key)) + self.dek_encrypt(self.pad_to_blocksize(key))
+        num_pad = len(key) % self.sk.blocksize
+        if num_pad:
+            return bertlv_encode_len(len(key)) + self.dek_encrypt(key + b'\x00'*num_pad)
         return self.dek_encrypt(key)
 
     def decrypt_key(self, encrypted_key:bytes) -> bytes:
@@ -264,8 +232,9 @@ class SCP(SecureChannel, abc.ABC):
             # Block provides the actual length of the key component value, which allows recovering the
             # clear-text key component value after decryption of the encrypted key component value and removal
             # of padding bytes.
-            key_len, remainder = bertlv_parse_len(encrypted_key)
-            return self.dek_decrypt(remainder)[:key_len]
+            decrypted = self.dek_decrypt(encrypted_key)
+            key_len, remainder = bertlv_parse_len(decrypted)
+            return remainder[:key_len]
         else:
             # If the length of the Key Component Block is a multiple of the block size of the encryption
             # algorithm (i.e.  8 bytes for DES, 16 bytes for AES), then it shall be assumed that no padding
@@ -291,8 +260,10 @@ class SCP02(SCP):
     # Key Version Number 0x70 is a non-spec special-case of sysmoISIM-SJA2/SJA5 and possibly more sysmocom products
     # Key Version Number 0x01 is a non-spec special-case of sysmoUSIM-SJS1
     kvn_ranges = [[0x01, 0x01], [0x20, 0x2f], [0x70, 0x70]]
-    # C-MAC (Single DES + final 3DES, B.1.2.2) is always one full DES block
-    mac_len = 8
+
+    def __init__(self, *args, **kwargs):
+        self.overhead = 8
+        super().__init__(*args, **kwargs)
 
     def dek_encrypt(self, plaintext:bytes) -> bytes:
         # See also GPC section B.1.1.2, E.4.7, and E.4.1
@@ -305,10 +276,10 @@ class SCP02(SCP):
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self, card_challenge: bytes, host_challenge: bytes):
-        log.debug("host_challenge(%s), card_challenge(%s)", b2h(host_challenge), b2h(card_challenge))
+        logger.debug("host_challenge(%s), card_challenge(%s)", b2h(host_challenge), b2h(card_challenge))
         self.host_cryptogram = self.sk.calc_mac_3des(self.sk.counter.to_bytes(2, 'big') + card_challenge + host_challenge)
         self.card_cryptogram = self.sk.calc_mac_3des(self.host_challenge + self.sk.counter.to_bytes(2, 'big') + card_challenge)
-        log.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
+        logger.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
 
     def gen_init_update_apdu(self, host_challenge: bytes = b'\x00'*8) -> bytes:
         """Generate INITIALIZE UPDATE APDU."""
@@ -320,7 +291,7 @@ class SCP02(SCP):
         resp = self.constr_iur.parse(resp_bin)
         self.card_challenge = resp['card_challenge']
         self.sk = Scp02SessionKeys(resp['seq_counter'], self.card_keys)
-        log.debug(self.sk)
+        logger.debug(self.sk)
         self._compute_cryptograms(self.card_challenge, self.host_challenge)
         if self.card_cryptogram != resp['card_cryptogram']:
             raise ValueError("card cryptogram doesn't match")
@@ -340,7 +311,7 @@ class SCP02(SCP):
 
     def _wrap_cmd_apdu(self, apdu: bytes, *args, **kwargs) -> bytes:
         """Wrap Command APDU for SCP02: calculate MAC and encrypt."""
-        log.debug("wrap_cmd_apdu(%s)", b2h(apdu))
+        logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
 
         if not self.do_cmac:
             return apdu
@@ -367,16 +338,10 @@ class SCP02(SCP):
             # CMAC on modified APDU
             mlc = lc + 8
             clac = cla | CLA_SM
-        if mlc >= 256:
-            raise ValueError('Modified Lc (%u) would exceed maximum when appending 8 bytes of mac' % mlc)
         mac = self.sk.calc_mac_1des(bytes([clac]) + apdu[1:4] + bytes([mlc]) + data)
         if self.do_cenc:
-            padded_data = pad80(data, 8)
-            if len(padded_data) + 8 >= 256:
-                raise ValueError('Modified Lc (%u) would exceed maximum when appending padding and mac' %
-                                 (len(padded_data) + 8))
             k = DES3.new(self.sk.enc, DES.MODE_CBC, b'\x00'*8)
-            data = k.encrypt(padded_data)
+            data = k.encrypt(pad80(data, 8))
             lc = len(data)
 
         lc += 8
@@ -413,7 +378,7 @@ def scp03_key_derivation(constant: bytes, context: bytes, base_key: bytes, l: Op
     if l is None:
         l = len(base_key) * 8
 
-    log.debug("scp03_kdf(constant=%s, context=%s, base_key=%s, l=%u)", b2h(constant), b2h(context), b2h(base_key), l)
+    logger.debug("scp03_kdf(constant=%s, context=%s, base_key=%s, l=%u)", b2h(constant), b2h(context), b2h(base_key), l)
     output_len = l // 8
     # SCP03 Section 4.1.5 defines a different parameter order than NIST SP 800-108, so we cannot use the
     # existing Cryptodome.Protocol.KDF.SP800_108_Counter function :(
@@ -486,7 +451,7 @@ class Scp03SessionKeys:
         # This block SHALL be encrypted with S-ENC to produce the ICV for command encryption.
         cipher = AES.new(self.s_enc, AES.MODE_CBC, iv)
         icv = cipher.encrypt(data)
-        log.debug("_get_icv(data=%s, is_resp=%s) -> icv=%s", b2h(data), is_response, b2h(icv))
+        logger.debug("_get_icv(data=%s, is_resp=%s) -> icv=%s", b2h(data), is_response, b2h(icv))
         return icv
 
     # TODO: Resolve duplication with pySim.esim.bsp.BspAlgoCryptAES128 which provides pad80-wrapping
@@ -512,13 +477,9 @@ class SCP03(SCP):
 
     def __init__(self, *args, **kwargs):
         self.s_mode = kwargs.pop('s_mode', 8)
+        self.overhead = self.s_mode
         super().__init__(*args, **kwargs)
 
-    @property
-    def mac_len(self) -> int:
-        # C-MAC truncated to 8 in S8 or 16 bytes in S16 mode
-        return self.s_mode
-
     def dek_encrypt(self, plaintext:bytes) -> bytes:
         cipher = AES.new(self.card_keys.dek, AES.MODE_CBC, b'\x00'*16)
         return cipher.encrypt(plaintext)
@@ -528,12 +489,12 @@ class SCP03(SCP):
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self):
-        log.debug("host_challenge(%s), card_challenge(%s)", b2h(self.host_challenge), b2h(self.card_challenge))
+        logger.debug("host_challenge(%s), card_challenge(%s)", b2h(self.host_challenge), b2h(self.card_challenge))
         # Card + Host Authentication Cryptogram: Section 6.2.2.2 + 6.2.2.3
         context = self.host_challenge + self.card_challenge
         self.card_cryptogram = scp03_key_derivation(self.sk.DERIV_CONST_AUTH_CGRAM_CARD, context, self.sk.s_mac, l=self.s_mode*8)
         self.host_cryptogram = scp03_key_derivation(self.sk.DERIV_CONST_AUTH_CGRAM_HOST, context, self.sk.s_mac, l=self.s_mode*8)
-        log.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
+        logger.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
 
     def gen_init_update_apdu(self, host_challenge: Optional[bytes] = None) -> bytes:
         """Generate INITIALIZE UPDATE APDU."""
@@ -553,7 +514,7 @@ class SCP03(SCP):
         self.i_param = resp['i_param']
         # derive session keys and compute cryptograms
         self.sk = Scp03SessionKeys(self.card_keys, self.host_challenge, self.card_challenge)
-        log.debug(self.sk)
+        logger.debug(self.sk)
         self._compute_cryptograms()
         # verify computed cryptogram matches received cryptogram
         if self.card_cryptogram != resp['card_cryptogram']:
@@ -568,7 +529,7 @@ class SCP03(SCP):
 
     def _wrap_cmd_apdu(self, apdu: bytes, skip_cenc: bool = False) -> bytes:
         """Wrap Command APDU for SCP03: calculate MAC and encrypt."""
-        log.debug("wrap_cmd_apdu(%s)", b2h(apdu))
+        logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
 
         if not self.do_cmac:
             return apdu
@@ -623,7 +584,7 @@ class SCP03(SCP):
         # status word: in this case only the status word shall be returned in the response. All status words
         # except '9000' and warning status words (i.e. '62xx' and '63xx') shall be interpreted as error status
         # words.
-        log.debug("unwrap_rsp_apdu(sw=%s, rsp_apdu=%s)", sw, rsp_apdu)
+        logger.debug("unwrap_rsp_apdu(sw=%s, rsp_apdu=%s)", sw, rsp_apdu)
         if not self.do_rmac:
             assert not self.do_renc
             return rsp_apdu
@@ -639,9 +600,9 @@ class SCP03(SCP):
         if self.do_renc:
             # decrypt response data
             decrypted = self.sk._decrypt(response_data)
-            log.debug("decrypted: %s", b2h(decrypted))
+            logger.debug("decrypted: %s", b2h(decrypted))
             # remove padding
             response_data = unpad80(decrypted)
-            log.debug("response_data: %s", b2h(response_data))
+            logger.debug("response_data: %s", b2h(response_data))
 
         return response_data

pySim/global_platform/uicc.py

@@ -91,7 +91,6 @@ class UiccSdInstallParams(TLV_IE_Collection, nested=[UiccScp, AcceptExtradAppsAn
 
 # Key Usage:
 # KVN 0x01 .. 0x0F reserved for SCP80
-# KVN 0x81 .. 0x8f reserved for SCP81
 # KVN 0x11 reserved for DAP specified in ETSI TS 102 226
 # KVN 0x20 .. 0x2F reserved for SCP02
 #   KID 0x01 = ENC; 0x02 = MAC; 0x03 = DEK

pySim/legacy/cards.py

@@ -152,8 +152,7 @@ class SimCard(SimCardBase):
         return sw
 
     def update_smsp(self, smsp):
-        print("using update_smsp")
-        data, sw = self._scc.update_record(EF['SMSP'], 1, smsp, leftpad=True)
+        data, sw = self._scc.update_record(EF['SMSP'], 1, rpad(smsp, 84))
         return sw
 
     def update_ad(self, mnc=None, opmode=None, ofm=None, path=EF['AD']):

pySim/log.py

@@ -24,15 +24,7 @@
 #
 
 import logging
-import cmd2
-from packaging import version
-
-if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
-    from cmd2 import stylize as _stylize # pylint: disable=no-name-in-module
-    def _style(text, fg=None): # pylint: disable=function-redefined
-        return _stylize(text, fg) if fg else text
-else: # cmd2>=2.6.2
-    from cmd2 import style as _style # pylint: disable=no-name-in-module
+from cmd2 import style
 
 class _PySimLogHandler(logging.Handler):
     def __init__(self, log_callback):
@@ -52,7 +44,7 @@ class PySimLogger:
     """
 
     LOG_FMTSTR = "%(levelname)s: %(message)s"
-    LOG_FMTSTR_VERBOSE = "%(name)s.%(lineno)d -- " + LOG_FMTSTR
+    LOG_FMTSTR_VERBOSE = "%(module)s.%(lineno)d -- " + LOG_FMTSTR
     __formatter = logging.Formatter(LOG_FMTSTR)
     __formatter_verbose = logging.Formatter(LOG_FMTSTR_VERBOSE)
 
@@ -129,7 +121,7 @@ class PySimLogger:
                 if isinstance(color, str):
                     PySimLogger.print_callback(color + formatted_message + "\033[0m")
                 else:
-                    PySimLogger.print_callback(_style(formatted_message, fg = color))
+                    PySimLogger.print_callback(style(formatted_message, fg = color))
             else:
                 PySimLogger.print_callback(formatted_message)
 

pySim/transport/__init__.py

@@ -307,48 +307,49 @@ class LinkBaseTpdu(LinkBase):
 
         # After sending the APDU/TPDU the UICC/eUICC or SIM may response with a status word that indicates that further
         # TPDUs have to be sent in order to complete the task.
-        if case == 4 or self.apdu_strict == False:
-            # In case the APDU is a case #4 APDU, the UICC/eUICC/SIM may indicate that there is response data
-            # available which has to be retrieved using a GET RESPONSE command TPDU.
-            #
-            # ETSI TS 102 221, section 7.3.1.1.4 is very cleare about the fact that the GET RESPONSE mechanism
-            # shall only apply on case #4 APDUs but unfortunately it is impossible to distinguish between case #3
-            # and case #4 when the APDU format is not strictly followed. In order to be able to detect case #4
-            # correctly the Le byte (usually 0x00) must be present, is often forgotten. To avoid problems with
-            # legacy scripts that use raw APDU strings, we will still loosely apply GET RESPONSE based on what
-            # the status word indicates. Unless the user explicitly enables the strict mode (set apdu_strict true)
-            while True:
-                if sw in ['9000', '9100']:
-                    # A status word of 9000 (or 9100 in case there is pending data from a proactive SIM command)
-                    # indicates that either no response data was returnd or all response data has been retrieved
-                    # successfully. We may discontinue the processing at this point.
-                    break;
-                if sw[0:2] in ['61', '9f']:
-                    # A status word of 61xx or 9fxx indicates that there is (still) response data available. We
-                    # send a GET RESPONSE command with the length value indicated in the second byte of the status
-                    # word. (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4a and 3GPP TS 51.011 9.4.1 and
-                    # ISO/IEC 7816-4, Table 5)
-                    le_gr = sw[2:4]
-                elif sw[0:2] in ['62', '63']:
-                    # There are corner cases (status word is 62xx or 63xx) where the UICC/eUICC/SIM asks us
-                    # to send a dummy GET RESPONSE command. We send a GET RESPONSE command with a length of 0.
-                    # (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4b and ETSI TS 151 011, section 9.4.1)
-                    le_gr = '00'
-                else:
-                    # A status word other then the ones covered by the above logic may indicate an error. In this
-                    # case we will discontinue the processing as well.
-                    # (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4c)
-                    break
-                tpdu_gr = tpdu[0:2] + 'c00000' + le_gr
-                prev_tpdu = tpdu_gr
-                data_gr, sw = self.send_tpdu(tpdu_gr)
-                log.debug("T0: GET RESPONSE TPDU: %s => %s %s", tpdu_gr, data_gr or "(no data)", sw or "(no status word)")
-                data += data_gr
-        if sw[0:2] == '6c':
-            # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
-            tpdu_gr = prev_tpdu[0:8] + sw[2:4]
-            data, sw = self.send_tpdu(tpdu_gr)
-            log.debug("T0: repated case #%u TPDU: %s => %s %s", case, tpdu_gr, data or "(no data)", sw or "(no status word)")
+        if sw is not None:
+            if case == 4 or self.apdu_strict == False:
+                # In case the APDU is a case #4 APDU, the UICC/eUICC/SIM may indicate that there is response data
+                # available which has to be retrieved using a GET RESPONSE command TPDU.
+                #
+                # ETSI TS 102 221, section 7.3.1.1.4 is very cleare about the fact that the GET RESPONSE mechanism
+                # shall only apply on case #4 APDUs but unfortunately it is impossible to distinguish between case #3
+                # and case #4 when the APDU format is not strictly followed. In order to be able to detect case #4
+                # correctly the Le byte (usually 0x00) must be present, is often forgotten. To avoid problems with
+                # legacy scripts that use raw APDU strings, we will still loosely apply GET RESPONSE based on what
+                # the status word indicates. Unless the user explicitly enables the strict mode (set apdu_strict true)
+                while True:
+                    if sw in ['9000', '9100']:
+                        # A status word of 9000 (or 9100 in case there is pending data from a proactive SIM command)
+                        # indicates that either no response data was returnd or all response data has been retrieved
+                        # successfully. We may discontinue the processing at this point.
+                        break;
+                    if sw[0:2] in ['61', '9f']:
+                        # A status word of 61xx or 9fxx indicates that there is (still) response data available. We
+                        # send a GET RESPONSE command with the length value indicated in the second byte of the status
+                        # word. (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4a and 3GPP TS 51.011 9.4.1 and
+                        # ISO/IEC 7816-4, Table 5)
+                        le_gr = sw[2:4]
+                    elif sw[0:2] in ['62', '63']:
+                        # There are corner cases (status word is 62xx or 63xx) where the UICC/eUICC/SIM asks us
+                        # to send a dummy GET RESPONSE command. We send a GET RESPONSE command with a length of 0.
+                        # (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4b and ETSI TS 151 011, section 9.4.1)
+                        le_gr = '00'
+                    else:
+                        # A status word other then the ones covered by the above logic may indicate an error. In this
+                        # case we will discontinue the processing as well.
+                        # (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4c)
+                        break
+                    tpdu_gr = tpdu[0:2] + 'c00000' + le_gr
+                    prev_tpdu = tpdu_gr
+                    data_gr, sw = self.send_tpdu(tpdu_gr)
+                    log.debug("T0: GET RESPONSE TPDU: %s => %s %s", tpdu_gr, data_gr or "(no data)", sw or "(no status word)")
+                    data += data_gr
+            if sw[0:2] == '6c':
+                # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
+                tpdu_gr = prev_tpdu[0:8] + sw[2:4]
+                data, sw = self.send_tpdu(tpdu_gr)
+                log.debug("T0: repated case #%u TPDU: %s => %s %s", case, tpdu_gr, data or "(no data)", sw or "(no status word)")
 
         return data, sw
 

pySim/ts_102_221.py

@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 from bidict import bidict
-import copy
 
 from construct import Select, Const, Bit, Struct, Int16ub, FlagsEnum, GreedyString, ValidationError
 from construct import Optional as COptional, Computed
@@ -336,8 +335,6 @@ class TerminalCapability(BER_TLV_IE, tag=0xa9, nested=[TerminalPowerSupply, Exte
 
 # ETSI TS 102 221 Section 9.2.7 + ISO7816-4 9.3.3/9.3.4
 class _AM_DO_DF(DataObject):
-    """ISO7816-4:2005 5.4.3.1 Table 16"""
-
     def __init__(self):
         super().__init__('access_mode', 'Access Mode', tag=0x80)
 
@@ -384,7 +381,7 @@ class _AM_DO_DF(DataObject):
 
 
 class _AM_DO_EF(DataObject):
-    """ISO7816-4:2005 5.4.3.1 Table 17"""
+    """ISO7816-4 9.3.2 Table 18 + 9.3.3.1 Table 31"""
 
     def __init__(self):
         super().__init__('access_mode', 'Access Mode', tag=0x80)
@@ -432,7 +429,7 @@ class _AM_DO_EF(DataObject):
 
 
 class _AM_DO_CHDR(DataObject):
-    """Command Header Access Mode DO according to ISO 7816-4:2005 5.4.3.2 Table 22."""
+    """Command Header Access Mode DO according to ISO 7816-4 Table 32."""
 
     def __init__(self, tag):
         super().__init__('command_header', 'Command Header Description', tag=tag)
@@ -546,9 +543,8 @@ class CRT_DO(DataObject):
         pin = pin_names.inverse[self.decoded]
         return b'\x83\x01' + pin.to_bytes(1, 'big') + b'\x95\x01\x08'
 
+# ISO7816-4 9.3.3 Table 33
 class SecCondByte_DO(DataObject):
-    """ISO7816-4:2005 5.4.3.1 Table 20"""
-
     def __init__(self, tag=0x9d):
         super().__init__('security_condition_byte', tag=tag)
 
@@ -736,57 +732,36 @@ class EF_ARR(LinFixedEF):
                 raise ValueError
         return by_mode
 
-    @staticmethod
-    def __get_do_sequence(decode_for_df : bool = False):
-        if decode_for_df:
-            return DataObjectSequence('arr', sequence=[AM_DO_DF, SC_DO])
-        else:
-            return DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
-
     def _decode_record_bin(self, raw_bin_data, **kwargs):
-        # we can only guess if we should decode for EF or DF here, but our caller may
-        # be able to pass us a hint:
-        arr_seq = self.__get_do_sequence(kwargs.get('decode_for_df', False))
+        # we can only guess if we should decode for EF or DF here :(
+        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
         dec = arr_seq.decode_multi(raw_bin_data)
         # we cannot pass the result through flatten() here, as we don't have a related
         # 'un-flattening' decoder, and hence would be unable to encode :(
         return dec[0]
 
     def _encode_record_bin(self, in_json, **kwargs):
-        # we can only guess if we should decode for EF or DF here, but our caller may
-        # be able to pass us a hint:
-        arr_seq = self.__get_do_sequence(kwargs.get('encode_for_df', False))
+        # we can only guess if we should decode for EF or DF here :(
+        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
         return arr_seq.encode_multi(in_json)
 
     @with_default_category('File-Specific Commands')
     class AddlShellCommands(CommandSet):
-        read_arr_argparser = copy.deepcopy(LinFixedEF.ShellCommands.read_rec_dec_parser)
-        read_arr_argparser.add_argument('--decode-for-df', action='store_true',
-                                        help='Decode EF.ARR record as if used by a DF (default: EF)')
-
-        @cmd2.with_argparser(read_arr_argparser)
+        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
         def do_read_arr_record(self, opts):
             """Read one EF.ARR record in flattened, human-friendly form."""
-            (hexdata, _sw) = self._cmd.lchan.read_record(opts.RECORD_NR)
-            data = self._cmd.lchan.selected_file._decode_record_bin(h2b(hexdata),
-                                                                    decode_for_df = opts.decode_for_df)
+            (data, _sw) = self._cmd.lchan.read_record_dec(opts.RECORD_NR)
             data = self._cmd.lchan.selected_file.flatten(data)
             self._cmd.poutput_json(data, opts.oneline)
 
-        read_arrs_argparser = copy.deepcopy(LinFixedEF.ShellCommands.read_recs_dec_parser)
-        read_arrs_argparser.add_argument('--decode-for-df', action='store_true',
-                                        help='Decode EF.ARR records as if used by a DF (default: EF)')
-
-        @cmd2.with_argparser(read_arrs_argparser)
+        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
         def do_read_arr_records(self, opts):
             """Read + decode all EF.ARR records in flattened, human-friendly form."""
             num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
             # collect all results in list so they are rendered as JSON list when printing
             data_list = []
             for recnr in range(1, 1 + num_of_rec):
-                (hexdata, _sw) = self._cmd.lchan.read_record(recnr)
-                data = self._cmd.lchan.selected_file._decode_record_bin(h2b(hexdata),
-                                                                        decode_for_df = opts.decode_for_df)
+                (data, _sw) = self._cmd.lchan.read_record_dec(recnr)
                 data = self._cmd.lchan.selected_file.flatten(data)
                 data_list.append(data)
             self._cmd.poutput_json(data_list, opts.oneline)

pySim/ts_31_102.py

@@ -285,14 +285,6 @@ class EF_SUCI_Calc_Info(TransparentEF):
                                 {"hnet_pubkey_identifier": 11, "hnet_pubkey":
                                  h2b("d1bc365f4997d17ce4374e72181431cbfeba9e1b98d7618f79d48561b144672a")}]} ),
     ]
-    _test_decode = [
-        ( 'A000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
-          {"prot_scheme_id_list": [],
-           "hnet_pubkey_list": []} ),
-        ( 'A000A100FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
-          {"prot_scheme_id_list": [],
-           "hnet_pubkey_list": []} ),
-    ]
     # 3GPP TS 31.102 Section 4.4.11.8
     class ProtSchemeIdList(BER_TLV_IE, tag=0xa0):
         # FIXME: 3GPP TS 24.501 Protection Scheme Identifier
@@ -335,7 +327,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         """conversion method to generate list of {hnet_pubkey_identifier, hnet_pubkey} dicts
         from flat [{hnet_pubkey_identifier: }, {net_pubkey: }, ...] list"""
         out = []
-        while l:
+        while len(l):
             a = l.pop(0)
             b = l.pop(0)
             z = {**a, **b}
@@ -397,7 +389,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         # remaining data holds Home Network Public Key Data Object
         hpkl = EF_SUCI_Calc_Info.HnetPubkeyList()
         hpkl.from_tlv(in_bytes[pos:])
-        hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'] or [])
+        hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'])
 
         return {
             'prot_scheme_id_list': prot_scheme_id_list,

pySim/ts_51_011.py

@@ -251,16 +251,6 @@ class EF_SMSP(LinFixedEF):
                                                       "numbering_plan_id": "isdn_e164" },
                             "call_number": "4915790109999" },
             "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
-        ( 'e1ffffffffffffffffffffffff0891945197109099f9ffffff0000a9',
-          { "alpha_id": "", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
-                                                          "tp_pid": True, "tp_dcs": True, "tp_vp": True },
-            "tp_dest_addr": { "length": 255, "ton_npi": { "ext": True, "type_of_number": "reserved_for_extension",
-                                                          "numbering_plan_id": "reserved_for_extension" },
-                              "call_number": "" },
-            "tp_sc_addr": { "length": 8, "ton_npi": { "ext": True, "type_of_number": "international",
-                                                      "numbering_plan_id": "isdn_e164" },
-                            "call_number": "4915790109999" },
-            "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
         ( '454e6574776f726b73fffffffffffffff1ffffffffffffffffffffffffffffffffffffffffffffffff0000a7',
           { "alpha_id": "ENetworks", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
                                                                "tp_pid": True, "tp_dcs": True, "tp_vp": False },
@@ -341,8 +331,7 @@ class EF_SMSP(LinFixedEF):
                         'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
         DestAddr = Struct('length'/Rebuild(Int8ub, lambda ctx: EF_SMSP.dest_addr_len(ctx)),
                         'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
-        # (see comment below)
-        self._construct = Struct('alpha_id'/GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28))),
+        self._construct = Struct('alpha_id'/COptional(GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28)))),
                                  'parameter_indicators'/InvertAdapter(BitStruct(
                                                                         Const(7, BitsInteger(3)),
                                                                         'tp_vp'/Flag,
@@ -356,25 +345,6 @@ class EF_SMSP(LinFixedEF):
                                  'tp_dcs'/Bytes(1),
                                  'tp_vp_minutes'/EF_SMSP.ValidityPeriodAdapter(Byte))
 
-    # Ensure 'alpha_id' is always present
-    def encode_record_hex(self, abstract_data: dict, record_nr: int, total_len: int = None) -> str:
-        # Problem: TS 51.011 Section 10.5.6 describes the 'alpha_id' field as optional. However, this is only true
-        # at the time when the record length of the file is set up in the file system. A card manufacturer may decide
-        # to remove the field by setting the record length to 28. Likewise, the card manaufacturer may also decide to
-        # set the field to a distinct length by setting the record length to a value greater than 28 (e.g. 14 bytes
-        # 'alpha_id' + 28 bytes). Due to the fixed nature of the record length, this eventually means that in practice
-        # 'alpha_id' is a mandatory field with a fixed length.
-        #
-        # Due to the problematic specification of 'alpha_id' as a pseudo-optional field at the beginning of a
-        # fixed-size memory, the construct definition in self._construct has been incorrectly implemented and the field
-        # has been marked as COptional. We may correct the problem by removing COptional. But to maintain compatibility,
-        # we then have to ensure that in case the field is not provided (None), it is set to an empty string ('').
-        #
-        # See also ts_31_102.py, class EF_OCI for a correct example.
-        if abstract_data['alpha_id'] is None:
-            abstract_data['alpha_id'] = ''
-        return super().encode_record_hex(abstract_data, record_nr, total_len)
-
 # TS 51.011 Section 10.5.7
 class EF_SMSS(TransparentEF):
     class MemCapAdapter(Adapter):
@@ -1263,11 +1233,9 @@ class CardProfileSIM(CardProfile):
 
     @staticmethod
     def decode_select_response(resp_hex: str) -> object:
-        """
-        Decode the select response to a dict representation, similar to the one of TS 102.221 (see ts_102_221.py,
-        class FcpTemplate), so that higher-level code only has to deal with one respresentation. See also
-        3GPP TS 51.011, section 9.2.1
-        """
+        # we try to build something that resembles a dict resulting from the TLV decoder
+        # of TS 102.221 (FcpTemplate), so that higher-level code only has to deal with one
+        # format of SELECT response
         resp_bin = h2b(resp_hex)
         struct_of_file_map = {
             0: 'transparent',
@@ -1305,24 +1273,13 @@ class CardProfileSIM(CardProfile):
                 record_len = resp_bin[14]
                 ret['file_descriptor']['record_len'] = record_len
                 ret['file_descriptor']['num_of_rec'] = ret['file_size'] // record_len
-            ret['access_conditions'] = b2h(resp_bin[8:11])
-
-            # Life cycle status integer, see also ETSI TS 102 221, table 11.7b
-            lcsi = resp_bin[11]
-            if lcsi == 0x00:
-                ret['life_cycle_status_int'] = 'no_information'
-            elif lcsi == 0x01:
-                ret['life_cycle_status_int'] = 'creation'
-            elif lcsi == 0x03:
-                ret['life_cycle_status_int'] = 'initialization'
-            elif lcsi & 0xFD == 0x05:
+            ret['access_conditions'] = b2h(resp_bin[8:10])
+            if resp_bin[11] & 0x01 == 0:
                 ret['life_cycle_status_int'] = 'operational_activated'
-            elif lcsi & 0xFD == 0x04:
+            elif resp_bin[11] & 0x04:
                 ret['life_cycle_status_int'] = 'operational_deactivated'
-            elif lcsi & 0xFC == 0x0C:
-                ret['life_cycle_status_int'] = 'termination'
             else:
-                ret['life_cycle_status_int'] = lcsi
+                ret['life_cycle_status_int'] = 'terminated'
         return ret
 
     @classmethod

pyproject.toml

@@ -4,7 +4,3 @@ build-backend = "setuptools.build_meta"
 
 [tool.pylint.main]
 ignored-classes = ["twisted.internet.reactor"]
-
-[tool.pylint.TYPECHECK]
-# SdKey subclasses are generated dynamically via SdKey.generate_sd_key_classes()
-generated-members = ["SdKey[A-Za-z0-9]+"]

requirements.txt

@@ -1,12 +1,12 @@
 pyscard
 pyserial
 pytlv
-cmd2>=2.6.2,<4.0
+cmd2>=2.6.2,<3.0
 jsonpath-ng
 construct>=2.10.70
 bidict
 pyosmocom>=0.0.12
-pyyaml>=5.4
+pyyaml>=5.1
 termcolor
 colorlog
 pycryptodomex

setup.py

@@ -21,12 +21,12 @@ setup(
         "pyscard",
         "pyserial",
         "pytlv",
-        "cmd2 >= 2.6.2, < 4.0",
+        "cmd2 >= 1.5.0, < 3.0",
         "jsonpath-ng",
         "construct >= 2.10.70",
         "bidict",
         "pyosmocom >= 0.0.12",
-        "pyyaml >= 5.4",
+        "pyyaml >= 5.1",
         "termcolor",
         "colorlog",
         "pycryptodomex",

tests/pySim-prog_test/Fairwaves-SIM.ok

@@ -5,7 +5,7 @@ ICCID: 8988219000000117833
 IMSI: 001010000000111
 GID1: ffffffffffffffff
 GID2: ffffffffffffffff
-SMSP: ffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
+SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
 SMSC: 0015555
 SPN: Fairwaves
 Show in HPLMN: False

tests/pySim-prog_test/Wavemobile-SIM.ok

@@ -5,7 +5,7 @@ ICCID: 89445310150011013678
 IMSI: 001010000000102
 GID1: Can't read file -- SW match failed! Expected 9000 and got 6a82.
 GID2: Can't read file -- SW match failed! Expected 9000 and got 6a82.
-SMSP: ffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
+SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
 SMSC: 0015555
 SPN: wavemobile
 Show in HPLMN: False

tests/pySim-smpp2sim_test/pySim-smpp2sim_test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Utility to verify the functionality of pySim-smpp2sim.py
+# Utility to verify the functionality of pySim-trace.py
 #
 # (C) 2026 by sysmocom - s.f.m.c. GmbH
 # All Rights Reserved

tests/unittests/smdpp_data

@@ -1 +0,0 @@
-../../smdpp-data

tests/unittests/test_card_key_provider.py

@@ -20,8 +20,7 @@ class TestCardKeyProviderCsv(unittest.TestCase):
                        "KIK3" : "00010204040506070809488B0C0D0E0F"}
 
         csv_file_path = os.path.dirname(os.path.abspath(__file__)) + "/test_card_key_provider.csv"
-        card_key_field_cryptor = CardKeyFieldCryptor(column_keys)
-        card_key_provider_register(CardKeyProviderCsv(csv_file_path, card_key_field_cryptor))
+        card_key_provider_register(CardKeyProviderCsv(csv_file_path, column_keys))
         super().__init__(*args, **kwargs)
 
     def test_card_key_provider_get(self):

tests/unittests/test_configurable_parameters.py

@@ -1,640 +0,0 @@
-#!/usr/bin/env python3
-
-# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
-#
-# Author: Neels Hofmeyr
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import enum
-import io
-import sys
-import unittest
-from importlib import resources
-from osmocom.utils import hexstr
-from pySim.esim.saip import ProfileElementSequence
-import pySim.esim.saip.personalization as p13n
-import smdpp_data.upp
-
-import xo
-update_expected_output = False
-
-def valstr(val):
-    if isinstance(val, io.BytesIO):
-        val = val.getvalue()
-    if isinstance(val, bytearray):
-        val = bytes(val)
-    return f'{val!r}'
-
-def valtypestr(val):
-    if isinstance(val, dict):
-        types = []
-        for v in val.values():
-            types.append(f'{type(v).__name__}')
-
-        val_type = '{' + ', '.join(types) + '}'
-    else:
-        val_type = f'{type(val).__name__}'
-    return f'{valstr(val)}:{val_type}'
-
-class ConfigurableParameterTest(unittest.TestCase):
-
-    def test_parameters(self):
-
-        upp_fnames = (
-                'TS48v5_SAIP2.1A_NoBERTLV.der',
-                'TS48v5_SAIP2.3_BERTLV_SUCI.der',
-               )
-
-        class Paramtest:
-            def __init__(self, param_cls, val, expect_val, expect_clean_val=None):
-                self.param_cls = param_cls
-                self.val = val
-                self.expect_clean_val = expect_clean_val
-                self.expect_val = expect_val
-
-        param_tests = [
-            Paramtest(param_cls=p13n.Imsi, val='123456',
-                      expect_clean_val=str('123456'),
-                      expect_val={'IMSI': hexstr('123456'),
-                                  'IMSI-ACC': '0040'}),
-            Paramtest(param_cls=p13n.Imsi, val=int(123456),
-                      expect_val={'IMSI': hexstr('123456'),
-                                  'IMSI-ACC': '0040'}),
-
-            Paramtest(param_cls=p13n.Imsi, val='123456789012345',
-                      expect_clean_val=str('123456789012345'),
-                      expect_val={'IMSI': hexstr('123456789012345'),
-                                  'IMSI-ACC': '0020'}),
-            Paramtest(param_cls=p13n.Imsi, val=int(123456789012345),
-                      expect_val={'IMSI': hexstr('123456789012345'),
-                                  'IMSI-ACC': '0020'}),
-
-            Paramtest(param_cls=p13n.Puk1,
-                      val='12345678',
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-            Paramtest(param_cls=p13n.Puk1,
-                      val=int(12345678),
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-
-            Paramtest(param_cls=p13n.Puk2,
-                      val='12345678',
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-
-            Paramtest(param_cls=p13n.Pin1,
-                      val='1234',
-                      expect_clean_val=b'1234\xff\xff\xff\xff',
-                      expect_val='1234'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val='123456',
-                      expect_clean_val=b'123456\xff\xff',
-                      expect_val='123456'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val='12345678',
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val=int(1234),
-                      expect_clean_val=b'1234\xff\xff\xff\xff',
-                      expect_val='1234'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val=int(123456),
-                      expect_clean_val=b'123456\xff\xff',
-                      expect_val='123456'),
-            Paramtest(param_cls=p13n.Pin1,
-                      val=int(12345678),
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-
-            Paramtest(param_cls=p13n.Adm1,
-                      val='1234',
-                      expect_clean_val=b'1234\xff\xff\xff\xff',
-                      expect_val='1234'),
-            Paramtest(param_cls=p13n.Adm1,
-                      val='123456',
-                      expect_clean_val=b'123456\xff\xff',
-                      expect_val='123456'),
-            Paramtest(param_cls=p13n.Adm1,
-                      val='12345678',
-                      expect_clean_val=b'12345678',
-                      expect_val='12345678'),
-            Paramtest(param_cls=p13n.Adm1,
-                      val=int(123456),
-                      expect_clean_val=b'123456\xff\xff',
-                      expect_val='123456'),
-
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val='Milenage',
-                      expect_clean_val=1,
-                      expect_val='Milenage'),
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val='TUAK',
-                      expect_clean_val=2,
-                      expect_val='TUAK'),
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val='usim-test',
-                      expect_clean_val=3,
-                      expect_val='usim_test'),
-
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val=1,
-                      expect_clean_val=1,
-                      expect_val='Milenage'),
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val=2,
-                      expect_clean_val=2,
-                      expect_val='TUAK'),
-            Paramtest(param_cls=p13n.AlgorithmID,
-                      val=3,
-                      expect_clean_val=3,
-                      expect_val='usim_test'),
-
-            Paramtest(param_cls=p13n.K,
-                      val='01020304050607080910111213141516',
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.K,
-                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.K,
-                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.K,
-                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.K,
-                      val=int(11020304050607080910111213141516),
-                      expect_clean_val=b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='11020304050607080910111213141516'),
-
-            Paramtest(param_cls=p13n.Opc,
-                      val='01020304050607080910111213141516',
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.Opc,
-                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.Opc,
-                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-            Paramtest(param_cls=p13n.Opc,
-                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
-                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
-                      expect_val='01020304050607080910111213141516'),
-
-            Paramtest(param_cls=p13n.SmspTpScAddr,
-                      val='+1234567',
-                      expect_clean_val=(True, '1234567'),
-                      expect_val='+1234567'),
-            Paramtest(param_cls=p13n.SmspTpScAddr,
-                      val=1234567,
-                      expect_clean_val=(False, '1234567'),
-                      expect_val='1234567'),
-
-            Paramtest(param_cls=p13n.TuakNumberOfKeccak,
-                      val='123',
-                      expect_clean_val=123,
-                      expect_val='123'),
-            Paramtest(param_cls=p13n.TuakNumberOfKeccak,
-                      val=123,
-                      expect_clean_val=123,
-                      expect_val='123'),
-
-            Paramtest(param_cls=p13n.MilenageRotationConstants,
-                      val='0a 0b 0c 01 02',
-                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
-                      expect_val='0a0b0c0102'),
-            Paramtest(param_cls=p13n.MilenageRotationConstants,
-                      val=b'\x0a\x0b\x0c\x01\x02',
-                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
-                      expect_val='0a0b0c0102'),
-            Paramtest(param_cls=p13n.MilenageRotationConstants,
-                      val=bytearray(b'\x0a\x0b\x0c\x01\x02'),
-                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
-                      expect_val='0a0b0c0102'),
-
-            Paramtest(param_cls=p13n.MilenageXoringConstants,
-                      val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
-                          ' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
-                          ' cccccccccccccccccccccccccccccccc'
-                          ' 11111111111111111111111111111111'
-                          ' 22222222222222222222222222222222',
-                      expect_clean_val=b'\xaa' * 16
-                                       + b'\xbb' * 16
-                                       + b'\xcc' * 16
-                                       + b'\x11' * 16
-                                       + b'\x22' * 16,
-                      expect_val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
-                                 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
-                                 'cccccccccccccccccccccccccccccccc'
-                                 '11111111111111111111111111111111'
-                                 '22222222222222222222222222222222'),
-            Paramtest(param_cls=p13n.MilenageXoringConstants,
-                      val=b'\xaa' * 16
-                          + b'\xbb' * 16
-                          + b'\xcc' * 16
-                          + b'\x11' * 16
-                          + b'\x22' * 16,
-                      expect_clean_val=b'\xaa' * 16
-                                       + b'\xbb' * 16
-                                       + b'\xcc' * 16
-                                       + b'\x11' * 16
-                                       + b'\x22' * 16,
-                      expect_val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
-                                 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
-                                 'cccccccccccccccccccccccccccccccc'
-                                 '11111111111111111111111111111111'
-                                 '22222222222222222222222222222222'),
-
-            Paramtest(param_cls=p13n.MncLen,
-                      val='2',
-                      expect_clean_val=2,
-                      expect_val='2'),
-            Paramtest(param_cls=p13n.MncLen,
-                      val=3,
-                      expect_clean_val=3,
-                      expect_val='3'),
-
-            ]
-
-        for sdkey_cls in (
-            # thin out the number of tests, as a compromise between completeness and test runtime
-            p13n.SdKeyScp02Kvn20AesDek,
-           #p13n.SdKeyScp02Kvn20AesEnc,
-           #p13n.SdKeyScp02Kvn20AesMac,
-           #p13n.SdKeyScp02Kvn21AesDek,
-            p13n.SdKeyScp02Kvn21AesEnc,
-           #p13n.SdKeyScp02Kvn21AesMac,
-           #p13n.SdKeyScp02Kvn22AesDek,
-           #p13n.SdKeyScp02Kvn22AesEnc,
-            p13n.SdKeyScp02Kvn22AesMac,
-           #p13n.SdKeyScp02KvnffAesDek,
-           #p13n.SdKeyScp02KvnffAesEnc,
-           #p13n.SdKeyScp02KvnffAesMac,
-            p13n.SdKeyScp03Kvn30AesDek,
-           #p13n.SdKeyScp03Kvn30AesEnc,
-           #p13n.SdKeyScp03Kvn30AesMac,
-           #p13n.SdKeyScp03Kvn31AesDek,
-            p13n.SdKeyScp03Kvn31AesEnc,
-           #p13n.SdKeyScp03Kvn31AesMac,
-           #p13n.SdKeyScp03Kvn32AesDek,
-           #p13n.SdKeyScp03Kvn32AesEnc,
-            p13n.SdKeyScp03Kvn32AesMac,
-           #p13n.SdKeyScp80Kvn01AesDek,
-           #p13n.SdKeyScp80Kvn01AesEnc,
-           #p13n.SdKeyScp80Kvn01AesMac,
-            p13n.SdKeyScp80Kvn01DesDek,
-           #p13n.SdKeyScp80Kvn01DesEnc,
-           #p13n.SdKeyScp80Kvn01DesMac,
-           #p13n.SdKeyScp80Kvn02AesDek,
-            p13n.SdKeyScp80Kvn02AesEnc,
-           #p13n.SdKeyScp80Kvn02AesMac,
-           #p13n.SdKeyScp80Kvn02DesDek,
-           #p13n.SdKeyScp80Kvn02DesEnc,
-            p13n.SdKeyScp80Kvn02DesMac,
-           #p13n.SdKeyScp80Kvn03AesDek,
-           #p13n.SdKeyScp80Kvn03AesEnc,
-           #p13n.SdKeyScp80Kvn03AesMac,
-            p13n.SdKeyScp80Kvn03DesDek,
-           #p13n.SdKeyScp80Kvn03DesEnc,
-           #p13n.SdKeyScp80Kvn03DesMac,
-            p13n.SdKeyScp81Kvn40AesDek,
-           #p13n.SdKeyScp81Kvn40Tlspsk,
-           #p13n.SdKeyScp81Kvn41AesDek,
-            p13n.SdKeyScp81Kvn41Tlspsk,
-           #p13n.SdKeyScp81Kvn42AesDek,
-           #p13n.SdKeyScp81Kvn42Tlspsk,
-            ):
-
-            for key_len in sdkey_cls.allow_len:
-                val = '0102030405060708091011121314151617181920212223242526272829303132'
-                expect_clean_val = (b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'
-                                    b'\x17\x18\x19\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x30\x31\x32')
-                expect_val = '0102030405060708091011121314151617181920212223242526272829303132'
-
-                val = val[:key_len*2]
-                expect_clean_val = expect_clean_val[:key_len]
-                expect_val = val
-
-                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
-
-                # test bytes input
-                val = expect_clean_val
-                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
-
-                # test bytearray input
-                val = bytearray(expect_clean_val)
-                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
-
-                # test BytesIO input
-                val = io.BytesIO(expect_clean_val)
-                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
-
-                if key_len == 16:
-                    # test huge integer input.
-                    # needs to start with nonzero.. stupid
-                    val = 11020304050607080910111213141516
-                    expect_clean_val = (b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16')
-                    expect_val = '11020304050607080910111213141516'
-                    param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
-
-        outputs = []
-
-        for upp_fname in upp_fnames:
-            test_idx = -1
-            try:
-
-                der = resources.read_binary(smdpp_data.upp, upp_fname)
-
-                for t in param_tests:
-                    test_idx += 1
-                    logloc = f'{upp_fname} {t.param_cls.__name__}(val={valtypestr(t.val)})'
-
-                    param = None
-                    try:
-                        param = t.param_cls()
-                        param.input_value = t.val
-                        param.validate()
-                    except ValueError as e:
-                        raise ValueError(f'{logloc}: {e}') from e
-
-                    clean_val = param.value
-                    logloc = f'{logloc} clean_val={valtypestr(clean_val)}'
-                    if t.expect_clean_val is not None and t.expect_clean_val != clean_val:
-                        raise ValueError(f'{logloc}: expected'
-                                         f' expect_clean_val={valtypestr(t.expect_clean_val)}')
-
-                    # on my laptop, deepcopy is about 30% slower than decoding the DER from scratch:
-                    # pes = copy.deepcopy(orig_pes)
-                    pes = ProfileElementSequence.from_der(der)
-                    try:
-                        param.apply(pes)
-                    except ValueError as e:
-                        raise ValueError(f'{logloc} apply_val(clean_val): {e}') from e
-
-                    changed_der = pes.to_der()
-
-                    pes2 = ProfileElementSequence.from_der(changed_der)
-
-                    read_back_val = t.param_cls.get_value_from_pes(pes2)
-
-                    # compose log string to show the precise type of dict values
-                    if isinstance(read_back_val, dict):
-                        types = set()
-                        for v in read_back_val.values():
-                            types.add(f'{type(v).__name__}')
-
-                        read_back_val_type = '{' + ', '.join(types) + '}'
-                    else:
-                        read_back_val_type = f'{type(read_back_val).__name__}'
-
-                    logloc = (f'{logloc} read_back_val={valtypestr(read_back_val)}')
-
-                    if isinstance(read_back_val, dict) and not t.param_cls.get_name() in read_back_val.keys():
-                        raise ValueError(f'{logloc}: expected to find name {t.param_cls.get_name()!r} in read_back_val')
-
-                    expect_val = t.expect_val
-                    if not isinstance(expect_val, dict):
-                        expect_val = { t.param_cls.get_name(): expect_val }
-                    if read_back_val != expect_val:
-                        raise ValueError(f'{logloc}: expected {expect_val=!r}:{type(t.expect_val).__name__}')
-
-                    ok = logloc.replace(' clean_val', '\n\tclean_val'
-                              ).replace(' read_back_val', '\n\tread_back_val'
-                              ).replace('=', '=\t'
-                              )
-                    output = f'\nok: {ok}'
-                    outputs.append(output)
-                    print(output)
-
-            except Exception as e:
-                raise RuntimeError(f'Error while testing UPP {upp_fname} {test_idx=}: {e}') from e
-
-        output = '\n'.join(outputs) + '\n'
-        xo_name = 'test_configurable_parameters'
-        if update_expected_output:
-            with resources.path(xo, xo_name) as xo_path:
-                with open(xo_path, 'w', encoding='utf-8') as f:
-                    f.write(output)
-        else:
-            xo_str = resources.read_text(xo, xo_name)
-            if xo_str != output:
-                at = 0
-                while at < len(output):
-                    if output[at] == xo_str[at]:
-                        at += 1
-                        continue
-                    break
-
-                raise RuntimeError(f'output differs from expected output at position {at}: "{output[at:at+20]}" != "{xo_str[at:at+20]}"')
-
-
-class TestValidateVal(unittest.TestCase):
-    """validate_val() tests for various ConfigurableParameter subclasses."""
-
-    def _ok(self, cls, val, expected=None):
-        result = cls.validate_val(val)
-        if expected is not None:
-            self.assertEqual(result, expected)
-        return result
-
-    def _err(self, cls, val):
-        with self.assertRaises(ValueError):
-            cls.validate_val(val)
-
-    # --- Iccid ---
-
-    def test_iccid_18digits_adds_luhn(self):
-        result = self._ok(p13n.Iccid, '998877665544332211')
-        self.assertIsInstance(result, str)
-        self.assertEqual(len(result), 19)
-        self.assertTrue(result.isdecimal())
-
-    def test_iccid_19digits_passthrough(self):
-        result = self._ok(p13n.Iccid, '9988776655443322110')
-        self.assertIsInstance(result, str)
-        self.assertEqual(len(result), 19)
-
-    def test_iccid_too_short(self):
-        self._err(p13n.Iccid, '12345678901234567')  # 17 digits
-
-    def test_iccid_too_long(self):
-        self._err(p13n.Iccid, '1' * 21)
-
-    def test_iccid_non_digits(self):
-        self._err(p13n.Iccid, '99887766554433221X')
-
-    # --- Imsi ---
-
-    def test_imsi_valid_short(self):
-        self._ok(p13n.Imsi, '001010', '001010')
-
-    def test_imsi_valid_long(self):
-        self._ok(p13n.Imsi, '001010123456789', '001010123456789')
-
-    def test_imsi_too_short(self):
-        self._err(p13n.Imsi, '12345')  # 5 digits, min is 6
-
-    def test_imsi_too_long(self):
-        self._err(p13n.Imsi, '1' * 16)
-
-    def test_imsi_non_digits(self):
-        self._err(p13n.Imsi, '00101A123456789')
-
-    # --- Pin1 ---
-
-    def test_pin1_4digits(self):
-        # DecimalHexParam encodes each digit as its ASCII byte, then rpad to 8 bytes with 0xff
-        self._ok(p13n.Pin1, '1234', b'1234\xff\xff\xff\xff')
-
-    def test_pin1_8digits(self):
-        self._ok(p13n.Pin1, '12345678', b'12345678')
-
-    def test_pin1_too_short(self):
-        self._err(p13n.Pin1, '123')
-
-    def test_pin1_too_long(self):
-        self._err(p13n.Pin1, '123456789')
-
-    def test_pin1_non_digits(self):
-        self._err(p13n.Pin1, '123A')
-
-    # --- Puk1 ---
-
-    def test_puk1_8digits(self):
-        self._ok(p13n.Puk1, '12345678', b'12345678')
-
-    def test_puk1_wrong_length(self):
-        self._err(p13n.Puk1, '1234567')   # 7 digits
-        self._err(p13n.Puk1, '123456789')  # 9 digits
-
-    def test_puk1_non_digits(self):
-        self._err(p13n.Puk1, '1234567X')
-
-    # --- K (BinaryParam) ---
-
-    def test_k_valid_hex_str(self):
-        self._ok(p13n.K, '000102030405060708090a0b0c0d0e0f',
-                 b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f')
-
-    def test_k_valid_bytes(self):
-        raw = bytes(range(16))
-        self._ok(p13n.K, raw, raw)
-
-    def test_k_wrong_length(self):
-        self._err(p13n.K, '00' * 15)  # 15 bytes, allow_len requires 16 or 32
-
-    def test_k_non_hex(self):
-        self._err(p13n.K, 'gg' * 16)
-
-    def test_k_odd_hex_digits(self):
-        self._err(p13n.K, '0' * 31)   # odd number of hex digits
-
-
-class TestEnumParam(unittest.TestCase):
-    """Tests for the EnumParam machinery, using AlgorithmID as the concrete subclass."""
-
-    # --- validate_val ---
-
-    def test_validate_by_name_exact(self):
-        self.assertEqual(p13n.AlgorithmID.validate_val('Milenage'), 1)
-        self.assertEqual(p13n.AlgorithmID.validate_val('TUAK'), 2)
-        self.assertEqual(p13n.AlgorithmID.validate_val('usim_test'), 3)
-
-    def test_validate_by_int(self):
-        self.assertEqual(p13n.AlgorithmID.validate_val(1), 1)
-        self.assertEqual(p13n.AlgorithmID.validate_val(2), 2)
-        self.assertEqual(p13n.AlgorithmID.validate_val(3), 3)
-
-    def test_validate_fuzzy_case(self):
-        self.assertEqual(p13n.AlgorithmID.validate_val('milenage'), 1)
-        self.assertEqual(p13n.AlgorithmID.validate_val('MILENAGE'), 1)
-        self.assertEqual(p13n.AlgorithmID.validate_val('tuak'), 2)
-
-    def test_validate_fuzzy_hyphen_underscore(self):
-        # 'usim-test' has a hyphen; enum member is 'usim_test' — must fuzzy-match
-        self.assertEqual(p13n.AlgorithmID.validate_val('usim-test'), 3)
-
-    def test_validate_invalid_name(self):
-        with self.assertRaises(ValueError):
-            p13n.AlgorithmID.validate_val('unknown')
-
-    def test_validate_invalid_int(self):
-        with self.assertRaises(ValueError):
-            p13n.AlgorithmID.validate_val(99)
-
-    def test_validate_returns_int(self):
-        result = p13n.AlgorithmID.validate_val('Milenage')
-        self.assertIsInstance(result, int)
-        self.assertNotIsInstance(result, enum.Enum)
-
-    # --- map_name_to_val ---
-
-    def test_map_name_exact(self):
-        self.assertEqual(p13n.AlgorithmID.map_name_to_val('Milenage'), 1)
-
-    def test_map_name_fuzzy(self):
-        self.assertEqual(p13n.AlgorithmID.map_name_to_val('milenage'), 1)
-        self.assertEqual(p13n.AlgorithmID.map_name_to_val('usim-test'), 3)
-
-    def test_map_name_strict_raises(self):
-        with self.assertRaises(ValueError):
-            p13n.AlgorithmID.map_name_to_val('unknown', strict=True)
-
-    def test_map_name_nonstrict_returns_none(self):
-        self.assertIsNone(p13n.AlgorithmID.map_name_to_val('unknown', strict=False))
-
-    # --- map_val_to_name ---
-
-    def test_map_val_known(self):
-        self.assertEqual(p13n.AlgorithmID.map_val_to_name(1), 'Milenage')
-        self.assertEqual(p13n.AlgorithmID.map_val_to_name(2), 'TUAK')
-        self.assertEqual(p13n.AlgorithmID.map_val_to_name(3), 'usim_test')
-
-    def test_map_val_unknown_nonstrict(self):
-        self.assertIsNone(p13n.AlgorithmID.map_val_to_name(99))
-
-    def test_map_val_unknown_strict(self):
-        with self.assertRaises(ValueError):
-            p13n.AlgorithmID.map_val_to_name(99, strict=True)
-
-    # --- name_normalize ---
-
-    def test_name_normalize(self):
-        self.assertEqual(p13n.AlgorithmID.name_normalize('Milenage'), 'Milenage')
-        self.assertEqual(p13n.AlgorithmID.name_normalize('milenage'), 'Milenage')
-        self.assertEqual(p13n.AlgorithmID.name_normalize('usim-test'), 'usim_test')
-
-    # --- clean_name_str ---
-
-    def test_clean_name_str(self):
-        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim-test'), 'usimtest')
-        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim_test'), 'usimtest')
-        self.assertEqual(p13n.AlgorithmID.clean_name_str('Milenage'), 'milenage')
-        self.assertEqual(p13n.AlgorithmID.clean_name_str('foo bar!'), 'foobar')
-
-
-if __name__ == "__main__":
-    if '-u' in sys.argv:
-        update_expected_output = True
-        sys.argv.remove('-u')
-    unittest.main()

tests/unittests/test_decode_select_resp.py

@@ -1,78 +0,0 @@
-#!/usr/bin/env python3
-
-# (C) 2026 by sysmocom - s.f.m.c. GmbH
-# All Rights Reserved
-#
-# Author: Philipp Maier <pmaier@sysmocom.de>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import unittest
-import os
-from pySim.profile import CardProfile
-from pySim.ts_51_011 import CardProfileSIM
-from pySim.ts_102_221 import CardProfileUICC
-
-class TestDecodeSelectResponse_CardProfile(unittest.TestCase):
-
-    def decode_select_response(self, card_Profile: CardProfile, testcases: list[dict]):
-        for testcase in testcases:
-            resp_hex = testcase['resp_hex']
-            decoded = card_Profile.decode_select_response(resp_hex)
-            if testcase['decoded']:
-                self.assertEqual(decoded, testcase['decoded'])
-            else:
-                print("no testvector to compare against, assuming the following output is correct:")
-                print("resp_hex:", resp_hex)
-                print("decoded:", decoded)
-
-    def test_CardProfileSIM(self):
-        testcases = [
-            # MF
-            {"resp_hex" : "000000003f000100000000000981020c0400838a838a",
-             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'mf'}}, 'proprietary_info': {'available_memory': 0}, 'file_id': '3f00', 'file_characteristics': '81', 'num_direct_child_df': 2, 'num_direct_child_ef': 12, 'num_chv_unblock_adm_codes': 4}},
-            # DF.TELECOM
-            {"resp_hex" : "000000007f100200000000000981000d0400838a838a",
-             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'df'}}, 'proprietary_info': {'available_memory': 0}, 'file_id': '7f10', 'file_characteristics': '81', 'num_direct_child_df': 0, 'num_direct_child_ef': 13, 'num_chv_unblock_adm_codes': 4}},
-            # EF.MSISDN
-            {"resp_hex" : "000000346f40040011ffff0102011a",
-             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'working_ef', 'structure': 'linear_fixed'}, 'record_len': 26, 'num_of_rec': 2}, 'proprietary_info': {}, 'file_id': '6f40', 'file_size': 52, 'access_conditions': '11ffff', 'life_cycle_status_int': 'creation'}},
-            # EF.ICCID
-            {"resp_hex" : "0000000a2fe204000cffff01020000",
-             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'working_ef', 'structure': 'transparent'}}, 'proprietary_info': {}, 'file_id': '2fe2', 'file_size': 10, 'access_conditions': '0cffff', 'life_cycle_status_int': 'creation'}},
-        ]
-        self.decode_select_response(CardProfileSIM, testcases)
-
-    def test_CardProfileUICC(self):
-        testcases = [
-            # MF
-            {"resp_hex" : "622c8202782183023f00a50c80017183040003a7388701018a01058b032f0601c60c90016083010183010a83010b",
-             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'df', 'structure': 'no_info_given'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'?\x00', 'proprietary_information': {'uicc_characteristics': b'q', 'available_memory': 239416, 'supported_filesystem_commands': {'terminal_capability': True}}, 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'/\x06', 'ef_arr_record_nr': 1}, 'pin_status_template_do': [{'ps_do': b'`'}, {'key_reference': 1}, {'key_reference': 10}, {'key_reference': 11}]}},
-            # ADF.USIM
-            {"resp_hex" : "623d8202782183027fd0840ca0000000871002ff49ff0589a50c80017183040003a7388701018a01058b032f0601c60f90017083010183018183010a83010b",
-             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'df', 'structure': 'no_info_given'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'\x7f\xd0', 'df_name': b'\xa0\x00\x00\x00\x87\x10\x02\xffI\xff\x05\x89', 'proprietary_information': {'uicc_characteristics': b'q', 'available_memory': 239416, 'supported_filesystem_commands': {'terminal_capability': True}}, 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'/\x06', 'ef_arr_record_nr': 1}, 'pin_status_template_do': [{'ps_do': b'p'}, {'key_reference': 1}, {'key_reference': 129}, {'key_reference': 10}, {'key_reference': 11}]}},
-            # ADF.ISIM
-            {"resp_hex" : "623d8202782183027fb0840ca0000000871004ff49ff0589a50c80017183040003a7388701018a01058b032f0601c60f90017083010183018183010a83010b",
-             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'df', 'structure': 'no_info_given'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'\x7f\xb0', 'df_name': b'\xa0\x00\x00\x00\x87\x10\x04\xffI\xff\x05\x89', 'proprietary_information': {'uicc_characteristics': b'q', 'available_memory': 239416, 'supported_filesystem_commands': {'terminal_capability': True}}, 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'/\x06', 'ef_arr_record_nr': 1}, 'pin_status_template_do': [{'ps_do': b'p'}, {'key_reference': 1}, {'key_reference': 129}, {'key_reference': 10}, {'key_reference': 11}]}},
-            # EF.IMSI
-            {"resp_hex" : "62178202412183026f078a01058b036f060a80020009880138",
-             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'working_ef', 'structure': 'transparent'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'o\x07', 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'o\x06', 'ef_arr_record_nr': 10}, 'file_size': 9, 'short_file_identifier': 7}},
-            # EF.ECC
-            {"resp_hex" : "621a82054221000e0283026fb78a01058b036f06088002001c880108",
-             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'working_ef', 'structure': 'linear_fixed'}, 'record_len': 14, 'num_of_rec': 2}, 'file_identifier': b'o\xb7', 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'o\x06', 'ef_arr_record_nr': 8}, 'file_size': 28, 'short_file_identifier': 1}},
-        ]
-        self.decode_select_response(CardProfileUICC, testcases)
-
-if __name__ == "__main__":
-	unittest.main()

tests/unittests/test_esim_saip.py

@@ -21,7 +21,7 @@ import copy
 from osmocom.utils import h2b, b2h
 
 from pySim.esim.saip import *
-from pySim.esim.saip import personalization
+from pySim.esim.saip.personalization import *
 from pprint import pprint as pp
 
 
@@ -55,56 +55,14 @@ class SaipTest(unittest.TestCase):
     def test_personalization(self):
         """Test some of the personalization operations."""
         pes = copy.deepcopy(self.pes)
-        params = [personalization.Puk1('01234567'),
-                  personalization.Puk2(98765432),
-                  personalization.Pin1('1111'),
-                  personalization.Pin2(2222),
-                  personalization.Adm1('11111111'),
-                  personalization.K(h2b('000102030405060708090a0b0c0d0e0f')),
-                  personalization.Opc(h2b('101112131415161718191a1b1c1d1e1f'))]
+        params = [Puk1('01234567'), Puk2(98765432), Pin1('1111'), Pin2(2222), Adm1('11111111'),
+                  K(h2b('000102030405060708090a0b0c0d0e0f')), Opc(h2b('101112131415161718191a1b1c1d1e1f'))]
         for p in params:
             p.validate()
             p.apply(pes)
         # TODO: we don't actually test the results here, but we just verify there is no exception
         pes.to_der()
 
-    def test_personalization2(self):
-        """Test some of the personalization operations."""
-        cls = personalization.SdKeyScp80Kvn01DesEnc
-        pes = ProfileElementSequence.from_der(self.per_input)
-        prev_val = tuple(cls.get_values_from_pes(pes))
-        print(f'{prev_val=}')
-        self.assertTrue(prev_val)
-
-        set_val = '42342342342342342342342342342342'
-        param = cls(set_val)
-        param.validate()
-        param.apply(pes)
-
-        get_val1 = tuple(cls.get_values_from_pes(pes))
-        print(f'{get_val1=} {set_val=}')
-        self.assertEqual(get_val1, ({cls.name: set_val},))
-
-        get_val1b = tuple(cls.get_values_from_pes(pes))
-        print(f'{get_val1b=} {set_val=}')
-        self.assertEqual(get_val1b, ({cls.name: set_val},))
-
-        der = pes.to_der()
-
-        get_val1c = tuple(cls.get_values_from_pes(pes))
-        print(f'{get_val1c=} {set_val=}')
-        self.assertEqual(get_val1c, ({cls.name: set_val},))
-
-        # assertTrue to not dump the entire der.
-        # Expecting the modified DER to be different. If this assertion fails, then no change has happened in the output
-        # DER and the ConfigurableParameter subclass is buggy.
-        self.assertTrue(der != self.per_input)
-
-        pes2 = ProfileElementSequence.from_der(der)
-        get_val2 = tuple(cls.get_values_from_pes(pes2))
-        print(f'{get_val2=} {set_val=}')
-        self.assertEqual(get_val2, ({cls.name: set_val},))
-
     def test_constructor_encode(self):
         """Test that DER-encoding of PE created by "empty" constructor works without raising exception."""
         for cls in [ProfileElementMF, ProfileElementPuk, ProfileElementPin, ProfileElementTelecom,

tests/unittests/test_globalplatform.py

@@ -17,10 +17,7 @@
 
 import unittest
 import logging
-import hashlib
-from types import SimpleNamespace
 from osmocom.utils import b2h, h2b
-from osmocom.tlv import bertlv_encode_len
 
 from pySim.global_platform import *
 from pySim.global_platform.scp import *
@@ -286,41 +283,6 @@ class SCP03_Test_AES256_33(SCP03_Test, unittest.TestCase):
 # FIXME: test auth with random (0x60) vs pseudo-random (0x70) challenge
 
 
-class KeyComponentBlock_Test(unittest.TestCase):
-    """Tests for the kcb of GP CardSpec v2.3
-    - Table 11-70 kcv that required padding, preceded by its clear-text length
-    - Table 11-71 no padding required"""
-
-    def setUp(self):
-        # SCP02 (3DES DEK, 8 byte blocks), same vectors as SCP02_Test
-        self.scp02 = SCP02(card_keys=ck_3des_70)
-        self.scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
-        self.scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
-        self.scp02.gen_ext_auth_apdu()
-        # SCP03 (AES DEK, 16 byte blocks), same vectors as SCP03_Test_AES128_11
-        self.scp03 = SCP03(card_keys=KEYSET_AES128)
-        self.scp03.gen_init_update_apdu(h2b('b13e5f938fc108c4'))
-        self.scp03.parse_init_update_resp(h2b('000000000000000000003003703eb51047495b249f66c484c1d2ef1948000002'))
-        self.scp03.gen_ext_auth_apdu(0x11)
-
-    def test_encrypt_decrypt_key(self):
-        for scp in (self.scp02, self.scp03):
-            bs = scp.sk.blocksize
-            for keylen in range(1, 3 * bs + 1):
-                with self.subTest(scp=type(scp).__name__, keylen=keylen):
-                    key = bytes(range(keylen))
-                    kcb = scp.encrypt_key(key)
-                    if keylen % bs:
-                        # Table 11-70: <length of clear key component> || <encrypted padded value>
-                        self.assertEqual(kcb[0], keylen)
-                        self.assertEqual((len(kcb) - 1) % bs, 0)
-                        self.assertEqual(len(kcb) - 1, keylen + (bs - keylen % bs))
-                    else:
-                        # Table 11-71: only the encrypted key component value
-                        self.assertEqual(len(kcb), keylen)
-                    self.assertEqual(scp.decrypt_key(kcb), key)
-
-
 class SCP03_KCV_Test(unittest.TestCase):
     def test_kcv(self):
         self.assertEqual(compute_kcv('aes', KEYSET_AES128.enc), h2b('C35280'))
@@ -328,208 +290,6 @@ class SCP03_KCV_Test(unittest.TestCase):
         self.assertEqual(compute_kcv('aes', KEYSET_AES128.dek), h2b('840DE5'))
 
 
-class PutKey_PSK_Test(unittest.TestCase):
-    """Tests for the PUT KEY command data field encoding, in particular the PSK TLS ('85') key data
-    field defined by GlobalPlatform Amendment B (Remote Application Management over HTTP) Table 3-13."""
-
-    # the PUT KEY encoder we exercise
-    C = ADF_SD.AddlShellCommands
-
-    # SCP80 TLS-PSK example key from the do_put_key docstring (16 bytes)
-    PSK_CLEAR = h2b('303132333435363738393a3b3c3d3e3f')
-    # its DEK ciphertext + Table 3-13 KCV with SCP02 session set up below
-    PSK_CIPHERED = h2b('15abf1fe16ccc5aa13743394442942cd')
-    PSK_KCV = h2b('06125d')  # = SHA-1(PSK_CLEAR)[:3]
-
-    def setUp(self):
-        # SCP02 with the same vectors as SCP02_Test, so that the whole PUT KEY data field is reproducible.
-        self.scp02 = SCP02(card_keys=ck_3des_70)
-        self.scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
-        self.scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
-        self.scp02.gen_ext_auth_apdu()
-
-    def test_psk_kcv_is_sha1(self):
-        # GP Amendment B Table 3-13: KCV = 3 most significant bytes of SHA-1(clear key)
-        self.assertEqual(compute_kcv('tls_psk', self.PSK_CLEAR), hashlib.sha1(self.PSK_CLEAR).digest()[:3])
-        self.assertEqual(compute_kcv('tls_psk', self.PSK_CLEAR), self.PSK_KCV)
-
-    def test_encode_psk_framing_golden(self):
-        # assert the exact Table 3-13 layout
-        #   85 | L1 | L2 | <ciphered> | 03 | <SHA-1(clear)[:3]>
-        clear = self.PSK_CLEAR
-        ciphered = h2b('aabbccddeeff00112233445566778899')  # arbitrary 16-byte ciphertext
-        kcv = hashlib.sha1(clear).digest()[:3]
-        field = self.C.encode_key_data_psk(clear, ciphered, kcv)
-        #                            85   L1   L2   <---------- ciphered ----------->  03  <-kcv->
-        self.assertEqual(b2h(field),'85' '11' '10' 'aabbccddeeff00112233445566778899' '03' + b2h(kcv))
-        self.assertEqual(b2h(field),'851110aabbccddeeff0011223344556677889903' + '06125d')
-
-    def test_psk_golden_over_scp02(self):
-        # Full PUT KEY data field (KVN 0x40 + single PSK key) enciphered with the SCP02 DEK.
-        keys = [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR,
-                 'kcv': compute_kcv('tls_psk', self.PSK_CLEAR)}]
-        data = self.C.build_put_key_data(0x40, keys, self.scp02)
-        self.assertEqual(b2h(data),
-                         '40' '85' '11' '10' + b2h(self.PSK_CIPHERED) + '03' + b2h(self.PSK_KCV))
-
-    def test_wrong_basic_format_differs(self):
-        # regression test, the generic "Basic format" does NOT match Table 3-13 for a PSK key
-        # rejected by card with with 6a88
-        wrong_basic = self.C.encode_key_data_basic('tls_psk', self.PSK_CIPHERED, b'')
-        right_psk = self.C.encode_key_data_psk(self.PSK_CLEAR, self.PSK_CIPHERED, self.PSK_KCV)
-        self.assertEqual(b2h(wrong_basic), '8510' + b2h(self.PSK_CIPHERED) + '00')
-        self.assertEqual(b2h(right_psk), '8511' '10' + b2h(self.PSK_CIPHERED) + '03' + b2h(self.PSK_KCV))
-        self.assertNotEqual(wrong_basic, right_psk)
-
-    def test_key_component_block_length_is_bertlv(self):
-        # GP CardSpec v2.3.1 Section 11.8.2.3.1: all lengths ofPUT KEY are always BER TLV coded
-        for kcb_len, exp_len_field in [(127, '7f'), (128, '8180'), (129, '8181'), (256, '820100')]:
-            with self.subTest(kcb_len=kcb_len):
-                kcb = bytes(kcb_len)
-                field = self.C.encode_key_data_basic('rsa_modulus_n', kcb, b'')
-                self.assertEqual(b2h(field), 'a2' + exp_len_field + b2h(kcb) + '00')
-                # 85 field of Amendment B Table 3-13 uses the same coding
-                # single byte inner length (clear key < 128) == block kcb_len bytes long
-                psk = self.C.encode_key_data_psk(bytes(120), bytes(kcb_len - 1), b'')
-                self.assertEqual(b2h(psk)[:2 + len(exp_len_field)], '85' + exp_len_field)
-
-    def test_basic_format_unchanged(self):
-        # as before
-        for kt, clear in [('des', h2b('404142434445464748494a4b4c4d4e4f')),
-                          ('aes', h2b('000102030405060708090a0b0c0d0e0f'))]:
-            ciph = self.scp02.encrypt_key(clear)
-            kcv = compute_kcv(kt, clear)
-            via_construct = build_construct(self.C.KeyDataBasic, {'key_type': kt, 'kcb': b2h(ciph), 'kcv': b2h(kcv)})
-            via_helper = self.C.encode_key_data_basic(kt, ciph, kcv)
-            self.assertEqual(via_helper, via_construct)
-
-    def test_psk_padding_no_double_length(self):
-        # A PSK key whose length is not a multiple of the DEK block size (DES: 8) is right-padded before
-        # ciphering. Table 3-13 states the clear key length (L2) in the '85' DO itself, so the ciphered
-        # key field is the bare cryptogram:
-        # - ciphered field == padded ciphertext (no duplicated length prefix),
-        # - clear key == first L2 bytes.
-        for keylen in (18, 20):
-            with self.subTest(keylen=keylen):
-                clear = bytes(range(keylen))
-                padded_len = keylen + (-keylen % 8)
-                field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': clear,
-                                                          'kcv': compute_kcv('tls_psk', clear)}], self.scp02)[1:]
-                self.assertEqual(field[0], 0x85)
-                l1 = field[1]
-                l2 = field[2]
-                self.assertEqual(l2, keylen)                # single-byte BER length of clear key
-                ciphered = field[3:3 + (l1 - 1)]            # value = L2 (1 byte) || ciphered key
-                self.assertEqual(len(ciphered), padded_len) # padded to the 8-byte DES block size
-                self.assertEqual(l1, 1 + padded_len)        # no duplicated length prefix
-                self.assertEqual(self.scp02.dek_decrypt(ciphered)[:keylen], clear)
-
-    def test_psk_clear_key_is_not_padded_in_place(self):
-        # padding the bytearray in place would make L2 the padded length,
-        # then stored as key material and rejected thanks to the KCV
-        clear = h2b('000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d')  # 30, not %8
-        kcv = compute_kcv('tls_psk', clear)
-        field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': clear,
-                                                  'kcv': kcv}], self.scp02)[1:]
-        self.assertEqual(len(clear), 30)
-        self.assertEqual(field[2], 30)                       # L2 == clear key length, not 32
-        self.assertEqual(self.scp02.dek_decrypt(field[3:3 + field[1] - 1])[:30], clear)
-
-    def test_kcv_suppressed(self):
-        # --suppress-key-check -> KCV length 00 and no KCV bytes
-        field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR,
-                                                  'kcv': b''}], self.scp02)[1:]
-        self.assertEqual(b2h(field), '8511' '10' + b2h(self.PSK_CIPHERED) + '00')
-
-    def test_multikey_psk_plus_des_dek(self):
-        # load a PSK TLS key (KID 1, Amendment B format) together with its DES DEK
-        # (KID 2, Basic format) in one PUT KEY.
-        # Verify the concatenated data field parses back into the two components with proper type formats.
-        dek = h2b('404142434445464748494a4b4c4d4e4f')
-        keys = [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR, 'kcv': compute_kcv('tls_psk', self.PSK_CLEAR)},
-                {'key_type': 'des', 'clear_key': dek, 'kcv': compute_kcv('des', dek)}]
-        data = self.C.build_put_key_data(0x40, keys, self.scp02)
-
-        b = data
-        self.assertEqual(b[0], 0x40)            # KVN
-        b = b[1:]
-        # component 1: PSK TLS (Table 3-13)
-        self.assertEqual(b[0], 0x85)
-        self.assertEqual(b[1], 0x11)            # L1 = 17
-        self.assertEqual(b[2], 0x10)            # L2 = 16 (clear key length)
-        self.assertEqual(b[3:3 + 16], self.PSK_CIPHERED)
-        self.assertEqual(b[3 + 16], 0x03)       # KCV length
-        self.assertEqual(b[3 + 16 + 1:3 + 16 + 1 + 3], self.PSK_KCV)
-        b = b[3 + 16 + 1 + 3:]
-        # component 2: DES DEK (Basic format)
-        self.assertEqual(b[0], 0x80)            # key type des
-        kcb_len = b[1]
-        self.assertEqual(kcb_len, 16)
-        self.assertEqual(b[2:2 + kcb_len], self.scp02.encrypt_key(dek))
-        b = b[2 + kcb_len:]
-        self.assertEqual(b[0], 0x03)            # KCV length
-        self.assertEqual(b[1:1 + 3], compute_kcv('des', dek))
-        self.assertEqual(b[1 + 3:], b'')        # no trailing bytes
-
-    def test_no_scp_leaves_key_clear(self):
-        # During personalization (no SCP) the key is not enciphered, framing still follows Table 3-13.
-        field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR,
-                                                  'kcv': self.PSK_KCV}], None)[1:]
-        self.assertEqual(b2h(field), '8511' '10' + b2h(self.PSK_CLEAR) + '03' + b2h(self.PSK_KCV))
-
-
-class PutKey_Length_Test(unittest.TestCase):
-    """Tests for the length of the PUT KEY command APDU.  Lc of GP CardSpec v2.3 Table 11-64 is a
-    single byte, so an oversized key data field cannot be sent."""
-
-    class PutKeyOnly(ADF_SD.AddlShellCommands):
-        """ADF_SD.AddlShellCommands with a canned scc to drive put_key()"""
-        def __init__(self, scp=None, max_cmd_len=255):
-            super().__init__()
-            self.sent = []
-            self.scc = SimpleNamespace(scp=scp, max_cmd_len=max_cmd_len,
-                                       send_apdu_checksw=lambda pdu: (self.sent.append(pdu), ('', '9000'))[1])
-
-        @property
-        def _cmd(self):
-            return SimpleNamespace(lchan=SimpleNamespace(scc=self.scc))
-
-    # KVN, key type, two byte BER length of the key component block, KCV length; KCV suppressed
-    FRAMING = 1 + 1 + 2 + 1
-
-    @staticmethod
-    def key(nbytes: int):
-        return [{'key_type': 'rsa_modulus_n', 'clear_key': bytes(nbytes), 'kcv': b''}]
-
-    def test_lc_matches_data_field(self):
-        # largest key component block that still fits without a secure channel
-        sd = self.PutKeyOnly()
-        sd.put_key(0, 0x40, 1, self.key(255 - self.FRAMING))
-        apdu = sd.sent[0]
-        self.assertEqual(apdu[:8], '80D80001')
-        lc = int(apdu[8:10], 16)
-        self.assertEqual(lc, 255)                       # Lc ...
-        self.assertEqual(len(apdu[10:-2]) // 2, lc)     # ... and it matches the actual data field
-
-    def test_oversized_key_data_raises(self):
-        # real world fat example: RSA-2048 modulus does not fit, led to 3 nibble Lc 106,
-        # which silently shifted and broke the whole APDU by half a byte.
-        sd = self.PutKeyOnly()
-        with self.assertRaises(ValueError) as ctx:
-            sd.put_key(0, 0x40, 1, self.key(256))
-        self.assertIn('262', str(ctx.exception))
-        self.assertIn('255', str(ctx.exception))
-        self.assertEqual(sd.sent, [])                   # nothing was sent to the card
-
-    def test_secure_channel_overhead_lowers_the_limit(self):
-        # scc.max_cmd_len shrinks by the C-MAC + encryption padding of active SCP
-        sd = self.PutKeyOnly(max_cmd_len=239)
-        sd.put_key(0, 0x40, 1, self.key(239 - self.FRAMING))
-        self.assertEqual(int(sd.sent[0][8:10], 16), 239)
-        with self.assertRaises(ValueError):
-            sd.put_key(0, 0x40, 1, self.key(239 - self.FRAMING + 1))
-
-
 class Install_param_Test(unittest.TestCase):
     def test_gen_install_parameters(self):
         load_parameters = gen_install_parameters(256, 256, '010001001505000000000000000000000000')
@@ -538,180 +298,5 @@ class Install_param_Test(unittest.TestCase):
         load_parameters = gen_install_parameters()
         self.assertEqual(load_parameters, 'c900')
 
-class SCP_Overhead_Test(unittest.TestCase):
-    """SCP.overhead varies according to the current security level:
-    C-MAC + at level >= 3 the worst-case padding!
-    """
-
-    def _scp02(self, security_level):
-        scp = SCP02(card_keys=ck_3des_70)
-        scp.sk = Scp02SessionKeys(0x0001, ck_3des_70)
-        scp.security_level = security_level
-        return scp
-
-    def _scp03(self, security_level, s_mode=8):
-        scp = SCP03(card_keys=KEYSET_AES128, s_mode=s_mode)
-        scp.sk = Scp03SessionKeys(KEYSET_AES128, b'\x00' * s_mode, b'\x11' * s_mode)
-        scp.security_level = security_level
-        return scp
-
-    def test_scp02(self):
-        self.assertEqual(self._scp02(0x00).overhead, 0)   # no wrapping at all
-        self.assertEqual(self._scp02(0x01).overhead, 8)   # C-MAC
-        self.assertEqual(self._scp02(0x03).overhead, 16)  # C-MAC + C-DEC: pad80 to 8, largest fit 239
-
-    def test_scp03_s8(self):
-        self.assertEqual(self._scp03(0x00).overhead, 0)
-        self.assertEqual(self._scp03(0x01).overhead, 8)
-        self.assertEqual(self._scp03(0x03).overhead, 16)  # pad80 to 16 within 247 -> 240, minus pad byte
-        self.assertEqual(self._scp03(0x33).overhead, 16)  # R-MAC/R-ENC add no *command* overhead
-
-    def test_scp03_s16(self):
-        self.assertEqual(self._scp03(0x01, s_mode=16).overhead, 16)
-        self.assertEqual(self._scp03(0x03, s_mode=16).overhead, 32)  # pad80 to 16 within 239 -> 224, minus pad byte
-
-
-class SCP_Lc_Limit_Test_Base(unittest.TestCase):
-    """Test wrap_cmd_apdu() boundary handling: data of (255 - overhead) must produce Lc <= 255 else ValueError"""
-
-    def _load_apdu(self, data_len):
-        return h2b('80E80000') + bytes([data_len]) + b'\xa5' * data_len
-
-    def _check_boundary(self, scp):
-        fits = 255 - scp.overhead
-        wrapped = scp.wrap_cmd_apdu(self._load_apdu(fits))
-        self.assertLessEqual(wrapped[4], 255)
-        self.assertEqual(len(wrapped), 5 + wrapped[4])  # case #3: header + Lc bytes, no Le
-        with self.assertRaises(ValueError) as ctx:
-            scp.wrap_cmd_apdu(self._load_apdu(fits + 1))
-        self.assertIn('Lc', str(ctx.exception))
-
-
-class SCP02_Lc_Limit_Test(SCP_Lc_Limit_Test_Base):
-    """Same session vectors as SCP02_Auth_Test"""
-
-    def setUp(self):
-        self.scp02 = SCP02(card_keys=ck_3des_70)
-        self.scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
-        self.scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
-        self.scp02.gen_ext_auth_apdu()
-
-    def test_cmac_only(self):
-        self.scp02.security_level = 0x01
-        self._check_boundary(self.scp02)  # 247 fits, 248 raises
-
-    def test_cmac_cdec(self):
-        self.scp02.security_level = 0x03
-        self._check_boundary(self.scp02)  # 239 fits (-> Lc 248), 240 raises (would be 256)
-
-    def test_cmac_cdec_wrapped_lc(self):
-        # my actual failing case: 240 bytes at level 3
-        self.scp02.security_level = 0x03
-        wrapped = self.scp02.wrap_cmd_apdu(self._load_apdu(239))
-        self.assertEqual(wrapped[4], 248)  # 239 -> pad80 -> 240 ciphertext + 8 mac
-
-
-class SCP03_Lc_Limit_Test(SCP_Lc_Limit_Test_Base):
-    """Session keys derived directly"""
-
-    def _scp03(self, security_level, s_mode):
-        scp = SCP03(card_keys=KEYSET_AES128, s_mode=s_mode)
-        scp.sk = Scp03SessionKeys(KEYSET_AES128, b'\x00' * s_mode, b'\x11' * s_mode)
-        scp.security_level = security_level
-        return scp
-
-    def test_s8_cmac_only(self):
-        self._check_boundary(self._scp03(0x01, 8))    # 247 fits, 248 raises
-
-    def test_s8_cmac_cdec(self):
-        self._check_boundary(self._scp03(0x03, 8))    # 239 fits, 240 raises
-
-    def test_s16_cmac_only(self):
-        self._check_boundary(self._scp03(0x01, 16))   # 239 fits, 240 raises
-
-    def test_s16_cmac_cdec(self):
-        self._check_boundary(self._scp03(0x03, 16))   # 223 fits, 224 raises
-
-
-class _FakeSccForLoad:
-    """mock lchan.scc: records LOAD APDUs, optionally wrapping them through a real SCP
-    instance first where the Lc overflow used to blow up"""
-
-    def __init__(self, max_cmd_len=255, scp=None):
-        self.max_cmd_len = max_cmd_len
-        self.scp = scp
-        self.sent = []
-        self.wrapped = []
-
-    def send_apdu_checksw(self, apdu, sw='9000'):
-        self.sent.append(apdu.lower())
-        if self.scp:
-            self.wrapped.append(self.scp.wrap_cmd_apdu(h2b(apdu)))
-        return ('', '9000')
-
-
-class Load_ChunkLen_Test(unittest.TestCase):
-    """ADF_SD.load() chunking: block size must use scc.max_cmd_len"""
-
-    payload = b'\xaa' * 500  # actual real world case LOAD TLV: C4 + 8201f4 + 500 = 504 total
-
-    def _sd(self, scc):
-        cmd = type('_Cmd', (), {'lchan': type('_Lchan', (), {'scc': scc})(),
-                                'poutput': lambda self, *args: None})()
-        # cmd2 CommandSet has a r/o _cmd property -> shadow it
-        _SD = type('_SD', (ADF_SD.AddlShellCommands,), {'_cmd': cmd})
-        return _SD.__new__(_SD)
-
-    def _blocks(self, scc):
-        """Get (p1, p2, lc) from LOAD APDU"""
-        for apdu in scc.sent:
-            self.assertEqual(apdu[0:4], '80e8')
-            yield int(apdu[4:6], 16), int(apdu[6:8], 16), int(apdu[8:10], 16)
-
-    def test_default_no_scp(self):
-        """Without SCP the old 240 byte block size is kept, no idea what else might rely on this number"""
-        scc = _FakeSccForLoad(max_cmd_len=255)
-        self._sd(scc).load(self.payload)
-        blocks = list(self._blocks(scc))
-        self.assertEqual([b[2] for b in blocks], [240, 240, 24])
-        self.assertEqual([b[0] for b in blocks], [0x00, 0x00, 0x80])  # P1: last block flagged
-        self.assertEqual([b[1] for b in blocks], [0, 1, 2])           # P2: block num
-
-    def test_default_scp02_level3(self):
-        """max_cmd_len 239 (SCP02 lvl 3) squeezes the blocks"""
-        scc = _FakeSccForLoad(max_cmd_len=239)
-        self._sd(scc).load(self.payload)
-        self.assertEqual([b[2] for b in list(self._blocks(scc))], [239, 239, 26])
-
-    def test_explicit_chunk_len(self):
-        scc = _FakeSccForLoad(max_cmd_len=255)
-        self._sd(scc).load(self.payload, chunk_len=100)
-        self.assertEqual([b[2] for b in list(self._blocks(scc))], [100] * 5 + [4])
-
-    def test_explicit_chunk_len_too_large(self):
-        scc = _FakeSccForLoad(max_cmd_len=239)
-        with self.assertRaises(ValueError):
-            self._sd(scc).load(self.payload, chunk_len=240)
-        self.assertEqual(scc.sent, [])  # nothing sent!
-
-    def test_explicit_chunk_len_zero(self):
-        scc = _FakeSccForLoad(max_cmd_len=255)
-        with self.assertRaises(ValueError):
-            self._sd(scc).load(self.payload, chunk_len=0)
-
-    def test_end_to_end_scp02_level3(self):
-        """original failure: 286 byte CAP + SCP02 lvl 3"""
-        scp02 = SCP02(card_keys=ck_3des_70)
-        scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
-        scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
-        scp02.gen_ext_auth_apdu()
-        scp02.security_level = 0x03
-        scc = _FakeSccForLoad(max_cmd_len=255 - scp02.overhead, scp=scp02)
-        self._sd(scc).load(b'\x5a' * 286)
-        self.assertEqual(len(scc.sent), 2)  # 289 byte TLV in blocks of 239
-        for wrapped in scc.wrapped:
-            self.assertLessEqual(wrapped[4], 255)
-
-
 if __name__ == "__main__":
 	unittest.main()

tests/unittests/test_log.py

@@ -37,17 +37,6 @@ expected_message = None
 
 class PySimLogger_Test(unittest.TestCase):
 
-    def setUp(self):
-        # PySimLogger.setup() is global, so a print callback left installed here fires for
-        # every PySimLogger message emitted by any test module that runs later in the same process
-        # ... where it asserts against a stale 'expected_message' and fails a test that has nothing
-        # to do with logging. Great fun!
-        # Restore before each test.
-        saved = (PySimLogger.print_callback, PySimLogger.verbose)
-        def _restore():
-            PySimLogger.print_callback, PySimLogger.verbose = saved
-        self.addCleanup(_restore)
-
     def __test_01_safe_defaults_one(self, callback, message:str):
         # When log messages are sent to an unconfigured PySimLogger class, we expect the unmodified message being
         # logged to stdout, just as if it were printed via a normal print() statement.

tests/unittests/test_param_src.py

@@ -1,206 +0,0 @@
-#!/usr/bin/env python3
-
-# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
-#
-# Author: Neels Hofmeyr
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import math
-from importlib import resources
-import unittest
-from pySim.esim.saip import param_source
-
-import xo
-update_expected_output = False
-
-class D:
-    mandatory = set()
-    optional = set()
-
-    def __init__(self, **kwargs):
-        if (set(kwargs.keys()) - set(self.optional)) != set(self.mandatory):
-            raise RuntimeError(f'{self.__class__.__name__}.__init__():'
-                               f' {set(kwargs.keys())=!r} - {self.optional=!r} != {self.mandatory=!r}')
-        for k, v in kwargs.items():
-            setattr(self, k, v)
-        for k in self.optional:
-            if not hasattr(self, k):
-                setattr(self, k, None)
-
-decimals = '0123456789'
-hexadecimals = '0123456789abcdefABCDEF'
-
-class FakeRandom:
-    vals = b'\xab\xcfm\xf0\x98J_\xcf\x96\x87fp5l\xe7f\xd1\xd6\x97\xc1\xf9]\x8c\x86+\xdb\t^ke\xc1r'
-    i = 0
-
-    @classmethod
-    def next(cls):
-        cls.i = (cls.i + 1) % len(cls.vals)
-        return cls.vals[cls.i]
-
-    @staticmethod
-    def randint(a, b):
-        d = b - a
-        n_bytes = math.ceil(math.log(d, 2))
-        r = int.from_bytes( bytes(FakeRandom.next() for i in range(n_bytes)) )
-        return a + (r % (b - a))
-
-    @staticmethod
-    def randbytes(n):
-        return bytes(FakeRandom.next() for i in range(n))
-
-
-class ParamSourceTest(unittest.TestCase):
-
-    def test_param_source(self):
-
-        class Paramtest(D):
-            mandatory = (
-                    'param_source',
-                    'n',
-                    'expect',
-                    )
-            optional = (
-                    'expect_arg',
-                    'csv_rows',
-                    )
-            param_source: param_source.ParamSource
-            n: int
-            expect: object
-            expect_arg: object
-            csv_rows: object
-
-        def expect_const(t, vals):
-            return tuple(t.expect_arg) == tuple(vals)
-
-        def expect_random(t, vals):
-            chars = t.expect_arg.get('digits')
-            repetitions = (t.n - len(set(vals)))
-            if repetitions:
-                raise RuntimeError(f'expect_random: there are {repetitions} repetitions in the returned values: {vals}')
-            for val_i in range(len(vals)):
-                v = vals[val_i]
-                val_minlen = t.expect_arg.get('val_minlen')
-                val_maxlen = t.expect_arg.get('val_maxlen')
-                if len(v) < val_minlen or len(v) > val_maxlen:
-                    raise RuntimeError(f'expect_random: invalid length {len(v)} for value [{val_i}]: {v!r}, expecting'
-                                       f' {val_minlen}..{val_maxlen}')
-
-                if chars is not None and not all(c in chars for c in v):
-                    raise RuntimeError(f'expect_random: invalid char in value [{val_i}]: {v!r}')
-            return True
-
-        param_source_tests = [
-            Paramtest(param_source=param_source.ConstantSource.from_str('123'),
-                      n=3,
-                      expect=expect_const,
-                      expect_arg=('123', '123', '123')),
-            Paramtest(param_source=param_source.RandomDigitSource.from_str('12345'),
-                      n=3,
-                      expect=expect_random,
-                      expect_arg={'digits': decimals,
-                                  'val_minlen': 5,
-                                  'val_maxlen': 5}),
-            Paramtest(param_source=param_source.RandomDigitSource.from_str('1..999'),
-                      n=10,
-                      expect=expect_random,
-                      expect_arg={'digits': decimals,
-                                  'val_minlen': 1,
-                                  'val_maxlen': 3}),
-            Paramtest(param_source=param_source.RandomDigitSource.from_str('001..999'),
-                      n=10,
-                      expect=expect_random,
-                      expect_arg={'digits': decimals,
-                                  'val_minlen': 3,
-                                  'val_maxlen': 3}),
-            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('12345678'),
-                      n=3,
-                      expect=expect_random,
-                      expect_arg={'digits': hexadecimals,
-                                  'val_minlen': 8,
-                                  'val_maxlen': 8}),
-            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('0*8'),
-                      n=3,
-                      expect=expect_random,
-                      expect_arg={'digits': hexadecimals,
-                                  'val_minlen': 8,
-                                  'val_maxlen': 8}),
-            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('00*4'),
-                      n=3,
-                      expect=expect_random,
-                      expect_arg={'digits': hexadecimals,
-                                  'val_minlen': 8,
-                                  'val_maxlen': 8}),
-            Paramtest(param_source=param_source.IncDigitSource.from_str('10001'),
-                      n=3,
-                      expect=expect_const,
-                      expect_arg=('10001', '10002', '10003')),
-            Paramtest(param_source=param_source.CsvSource('column_name'),
-                      n=3,
-                      expect=expect_const,
-                      expect_arg=('first val', 'second val', 'third val'),
-                      csv_rows=(
-                          {'column_name': 'first val'},
-                          {'column_name': 'second val'},
-                          {'column_name': 'third val'},
-                      )),
-            ]
-
-        outputs = []
-
-        for t in param_source_tests:
-            try:
-                if hasattr(t.param_source, 'random_impl'):
-                    t.param_source.random_impl = FakeRandom
-
-                vals = []
-                for i in range(t.n):
-                    csv_row = None
-                    if t.csv_rows is not None:
-                        csv_row = t.csv_rows[i]
-                    vals.append( t.param_source.get_next(csv_row=csv_row) )
-                if not t.expect(t, vals):
-                    raise RuntimeError(f'invalid values returned: returned {vals}')
-                output = f'ok: {t.param_source.__class__.__name__} {vals=!r}'
-                outputs.append(output)
-                print(output)
-            except RuntimeError as e:
-                raise RuntimeError(f'{t.param_source.__class__.__name__} {t.n=} {t.expect.__name__}({t.expect_arg!r}): {e}') from e
-
-        output = '\n'.join(outputs) + '\n'
-        xo_name = 'test_param_src'
-        if update_expected_output:
-            with resources.path(xo, xo_name) as xo_path:
-                with open(xo_path, 'w', encoding='utf-8') as f:
-                    f.write(output)
-        else:
-            xo_str = resources.read_text(xo, xo_name)
-            if xo_str != output:
-                at = 0
-                while at < len(output):
-                    if output[at] == xo_str[at]:
-                        at += 1
-                        continue
-                    break
-
-                raise RuntimeError(f'output differs from expected output at position {at}: {xo_str[at:at+128]!r}')
-
-if __name__ == "__main__":
-    if '-u' in sys.argv:
-        update_expected_output = True
-        sys.argv.remove('-u')
-    unittest.main()

tests/unittests/xo/test_param_src

@@ -1,9 +0,0 @@
-ok: ConstantSource vals=['123', '123', '123']
-ok: RandomDigitSource vals=['13987', '49298', '55670']
-ok: RandomDigitSource vals=['650', '580', '49', '885', '497', '195', '320', '137', '245', '663']
-ok: RandomDigitSource vals=['638', '025', '232', '779', '826', '972', '650', '580', '049', '885']
-ok: RandomHexDigitSource vals=['6b65c172', 'abcf6df0', '984a5fcf']
-ok: RandomHexDigitSource vals=['96876670', '356ce766', 'd1d697c1']
-ok: RandomHexDigitSource vals=['f95d8c86', '2bdb095e', '6b65c172']
-ok: IncDigitSource vals=['10001', '10002', '10003']
-ok: CsvSource vals=['first val', 'second val', 'third val']