fix

Change-Id: I1af2fd04ba935fc70a2306342eeb247d941c67af

README.md

@@ -97,7 +97,7 @@ Please install the following dependencies:
  - pyscard
  - pyserial
  - pytlv
- - pyyaml >= 5.4
+ - pyyaml >= 5.1
  - smpp.pdu (from `github.com/hologram-io/smpp.pdu`)
  - termcolor
 

contrib/jenkins.sh

@@ -10,11 +10,6 @@
 
 export PYTHONUNBUFFERED=1
 
-setup_venv() {
-	virtualenv -p python3 venv --system-site-packages
-	. venv/bin/activate
-}
-
 if [ ! -d "./tests/" ] ; then
 	echo "###############################################"
 	echo "Please call from pySim-prog top directory"
@@ -28,7 +23,8 @@ fi
 
 case "$JOB_TYPE" in
 "test")
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install -r requirements.txt
 	pip install pyshark
@@ -36,27 +32,23 @@ case "$JOB_TYPE" in
 	# Execute automatically discovered unit tests first
 	python -m unittest discover -v -s tests/unittests
 
-	# Run pySim-trace test
+	# Run pySim-prog integration tests (requires physical cards)
-	tests/pySim-trace_test/pySim-trace_test.sh
-	;;
-"card-test") # tests requiring physical cards
-	setup_venv
-
-	pip install -r requirements.txt
-
-	# Run pySim-prog integration tests
 	cd tests/pySim-prog_test/
         ./pySim-prog_test.sh
 	cd ../../
 
-	# Run pySim-shell integration tests
+	# Run pySim-trace test
+	tests/pySim-trace_test/pySim-trace_test.sh
+
+	# Run pySim-shell integration tests (requires physical cards)
 	python3 -m unittest discover -v -s ./tests/pySim-shell_test/
 
 	# Run pySim-smpp2sim test
 	tests/pySim-smpp2sim_test/pySim-smpp2sim_test.sh
 	;;
 "distcheck")
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install .
 	pip install pyshark
@@ -69,7 +61,8 @@ case "$JOB_TYPE" in
 	# Print pylint version
 	pip3 freeze | grep pylint
 
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install .
 
@@ -87,7 +80,8 @@ case "$JOB_TYPE" in
 		contrib/*.py
 	;;
 "docs")
-	setup_venv
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
 
 	pip install -r requirements.txt
 

pySim-prog.py

@@ -27,6 +27,7 @@
 import hashlib
 import argparse
 import os
+import random
 import re
 import sys
 import traceback
@@ -435,7 +436,7 @@ def gen_parameters(opts):
         if not re.match('^[0-9a-fA-F]{32}$', ki):
             raise ValueError('Ki needs to be 128 bits, in hex format')
     else:
-        ki = os.urandom(16).hex()
+        ki = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
 
     # OPC (random)
     if opts.opc is not None:
@@ -446,7 +447,7 @@ def gen_parameters(opts):
     elif opts.op is not None:
         opc = derive_milenage_opc(ki, opts.op)
     else:
-        opc = os.urandom(16).hex()
+        opc = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
 
     pin_adm = sanitize_pin_adm(opts.pin_adm, opts.pin_adm_hex)
 

pySim-shell.py

@@ -69,8 +69,8 @@ from pySim.ts_102_222 import Ts102222Commands
 from pySim.gsm_r import DF_EIRENE
 from pySim.cat import ProactiveCommand
 
-from pySim.card_key_provider import card_key_provider_argparse_add_args, card_key_provider_init
+from pySim.card_key_provider import CardKeyProviderCsv, CardKeyProviderPgsql
-from pySim.card_key_provider import card_key_provider_get_field, card_key_provider_get
+from pySim.card_key_provider import card_key_provider_register, card_key_provider_get_field, card_key_provider_get
 
 from pySim.app import init_card
 
@@ -1146,6 +1146,18 @@ global_group.add_argument("--skip-card-init", help="Skip all card/profile initia
 global_group.add_argument("--verbose", help="Enable verbose logging",
                           action='store_true', default=False)
 
+card_key_group = option_parser.add_argument_group('Card Key Provider Options')
+card_key_group.add_argument('--csv', metavar='FILE',
+                            default="~/.osmocom/pysim/card_data.csv",
+                            help='Read card data from CSV file')
+card_key_group.add_argument('--pgsql', metavar='FILE',
+                            default="~/.osmocom/pysim/card_data_pgsql.cfg",
+                            help='Read card data from PostgreSQL database (config file)')
+card_key_group.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                            help=argparse.SUPPRESS, dest='column_key')
+card_key_group.add_argument('--column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                            help='per-column AES transport key', dest='column_key')
+
 adm_group = global_group.add_mutually_exclusive_group()
 adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM1', dest='pin_adm', default=None,
                        help='ADM PIN used for provisioning (overwrites default)')
@@ -1158,7 +1170,6 @@ option_parser.add_argument("command", nargs='?',
                            help="A pySim-shell command that would optionally be executed at startup")
 option_parser.add_argument('command_args', nargs=argparse.REMAINDER,
                            help="Optional Arguments for command")
-card_key_provider_argparse_add_args(option_parser)
 
 if __name__ == '__main__':
     startup_errors = False
@@ -1167,8 +1178,16 @@ if __name__ == '__main__':
     # Ensure that we are able to print formatted warnings from the beginning.
     PySimLogger.setup(print, {logging.WARN: YELLOW}, opts.verbose)
 
-    # Init card key provider for automatic card key retrieval
+    # Register csv-file as card data provider, either from specified CSV
-    card_key_provider_init(opts)
+    # or from CSV file in home directory
+    column_keys = {}
+    for par in opts.column_key:
+        name, key = par.split(':')
+        column_keys[name] = key
+    if os.path.isfile(os.path.expanduser(opts.csv)):
+        card_key_provider_register(CardKeyProviderCsv(os.path.expanduser(opts.csv), column_keys))
+    if os.path.isfile(os.path.expanduser(opts.pgsql)):
+        card_key_provider_register(CardKeyProviderPgsql(os.path.expanduser(opts.pgsql), column_keys))
 
     # Init card reader driver
     sl = init_reader(opts, proactive_handler = Proact())

pySim/app.py

@@ -26,9 +26,6 @@ from pySim.cdma_ruim import CardProfileRUIM
 from pySim.ts_102_221 import CardProfileUICC
 from pySim.utils import all_subclasses
 from pySim.exceptions import SwMatchError
-from pySim.log import PySimLogger
-
-log = PySimLogger.get(__name__)
 
 # we need to import this module so that the SysmocomSJA2 sub-class of
 # CardModel is created, which will add the ATR-based matching and
@@ -57,7 +54,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
 
     # Wait up to three seconds for a card in reader and try to detect
     # the card type.
-    log.info("Waiting for card...")
+    print("Waiting for card...")
     sl.wait_for_card(3)
 
     # The user may opt to skip all card initialization. In this case only the
@@ -69,7 +66,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
     generic_card = False
     card = card_detect(scc)
     if card is None:
-        log.warning("Could not detect card type - assuming a generic card type...")
+        print("Warning: Could not detect card type - assuming a generic card type...")
         card = SimCardBase(scc)
         generic_card = True
 
@@ -79,7 +76,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
         # just means that pySim was unable to recognize the card profile. This
         # may happen in particular with unprovisioned cards that do not have
         # any files on them yet.
-        log.warning("Unsupported card type!")
+        print("Unsupported card type!")
         return None, card
 
     # ETSI TS 102 221, Table 9.3 specifies a default for the PIN key
@@ -90,7 +87,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
     if generic_card and isinstance(profile, CardProfileUICC):
         card._adm_chv_num = 0x0A
 
-    log.info("Card is of type: %s", str(profile))
+    print("Info: Card is of type: %s" % str(profile))
 
     # FIXME: this shouldn't really be here but somewhere else/more generic.
     # We cannot do it within pySim/profile.py as that would create circular

pySim/ara_m.py

@@ -334,10 +334,10 @@ class ADF_ARAM(CardADF):
         apdu_grp.add_argument(
             '--apdu-filter', help='APDU filter: multiple groups of 8 hex bytes (4 byte CLA/INS/P1/P2 followed by 4 byte mask)')
         nfc_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
-        nfc_grp.add_argument('--nfc-never', action='store_true',
-                             help='NFC event access is not allowed')
         nfc_grp.add_argument('--nfc-always', action='store_true',
                              help='NFC event access is allowed')
+        nfc_grp.add_argument('--nfc-never', action='store_true',
+                             help='NFC event access is not allowed')
         store_ref_ar_do_parse.add_argument(
             '--android-permissions', help='Android UICC Carrier Privilege Permissions (8 hex bytes)')
 

pySim/card_key_provider.py

@@ -33,12 +33,10 @@ from Cryptodome.Cipher import AES
 from osmocom.utils import h2b, b2h
 from pySim.log import PySimLogger
 
-import os
 import abc
 import csv
 import logging
 import yaml
-import argparse
 
 log = PySimLogger.get(__name__)
 
@@ -132,31 +130,6 @@ class CardKeyFieldCryptor:
         cipher = AES.new(h2b(self.transport_keys[field_name.upper()]), AES.MODE_CBC, self.__IV)
         return b2h(cipher.encrypt(h2b(plaintext_val)))
 
-    @staticmethod
-    def argparse_add_args(arg_parser: argparse.ArgumentParser):
-        arg_parser.add_argument('--column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
-                                help='per-column AES transport key', dest='column_key')
-        # Depprecated argument, replaced by --column-key (see above)
-        arg_parser.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
-                                help=argparse.SUPPRESS, dest='column_key')
-
-    @staticmethod
-    def transport_keys_from_opts(opts: argparse.Namespace) -> dict:
-        """
-        Transport keys are passed via the commandline using the '--column-key' option. Each column requires a
-        dedicated transport key. This method can be used to extract the column keys parameters from the commandline
-        options into a dict that can be directly passed to the construtor with the transport_keys argument.
-
-        Args:
-                opts: parsed commandline options (Namespace)
-        """
-
-        transport_keys = {}
-        for par in opts.column_key:
-            name, key = par.split(':')
-            transport_keys[name] = key
-        return transport_keys
-
 class CardKeyProvider(abc.ABC):
     """Base class, not containing any concrete implementation."""
 
@@ -175,33 +148,24 @@ class CardKeyProvider(abc.ABC):
                 fond None shall be returned.
         """
 
-    @staticmethod
-    def argparse_add_args(arg_parser: argparse.ArgumentParser):
-        """
-        Add the commandline arguments relevant for this card key provider.
-
-        Args:
-                arg_parser : argument parser group
-        """
-
     def __str__(self):
         return type(self).__name__
 
 class CardKeyProviderCsv(CardKeyProvider):
     """Card key provider implementation that allows to query against a specified CSV file."""
 
-    def __init__(self, csv_filename: str, field_cryptor: CardKeyFieldCryptor):
+    def __init__(self, csv_filename: str, transport_keys: dict):
         """
         Args:
                 csv_filename : file name (path) of CSV file containing card-individual key/data
-                field_cryptor : (see class CardKeyFieldCryptor)
+                transport_keys : (see class CardKeyFieldCryptor)
         """
         log.info("Using CSV file as card key data source: %s" % csv_filename)
         self.csv_file = open(csv_filename, 'r')
         if not self.csv_file:
             raise RuntimeError("Could not open CSV file '%s'" % csv_filename)
         self.csv_filename = csv_filename
-        self.crypt = field_cryptor
+        self.crypt = CardKeyFieldCryptor(transport_keys)
 
     def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         self.csv_file.seek(0)
@@ -224,20 +188,14 @@ class CardKeyProviderCsv(CardKeyProvider):
             return None
         return return_dict
 
-    @staticmethod
-    def argparse_add_args(arg_parser: argparse.ArgumentParser):
-        arg_parser.add_argument('--csv', metavar='FILE',
-                                default="~/.osmocom/pysim/card_data.csv",
-                                help='Read card data from CSV file')
-
 class CardKeyProviderPgsql(CardKeyProvider):
     """Card key provider implementation that allows to query against a specified PostgreSQL database table."""
 
-    def __init__(self, config_filename: str, field_cryptor: CardKeyFieldCryptor):
+    def __init__(self, config_filename: str, transport_keys: dict):
         """
         Args:
                 config_filename : file name (path) of CSV file containing card-individual key/data
-                field_cryptor : (see class CardKeyFieldCryptor)
+                transport_keys : (see class CardKeyFieldCryptor)
         """
         import psycopg2
         log.info("Using SQL database as card key data source: %s" % config_filename)
@@ -254,7 +212,7 @@ class CardKeyProviderPgsql(CardKeyProvider):
                                          host=config.get('host'))
             self.tables = config.get('table_names')
             log.info("Card key database tables: %s" % str(self.tables))
-            self.crypt = field_cryptor
+            self.crypt = CardKeyFieldCryptor(transport_keys)
 
     def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         import psycopg2
@@ -294,11 +252,6 @@ class CardKeyProviderPgsql(CardKeyProvider):
             result[k] = self.crypt.decrypt_field(k, result.get(k))
         return result
 
-    @staticmethod
-    def argparse_add_args(arg_parser: argparse.ArgumentParser):
-        arg_parser.add_argument('--pgsql', metavar='FILE',
-                                default="~/.osmocom/pysim/card_data_pgsql.cfg",
-                                help='Read card data from PostgreSQL database (config file)')
 
 def card_key_provider_register(provider: CardKeyProvider, provider_list=card_key_providers):
     """Register a new card key provider.
@@ -352,19 +305,3 @@ def card_key_provider_get_field(field: str, key: str, value: str, provider_list=
     fields = [field]
     result = card_key_provider_get(fields, key, value, card_key_providers)
     return result.get(field.upper())
-
-def card_key_provider_argparse_add_args(arg_parser: argparse.ArgumentParser):
-    """Add card key provider commandline options to the given argument parser"""
-    card_key_group = arg_parser.add_argument_group('Card Key Provider Options')
-    CardKeyProviderCsv.argparse_add_args(card_key_group)
-    CardKeyProviderPgsql.argparse_add_args(card_key_group)
-    CardKeyFieldCryptor.argparse_add_args(card_key_group)
-
-def card_key_provider_init(opts: argparse.Namespace):
-    """Initialize card key provider depending on the user provided commandline options"""
-    transport_keys = CardKeyFieldCryptor.transport_keys_from_opts(opts)
-    card_key_field_cryptor = CardKeyFieldCryptor(transport_keys)
-    if os.path.isfile(os.path.expanduser(opts.csv)):
-        card_key_provider_register(CardKeyProviderCsv(os.path.expanduser(opts.csv), card_key_field_cryptor))
-    if os.path.isfile(os.path.expanduser(opts.pgsql)):
-        card_key_provider_register(CardKeyProviderPgsql(os.path.expanduser(opts.pgsql), card_key_field_cryptor))

pySim/esim/saip/__init__.py

@@ -1079,13 +1079,6 @@ class SecurityDomainKey:
                 'keyVersionNumber': bytes([self.key_version_number]),
                 'keyComponents': [k.to_saip_dict() for k in self.key_components]}
 
-    def get_key_component(self, key_type):
-        for kc in self.key_components:
-            if kc.key_type == key_type:
-                return kc.key_data
-        return None
-
-
 class ProfileElementSD(ProfileElement):
     """Class representing a securityDomain ProfileElement."""
     type = 'securityDomain'

pySim/esim/saip/batch.py

@@ -1,120 +0,0 @@
-"""Implementation of Personalization of eSIM profiles in SimAlliance/TCA Interoperable Profile:
-   Run a batch of N personalizations"""
-
-# (C) 2025-2026 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
-#
-# Author: nhofmeyr@sysmocom.de
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import copy
-from typing import Generator
-from pySim.esim.saip.personalization import ConfigurableParameter
-from pySim.esim.saip import param_source
-from pySim.esim.saip import ProfileElementSequence
-
-class BatchPersonalization:
-    """Produce a series of eSIM profiles from predefined parameters.
-    Personalization parameters are derived from pysim.esim.saip.param_source.ParamSource.
-
-    Usage example:
-
-    der_input = open('some_file', 'rb').read()
-    pes = ProfileElementSequence.from_der(der_input)
-    p = BatchPersonalization(
-        n=10,
-        src_pes=pes,
-        csv_rows=get_csv_reader())
-
-    p.add_param_and_src(
-        personalization.Iccid(),
-        param_source.IncDigitSource(
-            num_digits=18,
-            first_value=123456789012340001,
-            last_value=123456789012340010))
-
-    # add more parameters here, using ConfigurableParameter and ParamSource subclass instances to define the profile
-    # ...
-
-    # generate all 10 profiles (from n=10 above)
-    for result_pes in p.generate_profiles():
-        upp = result_pes.to_der()
-        store_upp(upp)
-    """
-
-    class ParamAndSrc:
-        """tie a ConfigurableParameter to a source of actual values"""
-        def __init__(self, param: ConfigurableParameter, src: param_source.ParamSource):
-            if isinstance(param, type):
-                self.param_cls = param
-            else:
-                self.param_cls = param.__class__
-            self.src = src
-
-    def __init__(self,
-                 n: int,
-                 src_pes: ProfileElementSequence,
-                 params: list[ParamAndSrc]=None,
-                 csv_rows: Generator=None,
-                ):
-        """
-        n: number of eSIM profiles to generate.
-        src_pes: a decoded eSIM profile as ProfileElementSequence, to serve as template. This is not modified, only
-                 copied.
-        params: list of ParamAndSrc instances, defining a ConfigurableParameter and corresponding ParamSource to fill in
-                profile values.
-        csv_rows: A generator (e.g. iter(list_of_rows)) producing all CSV rows one at a time, starting with a row
-                  containing the column headers. This is compatible with the python csv.reader. Each row gets passed to
-                  ParamSource.get_next(), such that ParamSource implementations can access the row items. See
-                  param_source.CsvSource.
-        """
-        self.n = n
-        self.params = params or []
-        self.src_pes = src_pes
-        self.csv_rows = csv_rows
-
-    def add_param_and_src(self, param:ConfigurableParameter, src:param_source.ParamSource):
-        self.params.append(BatchPersonalization.ParamAndSrc(param, src))
-
-    def generate_profiles(self):
-        # get first row of CSV: column names
-        csv_columns = None
-        if self.csv_rows:
-            try:
-                csv_columns = next(self.csv_rows)
-            except StopIteration as e:
-                raise ValueError('the input CSV file appears to be empty') from e
-
-        for i in range(self.n):
-            csv_row = None
-            if self.csv_rows and csv_columns:
-                try:
-                    csv_row_list = next(self.csv_rows)
-                except StopIteration as e:
-                    raise ValueError(f'not enough rows in the input CSV for eSIM nr {i+1} of {self.n}') from e
-
-                csv_row = dict(zip(csv_columns, csv_row_list))
-
-            pes = copy.deepcopy(self.src_pes)
-
-            for p in self.params:
-                try:
-                    input_value = p.src.get_next(csv_row=csv_row)
-                    assert input_value is not None
-                    value = p.param_cls.validate_val(input_value)
-                    p.param_cls.apply_val(pes, value)
-                except Exception as e:
-                    raise ValueError(f'{p.param_cls.get_name()} fed by {p.src.name}: {e}') from e
-
-            yield pes

pySim/esim/saip/param_source.py

@@ -1,203 +0,0 @@
-# Implementation of SimAlliance/TCA Interoperable Profile handling: parameter sources for batch personalization.
-#
-# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
-#
-# Author: nhofmeyr@sysmocom.de
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import random
-import re
-from osmocom.utils import b2h
-
-class ParamSourceExn(Exception):
-    pass
-
-class ParamSourceExhaustedExn(ParamSourceExn):
-    pass
-
-class ParamSourceUndefinedExn(ParamSourceExn):
-    pass
-
-class ParamSource:
-    """abstract parameter source. For usage, see personalization.BatchPersonalization."""
-
-    # This name should be short but descriptive, useful for a user interface, like 'random decimal digits'.
-    name = "none"
-    numeric_base = None # or 10 or 16
-
-    def __init__(self, input_str:str):
-        """Subclasses should call super().__init__(input_str) before evaluating self.input_str. Each subclass __init__()
-        may in turn manipulate self.input_str to apply expansions or decodings."""
-        self.input_str = input_str
-
-    def get_next(self, csv_row:dict=None):
-        """Subclasses implement this: return the next value from the parameter source.
-           When there are no more values from the source, raise a ParamSourceExhaustedExn.
-           This default implementation is an empty source."""
-        raise ParamSourceExhaustedExn()
-
-    @classmethod
-    def from_str(cls, input_str:str):
-        """compatibility with earlier version of ParamSource. Just use the constructor."""
-        return cls(input_str)
-
-class ConstantSource(ParamSource):
-    """one value for all"""
-    name = "constant"
-
-    def get_next(self, csv_row:dict=None):
-        return self.input_str
-
-class InputExpandingParamSource(ParamSource):
-
-    def __init__(self, input_str:str):
-        super().__init__(input_str)
-        self.input_str = self.expand_input_str(self.input_str)
-
-    @classmethod
-    def expand_input_str(cls, input_str:str):
-        # user convenience syntax '0*32' becomes '00000000000000000000000000000000'
-        if "*" not in input_str:
-            return input_str
-        # re:                "XX            *        123" with optional spaces
-        tokens = re.split(r"([^ \t]+)[ \t]*\*[ \t]*([0-9]+)", input_str)
-        if len(tokens) < 3:
-            return input_str
-        parts = []
-        for unchanged, snippet, repeat_str in zip(tokens[0::3], tokens[1::3], tokens[2::3]):
-            parts.append(unchanged)
-            repeat = int(repeat_str)
-            parts.append(snippet * repeat)
-
-        return "".join(parts)
-
-class DecimalRangeSource(InputExpandingParamSource):
-    """abstract: decimal numbers with a value range"""
-
-    numeric_base = 10
-
-    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
-        """Constructor to set up values from a (user entered) string: DecimalRangeSource(input_str).
-        Constructor to set up values directly: DecimalRangeSource(num_digits=3, first_value=123, last_value=456)
-
-        num_digits produces leading zeros when first_value..last_value are shorter.
-        """
-        assert ((input_str is not None and (num_digits, first_value, last_value) == (None, None, None))
-                or (input_str is None and None not in (num_digits, first_value, last_value)))
-
-        if input_str is not None:
-            super().__init__(input_str)
-
-            input_str = self.input_str
-
-            if ".." in input_str:
-                first_str, last_str = input_str.split('..')
-                first_str = first_str.strip()
-                last_str = last_str.strip()
-            else:
-                first_str = input_str.strip()
-                last_str = None
-
-            num_digits = len(first_str)
-            first_value = int(first_str)
-            last_value = int(last_str if last_str is not None else "9" * num_digits)
-
-        assert num_digits > 0
-        assert first_value <= last_value
-        self.num_digits = num_digits
-        self.first_value = first_value
-        self.last_value = last_value
-
-    def val_to_digit(self, val:int):
-        return "%0*d" % (self.num_digits, val)  # pylint: disable=consider-using-f-string
-
-class RandomDigitSource(DecimalRangeSource):
-    """return a different sequence of random decimal digits each"""
-    name = "random decimal digits"
-
-    def get_next(self, csv_row:dict=None):
-        val = random.randint(self.first_value, self.last_value) # TODO secure random source?
-        return self.val_to_digit(val)
-
-class RandomHexDigitSource(InputExpandingParamSource):
-    """return a different sequence of random hexadecimal digits each"""
-    name = "random hexadecimal digits"
-    numeric_base = 16
-
-    def __init__(self, input_str:str):
-        super().__init__(input_str)
-        input_str = self.input_str
-
-        num_digits = len(input_str.strip())
-        if num_digits < 1:
-            raise ValueError("zero number of digits")
-        # hex digits always come in two
-        if (num_digits & 1) != 0:
-            raise ValueError(f"hexadecimal value should have even number of digits, not {num_digits}")
-        self.num_digits = num_digits
-
-    def get_next(self, csv_row:dict=None):
-        val = random.randbytes(self.num_digits // 2) # TODO secure random source?
-        return b2h(val)
-
-class IncDigitSource(DecimalRangeSource):
-    """incrementing sequence of digits"""
-    name = "incrementing decimal digits"
-
-    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
-        """input_str: the range of values to iterate. Format: 'FIRST..LAST' (e.g. '0001..9999') or
-        just 'FIRST' (iterates to the maximum value for the given digit width). Leading zeros in
-        FIRST determine the digit width and are preserved in returned values."""
-        super().__init__(input_str, num_digits, first_value, last_value)
-        self.next_val = None
-        self.reset()
-
-    def reset(self):
-        """Restart from the first value of the defined range passed to __init__()."""
-        self.next_val = self.first_value
-
-    def get_next(self, csv_row:dict=None):
-        val = self.next_val
-        if val is None:
-            raise ParamSourceExhaustedExn()
-
-        returnval = self.val_to_digit(val)
-
-        val += 1
-        if val > self.last_value:
-            self.next_val = None
-        else:
-            self.next_val = val
-
-        return returnval
-
-class CsvSource(ParamSource):
-    """apply a column from a CSV row, as passed in to ParamSource.get_next(csv_row)"""
-    name = "from CSV"
-
-    def __init__(self, input_str:str):
-        """input_str: the CSV column name to read values from.
-        The caller passes the current CSV row to get_next(), from which CsvSource picks the column matching
-        this name."""
-        super().__init__(input_str)
-        self.csv_column = self.input_str
-
-    def get_next(self, csv_row:dict=None):
-        val = None
-        if csv_row:
-            val = csv_row.get(self.csv_column)
-        if val is None:
-            raise ParamSourceUndefinedExn(f"no value for CSV column {self.csv_column!r}")
-        return val

pySim/esim/saip/personalization.py

@@ -16,22 +16,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import abc
-import enum
 import io
-import re
+from typing import List, Tuple
-from typing import List, Tuple, Generator, Optional
 
 from osmocom.tlv import camel_to_snake
-from osmocom.utils import hexstr
+from pySim.utils import enc_iccid, enc_imsi, h2b, rpad, sanitize_iccid
-from pySim.utils import enc_iccid, dec_iccid, enc_imsi, dec_imsi, h2b, b2h, rpad, sanitize_iccid
+from pySim.esim.saip import ProfileElement, ProfileElementSequence
 from pySim.ts_51_011 import EF_SMSP
-from pySim.esim.saip import param_source
-from pySim.esim.saip import ProfileElement, ProfileElementSD, ProfileElementSequence
-from pySim.esim.saip import SecurityDomainKey, SecurityDomainKeyComponent
-from pySim.global_platform import KeyUsageQualifier, KeyType
-
-def unrpad(s: hexstr, c='f') -> hexstr:
-    return hexstr(s.rstrip(c))
 
 def remove_unwanted_tuples_from_list(l: List[Tuple], unwanted_keys: List[str]) -> List[Tuple]:
     """In a list of tuples, remove all tuples whose first part equals 'unwanted_key'."""
@@ -126,7 +117,6 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
     max_len = None
     allow_len = None # a list of specific lengths
     example_input = None
-    default_source = None # a param_source.ParamSource subclass
 
     def __init__(self, input_value=None):
         self.input_value = input_value # the raw input value as given by caller
@@ -209,29 +199,6 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
         Write the given val in the right format in all the right places in pes."""
         pass
 
-    @classmethod
-    @abc.abstractmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence) -> Generator:
-        """This is what subclasses implement: yield all values from a decoded profile package.
-           Find all values in the pes, and yield them decoded to a valid cls.input_value format.
-           Should be a generator function, i.e. use 'yield' instead of 'return'.
-
-           Yielded value must be a dict(). Usually, an implementation will return only one key, like
-
-              { "ICCID": "1234567890123456789" }
-
-           Some implementations have more than one value to return, like
-
-              { "IMSI": "00101012345678", "IMSI-ACC" : "5" }
-
-           Implementation example:
-
-             for pe in pes:
-                if my_condition(pe):
-                    yield { cls.name: b2h(my_bin_value_from(pe)) }
-           """
-        pass
-
     @classmethod
     def get_len_range(cls):
         """considering all of min_len, max_len and allow_len, get a tuple of the resulting (min, max) of permitted
@@ -252,13 +219,6 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
             return (None, None)
         return (min(vals), max(vals))
 
-    @classmethod
-    def get_typical_input_len(cls):
-        '''return a good length to use as the visible width of a user interface input field.
-           May be overridden by subclasses.
-           This default implementation returns the maximum allowed value length -- a good fit for most subclasses.
-           '''
-        return cls.get_len_range()[1] or 16
 
 class DecimalParam(ConfigurableParameter):
     """Decimal digits. The input value may be a string of decimal digits like '012345', or an int. The output of
@@ -289,7 +249,6 @@ class DecimalHexParam(DecimalParam):
     @classmethod
     def validate_val(cls, val):
         val = super().validate_val(val)
-        assert isinstance(val, str)
         val = ''.join('%02x' % ord(x) for x in val)
         if cls.rpad is not None:
             c = cls.rpad_char
@@ -297,17 +256,6 @@ class DecimalHexParam(DecimalParam):
         # a DecimalHexParam subclass expects the apply_val() input to be a bytes instance ready for the pes
         return h2b(val)
 
-    @classmethod
-    def decimal_hex_to_str(cls, val):
-        """useful for get_values_from_pes() implementations of subclasses"""
-        if isinstance(val, bytes):
-            val = b2h(val)
-        assert isinstance(val, hexstr)
-        if cls.rpad is not None:
-            c = cls.rpad_char or 'f'
-            val = unrpad(val, c)
-        return val.to_bytes().decode('ascii')
-
 class IntegerParam(ConfigurableParameter):
     allow_types = (str, int)
     allow_chars = '0123456789'
@@ -331,19 +279,10 @@ class IntegerParam(ConfigurableParameter):
             raise ValueError(f'Value {val} is out of range, must be [{cls.min_val}..{cls.max_val}]')
         return val
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for valdict in super().get_values_from_pes(pes):
-            for key, val in valdict.items():
-                if isinstance(val, int):
-                    valdict[key] = str(val)
-            yield valdict
-
 class BinaryParam(ConfigurableParameter):
     allow_types = (str, io.BytesIO, bytes, bytearray)
     allow_chars = '0123456789abcdefABCDEF'
     strip_chars = ' \t\r\n'
-    default_source = param_source.RandomHexDigitSource
 
     @classmethod
     def validate_val(cls, val):
@@ -362,82 +301,6 @@ class BinaryParam(ConfigurableParameter):
         val = super().validate_val(val)
         return bytes(val)
 
-    @classmethod
-    def get_typical_input_len(cls):
-        # override to return twice the length, because of hex digits.
-        min_len, max_len = cls.get_len_range()
-        if max_len is None:
-            return None
-        # two hex characters per value octet.
-        # (maybe *3 to also allow for spaces?)
-        return max_len * 2
-
-
-class EnumParam(ConfigurableParameter):
-    """ConfigurableParameter for named integer enumeration values.
-
-    Subclasses must define a nested enum.IntEnum named 'Values' listing all valid names and their
-    integer codes.  apply_val() and get_values_from_pes() are not implemented here and this must
-    be inherited from another mixin."""
-
-    class Values(enum.IntEnum):
-        pass  # subclasses override this
-
-    @classmethod
-    def validate_val(cls, val) -> int:
-        if isinstance(val, int):
-            try:
-                return int(cls.Values(val))
-            except ValueError:
-                pass
-        elif isinstance(val, str):
-            member = cls.map_name_to_val(val, strict=False)
-            if member is not None:
-                return member
-
-        valid = ', '.join(m.name for m in cls.Values)
-        raise ValueError(f"{cls.get_name()}: invalid argument: {val!r}. Valid arguments are: {valid}")
-
-    @classmethod
-    def map_name_to_val(cls, name: str, strict=True) -> int:
-        """Return the integer value for a given enum member name. Performs an exact match first,
-        then falls back to fuzzy matching (case-insensitive, punctuation-insensitive)."""
-        try:
-            return int(cls.Values[name])
-        except KeyError:
-            pass
-
-        clean = cls.clean_name_str(name)
-        for member in cls.Values:
-            if cls.clean_name_str(member.name) == clean:
-                return int(member)
-
-        if strict:
-            valid = ', '.join(m.name for m in cls.Values)
-            raise ValueError(f"{cls.get_name()}: {name!r} is not a known value. Known values are: {valid}")
-        return None
-
-    @classmethod
-    def map_val_to_name(cls, val, strict=False) -> str:
-        """Return the enum member name for a given integer value."""
-        try:
-            return cls.Values(val).name
-        except ValueError:
-            if strict:
-                raise ValueError(f"{cls.get_name()}: {val!r} ({type(val).__name__}) is not a known value.")
-            return None
-
-    @classmethod
-    def name_normalize(cls, name: str) -> str:
-        """Map a (possibly fuzzy) name to its canonical enum member name."""
-        return cls.Values(cls.map_name_to_val(name)).name
-
-    @classmethod
-    def clean_name_str(cls, val: str) -> str:
-        """Strip punctuation and case for fuzzy name comparison.
-        Treats hyphens and underscores as equivalent (both removed)."""
-        return re.sub('[^0-9A-Za-z]', '', val).lower()
-
 
 class Iccid(DecimalParam):
     """ICCID Parameter. Input: string of decimal digits.
@@ -446,7 +309,6 @@ class Iccid(DecimalParam):
     min_len = 18
     max_len = 20
     example_input = '998877665544332211'
-    default_source = param_source.IncDigitSource
 
     @classmethod
     def validate_val(cls, val):
@@ -460,17 +322,6 @@ class Iccid(DecimalParam):
         # patch MF/EF.ICCID
         file_replace_content(pes.get_pe_for_type('mf').decoded['ef-iccid'], h2b(enc_iccid(val)))
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        padded = b2h(pes.get_pe_for_type('header').decoded['iccid'])
-        iccid = unrpad(padded)
-        yield { cls.name: iccid }
-
-        for pe in pes.get_pes_for_type('mf'):
-            iccid_f = pe.files.get('ef-iccid', None)
-            if iccid_f is not None:
-                yield { cls.name: dec_iccid(b2h(iccid_f.body)) }
-
 class Imsi(DecimalParam):
     """Configurable IMSI. Expects value to be a string of digits. Automatically sets the ACC to
     the last digit of the IMSI."""
@@ -479,7 +330,6 @@ class Imsi(DecimalParam):
     min_len = 6
     max_len = 15
     example_input = '00101' + ('0' * 10)
-    default_source = param_source.IncDigitSource
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -492,18 +342,6 @@ class Imsi(DecimalParam):
             file_replace_content(pe.decoded['ef-acc'], acc.to_bytes(2, 'big'))
         # TODO: DF.GSM_ACCESS if not linked?
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            imsi_f = pe.files.get('ef-imsi', None)
-            acc_f = pe.files.get('ef-acc', None)
-            y = {}
-            if imsi_f:
-                y[cls.name] = dec_imsi(b2h(imsi_f.body))
-            if acc_f:
-                y[cls.name + '-ACC'] = b2h(acc_f.body)
-            yield y
-
 class SmspTpScAddr(ConfigurableParameter):
     """Configurable SMSC (SMS Service Centre) TP-SC-ADDR. Expects to be a phone number in national or
     international format (designated by a leading +). Automatically sets the NPI to E.164 and the TON based on
@@ -515,41 +353,22 @@ class SmspTpScAddr(ConfigurableParameter):
     max_len = 21 # '+' and 20 digits
     min_len = 1
     example_input = '+49301234567'
-    default_source = param_source.ConstantSource
 
-    @staticmethod
+    @classmethod
-    def str_to_tuple(addr_str):
+    def validate_val(cls, val):
+        val = super().validate_val(val)
+        addr_str = str(val)
         if addr_str[0] == '+':
             digits = addr_str[1:]
             international = True
         else:
             digits = addr_str
             international = False
-        return (international, digits)
-
-    @staticmethod
-    def tuple_to_str(addr_tuple):
-        international, digits = addr_tuple
-        if international:
-            ret = '+'
-        else:
-            ret = ''
-        ret += digits
-        return ret
-
-    @classmethod
-    def validate_val(cls, val):
-        val = super().validate_val(val)
-
-        addr_tuple = cls.str_to_tuple(str(val))
-
-        international, digits = addr_tuple
         if len(digits) > 20:
             raise ValueError(f'TP-SC-ADDR must not exceed 20 digits: {digits!r}')
         if not digits.isdecimal():
             raise ValueError(f'TP-SC-ADDR must only contain decimal digits: {digits!r}')
-
+        return (international, digits)
-        return addr_tuple
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -579,32 +398,6 @@ class SmspTpScAddr(ConfigurableParameter):
             # re-generate the pe.decoded member from the File instance
             pe.file2pe(f_smsp)
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            f_smsp = pe.files['ef-smsp']
-            ef_smsp = EF_SMSP()
-            ef_smsp_dec = ef_smsp.decode_record_bin(f_smsp.body, 1)
-
-            tp_sc_addr = ef_smsp_dec.get('tp_sc_addr', None)
-            if not tp_sc_addr:
-                continue
-
-            digits = tp_sc_addr.get('call_number', None)
-            if not digits:
-                continue
-
-            ton_npi = tp_sc_addr.get('ton_npi', None)
-            if not ton_npi:
-                continue
-            international = ton_npi.get('type_of_number', None)
-            if international is None:
-                continue
-            international = (international == 'international')
-
-            yield { cls.name: cls.tuple_to_str((international, digits)) }
-
-
 class SdKey(BinaryParam, metaclass=ClassVarMeta):
     """Configurable Security Domain (SD) Key.  Value is presented as bytes."""
     # these will be set by subclasses
@@ -614,40 +407,28 @@ class SdKey(BinaryParam, metaclass=ClassVarMeta):
     key_usage_qual = None
 
     @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
+    def _apply_sd(cls, pe: ProfileElement, value):
-        set_components = [ SecurityDomainKeyComponent(cls.key_type, val) ]
+        assert pe.type == 'securityDomain'
-
+        for key in pe.decoded['keyList']:
-        for pe in pes.pe_list:
+            if key['keyIdentifier'][0] == cls.key_id and key['keyVersionNumber'][0] == cls.kvn:
-            if pe.type != 'securityDomain':
+                assert len(key['keyComponents']) == 1
-                continue
+                key['keyComponents'][0]['keyData'] = value
-            assert isinstance(pe, ProfileElementSD)
+                return
-
-            key = pe.find_key(key_version_number=cls.kvn, key_id=cls.key_id)
-            if not key:
         # Could not find matching key to patch, create a new one
-                key = SecurityDomainKey(
+        key = {
-                        key_version_number=cls.kvn,
+            'keyUsageQualifier': bytes([cls.key_usage_qual]),
-                        key_id=cls.key_id,
+            'keyIdentifier': bytes([cls.key_id]),
-                        key_usage_qualifier=KeyUsageQualifier.build(cls.key_usage_qual),
+            'keyVersionNumber': bytes([cls.kvn]),
-                        key_components=set_components,
+            'keyComponents': [
-                        )
+                { 'keyType': bytes([cls.key_type]), 'keyData': value },
-                pe.add_key(key)
+            ]
-            else:
+        }
-                key.key_components = set_components
+        pe.decoded['keyList'].append(key)
 
     @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
+    def apply_val(cls, pes: ProfileElementSequence, value):
-        for pe in pes.pe_list:
+        for pe in pes.get_pes_for_type('securityDomain'):
-            if pe.type != 'securityDomain':
+            cls._apply_sd(pe, value)
-                continue
-            assert isinstance(pe, ProfileElementSD)
-
-            key = pe.find_key(key_version_number=cls.kvn, key_id=cls.key_id)
-            if not key:
-                continue
-            kc = key.get_key_component(cls.key_type)
-            if kc:
-                yield { cls.name: b2h(kc) }
 
 class SdKeyScp80_01(SdKey, kvn=0x01, key_type=0x88, permitted_len=[16,24,32]): # AES key type
     pass
@@ -721,8 +502,7 @@ class Puk(DecimalHexParam):
     allow_len = 8
     rpad = 16
     keyReference = None
-    example_input = f'0*{allow_len}'
+    example_input = '0' * allow_len
-    default_source = param_source.RandomDigitSource
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -736,14 +516,6 @@ class Puk(DecimalHexParam):
         raise ValueError("input template UPP has unexpected structure:"
                          f" cannot find pukCode with keyReference={cls.keyReference}")
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        mf_pes = pes.pes_by_naa['mf'][0]
-        for pukCodes in obtain_all_pe_from_pelist(mf_pes, 'pukCodes'):
-            for pukCode in pukCodes.decoded['pukCodes']:
-                if pukCode['keyReference'] == cls.keyReference:
-                    yield { cls.name: cls.decimal_hex_to_str(pukCode['pukValue']) }
-
 class Puk1(Puk):
     name = 'PUK1'
     keyReference = 0x01
@@ -757,8 +529,7 @@ class Pin(DecimalHexParam):
     rpad = 16
     min_len = 4
     max_len = 8
-    example_input = f'0*{max_len}'
+    example_input = '0' * max_len
-    default_source = param_source.RandomDigitSource
     keyReference = None
 
     @staticmethod
@@ -780,24 +551,9 @@ class Pin(DecimalHexParam):
             raise ValueError('input template UPP has unexpected structure:'
                              + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference}')
 
-    @classmethod
-    def _read_all_pinvalues_from_pe(cls, pe: ProfileElement):
-        "This is a separate function because subclasses may feed different pe arguments."
-        for pinCodes in obtain_all_pe_from_pelist(pe, 'pinCodes'):
-            if pinCodes.decoded['pinCodes'][0] != 'pinconfig':
-                continue
-
-            for pinCode in pinCodes.decoded['pinCodes'][1]:
-                if pinCode['keyReference'] == cls.keyReference:
-                    yield { cls.name: cls.decimal_hex_to_str(pinCode['pinValue']) }
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        yield from cls._read_all_pinvalues_from_pe(pes.pes_by_naa['mf'][0])
-
 class Pin1(Pin):
     name = 'PIN1'
-    example_input = '0*4'  # PIN are usually 4 digits
+    example_input = '0' * 4  # PIN are usually 4 digits
     keyReference = 0x01
 
 class Pin2(Pin1):
@@ -816,14 +572,6 @@ class Pin2(Pin1):
                     raise ValueError('input template UPP has unexpected structure:'
                             + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference} in {naa=}')
 
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for naa in pes.pes_by_naa:
-            if naa not in ['usim','isim','csim','telecom']:
-                continue
-            for pe in pes.pes_by_naa[naa]:
-                yield from cls._read_all_pinvalues_from_pe(pe)
-
 class Adm1(Pin):
     name = 'ADM1'
     keyReference = 0x0A
@@ -848,59 +596,26 @@ class AlgoConfig(ConfigurableParameter):
             raise ValueError('input template UPP has unexpected structure:'
                              f' {cls.__name__} cannot find algoParameter with key={cls.algo_config_key}')
 
-    @classmethod
+class AlgorithmID(DecimalParam, AlgoConfig):
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('akaParameter'):
-            algoConfiguration = pe.decoded['algoConfiguration']
-            if len(algoConfiguration) < 2:
-                continue
-            if algoConfiguration[0] != 'algoParameter':
-                continue
-            if not algoConfiguration[1]:
-                continue
-            val = algoConfiguration[1].get(cls.algo_config_key, None)
-            if val is None:
-                continue
-            if isinstance(val, bytes):
-                val = b2h(val)
-            # if it is an int (algorithmID), just pass thru as int
-            yield { cls.name: val }
-
-class AlgorithmID(EnumParam, AlgoConfig):
-    """use validate_val() from EnumParam, and apply_val() from AlgoConfig.
-    In get_values_from_pes(), return enum value names, not raw values."""
-    name = "Algorithm"
     algo_config_key = 'algorithmID'
-    example_input = "Milenage"
+    allow_len = 1
-    default_source = param_source.ConstantSource
+    example_input = 1  # Milenage
-
-    # as in pySim/esim/asn1/saip/PE_Definitions-3.3.1.asn
-    class Values(enum.IntEnum):
-        Milenage  = 1
-        TUAK      = 2
-        usim_test = 3  # input 'usim-test' also accepted via fuzzy matching
-
-    # EnumParam.validate_val() returns the int values from Values
 
     @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
+    def validate_val(cls, val):
-        # return enum names, not raw values.
+        val = super().validate_val(val)
-        # use of super(): this intends to call AlgoConfig.get_values_from_pes() so that the cls argument is this cls
+        val = int(val)
-        # here (AlgorithmID); i.e. AlgoConfig.get_values_from_pes(pes) doesn't work, because AlgoConfig needs to look up
+        valid = (1, 2, 3)
-        # cls.algo_config_key.
+        if val not in valid:
-        for d in super(cls, cls).get_values_from_pes(pes):
+            raise ValueError(f'Invalid algorithmID {val!r}, must be one of {valid}')
-            if cls.name in d:
+        return val
-                # convert int to value string
-                val = d[cls.name]
-                d[cls.name] = cls.map_val_to_name(val, strict=True)
-            yield d
 
 class K(BinaryParam, AlgoConfig):
     """use validate_val() from BinaryParam, and apply_val() from AlgoConfig"""
     name = 'K'
     algo_config_key = 'key'
     allow_len = (128 // 8, 256 // 8) # length in bytes (from BinaryParam); TUAK also allows 256 bit
-    example_input = f'00*{allow_len[0]}'
+    example_input = '00' * allow_len[0]
 
 class Opc(K):
     name = 'OPc'
@@ -914,7 +629,6 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):
     algo_config_key = 'rotationConstants'
     allow_len = 5 # length in bytes (from BinaryParam)
     example_input = '40 00 20 40 60'
-    default_source = param_source.ConstantSource
 
     @classmethod
     def validate_val(cls, val):
@@ -945,7 +659,6 @@ class MilenageXoringConstants(BinaryParam, AlgoConfig):
                      ' 00000000000000000000000000000002'
                      ' 00000000000000000000000000000004'
                      ' 00000000000000000000000000000008')
-    default_source = param_source.ConstantSource
 
 class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
     """Number of iterations of Keccak-f[1600] permutation as recomended by Section 7.2 of 3GPP TS 35.231"""
@@ -954,4 +667,3 @@ class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
     min_val = 1
     max_val = 255
     example_input = '1'
-    default_source = param_source.ConstantSource

pySim/filesystem.py

@@ -44,7 +44,6 @@ from pySim.utils import sw_match, decomposeATR
 from pySim.jsonpath import js_path_modify
 from pySim.commands import SimCardCommands
 from pySim.exceptions import SwMatchError
-from pySim.log import PySimLogger
 
 # int: a single service is associated with this file
 # list: any of the listed services requires this file
@@ -53,8 +52,6 @@ CardFileService = Union[int, List[int], Tuple[int, ...]]
 
 Size = Tuple[int, Optional[int]]
 
-log = PySimLogger.get(__name__)
-
 class CardFile:
     """Base class for all objects in the smart card filesystem.
     Serve as a common ancestor to all other file types; rarely used directly.
@@ -1612,14 +1609,14 @@ class CardModel(abc.ABC):
         card_atr = scc.get_atr()
         for atr in cls._atrs:
             if atr == card_atr:
-                log.info("Detected CardModel: %s", cls.__name__)
+                print("Detected CardModel:", cls.__name__)
                 return True
         # if nothing found try to just compare the Historical Bytes of the ATR
         card_atr_hb = decomposeATR(card_atr)['hb']
         for atr in cls._atrs:
             atr_hb = decomposeATR(atr)['hb']
             if atr_hb == card_atr_hb:
-                log.info("Detected CardModel: %s", cls.__name__)
+                print("Detected CardModel:", cls.__name__)
                 return True
         return False
 

pySim/global_platform/scp.py

@@ -27,9 +27,9 @@ from osmocom.utils import b2h
 from osmocom.tlv import bertlv_parse_len, bertlv_encode_len
 from pySim.utils import parse_command_apdu
 from pySim.secure_channel import SecureChannel
-from pySim.log import PySimLogger
 
-log = PySimLogger.get(__name__)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
 
 def scp02_key_derivation(constant: bytes, counter: int, base_key: bytes) -> bytes:
     assert len(constant) == 2
@@ -75,7 +75,7 @@ class Scp02SessionKeys:
             h = e.encrypt(strxor(h, bytes(padded_data[8*i:8*(i+1)])))
         h = d.decrypt(h)
         h = e.encrypt(h)
-        log.debug("mac_1des(%s,icv=%s) -> %s", b2h(data), b2h(icv), b2h(h))
+        logger.debug("mac_1des(%s,icv=%s) -> %s", b2h(data), b2h(icv), b2h(h))
         if self.des_icv_enc:
             self.icv = self.des_icv_enc.encrypt(h)
         else:
@@ -89,7 +89,7 @@ class Scp02SessionKeys:
         h = b'\x00' * 8
         for i in range(q):
             h = e.encrypt(strxor(h, bytes(padded_data[8*i:8*(i+1)])))
-        log.debug("mac_3des(%s) -> %s", b2h(data), b2h(h))
+        logger.debug("mac_3des(%s) -> %s", b2h(data), b2h(h))
         return h
 
     def __init__(self, counter: int, card_keys: 'GpCardKeyset', icv_encrypt=True):
@@ -276,10 +276,10 @@ class SCP02(SCP):
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self, card_challenge: bytes, host_challenge: bytes):
-        log.debug("host_challenge(%s), card_challenge(%s)", b2h(host_challenge), b2h(card_challenge))
+        logger.debug("host_challenge(%s), card_challenge(%s)", b2h(host_challenge), b2h(card_challenge))
         self.host_cryptogram = self.sk.calc_mac_3des(self.sk.counter.to_bytes(2, 'big') + card_challenge + host_challenge)
         self.card_cryptogram = self.sk.calc_mac_3des(self.host_challenge + self.sk.counter.to_bytes(2, 'big') + card_challenge)
-        log.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
+        logger.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
 
     def gen_init_update_apdu(self, host_challenge: bytes = b'\x00'*8) -> bytes:
         """Generate INITIALIZE UPDATE APDU."""
@@ -291,7 +291,7 @@ class SCP02(SCP):
         resp = self.constr_iur.parse(resp_bin)
         self.card_challenge = resp['card_challenge']
         self.sk = Scp02SessionKeys(resp['seq_counter'], self.card_keys)
-        log.debug(self.sk)
+        logger.debug(self.sk)
         self._compute_cryptograms(self.card_challenge, self.host_challenge)
         if self.card_cryptogram != resp['card_cryptogram']:
             raise ValueError("card cryptogram doesn't match")
@@ -311,7 +311,7 @@ class SCP02(SCP):
 
     def _wrap_cmd_apdu(self, apdu: bytes, *args, **kwargs) -> bytes:
         """Wrap Command APDU for SCP02: calculate MAC and encrypt."""
-        log.debug("wrap_cmd_apdu(%s)", b2h(apdu))
+        logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
 
         if not self.do_cmac:
             return apdu
@@ -378,7 +378,7 @@ def scp03_key_derivation(constant: bytes, context: bytes, base_key: bytes, l: Op
     if l is None:
         l = len(base_key) * 8
 
-    log.debug("scp03_kdf(constant=%s, context=%s, base_key=%s, l=%u)", b2h(constant), b2h(context), b2h(base_key), l)
+    logger.debug("scp03_kdf(constant=%s, context=%s, base_key=%s, l=%u)", b2h(constant), b2h(context), b2h(base_key), l)
     output_len = l // 8
     # SCP03 Section 4.1.5 defines a different parameter order than NIST SP 800-108, so we cannot use the
     # existing Cryptodome.Protocol.KDF.SP800_108_Counter function :(
@@ -451,7 +451,7 @@ class Scp03SessionKeys:
         # This block SHALL be encrypted with S-ENC to produce the ICV for command encryption.
         cipher = AES.new(self.s_enc, AES.MODE_CBC, iv)
         icv = cipher.encrypt(data)
-        log.debug("_get_icv(data=%s, is_resp=%s) -> icv=%s", b2h(data), is_response, b2h(icv))
+        logger.debug("_get_icv(data=%s, is_resp=%s) -> icv=%s", b2h(data), is_response, b2h(icv))
         return icv
 
     # TODO: Resolve duplication with pySim.esim.bsp.BspAlgoCryptAES128 which provides pad80-wrapping
@@ -489,12 +489,12 @@ class SCP03(SCP):
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self):
-        log.debug("host_challenge(%s), card_challenge(%s)", b2h(self.host_challenge), b2h(self.card_challenge))
+        logger.debug("host_challenge(%s), card_challenge(%s)", b2h(self.host_challenge), b2h(self.card_challenge))
         # Card + Host Authentication Cryptogram: Section 6.2.2.2 + 6.2.2.3
         context = self.host_challenge + self.card_challenge
         self.card_cryptogram = scp03_key_derivation(self.sk.DERIV_CONST_AUTH_CGRAM_CARD, context, self.sk.s_mac, l=self.s_mode*8)
         self.host_cryptogram = scp03_key_derivation(self.sk.DERIV_CONST_AUTH_CGRAM_HOST, context, self.sk.s_mac, l=self.s_mode*8)
-        log.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
+        logger.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
 
     def gen_init_update_apdu(self, host_challenge: Optional[bytes] = None) -> bytes:
         """Generate INITIALIZE UPDATE APDU."""
@@ -514,7 +514,7 @@ class SCP03(SCP):
         self.i_param = resp['i_param']
         # derive session keys and compute cryptograms
         self.sk = Scp03SessionKeys(self.card_keys, self.host_challenge, self.card_challenge)
-        log.debug(self.sk)
+        logger.debug(self.sk)
         self._compute_cryptograms()
         # verify computed cryptogram matches received cryptogram
         if self.card_cryptogram != resp['card_cryptogram']:
@@ -529,7 +529,7 @@ class SCP03(SCP):
 
     def _wrap_cmd_apdu(self, apdu: bytes, skip_cenc: bool = False) -> bytes:
         """Wrap Command APDU for SCP03: calculate MAC and encrypt."""
-        log.debug("wrap_cmd_apdu(%s)", b2h(apdu))
+        logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
 
         if not self.do_cmac:
             return apdu
@@ -584,7 +584,7 @@ class SCP03(SCP):
         # status word: in this case only the status word shall be returned in the response. All status words
         # except '9000' and warning status words (i.e. '62xx' and '63xx') shall be interpreted as error status
         # words.
-        log.debug("unwrap_rsp_apdu(sw=%s, rsp_apdu=%s)", sw, rsp_apdu)
+        logger.debug("unwrap_rsp_apdu(sw=%s, rsp_apdu=%s)", sw, rsp_apdu)
         if not self.do_rmac:
             assert not self.do_renc
             return rsp_apdu
@@ -600,9 +600,9 @@ class SCP03(SCP):
         if self.do_renc:
             # decrypt response data
             decrypted = self.sk._decrypt(response_data)
-            log.debug("decrypted: %s", b2h(decrypted))
+            logger.debug("decrypted: %s", b2h(decrypted))
             # remove padding
             response_data = unpad80(decrypted)
-            log.debug("response_data: %s", b2h(response_data))
+            logger.debug("response_data: %s", b2h(response_data))
 
         return response_data

pySim/global_platform/uicc.py

@@ -91,7 +91,6 @@ class UiccSdInstallParams(TLV_IE_Collection, nested=[UiccScp, AcceptExtradAppsAn
 
 # Key Usage:
 # KVN 0x01 .. 0x0F reserved for SCP80
-# KVN 0x81 .. 0x8f reserved for SCP81
 # KVN 0x11 reserved for DAP specified in ETSI TS 102 226
 # KVN 0x20 .. 0x2F reserved for SCP02
 #   KID 0x01 = ENC; 0x02 = MAC; 0x03 = DEK

pySim/legacy/cards.py

@@ -152,8 +152,7 @@ class SimCard(SimCardBase):
         return sw
 
     def update_smsp(self, smsp):
-        print("using update_smsp")
+        data, sw = self._scc.update_record(EF['SMSP'], 1, rpad(smsp, 84))
-        data, sw = self._scc.update_record(EF['SMSP'], 1, smsp, leftpad=True)
         return sw
 
     def update_ad(self, mnc=None, opmode=None, ofm=None, path=EF['AD']):

pySim/log.py

@@ -44,7 +44,7 @@ class PySimLogger:
     """
 
     LOG_FMTSTR = "%(levelname)s: %(message)s"
-    LOG_FMTSTR_VERBOSE = "%(name)s.%(lineno)d -- " + LOG_FMTSTR
+    LOG_FMTSTR_VERBOSE = "%(module)s.%(lineno)d -- " + LOG_FMTSTR
     __formatter = logging.Formatter(LOG_FMTSTR)
     __formatter_verbose = logging.Formatter(LOG_FMTSTR_VERBOSE)
 

pySim/transport/__init__.py

@@ -307,6 +307,7 @@ class LinkBaseTpdu(LinkBase):
 
         # After sending the APDU/TPDU the UICC/eUICC or SIM may response with a status word that indicates that further
         # TPDUs have to be sent in order to complete the task.
+        if sw is not None:
             if case == 4 or self.apdu_strict == False:
                 # In case the APDU is a case #4 APDU, the UICC/eUICC/SIM may indicate that there is response data
                 # available which has to be retrieved using a GET RESPONSE command TPDU.

pySim/ts_51_011.py

@@ -251,16 +251,6 @@ class EF_SMSP(LinFixedEF):
                                                       "numbering_plan_id": "isdn_e164" },
                             "call_number": "4915790109999" },
             "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
-        ( 'e1ffffffffffffffffffffffff0891945197109099f9ffffff0000a9',
-          { "alpha_id": "", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
-                                                          "tp_pid": True, "tp_dcs": True, "tp_vp": True },
-            "tp_dest_addr": { "length": 255, "ton_npi": { "ext": True, "type_of_number": "reserved_for_extension",
-                                                          "numbering_plan_id": "reserved_for_extension" },
-                              "call_number": "" },
-            "tp_sc_addr": { "length": 8, "ton_npi": { "ext": True, "type_of_number": "international",
-                                                      "numbering_plan_id": "isdn_e164" },
-                            "call_number": "4915790109999" },
-            "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
         ( '454e6574776f726b73fffffffffffffff1ffffffffffffffffffffffffffffffffffffffffffffffff0000a7',
           { "alpha_id": "ENetworks", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
                                                                "tp_pid": True, "tp_dcs": True, "tp_vp": False },
@@ -341,8 +331,7 @@ class EF_SMSP(LinFixedEF):
                         'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
         DestAddr = Struct('length'/Rebuild(Int8ub, lambda ctx: EF_SMSP.dest_addr_len(ctx)),
                         'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
-        # (see comment below)
+        self._construct = Struct('alpha_id'/COptional(GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28)))),
-        self._construct = Struct('alpha_id'/GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28))),
                                  'parameter_indicators'/InvertAdapter(BitStruct(
                                                                         Const(7, BitsInteger(3)),
                                                                         'tp_vp'/Flag,
@@ -356,25 +345,6 @@ class EF_SMSP(LinFixedEF):
                                  'tp_dcs'/Bytes(1),
                                  'tp_vp_minutes'/EF_SMSP.ValidityPeriodAdapter(Byte))
 
-    # Ensure 'alpha_id' is always present
-    def encode_record_hex(self, abstract_data: dict, record_nr: int, total_len: int = None) -> str:
-        # Problem: TS 51.011 Section 10.5.6 describes the 'alpha_id' field as optional. However, this is only true
-        # at the time when the record length of the file is set up in the file system. A card manufacturer may decide
-        # to remove the field by setting the record length to 28. Likewise, the card manaufacturer may also decide to
-        # set the field to a distinct length by setting the record length to a value greater than 28 (e.g. 14 bytes
-        # 'alpha_id' + 28 bytes). Due to the fixed nature of the record length, this eventually means that in practice
-        # 'alpha_id' is a mandatory field with a fixed length.
-        #
-        # Due to the problematic specification of 'alpha_id' as a pseudo-optional field at the beginning of a
-        # fixed-size memory, the construct definition in self._construct has been incorrectly implemented and the field
-        # has been marked as COptional. We may correct the problem by removing COptional. But to maintain compatibility,
-        # we then have to ensure that in case the field is not provided (None), it is set to an empty string ('').
-        #
-        # See also ts_31_102.py, class EF_OCI for a correct example.
-        if abstract_data['alpha_id'] is None:
-            abstract_data['alpha_id'] = ''
-        return super().encode_record_hex(abstract_data, record_nr, total_len)
-
 # TS 51.011 Section 10.5.7
 class EF_SMSS(TransparentEF):
     class MemCapAdapter(Adapter):

requirements.txt

@@ -6,7 +6,7 @@ jsonpath-ng
 construct>=2.10.70
 bidict
 pyosmocom>=0.0.12
-pyyaml>=5.4
+pyyaml>=5.1
 termcolor
 colorlog
 pycryptodomex

setup.py

@@ -26,7 +26,7 @@ setup(
         "construct >= 2.10.70",
         "bidict",
         "pyosmocom >= 0.0.12",
-        "pyyaml >= 5.4",
+        "pyyaml >= 5.1",
         "termcolor",
         "colorlog",
         "pycryptodomex",

tests/pySim-prog_test/Fairwaves-SIM.ok

@@ -5,7 +5,7 @@ ICCID: 8988219000000117833
 IMSI: 001010000000111
 GID1: ffffffffffffffff
 GID2: ffffffffffffffff
-SMSP: ffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
+SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
 SMSC: 0015555
 SPN: Fairwaves
 Show in HPLMN: False

tests/pySim-prog_test/Wavemobile-SIM.ok

@@ -5,7 +5,7 @@ ICCID: 89445310150011013678
 IMSI: 001010000000102
 GID1: Can't read file -- SW match failed! Expected 9000 and got 6a82.
 GID2: Can't read file -- SW match failed! Expected 9000 and got 6a82.
-SMSP: ffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
+SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
 SMSC: 0015555
 SPN: wavemobile
 Show in HPLMN: False

tests/pySim-smpp2sim_test/pySim-smpp2sim_test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Utility to verify the functionality of pySim-smpp2sim.py
+# Utility to verify the functionality of pySim-trace.py
 #
 # (C) 2026 by sysmocom - s.f.m.c. GmbH
 # All Rights Reserved

tests/unittests/test_card_key_provider.py

@@ -20,8 +20,7 @@ class TestCardKeyProviderCsv(unittest.TestCase):
                        "KIK3" : "00010204040506070809488B0C0D0E0F"}
 
         csv_file_path = os.path.dirname(os.path.abspath(__file__)) + "/test_card_key_provider.csv"
-        card_key_field_cryptor = CardKeyFieldCryptor(column_keys)
+        card_key_provider_register(CardKeyProviderCsv(csv_file_path, column_keys))
-        card_key_provider_register(CardKeyProviderCsv(csv_file_path, card_key_field_cryptor))
         super().__init__(*args, **kwargs)
 
     def test_card_key_provider_get(self):