Allow saving unknown files with raw data

When a file has no custom encoder, the decode path returns {'raw': ...} but the encode path raised NotImplementedError. Add a fallback in all encoder methods to return the raw data directly when present in the abstract_data dict, allowing unknown files to round-trip.
pySim.log: fix E0611: No name 'style' in module 'cmd2'
2026-07-13 22:45:42 +03:00 · 2026-07-04 11:48:53 +03:00 · 2026-07-01 16:47:08 +07:00 · 2026-06-26 02:47:45 +07:00 · 2026-06-25 22:51:03 +07:00 · 2026-06-25 21:43:52 +07:00
15 changed files with 3318 additions and 3770 deletions
@@ -24,21 +24,21 @@ import traceback
 import re
 import cmd2
 from packaging import version
-from cmd2 import style
-
 import logging
 from pySim.log import PySimLogger
 from osmocom.utils import auto_uint8

-# cmd2 >= 2.3.0 has deprecated the bg/fg in favor of Bg/Fg :(
-if version.parse(cmd2.__version__) < version.parse("2.3.0"):
-    from cmd2 import fg, bg # pylint: disable=no-name-in-module
-    RED = fg.red
-    YELLOW = fg.yellow
-    LIGHT_RED = fg.bright_red
-    LIGHT_GREEN = fg.bright_green
+# cmd2 >= 3.0 replaced Fg + style() with Color + stylize()
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
+    from cmd2 import Color, stylize # pylint: disable=no-name-in-module
+    RED = Color.RED
+    YELLOW = Color.YELLOW
+    LIGHT_RED = Color.BRIGHT_RED
+    LIGHT_GREEN = Color.BRIGHT_GREEN
+    def style(text, fg=None, bg=None, bold=False): # pylint: disable=function-redefined
+        return stylize(text, fg) if fg else text
 else:
-    from cmd2 import Fg, Bg # pylint: disable=no-name-in-module
+    from cmd2 import style, Fg # pylint: disable=no-name-in-module
    RED = Fg.RED
    YELLOW = Fg.YELLOW
    LIGHT_RED = Fg.LIGHT_RED
@@ -76,43 +76,19 @@ from pySim.app import init_card

 log = PySimLogger.get(Path(__file__).stem)

-class Cmd2Compat(cmd2.Cmd):
-    """Backwards-compatibility wrapper around cmd2.Cmd to support older and newer
-    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
-    def run_editor(self, file_path: Optional[str] = None) -> None:
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            return self._run_editor(file_path) # pylint: disable=no-member
-        else:
-            return super().run_editor(file_path) # pylint: disable=no-member
-
-class Settable2Compat(cmd2.Settable):
-    """Backwards-compatibility wrapper around cmd2.Settable to support older and newer
-    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
-    def __init__(self, name, val_type, description, settable_object, **kwargs):
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            super().__init__(name, val_type, description, **kwargs) # pylint: disable=no-value-for-parameter
-        else:
-            super().__init__(name, val_type, description, settable_object, **kwargs) # pylint: disable=too-many-function-args
-
-class PysimApp(Cmd2Compat):
+class PysimApp(cmd2.Cmd):
    CUSTOM_CATEGORY = 'pySim Commands'
    BANNER = """Welcome to pySim-shell!
 (C) 2021-2023 by Harald Welte, sysmocom - s.f.m.c. GmbH and contributors
 Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/shell.html """

    def __init__(self, verbose, card, rs, sl, ch, script=None):
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            kwargs = {'use_ipython': True}
-        else:
-            kwargs = {'include_ipy': True}
-
        self.verbose = verbose
        PySimLogger.setup(self.poutput, {logging.WARN: YELLOW})
        self._onchange_verbose('verbose', False, self.verbose)

-        # pylint: disable=unexpected-keyword-arg
        super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
-                         auto_load_commands=False, startup_script=script, **kwargs)
+                         auto_load_commands=False, startup_script=script, include_ipy=True)
        self.intro = style(self.BANNER, fg=RED)
        self.default_category = 'pySim-shell built-in commands'
        self.card = None
@@ -128,18 +104,24 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
        self.apdu_trace = False
        self.apdu_strict = False

-        self.add_settable(Settable2Compat('numeric_path', bool, 'Print File IDs instead of names', self,
-                                          onchange_cb=self._onchange_numeric_path))
-        self.add_settable(Settable2Compat('conserve_write', bool, 'Read and compare before write', self,
-                                          onchange_cb=self._onchange_conserve_write))
-        self.add_settable(Settable2Compat('json_pretty_print', bool, 'Pretty-Print JSON output', self))
-        self.add_settable(Settable2Compat('apdu_trace', bool, 'Trace and display APDUs exchanged with card', self,
-                                          onchange_cb=self._onchange_apdu_trace))
-        self.add_settable(Settable2Compat('apdu_strict', bool,
-                                          'Strictly apply APDU format according to ISO/IEC 7816-3, table 12', self))
-        self.add_settable(Settable2Compat('verbose', bool,
-                                          'Enable/disable verbose logging', self,
-                                          onchange_cb=self._onchange_verbose))
+        self.add_settable(cmd2.Settable('numeric_path', bool,
+                                        'Print File IDs instead of names',
+                                        self, onchange_cb=self._onchange_numeric_path))
+        self.add_settable(cmd2.Settable('conserve_write', bool,
+                                        'Read and compare before write',
+                                        self, onchange_cb=self._onchange_conserve_write))
+        self.add_settable(cmd2.Settable('json_pretty_print', bool,
+                                        'Pretty-Print JSON output',
+                                        self))
+        self.add_settable(cmd2.Settable('apdu_trace', bool,
+                                        'Trace and display APDUs exchanged with card',
+                                        self, onchange_cb=self._onchange_apdu_trace))
+        self.add_settable(cmd2.Settable('apdu_strict', bool,
+                                        'Strictly apply APDU format according to ISO/IEC 7816-3, table 12',
+                                        self))
+        self.add_settable(cmd2.Settable('verbose', bool,
+                                        'Enable/disable verbose logging',
+                                        self, onchange_cb=self._onchange_verbose))
        self.equip(card, rs)

    def equip(self, card, rs):
@@ -16,6 +16,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

 import requests
+from klein import Klein
+from twisted.internet import defer, protocol, ssl, task, endpoints, reactor
+from twisted.internet.posixbase import PosixReactorBase
+from pathlib import Path
+from twisted.web.server import Site, Request
+
 import logging
 from datetime import datetime
 import time
@@ -123,10 +129,12 @@ class Es2PlusApiFunction(JsonHttpApiFunction):
 class DownloadOrder(Es2PlusApiFunction):
    path = '/gsma/rsp2/es2plus/downloadOrder'
    input_params = {
+        'header': JsonRequestHeader,
        'eid': param.Eid,
        'iccid': param.Iccid,
        'profileType': param.ProfileType
      }
+    input_mandatory = ['header']
    output_params = {
        'header': JsonResponseHeader,
        'iccid': param.Iccid,
@@ -137,6 +145,7 @@ class DownloadOrder(Es2PlusApiFunction):
 class ConfirmOrder(Es2PlusApiFunction):
    path = '/gsma/rsp2/es2plus/confirmOrder'
    input_params = {
+        'header': JsonRequestHeader,
        'iccid': param.Iccid,
        'eid': param.Eid,
        'matchingId': param.MatchingId,
@@ -144,7 +153,7 @@ class ConfirmOrder(Es2PlusApiFunction):
        'smdsAddress': param.SmdsAddress,
        'releaseFlag': param.ReleaseFlag,
      }
-    input_mandatory = ['iccid', 'releaseFlag']
+    input_mandatory = ['header', 'iccid', 'releaseFlag']
    output_params = {
        'header': JsonResponseHeader,
        'eid': param.Eid,
@@ -157,12 +166,13 @@ class ConfirmOrder(Es2PlusApiFunction):
 class CancelOrder(Es2PlusApiFunction):
    path = '/gsma/rsp2/es2plus/cancelOrder'
    input_params = {
+        'header': JsonRequestHeader,
        'iccid': param.Iccid,
        'eid': param.Eid,
        'matchingId': param.MatchingId,
        'finalProfileStatusIndicator': param.FinalProfileStatusIndicator,
      }
-    input_mandatory = ['finalProfileStatusIndicator', 'iccid']
+    input_mandatory = ['header', 'finalProfileStatusIndicator', 'iccid']
    output_params = {
        'header': JsonResponseHeader,
      }
@@ -172,9 +182,10 @@ class CancelOrder(Es2PlusApiFunction):
 class ReleaseProfile(Es2PlusApiFunction):
    path = '/gsma/rsp2/es2plus/releaseProfile'
    input_params = {
+        'header': JsonRequestHeader,
        'iccid': param.Iccid,
      }
-    input_mandatory = ['iccid']
+    input_mandatory = ['header', 'iccid']
    output_params = {
        'header': JsonResponseHeader,
      }
@@ -184,6 +195,7 @@ class ReleaseProfile(Es2PlusApiFunction):
 class HandleDownloadProgressInfo(Es2PlusApiFunction):
    path = '/gsma/rsp2/es2plus/handleDownloadProgressInfo'
    input_params = {
+        'header': JsonRequestHeader,
        'eid': param.Eid,
        'iccid': param.Iccid,
        'profileType': param.ProfileType,
@@ -192,10 +204,9 @@ class HandleDownloadProgressInfo(Es2PlusApiFunction):
        'notificationPointStatus': param.NotificationPointStatus,
        'resultData': param.ResultData,
    }
-    input_mandatory = ['iccid', 'profileType', 'timestamp', 'notificationPointId', 'notificationPointStatus']
+    input_mandatory = ['header', 'iccid', 'profileType', 'timestamp', 'notificationPointId', 'notificationPointStatus']
    expected_http_status = 204

-
 class Es2pApiClient:
    """Main class representing a full ES2+ API client. Has one method for each API function."""
    def __init__(self, url_prefix:str, func_req_id:str, server_cert_verify: str = None, client_cert: str = None):
@@ -206,18 +217,17 @@ class Es2pApiClient:
        if client_cert:
            self.session.cert = client_cert

-        self.downloadOrder = DownloadOrder(url_prefix, func_req_id, self.session)
-        self.confirmOrder = ConfirmOrder(url_prefix, func_req_id, self.session)
-        self.cancelOrder = CancelOrder(url_prefix, func_req_id, self.session)
-        self.releaseProfile = ReleaseProfile(url_prefix, func_req_id, self.session)
-        self.handleDownloadProgressInfo = HandleDownloadProgressInfo(url_prefix, func_req_id, self.session)
+        self.downloadOrder = JsonHttpApiClient(DownloadOrder(), url_prefix, func_req_id, self.session)
+        self.confirmOrder = JsonHttpApiClient(ConfirmOrder(), url_prefix, func_req_id, self.session)
+        self.cancelOrder = JsonHttpApiClient(CancelOrder(), url_prefix, func_req_id, self.session)
+        self.releaseProfile = JsonHttpApiClient(ReleaseProfile(), url_prefix, func_req_id, self.session)
+        self.handleDownloadProgressInfo = JsonHttpApiClient(HandleDownloadProgressInfo(), url_prefix, func_req_id, self.session)

    def _gen_func_id(self) -> str:
        """Generate the next function call id."""
        self.func_id += 1
        return 'FCI-%u-%u' % (time.time(), self.func_id)

-
    def call_downloadOrder(self, data: dict) -> dict:
        """Perform ES2+ DownloadOrder function (SGP.22 section 5.3.1)."""
        return self.downloadOrder.call(data, self._gen_func_id())
@@ -237,3 +247,116 @@ class Es2pApiClient:
    def call_handleDownloadProgressInfo(self, data: dict) -> dict:
        """Perform ES2+ HandleDownloadProgressInfo function (SGP.22 section 5.3.5)."""
        return self.handleDownloadProgressInfo.call(data, self._gen_func_id())
+
+class Es2pApiServerHandlerSmdpp(abc.ABC):
+    """ES2+ (SMDP+ side) API Server handler class. The API user is expected to override the contained methods."""
+
+    @abc.abstractmethod
+    def call_downloadOrder(self, data: dict) -> (dict, str):
+        """Perform ES2+ DownloadOrder function (SGP.22 section 5.3.1)."""
+        pass
+
+    @abc.abstractmethod
+    def call_confirmOrder(self, data: dict) -> (dict, str):
+        """Perform ES2+ ConfirmOrder function (SGP.22 section 5.3.2)."""
+        pass
+
+    @abc.abstractmethod
+    def call_cancelOrder(self, data: dict) -> (dict, str):
+        """Perform ES2+ CancelOrder function (SGP.22 section 5.3.3)."""
+        pass
+
+    @abc.abstractmethod
+    def call_releaseProfile(self, data: dict) -> (dict, str):
+        """Perform ES2+ CancelOrder function (SGP.22 section 5.3.4)."""
+        pass
+
+class Es2pApiServerHandlerMno(abc.ABC):
+    """ES2+ (MNO side) API Server handler class. The API user is expected to override the contained methods."""
+
+    @abc.abstractmethod
+    def call_handleDownloadProgressInfo(self, data: dict) -> (dict, str):
+        """Perform ES2+ HandleDownloadProgressInfo function (SGP.22 section 5.3.5)."""
+        pass
+
+class Es2pApiServer(abc.ABC):
+    """Main class representing a full ES2+ API server. Has one method for each API function."""
+    app = None
+
+    def __init__(self, port: int, interface: str, server_cert: str = None, client_cert_verify: str = None):
+        logger.debug("HTTP SRV: starting ES2+ API server on %s:%s" % (interface, port))
+        self.port = port
+        self.interface = interface
+        if server_cert:
+            self.server_cert = ssl.PrivateCertificate.loadPEM(Path(server_cert).read_text())
+        else:
+            self.server_cert = None
+        if client_cert_verify:
+            self.client_cert_verify = ssl.Certificate.loadPEM(Path(client_cert_verify).read_text())
+        else:
+            self.client_cert_verify = None
+
+    def reactor(self, reactor: PosixReactorBase):
+        logger.debug("HTTP SRV: listen on %s:%s" % (self.interface, self.port))
+        if self.server_cert:
+            if self.client_cert_verify:
+                reactor.listenSSL(self.port, Site(self.app.resource()), self.server_cert.options(self.client_cert_verify),
+                                  interface=self.interface)
+            else:
+                reactor.listenSSL(self.port, Site(self.app.resource()), self.server_cert.options(),
+                                  interface=self.interface)
+        else:
+            reactor.listenTCP(self.port, Site(self.app.resource()), interface=self.interface)
+        return defer.Deferred()
+
+class Es2pApiServerSmdpp(Es2pApiServer):
+    """ES2+ (SMDP+ side) API Server."""
+    app = Klein()
+
+    def __init__(self, port: int, interface: str, handler: Es2pApiServerHandlerSmdpp,
+                 server_cert: str = None, client_cert_verify: str = None):
+        super().__init__(port, interface, server_cert, client_cert_verify)
+        self.handler = handler
+        self.downloadOrder = JsonHttpApiServer(DownloadOrder(), handler.call_downloadOrder)
+        self.confirmOrder = JsonHttpApiServer(ConfirmOrder(), handler.call_confirmOrder)
+        self.cancelOrder = JsonHttpApiServer(CancelOrder(), handler.call_cancelOrder)
+        self.releaseProfile = JsonHttpApiServer(ReleaseProfile(), handler.call_releaseProfile)
+        task.react(self.reactor)
+
+    @app.route(DownloadOrder.path)
+    def call_downloadOrder(self, request: Request) -> dict:
+        """Perform ES2+ DownloadOrder function (SGP.22 section 5.3.1)."""
+        return self.downloadOrder.call(request)
+
+    @app.route(ConfirmOrder.path)
+    def call_confirmOrder(self, request: Request) -> dict:
+        """Perform ES2+ ConfirmOrder function (SGP.22 section 5.3.2)."""
+        return self.confirmOrder.call(request)
+
+    @app.route(CancelOrder.path)
+    def call_cancelOrder(self, request: Request) -> dict:
+        """Perform ES2+ CancelOrder function (SGP.22 section 5.3.3)."""
+        return self.cancelOrder.call(request)
+
+    @app.route(ReleaseProfile.path)
+    def call_releaseProfile(self, request: Request) -> dict:
+        """Perform ES2+ CancelOrder function (SGP.22 section 5.3.4)."""
+        return self.releaseProfile.call(request)
+
+class Es2pApiServerMno(Es2pApiServer):
+    """ES2+ (MNO side) API Server."""
+
+    app = Klein()
+
+    def __init__(self, port: int, interface: str, handler: Es2pApiServerHandlerMno,
+                 server_cert: str = None, client_cert_verify: str = None):
+        super().__init__(port, interface, server_cert, client_cert_verify)
+        self.handler = handler
+        self.handleDownloadProgressInfo = JsonHttpApiServer(HandleDownloadProgressInfo(),
+                                                            handler.call_handleDownloadProgressInfo)
+        task.react(self.reactor)
+
+    @app.route(HandleDownloadProgressInfo.path)
+    def call_handleDownloadProgressInfo(self, request: Request) -> dict:
+        """Perform ES2+ HandleDownloadProgressInfo function (SGP.22 section 5.3.5)."""
+        return self.handleDownloadProgressInfo.call(request)
@@ -155,11 +155,11 @@ class Es9pApiClient:
        if server_cert_verify:
            self.session.verify = server_cert_verify

-        self.initiateAuthentication = InitiateAuthentication(url_prefix, '', self.session)
-        self.authenticateClient = AuthenticateClient(url_prefix, '', self.session)
-        self.getBoundProfilePackage = GetBoundProfilePackage(url_prefix, '', self.session)
-        self.handleNotification = HandleNotification(url_prefix, '', self.session)
-        self.cancelSession = CancelSession(url_prefix, '', self.session)
+        self.initiateAuthentication = JsonHttpApiClient(InitiateAuthentication(), url_prefix, '', self.session)
+        self.authenticateClient = JsonHttpApiClient(AuthenticateClient(), url_prefix, '', self.session)
+        self.getBoundProfilePackage = JsonHttpApiClient(GetBoundProfilePackage(), url_prefix, '', self.session)
+        self.handleNotification = JsonHttpApiClient(HandleNotification(), url_prefix, '', self.session)
+        self.cancelSession = JsonHttpApiClient(CancelSession(), url_prefix, '', self.session)

    def call_initiateAuthentication(self, data: dict) -> dict:
        return self.initiateAuthentication.call(data)
@@ -19,8 +19,10 @@ import abc
 import requests
 import logging
 import json
-from typing import Optional
+from typing import Optional, Tuple
 import base64
+from twisted.web.server import Request
+

 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -131,6 +133,16 @@ class JsonResponseHeader(ApiParam):
        if status not in ['Executed-Success', 'Executed-WithWarning', 'Failed', 'Expired']:
            raise ValueError('Unknown/unspecified status "%s"' % status)

+class JsonRequestHeader(ApiParam):
+    """SGP.22 section 6.5.1.3."""
+    @classmethod
+    def verify_decoded(cls, data):
+        func_req_id = data.get('functionRequesterIdentifier')
+        if not func_req_id:
+            raise ValueError('Missing mandatory functionRequesterIdentifier in header')
+        func_call_id = data.get('functionCallIdentifier')
+        if not func_call_id:
+            raise ValueError('Missing mandatory functionCallIdentifier in header')

 class HttpStatusError(Exception):
    pass
@@ -161,65 +173,118 @@ class ApiError(Exception):

 class JsonHttpApiFunction(abc.ABC):
    """Base class for representing an HTTP[s] API Function."""
-     # the below class variables are expected to be overridden in derived classes
+    # The below class variables are used to describe the properties of the API function. Derived classes are expected
+    # to orverride those class properties with useful values. The prefixes "input_" and "output_" refer to the API
+    # function from an abstract point of view. Seen from the client perspective, "input_" will refer to parameters the
+    # client sends to a HTTP server. Seen from the server perspective, "input_" will refer to parameters the server
+    # receives from the a requesting client. The same applies vice versa to class variables that have an "output_"
+    # prefix.

+    # path of the API function (e.g. '/gsma/rsp2/es2plus/confirmOrder', see also method rewrite_url).
    path = None
+
    # dictionary of input parameters. key is parameter name, value is ApiParam class
    input_params = {}
+
    # list of mandatory input parameters
    input_mandatory = []
+
    # dictionary of output parameters. key is parameter name, value is ApiParam class
    output_params = {}
+
    # list of mandatory output parameters (for successful response)
    output_mandatory = []
+
+    # list of mandatory output parameters (for failed response)
+    output_mandatory_failed = []
+
    # expected HTTP status code of the response
    expected_http_status = 200
+
    # the HTTP method used (GET, OPTIONS, HEAD, POST, PUT, PATCH or DELETE)
    http_method = 'POST'
+
+    # additional custom HTTP headers (client requests)
    extra_http_req_headers = {}

-    def __init__(self, url_prefix: str, func_req_id: Optional[str], session: requests.Session):
-        self.url_prefix = url_prefix
-        self.func_req_id = func_req_id
-        self.session = session
+    # additional custom HTTP headers (server responses)
+    extra_http_res_headers = {}

-    def encode(self, data: dict, func_call_id: Optional[str] = None) -> dict:
+    def __new__(cls, *args, role = 'legacy_client', **kwargs):
+        """
+        Args:
+                args: (see JsonHttpApiClient and JsonHttpApiServer)
+                role: role ('server' or 'client') in which the JsonHttpApiFunction should be created.
+                kwargs: (see JsonHttpApiClient and JsonHttpApiServer)
+        """
+
+        # Create a dictionary with the class attributes of this class (the properties listed above and the encode_
+        # decode_ methods below). The dictionary will not include any dunder/magic methods
+        cls_attr = {attr_name: getattr(cls, attr_name) for attr_name in dir(cls) if not attr_name.startswith('__')}
+
+        # Normal instantiation as JsonHttpApiFunction:
+        if len(args) == 0 and len(kwargs) == 0:
+            return type(cls.__name__, (abc.ABC,), cls_attr)()
+
+        # Instantiation as as JsonHttpApiFunction with a JsonHttpApiClient or JsonHttpApiServer base
+        if role == 'legacy_client':
+            # Deprecated: With the advent of the server role (JsonHttpApiServer) the API had to be changed. To maintain
+            # compatibility with existing code (out-of-tree) the original behaviour and API interface and behaviour had
+            # to be preserved. Already existing JsonHttpApiFunction definitions will still work and the related objects
+            # may still be created on the original way: my_api_func = MyApiFunc(url_prefix, func_req_id, self.session)
+            logger.warning('implicit role (falling back to legacy JsonHttpApiClient) is deprecated, please specify role explcitly')
+            result = type(cls.__name__, (JsonHttpApiClient,), cls_attr)(None, *args, **kwargs)
+            result.api_func = result
+            result.legacy = True
+            return result
+        elif role == 'client':
+            # Create a JsonHttpApiFunction in client role
+            # Example: my_api_func = MyApiFunc(url_prefix, func_req_id, self.session, role='client')
+            result = type(cls.__name__, (JsonHttpApiClient,), cls_attr)(None, *args, **kwargs)
+            result.api_func = result
+            return result
+        elif role == 'server':
+            # Create a JsonHttpApiFunction in server role
+            # Example: my_api_func = MyApiFunc(url_prefix, func_req_id, self.session, role='server')
+            result = type(cls.__name__, (JsonHttpApiServer,), cls_attr)(None, *args, **kwargs)
+            result.api_func = result
+            return result
+        else:
+            raise ValueError('Invalid role \'%s\' specified' % role)
+
+    def encode_client(self, data: dict) -> dict:
        """Validate an encode input dict into JSON-serializable dict for request body."""
        output = {}
-        if func_call_id:
-            output['header'] = {
-                    'functionRequesterIdentifier': self.func_req_id,
-                    'functionCallIdentifier': func_call_id
-                }
-
        for p in self.input_mandatory:
            if not p in data:
                raise ValueError('Mandatory input parameter %s missing' % p)
        for p, v in data.items():
            p_class = self.input_params.get(p)
            if not p_class:
-                logger.warning('Unexpected/unsupported input parameter %s=%s', p, v)
-                output[p] = v
+                # pySim/esim/http_json_api.py:269:47: E1101: Instance of 'JsonHttpApiFunction' has no 'legacy' member (no-member)
+                # pylint: disable=no-member
+                if hasattr(self, 'legacy') and self.legacy:
+                    output[p] = JsonRequestHeader.encode(v)
+                else:
+                    logger.warning('Unexpected/unsupported input parameter %s=%s', p, v)
+                    output[p] = v
            else:
                output[p] = p_class.encode(v)
        return output

-    def decode(self, data: dict) -> dict:
+    def decode_client(self, data: dict) -> dict:
        """[further] Decode and validate the JSON-Dict of the response body."""
        output = {}
-        if 'header' in self.output_params:
-            # let's first do the header, it's special
-            if not 'header' in data:
-                raise ValueError('Mandatory output parameter "header" missing')
-            hdr_class = self.output_params.get('header')
-            output['header'] = hdr_class.decode(data['header'])
+        output_mandatory = self.output_mandatory

-            if output['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
-                raise ApiError(output['header']['functionExecutionStatus'])
-        # we can only expect mandatory parameters to be present in case of successful execution
-        for p in self.output_mandatory:
-            if p == 'header':
-                continue
+        # In case a provided header (may be optional) indicates that the API function call was unsuccessful, a
+        # different set of mandatory parameters applies.
+        header = data.get('header')
+        if header:
+            if data['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
+                output_mandatory = self.output_mandatory_failed
+
+        for p in output_mandatory:
            if not p in data:
                raise ValueError('Mandatory output parameter "%s" missing' % p)
        for p, v in data.items():
@@ -231,37 +296,195 @@ class JsonHttpApiFunction(abc.ABC):
                output[p] = p_class.decode(v)
        return output

+    def encode_server(self, data: dict) -> dict:
+        """Validate an encode input dict into JSON-serializable dict for response body."""
+        output = {}
+        output_mandatory = self.output_mandatory
+
+        # In case a provided header (may be optional) indicates that the API function call was unsuccessful, a
+        # different set of mandatory parameters applies.
+        header = data.get('header')
+        if header:
+            if data['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
+                output_mandatory = self.output_mandatory_failed
+
+        for p in output_mandatory:
+            if not p in data:
+                raise ValueError('Mandatory output parameter %s missing' % p)
+        for p, v in data.items():
+            p_class = self.output_params.get(p)
+            if not p_class:
+                logger.warning('Unexpected/unsupported output parameter %s=%s', p, v)
+                output[p] = v
+            else:
+                output[p] = p_class.encode(v)
+        return output
+
+    def decode_server(self, data: dict) -> dict:
+        """[further] Decode and validate the JSON-Dict of the request body."""
+        output = {}
+
+        for p in self.input_mandatory:
+            if not p in data:
+                raise ValueError('Mandatory input parameter "%s" missing' % p)
+        for p, v in data.items():
+            p_class = self.input_params.get(p)
+            if not p_class:
+                logger.warning('Unexpected/unsupported input parameter "%s"="%s"', p, v)
+                output[p] = v
+            else:
+                output[p] = p_class.decode(v)
+        return output
+
+    def rewrite_url(self, data: dict, url: str) -> Tuple[dict, str]:
+        """
+        Rewrite a static URL using information passed in the data dict. This method may be overloaded by a derived
+        class to allow fully dynamic URLs. The input parameters required for the URL rewriting may be passed using
+        data parameter. In case those parameters are additional parameters that are not intended to be passed to
+        the encode_client method later, they must be removed explcitly.
+
+        Args:
+                data: (see JsonHttpApiClient and JsonHttpApiServer)
+                url: statically generated URL string (see comment in JsonHttpApiClient)
+        """
+
+        # This implementation is a placeholder in which we do not perform any URL rewriting. We just pass through data
+        # and url unmodified.
+        return data, url
+
+class JsonHttpApiClient():
+    def __init__(self, api_func: JsonHttpApiFunction, url_prefix: str, func_req_id: Optional[str],
+                 session: requests.Session):
+        """
+        Args:
+                api_func : API function definition (JsonHttpApiFunction)
+                url_prefix : prefix to be put in front of the API function path (see JsonHttpApiFunction)
+                func_req_id : function requestor id to use for requests
+                session : session object (requests)
+        """
+        self.api_func = api_func
+        self.url_prefix = url_prefix
+        self.func_req_id = func_req_id
+        self.session = session
+
    def call(self, data: dict, func_call_id: Optional[str] = None, timeout=10) -> Optional[dict]:
-        """Make an API call to the HTTP API endpoint represented by this object.
-        Input data is passed in `data` as json-serializable dict.  Output data
-        is returned as json-deserialized dict."""
-        url = self.url_prefix + self.path
-        encoded = json.dumps(self.encode(data, func_call_id))
+        """
+        Make an API call to the HTTP API endpoint represented by this object. Input data is passed in `data` as
+        json-serializable fields. `data` may also contain additional parameters required for URL rewriting (see
+        rewrite_url in class JsonHttpApiFunction). Output data is returned as json-deserialized dict.
+
+        Args:
+                data: Input data required to perform the request.
+                func_call_id: Function Call Identifier, if present a header field is generated automatically.
+                timeout: Maximum amount of time to wait for the request to complete.
+        """
+
+        # In case a function caller ID is supplied, use it together with the stored function requestor ID to generate
+        # and prepend the header field according to SGP.22, section 6.5.1.1 and 6.5.1.3. (the presence of the header
+        # field is checked by the encode_client method)
+        if func_call_id:
+            data = {'header' : {'functionRequesterIdentifier': self.func_req_id,
+                                'functionCallIdentifier': func_call_id}} | data
+
+        # The URL used for the HTTP request (see below) normally consists of the initially given url_prefix
+        # concatenated with the path defined by the JsonHttpApiFunction definition. This static URL path may be
+        # rewritten by rewrite_url method defined in the JsonHttpApiFunction.
+        data, url = self.api_func.rewrite_url(data, self.url_prefix + self.api_func.path)
+
+        # Encode the message (the presence of mandatory fields is checked during encoding)
+        encoded = json.dumps(self.api_func.encode_client(data))
+
+        # Apply HTTP request headers according to SGP.22, section 6.5.1
        req_headers = {
            'Content-Type': 'application/json',
            'X-Admin-Protocol': 'gsma/rsp/v2.5.0',
        }
-        req_headers.update(self.extra_http_req_headers)
+        req_headers.update(self.api_func.extra_http_req_headers)

+        # Perform HTTP request
        logger.debug("HTTP REQ %s - hdr: %s '%s'" % (url, req_headers, encoded))
-        response = self.session.request(self.http_method, url, data=encoded, headers=req_headers, timeout=timeout)
+        response = self.session.request(self.api_func.http_method, url, data=encoded, headers=req_headers, timeout=timeout)
        logger.debug("HTTP RSP-STS: [%u] hdr: %s" % (response.status_code, response.headers))
        logger.debug("HTTP RSP: %s" % (response.content))

-        if response.status_code != self.expected_http_status:
+        # Check HTTP response status code and make sure that the returned HTTP headers look plausible (according to
+        # SGP.22, section 6.5.1)
+        if response.status_code != self.api_func.expected_http_status:
            raise HttpStatusError(response)
-
-        resp_content_type = response.headers.get('Content-Type')
-        if not resp_content_type.startswith(req_headers['Content-Type']):
+        if response.content and not response.headers.get('Content-Type').startswith(req_headers['Content-Type']):
            raise HttpHeaderError(response)
-
        if not response.headers.get('X-Admin-Protocol', 'gsma/rsp/v2.unknown').startswith('gsma/rsp/v2.'):
            raise HttpHeaderError(response)

+        # Decode response and return the result back to the caller
        if response.content:
-            if resp_content_type.startswith('application/json'):
-                return self.decode(response.json())
-            elif resp_content_type.startswith('text/plain;charset=UTF-8'):
-                return { 'data': response.content.decode('utf-8') }
-            raise HttpHeaderError(f'unimplemented response Content-Type: {response.headers=!r}')
+            output = self.api_func.decode_client(response.json())
+            # In case the response contains a header, check it to make sure that the API call was executed successfully
+            # (the presence of the header field is checked by the decode_client method)
+            if 'header' in output:
+                if output['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
+                    raise ApiError(output['header']['functionExecutionStatus'])
+            return output
        return None
+
+class JsonHttpApiServer():
+    def __init__(self, api_func: JsonHttpApiFunction, call_handler = None):
+        """
+        Args:
+                api_func : API function definition (JsonHttpApiFunction)
+                call_handler : handler function to process the request. This function must accept the
+                               decoded request as a dictionary. The handler function must return a tuple consisting
+                               of the response in the form of a dictionary (may be empty), and a function execution
+                               status string ('Executed-Success', 'Executed-WithWarning', 'Failed' or 'Expired')
+        """
+        self.api_func = api_func
+        if call_handler:
+            self.call_handler = call_handler
+        else:
+            self.call_handler = self.default_handler
+
+    def default_handler(self, data: dict) -> (dict, str):
+        """default handler, used in case no call handler is provided."""
+        logger.error("no handler function for request: %s" % str(data))
+        return {}, 'Failed'
+
+    def call(self, request: Request) -> str:
+        """ Process an incoming request.
+        Args:
+                request : request object as received using twisted.web.server
+        Returns:
+                encoded JSON string (HTTP response code and headers are set by calling the appropriate methods on the
+                provided the request object)
+        """
+
+        # Make sure the request is done with the correct HTTP method
+        if (request.method.decode() != self.api_func.http_method):
+            raise ValueError('Wrong HTTP method %s!=%s' % (request.method.decode(), self.api_func.http_method))
+
+        # Decode the request
+        decoded_request = self.api_func.decode_server(json.loads(request.content.read()))
+
+        # Run call handler (see above)
+        data, fe_status = self.call_handler(decoded_request)
+
+        # In case a function execution status is returned, use it to generate and prepend the header field according to
+        # SGP.22, section 6.5.1.2 and 6.5.1.4 (the presence of the header filed is checked by the encode_server method)
+        if fe_status:
+            data = {'header' : {'functionExecutionStatus': {'status' : fe_status}}} | data
+
+        # Encode the message (the presence of mandatory fields is checked during encoding)
+        encoded = json.dumps(self.api_func.encode_server(data))
+
+        # Apply HTTP request headers according to SGP.22, section 6.5.1
+        res_headers = {
+            'Content-Type': 'application/json',
+            'X-Admin-Protocol': 'gsma/rsp/v2.5.0',
+        }
+        res_headers.update(self.api_func.extra_http_res_headers)
+        for header, value in res_headers.items():
+            request.setHeader(header, value)
+        request.setResponseCode(self.api_func.expected_http_status)
+
+        # Return the encoded result back to the caller for sending (using twisted/klein)
+        return encoded
+
@@ -34,7 +34,7 @@ from pySim import ts_102_222
 from pySim.utils import dec_imsi
 from pySim.ts_102_221 import FileDescriptor
 from pySim.filesystem import CardADF, Path
-from pySim.ts_31_102 import ADF_USIM, EF_UST, EF_SUCI_Calc_Info
+from pySim.ts_31_102 import ADF_USIM
 from pySim.ts_31_103 import ADF_ISIM
 from pySim.esim import compile_asn1_subdir
 from pySim.esim.saip import templates
@@ -1517,11 +1517,8 @@ class ProfileElementHeader(ProfileElement):
    def mandatory_service_add(self, service_name):
        self.decoded['eUICC-Mandatory-services'][service_name] = None

-    def mandatory_service_present(self, service_name):
-        return service_name in self.decoded['eUICC-Mandatory-services'].keys()
-
    def mandatory_service_remove(self, service_name):
-        if self.mandatory_service_present(service_name):
+        if service_name in self.decoded['eUICC-Mandatory-services'].keys():
            del self.decoded['eUICC-Mandatory-services'][service_name]
        else:
            raise ValueError("service not in eUICC-Mandatory-services list, cannot remove")
@@ -1729,61 +1726,12 @@ class ProfileElementSequence:
        if 'BT' in ftype_list:
            svc_set.add('ber-tlv')
        # FIXME:dfLinked files (scan all files, check for non-empty Fcp.linkPath presence of DFs)
-
-        # 5G:
-        # - When SUCI is:
-        #   - enabled (EF.UST 124 = true)
-        #   AND
-        #   - calculated in the USIM (EF.UST 125 = true),
-        #   then eUICC-Mandatory-services needs 'get-identity'.
-        # - 'get-identity' implies that the eUICC must support ONE OF profile-A OR profile-B.
-        #   (One might assume from this that, when SUCI-CalcInfo for USIM in DF.SAIP contains both key types, then no
-        #   profile-A or B services need to be requested explicitly. However, the correct logic is:)
-        # - Iff the SUCI-CalcInfo for USIM (DF.SAIP) contains a key of profile-A ("identifier": 1),
-        #   then eUICC-Mandatory-services needs 'profile-a-x25519'.
-        # - Same: profile-B ("identifier": 2) needs 'profile-b-p256'.
-        # - (When SUCI is calculated in the UE, then the eUICC does not need to provide any of these services.)
-        suci_in_usim_enabled = False
-        try:
-            f_ust = self.get_pe_for_type("usim").files["ef-ust"]
-            ust = EF_UST().decode_bin(f_ust.body)
-            suci_in_usim_enabled = ust[124]['activated'] and ust[125]['activated']
-        except (KeyError, AttributeError):
-            pass
-        if suci_in_usim_enabled:
-            svc_set.add('get-identity')
-            # now check for profile-a and profile-b presence
-            suci_calcinfo_has_profile_a = False
-            suci_calcinfo_has_profile_b = False
-            try:
-                f_sucici = self.get_pe_for_type("df-saip").files["ef-suci-calc-info-usim"]
-                sucici = EF_SUCI_Calc_Info().decode_bin(f_sucici.body) or {}
-                for prot_scheme in sucici['prot_scheme_id_list']:
-                    if not isinstance(prot_scheme, dict):
-                        continue
-                    ps_id = prot_scheme["identifier"]
-                    if ps_id == 1:
-                        suci_calcinfo_has_profile_a = True
-                    elif ps_id == 2:
-                        suci_calcinfo_has_profile_b = True
-            except (KeyError, AttributeError):
-                pass
-            if suci_calcinfo_has_profile_a:
-                # The profile has a profile-A key, so require that
-                svc_set.add('profile-a-x25519')
-            if suci_calcinfo_has_profile_b:
-                # The profile has a profile-B key, so require that
-                svc_set.add('profile-b-p256')
-
+        # TODO: 5G related bits (derive from EF.UST or file presence?)
        hdr_pe = self.get_pe_for_type('header')
        # patch in the 'manual' services from the existing list:
-        old_svc_set = set()
        for old_svc in hdr_pe.decoded['eUICC-Mandatory-services'].keys():
            if old_svc in manual_services:
-                old_svc_set.add(old_svc)
-        logger.debug(f"{svc_set=}  +  {old_svc_set=}")
-        svc_set = svc_set.union(old_svc_set)
-        logger.debug(f"{svc_set=}")
+                svc_set.add(old_svc)
        hdr_pe.decoded['eUICC-Mandatory-services'] = {x: None for x in svc_set}

    def rebuild_mandatory_gfstelist(self):
@@ -20,9 +20,6 @@

 import copy
 import pprint
-import logging
-import traceback
-import inspect
 from typing import Generator, Union
 from pySim.esim.saip.personalization import ConfigurableParameter
 from pySim.esim.saip import param_source
@@ -30,10 +27,6 @@ from pySim.esim.saip import ProfileElementSequence, ProfileElementSD
 from pySim.global_platform import KeyUsageQualifier
 from osmocom.utils import b2h

-logger = logging.getLogger(__name__)
-def _func_():
-    return inspect.currentframe().f_back.f_code.co_name
-
 # a list of ConfigurableParameter classes and/or ConfigurableParameter class instances
 ParamList = list[Union[type[ConfigurableParameter], ConfigurableParameter]]

@@ -128,12 +121,8 @@ class BatchPersonalization:
                    value = p.param_cls.validate_val(input_value)
                    p.param_cls.apply_val(pes, value)
                except Exception as e:
-                    print(traceback.format_exc())
-                    logger.error('during %s: %r', _func_(), e)
                    raise ValueError(f'{p.param_cls.get_name()} fed by {p.src.name}: {e}') from e

-            pes.rebuild_mandatory_services()
-
            yield pes


@@ -333,15 +322,12 @@ class BatchAudit(list):

        return batch_audit

-    def to_csv_rows(self, headers=True, sort_key=None, column_blacklist=None):
+    def to_csv_rows(self, headers=True, sort_key=None):
        """generator that yields all audits' values as rows, useful feed to a csv.writer."""
        columns = set()
        for audit in self:
            columns.update(audit.keys())

-        if column_blacklist:
-            columns.difference_update(set(column_blacklist))
-
        columns = tuple(sorted(columns, key=sort_key))

        if headers:
@@ -19,26 +19,17 @@ import abc
 import enum
 import io
 import re
-import json
 from typing import List, Tuple, Generator, Optional

-from construct.core import StreamError
 from osmocom.tlv import camel_to_snake
 from osmocom.utils import hexstr
 from pySim.utils import enc_iccid, dec_iccid, enc_imsi, dec_imsi, h2b, b2h, rpad, sanitize_iccid
-from pySim.ts_31_102 import EF_AD, EF_UST, EF_Routing_Indicator, EF_SUCI_Calc_Info, DF_USIM_5GS
 from pySim.ts_51_011 import EF_SMSP
 from pySim.esim.saip import param_source
 from pySim.esim.saip import ProfileElement, ProfileElementSD, ProfileElementSequence
-from pySim.esim.saip import ProfileElementHeader
 from pySim.esim.saip import SecurityDomainKey, SecurityDomainKeyComponent
 from pySim.global_platform import KeyUsageQualifier, KeyType

-# optimization: instantiate class instance to get the fid only once.
-file_path_df_5gs = bytes.fromhex(DF_USIM_5GS().fid)
-fid_ri = bytes.fromhex(EF_Routing_Indicator().fid)
-fid_sucici = bytes.fromhex(EF_SUCI_Calc_Info().fid)
-
 def unrpad(s: hexstr, c='f') -> hexstr:
    return hexstr(s.rstrip(c))

@@ -297,9 +288,7 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
           May be overridden by subclasses.
           This default implementation returns the maximum allowed value length -- a good fit for most subclasses.
           '''
-        l = cls.get_len_range()[1] or 16
-        l = min(10*80, l)
-        return l
+        return cls.get_len_range()[1] or 16

    @classmethod
    def is_super_of(cls, other_class):
@@ -429,71 +418,69 @@ class BinaryParam(ConfigurableParameter):


 class EnumParam(ConfigurableParameter):
-    """ConfigurableParameter for named value enumerations.
+    """ConfigurableParameter for named integer enumeration values.

-    Subclasses define an own value_map, and implement their own apply_val() and get_values_from_pes().
-    """
-    value_map = {
-            # For example:
-            #'Meaningful label for value 23': 0x23,
-            # Where 0x23 is a valid value to use for apply_val(), of any valid type.
-        }
-    _value_map_reverse = None
+    Subclasses must define a nested enum.IntEnum named 'Values' listing all valid names and their
+    integer codes.  apply_val() and get_values_from_pes() are not implemented here and this must
+    be inherited from another mixin."""
+
+    class Values(enum.IntEnum):
+        pass  # subclasses override this

    @classmethod
-    def validate_val(cls, val):
-        orig_val = val
-        enum_val = None
-        if isinstance(val, str):
-            enum_name = val
-            enum_val = cls.map_name_to_val(enum_name)
+    def validate_val(cls, val) -> int:
+        if isinstance(val, int):
+            try:
+                return int(cls.Values(val))
+            except ValueError:
+                pass
+        elif isinstance(val, str):
+            member = cls.map_name_to_val(val, strict=False)
+            if member is not None:
+                return member

-        # if the str is not one of the known value_map.keys(), is it maybe one of value_map.keys()?
-        if enum_val is None and val in cls.value_map.values():
-            enum_val = val
-
-        if enum_val not in cls.value_map.values():
-            raise ValueError(f"{cls.get_name()}: invalid argument: {orig_val!r}. Valid arguments are:"
-                             f" {', '.join(cls.value_map.keys())}")
-
-        return enum_val
+        valid = ', '.join(m.name for m in cls.Values)
+        raise ValueError(f"{cls.get_name()}: invalid argument: {val!r}. Valid arguments are: {valid}")

    @classmethod
-    def map_name_to_val(cls, name:str, strict=True):
-        val = cls.value_map.get(name)
-        if val is not None:
-            return val
+    def map_name_to_val(cls, name: str, strict=True) -> int:
+        """Return the integer value for a given enum member name. Performs an exact match first,
+        then falls back to fuzzy matching (case-insensitive, punctuation-insensitive)."""
+        try:
+            return int(cls.Values[name])
+        except KeyError:
+            pass

-        clean_name = cls.clean_name_str(name)
-        for k, v in cls.value_map.items():
-            if clean_name == cls.clean_name_str(k):
-                return v
+        clean = cls.clean_name_str(name)
+        for member in cls.Values:
+            if cls.clean_name_str(member.name) == clean:
+                return int(member)

        if strict:
-            raise ValueError(f"Problem in {cls.get_name()}: {name!r} is not a known value."
-                    f" Known values are: {cls.value_map.keys()!r}")
+            valid = ', '.join(m.name for m in cls.Values)
+            raise ValueError(f"{cls.get_name()}: {name!r} is not a known value. Known values are: {valid}")
        return None

    @classmethod
    def map_val_to_name(cls, val, strict=False) -> str:
-        if cls._value_map_reverse is None:
-            cls._value_map_reverse = dict((v, k) for k, v in cls.value_map.items())
-
-        name = cls._value_map_reverse.get(val)
-        if name:
-            return name
-        if strict:
-            raise ValueError(f"Problem in {cls.get_name()}: {val!r} ({type(val)}) is not a known value."
-                    f" Known values are: {cls.value_map.values()!r}")
-        return None
+        """Return the enum member name for a given integer value."""
+        try:
+            return cls.Values(val).name
+        except ValueError:
+            if strict:
+                raise ValueError(f"{cls.get_name()}: {val!r} ({type(val).__name__}) is not a known value.")
+            return None

    @classmethod
-    def name_normalize(cls, name:str) -> str:
-        return cls.map_val_to_name(cls.map_name_to_val(name))
+    def name_normalize(cls, name: str) -> str:
+        """Map a (possibly fuzzy) name to its canonical enum member name."""
+        return cls.Values(cls.map_name_to_val(name)).name

    @classmethod
-    def clean_name_str(cls, val):
-        return re.sub('[^0-9A-Za-z-_]', '', val).lower()
+    def clean_name_str(cls, val: str) -> str:
+        """Strip punctuation and case for fuzzy name comparison.
+        Treats hyphens and underscores as equivalent (both removed)."""
+        return re.sub('[^0-9A-Za-z]', '', val).lower()


 class Iccid(DecimalParam):
@@ -646,14 +633,13 @@ class SmspTpScAddr(ConfigurableParameter):
            # - To generate the right amount of fillFileContent, pass total_len=42 to encode_record_bin().
            # - To show the right size in the PES, set f_smsp.rec_len = 42
            ef_smsp_dec['alpha_id'] = ''
-
-            # we can set this to choose a fixed length:
-            #f_smsp.rec_len = 42
-            # but leave rec_len unchanged to keep the same length as was found in the eSIM template.
+            f_smsp.rec_len = 42

            # re-encode into the File body.
-            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1, total_len=f_smsp.rec_len)
+            #
+            #print("SMSP  (new): %s" % f_smsp.body)
            # re-generate the pe.decoded member from the File instance
+            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1, total_len=f_smsp.rec_len)
            pe.file2pe(f_smsp)

    @classmethod
@@ -674,60 +660,6 @@ class SmspTpScAddr(ConfigurableParameter):
            yield { cls.name: cls.tuple_to_str((international, digits)) }


-class MncLen(EnumParam):
-    """MNC length. Must be either 2 or 3. Sets only the MNC length field in EF-AD (Administrative Data)."""
-    name = 'MNC-LEN'
-    value_map = { '2': 2, '3': 3 }
-    default_source = param_source.ConstantSource
-    example_input = '2'
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        """val must be an int: either 2 or 3"""
-        for pe in pes.get_pes_for_type('usim'):
-            if not hasattr(pe, 'files'):
-                continue
-            f_ad = pe.files.get('ef-ad')
-            if not f_ad:
-                continue
-            # decode existing values
-            if not f_ad.body:
-                continue
-            try:
-                ef_ad = EF_AD()
-                ef_ad_dec = ef_ad.decode_bin(f_ad.body)
-            except StreamError:
-                continue
-            if 'mnc_len' not in ef_ad_dec:
-                continue
-            # change mnc_len
-            ef_ad_dec['mnc_len'] = val
-            # re-encode into the File body
-            f_ad.body = ef_ad.encode_bin(ef_ad_dec)
-            pe.file2pe(f_ad)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            if not hasattr(pe, 'files'):
-                continue
-            f_ad = pe.files.get('ef-ad', None)
-            if f_ad is None:
-                continue
-
-            try:
-                ef_ad = EF_AD()
-                ef_ad_dec = ef_ad.decode_bin(f_ad.body)
-            except StreamError:
-                continue
-
-            mnc_len = ef_ad_dec.get('mnc_len', None)
-            if mnc_len is None:
-                continue
-
-            yield { cls.name: cls.map_val_to_name(int(mnc_len)) }
-
-
 class SdKey(BinaryParam):
    """Configurable Security Domain (SD) Key.  Value is presented as bytes.
       Non-abstract implementations are generated in SdKey.generate_sd_key_classes"""
@@ -1099,17 +1031,17 @@ class AlgorithmID(EnumParam, AlgoConfig):
    """use validate_val() from EnumParam, and apply_val() from AlgoConfig.
    In get_values_from_pes(), return enum value names, not raw values."""
    name = "Algorithm"
-    # as in pySim/esim/asn1/saip/PE_Definitions-3.3.1.asn
-    value_map = {
-            "Milenage" : 1,
-            "TUAK" : 2,
-            "usim-test" : 3,
-        }
    algo_config_key = 'algorithmID'
    example_input = "Milenage"
    default_source = param_source.ConstantSource

-    # EnumParam.validate_val() returns the int values from value_map
+    # as in pySim/esim/asn1/saip/PE_Definitions-3.3.1.asn
+    class Values(enum.IntEnum):
+        Milenage  = 1
+        TUAK      = 2
+        usim_test = 3  # input 'usim-test' also accepted via fuzzy matching
+
+    # EnumParam.validate_val() returns the int values from Values

    @classmethod
    def get_values_from_pes(cls, pes: ProfileElementSequence):
@@ -1156,7 +1088,7 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):

 class MilenageXoringConstants(BinaryParam, AlgoConfig):
    """XOR-ing constants c1,c2,c3,c4,c5 of Milenage, 128bit each. See 3GPP TS 35.206 Sections 2.3 + 5.3.
-    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concatenation
+    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concetenation
    of::

     00000000000000000000000000000000
@@ -1184,411 +1116,3 @@ class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
    max_val = 255
    example_input = '1'
    default_source = param_source.ConstantSource
-    numeric_base = None  # indicate that this won't need random number sources
-
-
-class EfUstServiceParam(EnumParam):
-    """superclass for EF-UST service flag parameters"""
-    service_idx = 0
-    value_map = { 'enabled': True, 'disabled': False }
-    default_source = param_source.ConstantSource
-    example_input = sorted(value_map.keys())[0]
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ust = pe.files['ef-ust']
-            ef_ust = EF_UST()
-            ust = ef_ust.decode_bin(f_ust.body)
-
-            ust[cls.service_idx]['activated'] = val
-
-            f_ust.body = ef_ust.encode_bin(ust)
-            pe.file2pe(f_ust)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ust = pe.files.get('ef-ust', None)
-            if not f_ust:
-                continue
-            ef_ust = EF_UST()
-            try:
-                ust = ef_ust.decode_bin(f_ust.body)
-
-                service_flag = ust[cls.service_idx]['activated']
-                yield { cls.name: cls.map_val_to_name(service_flag) }
-            except:
-                pass
-
-class SuciActive(EfUstServiceParam):
-    """EF-UST service nr 124: enable or disable the SUCI service."""
-    service_idx = 124
-    name = '5G-SUCI-active'
-    value_map = { 'SUCI-off': False, 'SUCI-on': True }
-    example_input = 'SUCI-on'
-
-class SuciInUsim(EfUstServiceParam):
-    """EF-UST service nr 125: calculate SUCI in UE or in USIM"""
-    service_idx = 125
-    name = '5G-SUCI-in-USIM'
-    value_map = { 'SUCI-in-UE': False, 'SUCI-in-USIM': True }
-    example_input = 'SUCI-in-USIM'
-
-class SuciRi(ConfigurableParameter):
-    """SUCI Routing Indicator as in section 4.4.11.11 of 3GPP TS 31.102"""
-    name = '5G-SUCI-RI'
-    allow_chars = '0123456789'
-    min_len = 1
-    max_len = 4
-    allow_types = (str,)
-    example_input = '0'
-    default_source = param_source.ConstantSource
-
-    KEY_RI = "routing_indicator"
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        for pe in pes.get_pes_for_type('df-5gs'):
-            f_ri = pe.files.get('ef-routing-indicator', None)
-            if f_ri is None:
-                continue
-            ef_ri = EF_Routing_Indicator()
-            ri = ef_ri.decode_bin(f_ri.body)
-
-            ri[cls.KEY_RI] = str(val)
-
-            f_ri.body = ef_ri.encode_bin(ri)
-            pe.file2pe(f_ri)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('df-5gs'):
-            f_ri = pe.files.get('ef-routing-indicator', None)
-            if f_ri is None:
-                continue
-            ef_ri = EF_Routing_Indicator()
-            try:
-                ri = ef_ri.decode_bin(f_ri.body)
-                yield { cls.name: ri.get(cls.KEY_RI) }
-            except:
-                pass
-
-class SuciCalcInfoParameter(ConfigurableParameter):
-    """SUCI Calculation Information as in section 4.4.11.8 of 3GPP TS 31.102"""
-    name = '5G-SUCI-CalcInfo'
-    default_source = param_source.ConstantSource
-    allow_types = (str,)
-    max_len = 4096  # to indicate a large input field to UI renderers
-    example_input = '{"prot_scheme_id_list": [{"priority": 0, "identifier": 0, "key_index": 0}], "hnet_pubkey_list": []}'
-
-    PE_IN_UE = ("df-5gs", "ef-suci-calc-info")
-    PE_IN_USIM = ("df-saip", "ef-suci-calc-info-usim")
-    suci_calc_info_pe = None
-
-    @classmethod
-    def validate_val(cls, val):
-        val = super().validate_val(val)
-
-        if not val:
-            val = "{}"
-
-        # check that it is a dict something like
-        # {
-        #     "prot_scheme_id_list": [
-        #         {"priority": 0, "identifier": 2, "key_index": 1},
-        #         {"priority": 1, "identifier": 1, "key_index": 2},
-        #         ],
-        #     "hnet_pubkey_list": [
-        #         {"hnet_pubkey_identifier": 27,
-        #          "hnet_pubkey": "0472DA71976234CE833A6907425867B82E074D44EF907DFB4B3E21C1C2256EBCD15A7DED52FCBB097A4ED250E036C7B9C8C7004C4EEDC4F068CD7BF8D3F900E3B4"},
-        #         {"hnet_pubkey_identifier": 30,
-        #          "hnet_pubkey": "5A8D38864820197C3394B92613B20B91633CBD897119273BF8E4A6F4EEC0A650"},
-        #          ],
-        # }
-
-        try:
-            d = json.loads(val)
-        except json.decoder.JSONDecodeError as e:
-            raise ValueError(f"Cannot parse SUCI Calc Info: {e}") from e
-
-        KEY_PSI_LIST = 'prot_scheme_id_list'
-        KEY_HPK_LIST = 'hnet_pubkey_list'
-        KEYS_D = set((KEY_HPK_LIST, KEY_PSI_LIST))
-        KEYS_PSI = set(('identifier', 'key_index', 'priority'))
-        KEYS_HPK = set(('hnet_pubkey_identifier', 'hnet_pubkey'))
-
-        if not d:
-            d = { KEY_PSI_LIST: [], KEY_HPK_LIST: [] }
-
-        if not (isinstance(d, dict)
-                and set(d.keys()) == KEYS_D):
-            raise ValueError(f"Unexpected structure in SUCI Calc Info: expected dict with entries {KEYS_D}")
-
-        psi = d.get(KEY_PSI_LIST, None)
-        if not all((set(e.keys()) == KEYS_PSI) for e in psi):
-            raise ValueError("Unexpected structure in SUCI Calc Info:"
-                             f" in {KEY_PSI_LIST}, expected dict with entries {KEYS_PSI}")
-
-        hpk = d.get(KEY_HPK_LIST, None)
-        if not all((set(e.keys()) == KEYS_HPK) for e in hpk):
-            raise ValueError("Unexpected structure in SUCI Calc Info:"
-                             f" in {KEY_HPK_LIST}, expected dict with entries {KEYS_HPK}")
-        return d
-
-    @classmethod
-    def _apply_suci(cls, pes: ProfileElementSequence, val, pe_type="df-5gs", pe_file="ef-suci-calc-info"):
-        for pe in pes.get_pes_for_type(pe_type):
-            f_sucici = pe.files.get(pe_file, None)
-            if not f_sucici:
-                continue
-            ef_sucici = EF_SUCI_Calc_Info()
-            body = ef_sucici.encode_bin(val)
-
-            # 0xff pad up to the existing file size, so that the underlying template doesn't come through
-            is_size = f_sucici.file_size
-            pad_n = is_size - len(body)
-            if pad_n > 0:
-                body = body + b'\xff' * pad_n
-
-            f_sucici.body = body
-            pe.file2pe(f_sucici)
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        cls._apply_suci(pes, val, *cls.suci_calc_info_pe)
-
-    @staticmethod
-    def normalize_sucici(sucici:dict):
-        """Normalize the CalcInfo dict so it can be json encoded:
-           convert bytes to hex strings."""
-        if not sucici:
-            sucici = {}
-
-        for hnet_pubkey in sucici.get('hnet_pubkey_list', ()):
-            val = hnet_pubkey['hnet_pubkey']
-            if isinstance(val, bytes):
-                val = b2h(val)
-                hnet_pubkey['hnet_pubkey'] = val
-
-        return sucici
-
-    @classmethod
-    def _get_suci(cls, pes: ProfileElementSequence, pe_type="df-5gs", pe_file="ef-suci-calc-info"):
-        for pe in pes.get_pes_for_type(pe_type):
-            f_sucici = pe.files.get(pe_file, None)
-            if not f_sucici:
-                continue
-            ef_sucici = EF_SUCI_Calc_Info()
-            sucici = ef_sucici.decode_bin(f_sucici.body)
-
-            # normalize to string (bytes cannot go into json)
-            sucici = cls.normalize_sucici(sucici)
-
-            yield { cls.name: json.dumps(sucici) }
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        yield from cls._get_suci(pes, *cls.suci_calc_info_pe)
-
-class SuciCalcInfoUe(SuciCalcInfoParameter):
-    """SUCI Calculation Information as in section 4.4.11.8 of 3GPP TS 31.102, readable by UE (DF-5GS)"""
-    name = '5G-SUCI-CalcInfo-UE'
-    suci_calc_info_pe = SuciCalcInfoParameter.PE_IN_UE
-
-class SuciCalcInfoUsim(SuciCalcInfoParameter):
-    """SUCI Calculation Information as in section 4.4.11.8 of 3GPP TS 31.102, readable only by USIM (DF-SAIP)"""
-    name = '5G-SUCI-CalcInfo-USIM'
-    suci_calc_info_pe = SuciCalcInfoParameter.PE_IN_USIM
-
-def gfm_find(pes: ProfileElementSequence, file_path:bytes, ef_fid:bytes):
-    """look through genericFileManagement PE and return the fmc list with start and end indexes as
-       (fmc_list, first_idx, after_last_idx)
-       so that fmc_list[first_idx:after_last_idx] is the slice of file management commands relevant to the given
-       file_path/ef_fid.
-    """
-    for pe in pes.get_pes_for_type('genericFileManagement'):
-        path_match = False
-        creating_fid = False
-
-        for fmc in pe.decoded['fileManagementCMD']:
-            first = None
-            last = None
-
-            for idx in range(len(fmc)):
-                cmd, arg = fmc[idx]
-
-                if cmd == 'filePath':
-                    path_match = (arg == file_path)
-                    if not path_match:
-                        creating_fid = False
-                elif path_match and cmd == 'createFCP':
-                    creating_fid = (arg.get('fileID') == ef_fid)
-                if creating_fid:
-                    if first is None:
-                        first = idx
-                        last = idx
-                    first = min(first, idx)
-                    last = max(last, idx)
-
-            if first is not None:
-                yield fmc, first, last + 1
-
-# genericFileManagement 5G params
-
-def pes_get_adf_fid(pes:ProfileElementSequence, naa_name="usim", adf_name="adf-usim"):
-    adf = pes.get_pe_for_type(naa_name)
-    return adf.decoded[adf_name][0][1]['fileID']
-
-def mk_adf_df_path(pes, naa:str, adf:str, file_path:bytes) -> bytes:
-    adf_file_id = pes_get_adf_fid(pes, naa, adf)
-    return b''.join((adf_file_id, file_path))
-
-def gfm_get_file_content(pes: ProfileElementSequence, naa:str, adf:str, file_path:bytes, ef_fid:bytes) -> bytes:
-    '''find a given file in the genericFileManagement section, and return the bytes from the first fillFileContent
-       item.
-       TODO: implement File.from_gfm() and return the full resulting bytes?
-    '''
-    adf_df_path = mk_adf_df_path(pes, naa, adf, file_path)
-
-    data = []
-    for fmc, first_idx, after_last_idx in gfm_find(pes, adf_df_path, ef_fid):
-        assert fmc[first_idx][0] == 'createFCP'
-        assert after_last_idx > first_idx
-
-        idx = first_idx + 1
-        while idx < after_last_idx:
-            if fmc[idx][0] == 'fillFileContent':
-                data.append(fmc[idx][1])
-            idx += 1
-
-    return data
-
-def gfm_set_file_content(pes: ProfileElementSequence, naa:str, adf:str, file_path:bytes, ef_fid:bytes, file_content:bytes) -> int:
-    adf_df_path = mk_adf_df_path(pes, naa, adf, file_path)
-
-    found = 0
-    for fmc, first_idx, after_last_idx in gfm_find(pes, adf_df_path, ef_fid):
-        assert fmc[first_idx][0] == 'createFCP'
-        assert after_last_idx > first_idx
-
-        new_fmc = [
-                fmc[first_idx],
-                ('fillFileContent', file_content),
-            ]
-        new_fmc[0][1]['efFileSize'] = bytes((len(file_content), ))
-
-        fmc[first_idx:after_last_idx] = new_fmc
-
-        found += 1
-    return found
-
-class GfmSuciRi(SuciRi):
-    """SUCI Routing Indicator as in section 4.4.11.11 of 3GPP TS 31.102,
-       applied via General File Management. Intended for SAIP 2.1 profiles."""
-    name = 'GFM-5G-SUCI-RI'
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        ri = {
-            "routing_indicator": str(val),
-            "rfu": "ffff"
-        }
-        ef_ri = EF_Routing_Indicator()
-        found = gfm_set_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_ri,
-                                     ef_ri.encode_bin(ri))
-        if not found:
-            raise ValueError(f"No target file found, Cannot apply {cls.name} = {ri}")
-
-        data = gfm_get_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_ri)
-        val = ef_ri.decode_bin(b''.join(data))
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        data = gfm_get_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_ri)
-        if not data:
-            return
-
-        data = b''.join(data)
-        if not data:
-            return
-
-        ef_ri = EF_Routing_Indicator()
-        ri = ef_ri.decode_bin(data)
-        yield { cls.name: ri.get(cls.KEY_RI) }
-
-class GfmSuciCalcInfoUe(SuciCalcInfoUe):
-    """SUCI Calculation Information as in section 4.4.11.8 of 3GPP TS 31.102, readable by UE (DF-5GS),
-       applied via General File Management. Intended for SAIP 2.1 profiles."""
-    name = 'GFM-5G-SUCI-CalcInfo-UE'
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        if not isinstance(val, dict):
-            raise ValueError("val should be a dict, after 'val = SuciCalcInfoParameter.validate_val(val)'")
-
-        ef_sucici = EF_SUCI_Calc_Info()
-        body = ef_sucici.encode_bin(val)
-        gfm_set_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_sucici,
-                             body)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        data = gfm_get_file_content(pes, 'usim', 'adf-usim', file_path_df_5gs, fid_sucici)
-        if not data:
-            return
-
-        data = b''.join(data)
-        if not data:
-            return
-
-        ef_sucici = EF_SUCI_Calc_Info()
-        sucici = ef_sucici.decode_bin(data)
-        sucici = cls.normalize_sucici(sucici)
-        yield { cls.name: json.dumps(sucici) }
-
-
-class EuiccMandatoryServiceParam(EnumParam):
-    """superclass for managing items of the ProfileHeader / eUICC-Mandatory-services ServicesList"""
-    service_name = None
-    value_map = { 'mandatory': True, 'optional': False }
-    default_source = param_source.ConstantSource
-    example_input = sorted(value_map.keys())[0]
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val):
-        for pe in pes.get_pes_for_type('header'):
-            assert isinstance(pe, ProfileElementHeader)
-            if val:
-                pe.mandatory_service_add(cls.service_name)
-            else:
-                # explicitly check to avoid exception when then service is already not present
-                if pe.mandatory_service_present(cls.service_name):
-                    pe.mandatory_service_remove(cls.service_name)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('header'):
-            assert isinstance(pe, ProfileElementHeader)
-            val = bool(pe.mandatory_service_present(cls.service_name))
-            yield { cls.name: cls.map_val_to_name(val) }
-
-class EuiccMandatoryServiceGetIdentity(EuiccMandatoryServiceParam):
-    """eUICC Mandatory Services: get-identity. The eUICC must be capable of providing a 5G identity using SUCI-CalcInfo
-    located in the USIM's DF-SAIP, see parameter 5G-SUCI-CalcInfo-USIM."""
-    name = '5G-eUICC-get-identity'
-    service_name = 'get-identity'
-
-class EuiccMandatoryServiceProfileA(EuiccMandatoryServiceParam):
-    """eUICC Mandatory Services: profile-a-x25519. The eUICC must be able to estblish a 5G identity using an X25519 key,
-    as provided in a profile-A ("identifier": 1) key in SUCI-CalcInfo located in the USIM's DF-SAIP, see parameter
-    5G-SUCI-CalcInfo-USIM."""
-    name = '5G-eUICC-profile-a-x25519'
-    service_name = 'profile-a-x25519'
-
-class EuiccMandatoryServiceProfileB(EuiccMandatoryServiceParam):
-    """eUICC Mandatory Services: profile-b-p256. The eUICC must be able to estblish a 5G identity using a P256 key, as
-    provided in a profile-B ("identifier": 2) key in SUCI-CalcInfo located in the USIM's DF-SAIP, see parameter
-    5G-SUCI-CalcInfo-USIM."""
-    name = '5G-eUICC-profile-b-p256'
-    service_name = 'profile-b-p256'
@@ -863,6 +863,8 @@ class TransparentEF(CardEF):
            t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
            t.from_dict(abstract_data)
            return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
        raise NotImplementedError(
            "%s encoder not yet implemented. Patches welcome." % self)

@@ -892,6 +894,8 @@ class TransparentEF(CardEF):
            t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
            t.from_dict(abstract_data)
            return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
        raise NotImplementedError(
            "%s encoder not yet implemented. Patches welcome." % self)

@@ -1166,6 +1170,8 @@ class LinFixedEF(CardEF):
            t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
            t.from_dict(abstract_data)
            return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
        raise NotImplementedError(
            "%s encoder not yet implemented. Patches welcome." % self)

@@ -1195,6 +1201,8 @@ class LinFixedEF(CardEF):
            t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
            t.from_dict(abstract_data)
            return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
        raise NotImplementedError(
            "%s encoder not yet implemented. Patches welcome." % self)

@@ -1386,6 +1394,8 @@ class TransRecEF(TransparentEF):
            t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
            t.from_dict(abstract_data)
            return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
        raise NotImplementedError(
            "%s encoder not yet implemented. Patches welcome." % self)

@@ -1415,6 +1425,8 @@ class TransRecEF(TransparentEF):
            t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
            t.from_dict(abstract_data)
            return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
        raise NotImplementedError(
            "%s encoder not yet implemented. Patches welcome." % self)

@@ -24,7 +24,15 @@
 #

 import logging
-from cmd2 import style
+import cmd2
+from packaging import version
+
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
+    from cmd2 import stylize as _stylize # pylint: disable=no-name-in-module
+    def _style(text, fg=None): # pylint: disable=function-redefined
+        return _stylize(text, fg) if fg else text
+else: # cmd2>=2.6.2
+    from cmd2 import style as _style # pylint: disable=no-name-in-module

 class _PySimLogHandler(logging.Handler):
    def __init__(self, log_callback):
@@ -121,7 +129,7 @@ class PySimLogger:
                if isinstance(color, str):
                    PySimLogger.print_callback(color + formatted_message + "\033[0m")
                else:
-                    PySimLogger.print_callback(style(formatted_message, fg = color))
+                    PySimLogger.print_callback(_style(formatted_message, fg = color))
            else:
                PySimLogger.print_callback(formatted_message)

@@ -327,7 +327,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
        """conversion method to generate list of {hnet_pubkey_identifier, hnet_pubkey} dicts
        from flat [{hnet_pubkey_identifier: }, {net_pubkey: }, ...] list"""
        out = []
-        while l:
+        while len(l):
            a = l.pop(0)
            b = l.pop(0)
            z = {**a, **b}
@@ -1,7 +1,7 @@
 pyscard
 pyserial
 pytlv
-cmd2>=2.6.2,<3.0
+cmd2>=2.6.2,<4.0
 jsonpath-ng
 construct>=2.10.70
 bidict
@@ -21,7 +21,7 @@ setup(
        "pyscard",
        "pyserial",
        "pytlv",
-        "cmd2 >= 1.5.0, < 3.0",
+        "cmd2 >= 2.6.2, < 4.0",
        "jsonpath-ng",
        "construct >= 2.10.70",
        "bidict",
@@ -21,7 +21,6 @@ import enum
 import io
 import sys
 import unittest
-import json
 from importlib import resources
 from osmocom.utils import hexstr
 from pySim.esim.saip import ProfileElementSequence
@@ -54,21 +53,18 @@ class ConfigurableParameterTest(unittest.TestCase):
    def test_parameters(self):

        upp_fnames = (
-                'SAIP2.1_gfmsuci.der',
+                'TS48v5_SAIP2.1A_NoBERTLV.der',
+                'TS48v5_SAIP2.3_BERTLV_SUCI.der',
                'TS48v5_SAIP2.1B_NoBERTLV.der',
                'TS48v5_SAIP2.3_NoBERTLV.der',
               )

        class Paramtest:
-            iff_present_default = False
-            def __init__(self, param_cls, val, expect_val, expect_clean_val=None, iff_present=None):
+            def __init__(self, param_cls, val, expect_val, expect_clean_val=None):
                self.param_cls = param_cls
                self.val = val
                self.expect_clean_val = expect_clean_val
                self.expect_val = expect_val
-                if iff_present is None:
-                    iff_present = Paramtest.iff_present_default
-                self.iff_present = iff_present

        param_tests = [
            Paramtest(param_cls=p13n.Imsi, val='123456',
@@ -154,7 +150,7 @@ class ConfigurableParameterTest(unittest.TestCase):
            Paramtest(param_cls=p13n.AlgorithmID,
                      val='usim-test',
                      expect_clean_val=3,
-                      expect_val='usim-test'),
+                      expect_val='usim_test'),

            Paramtest(param_cls=p13n.AlgorithmID,
                      val=1,
@@ -167,7 +163,7 @@ class ConfigurableParameterTest(unittest.TestCase):
            Paramtest(param_cls=p13n.AlgorithmID,
                      val=3,
                      expect_clean_val=3,
-                      expect_val='usim-test'),
+                      expect_val='usim_test'),

            Paramtest(param_cls=p13n.K,
                      val='01020304050607080910111213141516',
@@ -271,112 +267,7 @@ class ConfigurableParameterTest(unittest.TestCase):
                                 '11111111111111111111111111111111'
                                 '22222222222222222222222222222222'),

-
-            Paramtest(param_cls=p13n.MncLen,
-                      val='2',
-                      expect_clean_val=2,
-                      expect_val='2'),
-            Paramtest(param_cls=p13n.MncLen,
-                      val=3,
-                      expect_clean_val=3,
-                      expect_val='3'),
-
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceGetIdentity,
-                      val='mandatory',
-                      expect_clean_val=True,
-                      expect_val='mandatory'),
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceGetIdentity,
-                      val='optional',
-                      expect_clean_val=False,
-                      expect_val='optional'),
-
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileA,
-                      val='mandatory',
-                      expect_clean_val=True,
-                      expect_val='mandatory'),
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileA,
-                      val='optional',
-                      expect_clean_val=False,
-                      expect_val='optional'),
-
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileB,
-                      val='mandatory',
-                      expect_clean_val=True,
-                      expect_val='mandatory'),
-            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileB,
-                      val='optional',
-                      expect_clean_val=False,
-                      expect_val='optional'),
-        ]
-
-        Paramtest.iff_present_default = True
-
-        sucici = {
-             "prot_scheme_id_list": [
-                 {"priority": 0, "identifier": 2, "key_index": 1},
-                 {"priority": 1, "identifier": 1, "key_index": 2},
-                 ],
-             "hnet_pubkey_list": [
-                 {"hnet_pubkey_identifier": 27,
-                  "hnet_pubkey": "0472da71976234ce833a6907425867b82e074d44ef907dfb4b3e21c1c2256ebcd15a7ded52fcbb097a4ed250e036c7b9c8c7004c4eedc4f068cd7bf8d3f900e3b4"},
-                 {"hnet_pubkey_identifier": 30,
-                  "hnet_pubkey": "5a8d38864820197c3394b92613b20b91633cbd897119273bf8e4a6f4eec0a650"},
-                  ],
-        }
-
-        param_tests.extend([
-            Paramtest(param_cls=p13n.SuciActive, val='SUCI-on',
-                      expect_clean_val=True,
-                      expect_val={'5G-SUCI-active': 'SUCI-on'}),
-            Paramtest(param_cls=p13n.SuciActive, val='SUCI-off',
-                      expect_clean_val=False,
-                      expect_val={'5G-SUCI-active': 'SUCI-off'}),
-
-            Paramtest(param_cls=p13n.SuciInUsim, val='SUCI-in-UE',
-                      expect_clean_val=False,
-                      expect_val={'5G-SUCI-in-USIM': 'SUCI-in-UE'}),
-            Paramtest(param_cls=p13n.SuciInUsim, val='SUCI-in-USIM',
-                      expect_clean_val=True,
-                      expect_val={'5G-SUCI-in-USIM': 'SUCI-in-USIM'}),
-
-            Paramtest(param_cls=p13n.SuciRi, val='123',
-                      expect_clean_val='123',
-                      expect_val={'5G-SUCI-RI': '123'}),
-            Paramtest(param_cls=p13n.SuciRi, val='0',
-                      expect_clean_val='0',
-                      expect_val={'5G-SUCI-RI': '0'}),
-            Paramtest(param_cls=p13n.SuciRi, val='9999',
-                      expect_clean_val='9999',
-                      expect_val={'5G-SUCI-RI': '9999'}),
-
-            Paramtest(param_cls=p13n.SuciCalcInfoUe,
-                      val=json.dumps(sucici),
-                      expect_clean_val=sucici,
-                      expect_val={'5G-SUCI-CalcInfo-UE': json.dumps(sucici)}),
-
-            Paramtest(param_cls=p13n.SuciCalcInfoUsim,
-                      val=json.dumps(sucici),
-                      expect_clean_val=sucici,
-                      expect_val={'5G-SUCI-CalcInfo-USIM': json.dumps(sucici)}),
-
-            Paramtest(param_cls=p13n.GfmSuciRi, val='123',
-                      expect_clean_val='123',
-                      expect_val={'GFM-5G-SUCI-RI': '123'}),
-            Paramtest(param_cls=p13n.GfmSuciRi, val='0',
-                      expect_clean_val='0',
-                      expect_val={'GFM-5G-SUCI-RI': '0'}),
-            Paramtest(param_cls=p13n.GfmSuciRi, val='9999',
-                      expect_clean_val='9999',
-                      expect_val={'GFM-5G-SUCI-RI': '9999'}),
-
-            Paramtest(param_cls=p13n.GfmSuciCalcInfoUe,
-                      val=json.dumps(sucici),
-                      expect_clean_val=sucici,
-                      expect_val={'GFM-5G-SUCI-CalcInfo-UE': json.dumps(sucici)}),
-
-        ])
-
-        Paramtest.iff_present_default = False
+            ]

        for sdkey_cls in (
            # thin out the number of tests, as a compromise between completeness and test runtime
@@ -469,8 +360,7 @@ class ConfigurableParameterTest(unittest.TestCase):

                for t in param_tests:
                    test_idx += 1
-                    testlog = []
-                    testlog.append(f'{upp_fname} {t.param_cls.__name__}(val={valtypestr(t.val)})')
+                    logloc = f'{upp_fname} {t.param_cls.__name__}(val={valtypestr(t.val)})'

                    param = None
                    try:
@@ -478,32 +368,21 @@ class ConfigurableParameterTest(unittest.TestCase):
                        param.input_value = t.val
                        param.validate()
                    except ValueError as e:
-                        raise ValueError(f'{" ".join(testlog)}: {e}') from e
+                        raise ValueError(f'{logloc}: {e}') from e

                    clean_val = param.value
-                    testlog.append(f'clean_val={valtypestr(clean_val)}')
+                    logloc = f'{logloc} clean_val={valtypestr(clean_val)}'
                    if t.expect_clean_val is not None and t.expect_clean_val != clean_val:
-                        raise ValueError(f'{" ".join(testlog)}: expected'
+                        raise ValueError(f'{logloc}: expected'
                                         f' expect_clean_val={valtypestr(t.expect_clean_val)}')

                    # on my laptop, deepcopy is about 30% slower than decoding the DER from scratch:
                    # pes = copy.deepcopy(orig_pes)
                    pes = ProfileElementSequence.from_der(der)
-
-                    found = list((t.param_cls.get_value_from_pes(pes) or {}).values())
-                    testlog.append(f"previous value: {found}")
-
-                    if t.iff_present and not found:
-                        testlog.append("skipping, param not in template.")
-                        output = "\nskip: " + "\n  ".join(testlog)
-                        outputs.append(output)
-                        print(output)
-                        continue
-
                    try:
                        param.apply(pes)
                    except ValueError as e:
-                        raise ValueError(f'{" ".join(testlog)} apply_val(clean_val): {e}') from e
+                        raise ValueError(f'{logloc} apply_val(clean_val): {e}') from e

                    changed_der = pes.to_der()

@@ -521,18 +400,22 @@ class ConfigurableParameterTest(unittest.TestCase):
                    else:
                        read_back_val_type = f'{type(read_back_val).__name__}'

-                    testlog.append(f'read_back_val={valtypestr(read_back_val)}')
+                    logloc = (f'{logloc} read_back_val={valtypestr(read_back_val)}')

                    if isinstance(read_back_val, dict) and not t.param_cls.get_name() in read_back_val.keys():
-                        raise ValueError(f'{" ".join(testlog)}: expected to find name {t.param_cls.get_name()!r} in read_back_val')
+                        raise ValueError(f'{logloc}: expected to find name {t.param_cls.get_name()!r} in read_back_val')

                    expect_val = t.expect_val
                    if not isinstance(expect_val, dict):
                        expect_val = { t.param_cls.get_name(): expect_val }
                    if read_back_val != expect_val:
-                        raise ValueError(f'{" ".join(testlog)}: expected {expect_val=!r}:{type(t.expect_val).__name__}')
+                        raise ValueError(f'{logloc}: expected {expect_val=!r}:{type(t.expect_val).__name__}')

-                    output = "\nok: " + "\n  ".join(testlog)
+                    ok = logloc.replace(' clean_val', '\n\tclean_val'
+                              ).replace(' read_back_val', '\n\tread_back_val'
+                              ).replace('=', '=\t'
+                              )
+                    output = f'\nok: {ok}'
                    outputs.append(output)
                    print(output)

@@ -668,7 +551,7 @@ class TestEnumParam(unittest.TestCase):
    def test_validate_by_name_exact(self):
        self.assertEqual(p13n.AlgorithmID.validate_val('Milenage'), 1)
        self.assertEqual(p13n.AlgorithmID.validate_val('TUAK'), 2)
-        self.assertEqual(p13n.AlgorithmID.validate_val('usim-test'), 3)
+        self.assertEqual(p13n.AlgorithmID.validate_val('usim_test'), 3)

    def test_validate_by_int(self):
        self.assertEqual(p13n.AlgorithmID.validate_val(1), 1)
@@ -681,6 +564,7 @@ class TestEnumParam(unittest.TestCase):
        self.assertEqual(p13n.AlgorithmID.validate_val('tuak'), 2)

    def test_validate_fuzzy_hyphen_underscore(self):
+        # 'usim-test' has a hyphen; enum member is 'usim_test' — must fuzzy-match
        self.assertEqual(p13n.AlgorithmID.validate_val('usim-test'), 3)

    def test_validate_invalid_name(self):
@@ -717,7 +601,7 @@ class TestEnumParam(unittest.TestCase):
    def test_map_val_known(self):
        self.assertEqual(p13n.AlgorithmID.map_val_to_name(1), 'Milenage')
        self.assertEqual(p13n.AlgorithmID.map_val_to_name(2), 'TUAK')
-        self.assertEqual(p13n.AlgorithmID.map_val_to_name(3), 'usim-test')
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(3), 'usim_test')

    def test_map_val_unknown_nonstrict(self):
        self.assertIsNone(p13n.AlgorithmID.map_val_to_name(99))
@@ -731,13 +615,13 @@ class TestEnumParam(unittest.TestCase):
    def test_name_normalize(self):
        self.assertEqual(p13n.AlgorithmID.name_normalize('Milenage'), 'Milenage')
        self.assertEqual(p13n.AlgorithmID.name_normalize('milenage'), 'Milenage')
-        self.assertEqual(p13n.AlgorithmID.name_normalize('usim-test'), 'usim-test')
+        self.assertEqual(p13n.AlgorithmID.name_normalize('usim-test'), 'usim_test')

    # --- clean_name_str ---

    def test_clean_name_str(self):
-        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim-test'), 'usim-test')
-        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim_test'), 'usim_test')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim-test'), 'usimtest')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim_test'), 'usimtest')
        self.assertEqual(p13n.AlgorithmID.clean_name_str('Milenage'), 'milenage')
        self.assertEqual(p13n.AlgorithmID.clean_name_str('foo bar!'), 'foobar')
Author	SHA1	Message	Date
catarrh	1b2fc595ce	Allow saving unknown files with raw data When a file has no custom encoder, the decode path returns {'raw': ...} but the encode path raised NotImplementedError. Add a fallback in all encoder methods to return the raw data directly when present in the abstract_data dict, allowing unknown files to round-trip.	2026-07-04 11:48:53 +03:00
Vadim Yanitskiy	973d6eb2cc	pySim.log: fix E0611: No name 'style' in module 'cmd2' Change-Id: I191ea56f4c6e4e1916369f69fe2e1653e1d92df1 Fixes: `597f1e0` ("pySim.log, pySim-shell: fix compatibility with cmd2 >= 3.0.0")	2026-07-01 16:47:08 +07:00
Vadim Yanitskiy	597f1e0398	pySim.log, pySim-shell: fix compatibility with cmd2 >= 3.0.0 Some Linux distributions (e.g. Arch Linux) already ship cmd2 3.x.x, which removed the style()/Fg/Bg API in favor of stylize()/Color. Add a version guard to select the right API at runtime. Adjust the upper bound cap in requirements.txt and setup.py. Change-Id: Ibf2ac7847933296fb06665c87f53ed6e1f315d27	2026-06-26 02:47:45 +07:00
Vadim Yanitskiy	45d37ed959	pySim-shell: drop backwards compat quirks for cmd2 < 2.6.2 Remove version guards for cmd2 < 2.0.0 and < 2.3.0, the Cmd2Compat and Settable2Compat wrapper classes, and the old fg/bg color API - none of these are needed since both requirements.txt and setup.py already mandate cmd2 >= 2.6.2. Change-Id: Ifd1c484ab66d74323d10e946347daa637cf6f5d8	2026-06-25 22:51:03 +07:00
Harald Welte	757c7d048e	setup.py: Align cmd2 minimum version with requirements.txt As pointed out in the commit-log of Change-Id I5186f242dbc1b770e3ab8cdca7f27d2a1029fff6 we had different minimum versions for cmd2 in requirements.txt vs setup.py. Let's align that. Change-Id: I71cee0ec3ed2abec68ec567beaab13c868721dad	2026-06-25 21:43:52 +07:00