Revert "esim/http_json_api: extend JSON API with server functionality"

This reverts commit e00c0becca.

pySim/esim/es2p.py

@@ -16,12 +16,6 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import requests
-from klein import Klein
-from twisted.internet import defer, protocol, ssl, task, endpoints, reactor
-from twisted.internet.posixbase import PosixReactorBase
-from pathlib import Path
-from twisted.web.server import Site, Request
-
 import logging
 from datetime import datetime
 import time
@@ -129,12 +123,10 @@ class Es2PlusApiFunction(JsonHttpApiFunction):
 class DownloadOrder(Es2PlusApiFunction):
     path = '/gsma/rsp2/es2plus/downloadOrder'
     input_params = {
-        'header': JsonRequestHeader,
         'eid': param.Eid,
         'iccid': param.Iccid,
         'profileType': param.ProfileType
       }
-    input_mandatory = ['header']
     output_params = {
         'header': JsonResponseHeader,
         'iccid': param.Iccid,
@@ -145,7 +137,6 @@ class DownloadOrder(Es2PlusApiFunction):
 class ConfirmOrder(Es2PlusApiFunction):
     path = '/gsma/rsp2/es2plus/confirmOrder'
     input_params = {
-        'header': JsonRequestHeader,
         'iccid': param.Iccid,
         'eid': param.Eid,
         'matchingId': param.MatchingId,
@@ -153,7 +144,7 @@ class ConfirmOrder(Es2PlusApiFunction):
         'smdsAddress': param.SmdsAddress,
         'releaseFlag': param.ReleaseFlag,
       }
-    input_mandatory = ['header', 'iccid', 'releaseFlag']
+    input_mandatory = ['iccid', 'releaseFlag']
     output_params = {
         'header': JsonResponseHeader,
         'eid': param.Eid,
@@ -166,13 +157,12 @@ class ConfirmOrder(Es2PlusApiFunction):
 class CancelOrder(Es2PlusApiFunction):
     path = '/gsma/rsp2/es2plus/cancelOrder'
     input_params = {
-        'header': JsonRequestHeader,
         'iccid': param.Iccid,
         'eid': param.Eid,
         'matchingId': param.MatchingId,
         'finalProfileStatusIndicator': param.FinalProfileStatusIndicator,
       }
-    input_mandatory = ['header', 'finalProfileStatusIndicator', 'iccid']
+    input_mandatory = ['finalProfileStatusIndicator', 'iccid']
     output_params = {
         'header': JsonResponseHeader,
       }
@@ -182,10 +172,9 @@ class CancelOrder(Es2PlusApiFunction):
 class ReleaseProfile(Es2PlusApiFunction):
     path = '/gsma/rsp2/es2plus/releaseProfile'
     input_params = {
-        'header': JsonRequestHeader,
         'iccid': param.Iccid,
       }
-    input_mandatory = ['header', 'iccid']
+    input_mandatory = ['iccid']
     output_params = {
         'header': JsonResponseHeader,
       }
@@ -195,7 +184,6 @@ class ReleaseProfile(Es2PlusApiFunction):
 class HandleDownloadProgressInfo(Es2PlusApiFunction):
     path = '/gsma/rsp2/es2plus/handleDownloadProgressInfo'
     input_params = {
-        'header': JsonRequestHeader,
         'eid': param.Eid,
         'iccid': param.Iccid,
         'profileType': param.ProfileType,
@@ -204,9 +192,10 @@ class HandleDownloadProgressInfo(Es2PlusApiFunction):
         'notificationPointStatus': param.NotificationPointStatus,
         'resultData': param.ResultData,
     }
-    input_mandatory = ['header', 'iccid', 'profileType', 'timestamp', 'notificationPointId', 'notificationPointStatus']
+    input_mandatory = ['iccid', 'profileType', 'timestamp', 'notificationPointId', 'notificationPointStatus']
     expected_http_status = 204
 
+
 class Es2pApiClient:
     """Main class representing a full ES2+ API client. Has one method for each API function."""
     def __init__(self, url_prefix:str, func_req_id:str, server_cert_verify: str = None, client_cert: str = None):
@@ -217,17 +206,18 @@ class Es2pApiClient:
         if client_cert:
             self.session.cert = client_cert
 
-        self.downloadOrder = JsonHttpApiClient(DownloadOrder(), url_prefix, func_req_id, self.session)
+        self.downloadOrder = DownloadOrder(url_prefix, func_req_id, self.session)
-        self.confirmOrder = JsonHttpApiClient(ConfirmOrder(), url_prefix, func_req_id, self.session)
+        self.confirmOrder = ConfirmOrder(url_prefix, func_req_id, self.session)
-        self.cancelOrder = JsonHttpApiClient(CancelOrder(), url_prefix, func_req_id, self.session)
+        self.cancelOrder = CancelOrder(url_prefix, func_req_id, self.session)
-        self.releaseProfile = JsonHttpApiClient(ReleaseProfile(), url_prefix, func_req_id, self.session)
+        self.releaseProfile = ReleaseProfile(url_prefix, func_req_id, self.session)
-        self.handleDownloadProgressInfo = JsonHttpApiClient(HandleDownloadProgressInfo(), url_prefix, func_req_id, self.session)
+        self.handleDownloadProgressInfo = HandleDownloadProgressInfo(url_prefix, func_req_id, self.session)
 
     def _gen_func_id(self) -> str:
         """Generate the next function call id."""
         self.func_id += 1
         return 'FCI-%u-%u' % (time.time(), self.func_id)
 
+
     def call_downloadOrder(self, data: dict) -> dict:
         """Perform ES2+ DownloadOrder function (SGP.22 section 5.3.1)."""
         return self.downloadOrder.call(data, self._gen_func_id())
@@ -247,116 +237,3 @@ class Es2pApiClient:
     def call_handleDownloadProgressInfo(self, data: dict) -> dict:
         """Perform ES2+ HandleDownloadProgressInfo function (SGP.22 section 5.3.5)."""
         return self.handleDownloadProgressInfo.call(data, self._gen_func_id())
-
-class Es2pApiServerHandlerSmdpp(abc.ABC):
-    """ES2+ (SMDP+ side) API Server handler class. The API user is expected to override the contained methods."""
-
-    @abc.abstractmethod
-    def call_downloadOrder(self, data: dict) -> (dict, str):
-        """Perform ES2+ DownloadOrder function (SGP.22 section 5.3.1)."""
-        pass
-
-    @abc.abstractmethod
-    def call_confirmOrder(self, data: dict) -> (dict, str):
-        """Perform ES2+ ConfirmOrder function (SGP.22 section 5.3.2)."""
-        pass
-
-    @abc.abstractmethod
-    def call_cancelOrder(self, data: dict) -> (dict, str):
-        """Perform ES2+ CancelOrder function (SGP.22 section 5.3.3)."""
-        pass
-
-    @abc.abstractmethod
-    def call_releaseProfile(self, data: dict) -> (dict, str):
-        """Perform ES2+ CancelOrder function (SGP.22 section 5.3.4)."""
-        pass
-
-class Es2pApiServerHandlerMno(abc.ABC):
-    """ES2+ (MNO side) API Server handler class. The API user is expected to override the contained methods."""
-
-    @abc.abstractmethod
-    def call_handleDownloadProgressInfo(self, data: dict) -> (dict, str):
-        """Perform ES2+ HandleDownloadProgressInfo function (SGP.22 section 5.3.5)."""
-        pass
-
-class Es2pApiServer(abc.ABC):
-    """Main class representing a full ES2+ API server. Has one method for each API function."""
-    app = None
-
-    def __init__(self, port: int, interface: str, server_cert: str = None, client_cert_verify: str = None):
-        logger.debug("HTTP SRV: starting ES2+ API server on %s:%s" % (interface, port))
-        self.port = port
-        self.interface = interface
-        if server_cert:
-            self.server_cert = ssl.PrivateCertificate.loadPEM(Path(server_cert).read_text())
-        else:
-            self.server_cert = None
-        if client_cert_verify:
-            self.client_cert_verify = ssl.Certificate.loadPEM(Path(client_cert_verify).read_text())
-        else:
-            self.client_cert_verify = None
-
-    def reactor(self, reactor: PosixReactorBase):
-        logger.debug("HTTP SRV: listen on %s:%s" % (self.interface, self.port))
-        if self.server_cert:
-            if self.client_cert_verify:
-                reactor.listenSSL(self.port, Site(self.app.resource()), self.server_cert.options(self.client_cert_verify),
-                                  interface=self.interface)
-            else:
-                reactor.listenSSL(self.port, Site(self.app.resource()), self.server_cert.options(),
-                                  interface=self.interface)
-        else:
-            reactor.listenTCP(self.port, Site(self.app.resource()), interface=self.interface)
-        return defer.Deferred()
-
-class Es2pApiServerSmdpp(Es2pApiServer):
-    """ES2+ (SMDP+ side) API Server."""
-    app = Klein()
-
-    def __init__(self, port: int, interface: str, handler: Es2pApiServerHandlerSmdpp,
-                 server_cert: str = None, client_cert_verify: str = None):
-        super().__init__(port, interface, server_cert, client_cert_verify)
-        self.handler = handler
-        self.downloadOrder = JsonHttpApiServer(DownloadOrder(), handler.call_downloadOrder)
-        self.confirmOrder = JsonHttpApiServer(ConfirmOrder(), handler.call_confirmOrder)
-        self.cancelOrder = JsonHttpApiServer(CancelOrder(), handler.call_cancelOrder)
-        self.releaseProfile = JsonHttpApiServer(ReleaseProfile(), handler.call_releaseProfile)
-        task.react(self.reactor)
-
-    @app.route(DownloadOrder.path)
-    def call_downloadOrder(self, request: Request) -> dict:
-        """Perform ES2+ DownloadOrder function (SGP.22 section 5.3.1)."""
-        return self.downloadOrder.call(request)
-
-    @app.route(ConfirmOrder.path)
-    def call_confirmOrder(self, request: Request) -> dict:
-        """Perform ES2+ ConfirmOrder function (SGP.22 section 5.3.2)."""
-        return self.confirmOrder.call(request)
-
-    @app.route(CancelOrder.path)
-    def call_cancelOrder(self, request: Request) -> dict:
-        """Perform ES2+ CancelOrder function (SGP.22 section 5.3.3)."""
-        return self.cancelOrder.call(request)
-
-    @app.route(ReleaseProfile.path)
-    def call_releaseProfile(self, request: Request) -> dict:
-        """Perform ES2+ CancelOrder function (SGP.22 section 5.3.4)."""
-        return self.releaseProfile.call(request)
-
-class Es2pApiServerMno(Es2pApiServer):
-    """ES2+ (MNO side) API Server."""
-
-    app = Klein()
-
-    def __init__(self, port: int, interface: str, handler: Es2pApiServerHandlerMno,
-                 server_cert: str = None, client_cert_verify: str = None):
-        super().__init__(port, interface, server_cert, client_cert_verify)
-        self.handler = handler
-        self.handleDownloadProgressInfo = JsonHttpApiServer(HandleDownloadProgressInfo(),
-                                                            handler.call_handleDownloadProgressInfo)
-        task.react(self.reactor)
-
-    @app.route(HandleDownloadProgressInfo.path)
-    def call_handleDownloadProgressInfo(self, request: Request) -> dict:
-        """Perform ES2+ HandleDownloadProgressInfo function (SGP.22 section 5.3.5)."""
-        return self.handleDownloadProgressInfo.call(request)

pySim/esim/es9p.py

@@ -155,11 +155,11 @@ class Es9pApiClient:
         if server_cert_verify:
             self.session.verify = server_cert_verify
 
-        self.initiateAuthentication = JsonHttpApiClient(InitiateAuthentication(), url_prefix, '', self.session)
+        self.initiateAuthentication = InitiateAuthentication(url_prefix, '', self.session)
-        self.authenticateClient = JsonHttpApiClient(AuthenticateClient(), url_prefix, '', self.session)
+        self.authenticateClient = AuthenticateClient(url_prefix, '', self.session)
-        self.getBoundProfilePackage = JsonHttpApiClient(GetBoundProfilePackage(), url_prefix, '', self.session)
+        self.getBoundProfilePackage = GetBoundProfilePackage(url_prefix, '', self.session)
-        self.handleNotification = JsonHttpApiClient(HandleNotification(), url_prefix, '', self.session)
+        self.handleNotification = HandleNotification(url_prefix, '', self.session)
-        self.cancelSession = JsonHttpApiClient(CancelSession(), url_prefix, '', self.session)
+        self.cancelSession = CancelSession(url_prefix, '', self.session)
 
     def call_initiateAuthentication(self, data: dict) -> dict:
         return self.initiateAuthentication.call(data)

pySim/esim/http_json_api.py

@@ -19,10 +19,8 @@ import abc
 import requests
 import logging
 import json
-from typing import Optional, Tuple
+from typing import Optional
 import base64
-from twisted.web.server import Request
-
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -133,16 +131,6 @@ class JsonResponseHeader(ApiParam):
         if status not in ['Executed-Success', 'Executed-WithWarning', 'Failed', 'Expired']:
             raise ValueError('Unknown/unspecified status "%s"' % status)
 
-class JsonRequestHeader(ApiParam):
-    """SGP.22 section 6.5.1.3."""
-    @classmethod
-    def verify_decoded(cls, data):
-        func_req_id = data.get('functionRequesterIdentifier')
-        if not func_req_id:
-            raise ValueError('Missing mandatory functionRequesterIdentifier in header')
-        func_call_id = data.get('functionCallIdentifier')
-        if not func_call_id:
-            raise ValueError('Missing mandatory functionCallIdentifier in header')
 
 class HttpStatusError(Exception):
     pass
@@ -173,118 +161,65 @@ class ApiError(Exception):
 
 class JsonHttpApiFunction(abc.ABC):
     """Base class for representing an HTTP[s] API Function."""
-    # The below class variables are used to describe the properties of the API function. Derived classes are expected
+     # the below class variables are expected to be overridden in derived classes
-    # to orverride those class properties with useful values. The prefixes "input_" and "output_" refer to the API
-    # function from an abstract point of view. Seen from the client perspective, "input_" will refer to parameters the
-    # client sends to a HTTP server. Seen from the server perspective, "input_" will refer to parameters the server
-    # receives from the a requesting client. The same applies vice versa to class variables that have an "output_"
-    # prefix.
 
-    # path of the API function (e.g. '/gsma/rsp2/es2plus/confirmOrder', see also method rewrite_url).
     path = None
-
     # dictionary of input parameters. key is parameter name, value is ApiParam class
     input_params = {}
-
     # list of mandatory input parameters
     input_mandatory = []
-
     # dictionary of output parameters. key is parameter name, value is ApiParam class
     output_params = {}
-
     # list of mandatory output parameters (for successful response)
     output_mandatory = []
-
-    # list of mandatory output parameters (for failed response)
-    output_mandatory_failed = []
-
     # expected HTTP status code of the response
     expected_http_status = 200
-
     # the HTTP method used (GET, OPTIONS, HEAD, POST, PUT, PATCH or DELETE)
     http_method = 'POST'
-
-    # additional custom HTTP headers (client requests)
     extra_http_req_headers = {}
 
-    # additional custom HTTP headers (server responses)
+    def __init__(self, url_prefix: str, func_req_id: Optional[str], session: requests.Session):
-    extra_http_res_headers = {}
+        self.url_prefix = url_prefix
+        self.func_req_id = func_req_id
+        self.session = session
 
-    def __new__(cls, *args, role = 'legacy_client', **kwargs):
+    def encode(self, data: dict, func_call_id: Optional[str] = None) -> dict:
-        """
-        Args:
-                args: (see JsonHttpApiClient and JsonHttpApiServer)
-                role: role ('server' or 'client') in which the JsonHttpApiFunction should be created.
-                kwargs: (see JsonHttpApiClient and JsonHttpApiServer)
-        """
-
-        # Create a dictionary with the class attributes of this class (the properties listed above and the encode_
-        # decode_ methods below). The dictionary will not include any dunder/magic methods
-        cls_attr = {attr_name: getattr(cls, attr_name) for attr_name in dir(cls) if not attr_name.startswith('__')}
-
-        # Normal instantiation as JsonHttpApiFunction:
-        if len(args) == 0 and len(kwargs) == 0:
-            return type(cls.__name__, (abc.ABC,), cls_attr)()
-
-        # Instantiation as as JsonHttpApiFunction with a JsonHttpApiClient or JsonHttpApiServer base
-        if role == 'legacy_client':
-            # Deprecated: With the advent of the server role (JsonHttpApiServer) the API had to be changed. To maintain
-            # compatibility with existing code (out-of-tree) the original behaviour and API interface and behaviour had
-            # to be preserved. Already existing JsonHttpApiFunction definitions will still work and the related objects
-            # may still be created on the original way: my_api_func = MyApiFunc(url_prefix, func_req_id, self.session)
-            logger.warning('implicit role (falling back to legacy JsonHttpApiClient) is deprecated, please specify role explcitly')
-            result = type(cls.__name__, (JsonHttpApiClient,), cls_attr)(None, *args, **kwargs)
-            result.api_func = result
-            result.legacy = True
-            return result
-        elif role == 'client':
-            # Create a JsonHttpApiFunction in client role
-            # Example: my_api_func = MyApiFunc(url_prefix, func_req_id, self.session, role='client')
-            result = type(cls.__name__, (JsonHttpApiClient,), cls_attr)(None, *args, **kwargs)
-            result.api_func = result
-            return result
-        elif role == 'server':
-            # Create a JsonHttpApiFunction in server role
-            # Example: my_api_func = MyApiFunc(url_prefix, func_req_id, self.session, role='server')
-            result = type(cls.__name__, (JsonHttpApiServer,), cls_attr)(None, *args, **kwargs)
-            result.api_func = result
-            return result
-        else:
-            raise ValueError('Invalid role \'%s\' specified' % role)
-
-    def encode_client(self, data: dict) -> dict:
         """Validate an encode input dict into JSON-serializable dict for request body."""
         output = {}
+        if func_call_id:
+            output['header'] = {
+                    'functionRequesterIdentifier': self.func_req_id,
+                    'functionCallIdentifier': func_call_id
+                }
+
         for p in self.input_mandatory:
             if not p in data:
                 raise ValueError('Mandatory input parameter %s missing' % p)
         for p, v in data.items():
             p_class = self.input_params.get(p)
             if not p_class:
-                # pySim/esim/http_json_api.py:269:47: E1101: Instance of 'JsonHttpApiFunction' has no 'legacy' member (no-member)
+                logger.warning('Unexpected/unsupported input parameter %s=%s', p, v)
-                # pylint: disable=no-member
+                output[p] = v
-                if hasattr(self, 'legacy') and self.legacy:
-                    output[p] = JsonRequestHeader.encode(v)
-                else:
-                    logger.warning('Unexpected/unsupported input parameter %s=%s', p, v)
-                    output[p] = v
             else:
                 output[p] = p_class.encode(v)
         return output
 
-    def decode_client(self, data: dict) -> dict:
+    def decode(self, data: dict) -> dict:
         """[further] Decode and validate the JSON-Dict of the response body."""
         output = {}
-        output_mandatory = self.output_mandatory
+        if 'header' in self.output_params:
+            # let's first do the header, it's special
+            if not 'header' in data:
+                raise ValueError('Mandatory output parameter "header" missing')
+            hdr_class = self.output_params.get('header')
+            output['header'] = hdr_class.decode(data['header'])
 
-        # In case a provided header (may be optional) indicates that the API function call was unsuccessful, a
+            if output['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
-        # different set of mandatory parameters applies.
+                raise ApiError(output['header']['functionExecutionStatus'])
-        header = data.get('header')
+        # we can only expect mandatory parameters to be present in case of successful execution
-        if header:
+        for p in self.output_mandatory:
-            if data['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
+            if p == 'header':
-                output_mandatory = self.output_mandatory_failed
+                continue
-
-        for p in output_mandatory:
             if not p in data:
                 raise ValueError('Mandatory output parameter "%s" missing' % p)
         for p, v in data.items():
@@ -296,195 +231,37 @@ class JsonHttpApiFunction(abc.ABC):
                 output[p] = p_class.decode(v)
         return output
 
-    def encode_server(self, data: dict) -> dict:
-        """Validate an encode input dict into JSON-serializable dict for response body."""
-        output = {}
-        output_mandatory = self.output_mandatory
-
-        # In case a provided header (may be optional) indicates that the API function call was unsuccessful, a
-        # different set of mandatory parameters applies.
-        header = data.get('header')
-        if header:
-            if data['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
-                output_mandatory = self.output_mandatory_failed
-
-        for p in output_mandatory:
-            if not p in data:
-                raise ValueError('Mandatory output parameter %s missing' % p)
-        for p, v in data.items():
-            p_class = self.output_params.get(p)
-            if not p_class:
-                logger.warning('Unexpected/unsupported output parameter %s=%s', p, v)
-                output[p] = v
-            else:
-                output[p] = p_class.encode(v)
-        return output
-
-    def decode_server(self, data: dict) -> dict:
-        """[further] Decode and validate the JSON-Dict of the request body."""
-        output = {}
-
-        for p in self.input_mandatory:
-            if not p in data:
-                raise ValueError('Mandatory input parameter "%s" missing' % p)
-        for p, v in data.items():
-            p_class = self.input_params.get(p)
-            if not p_class:
-                logger.warning('Unexpected/unsupported input parameter "%s"="%s"', p, v)
-                output[p] = v
-            else:
-                output[p] = p_class.decode(v)
-        return output
-
-    def rewrite_url(self, data: dict, url: str) -> Tuple[dict, str]:
-        """
-        Rewrite a static URL using information passed in the data dict. This method may be overloaded by a derived
-        class to allow fully dynamic URLs. The input parameters required for the URL rewriting may be passed using
-        data parameter. In case those parameters are additional parameters that are not intended to be passed to
-        the encode_client method later, they must be removed explcitly.
-
-        Args:
-                data: (see JsonHttpApiClient and JsonHttpApiServer)
-                url: statically generated URL string (see comment in JsonHttpApiClient)
-        """
-
-        # This implementation is a placeholder in which we do not perform any URL rewriting. We just pass through data
-        # and url unmodified.
-        return data, url
-
-class JsonHttpApiClient():
-    def __init__(self, api_func: JsonHttpApiFunction, url_prefix: str, func_req_id: Optional[str],
-                 session: requests.Session):
-        """
-        Args:
-                api_func : API function definition (JsonHttpApiFunction)
-                url_prefix : prefix to be put in front of the API function path (see JsonHttpApiFunction)
-                func_req_id : function requestor id to use for requests
-                session : session object (requests)
-        """
-        self.api_func = api_func
-        self.url_prefix = url_prefix
-        self.func_req_id = func_req_id
-        self.session = session
-
     def call(self, data: dict, func_call_id: Optional[str] = None, timeout=10) -> Optional[dict]:
-        """
+        """Make an API call to the HTTP API endpoint represented by this object.
-        Make an API call to the HTTP API endpoint represented by this object. Input data is passed in `data` as
+        Input data is passed in `data` as json-serializable dict.  Output data
-        json-serializable fields. `data` may also contain additional parameters required for URL rewriting (see
+        is returned as json-deserialized dict."""
-        rewrite_url in class JsonHttpApiFunction). Output data is returned as json-deserialized dict.
+        url = self.url_prefix + self.path
-
+        encoded = json.dumps(self.encode(data, func_call_id))
-        Args:
-                data: Input data required to perform the request.
-                func_call_id: Function Call Identifier, if present a header field is generated automatically.
-                timeout: Maximum amount of time to wait for the request to complete.
-        """
-
-        # In case a function caller ID is supplied, use it together with the stored function requestor ID to generate
-        # and prepend the header field according to SGP.22, section 6.5.1.1 and 6.5.1.3. (the presence of the header
-        # field is checked by the encode_client method)
-        if func_call_id:
-            data = {'header' : {'functionRequesterIdentifier': self.func_req_id,
-                                'functionCallIdentifier': func_call_id}} | data
-
-        # The URL used for the HTTP request (see below) normally consists of the initially given url_prefix
-        # concatenated with the path defined by the JsonHttpApiFunction definition. This static URL path may be
-        # rewritten by rewrite_url method defined in the JsonHttpApiFunction.
-        data, url = self.api_func.rewrite_url(data, self.url_prefix + self.api_func.path)
-
-        # Encode the message (the presence of mandatory fields is checked during encoding)
-        encoded = json.dumps(self.api_func.encode_client(data))
-
-        # Apply HTTP request headers according to SGP.22, section 6.5.1
         req_headers = {
             'Content-Type': 'application/json',
             'X-Admin-Protocol': 'gsma/rsp/v2.5.0',
         }
-        req_headers.update(self.api_func.extra_http_req_headers)
+        req_headers.update(self.extra_http_req_headers)
 
-        # Perform HTTP request
         logger.debug("HTTP REQ %s - hdr: %s '%s'" % (url, req_headers, encoded))
-        response = self.session.request(self.api_func.http_method, url, data=encoded, headers=req_headers, timeout=timeout)
+        response = self.session.request(self.http_method, url, data=encoded, headers=req_headers, timeout=timeout)
         logger.debug("HTTP RSP-STS: [%u] hdr: %s" % (response.status_code, response.headers))
         logger.debug("HTTP RSP: %s" % (response.content))
 
-        # Check HTTP response status code and make sure that the returned HTTP headers look plausible (according to
+        if response.status_code != self.expected_http_status:
-        # SGP.22, section 6.5.1)
-        if response.status_code != self.api_func.expected_http_status:
             raise HttpStatusError(response)
-        if response.content and not response.headers.get('Content-Type').startswith(req_headers['Content-Type']):
+
+        resp_content_type = response.headers.get('Content-Type')
+        if not resp_content_type.startswith(req_headers['Content-Type']):
             raise HttpHeaderError(response)
+
         if not response.headers.get('X-Admin-Protocol', 'gsma/rsp/v2.unknown').startswith('gsma/rsp/v2.'):
             raise HttpHeaderError(response)
 
-        # Decode response and return the result back to the caller
         if response.content:
-            output = self.api_func.decode_client(response.json())
+            if resp_content_type.startswith('application/json'):
-            # In case the response contains a header, check it to make sure that the API call was executed successfully
+                return self.decode(response.json())
-            # (the presence of the header field is checked by the decode_client method)
+            elif resp_content_type.startswith('text/plain;charset=UTF-8'):
-            if 'header' in output:
+                return { 'data': response.content.decode('utf-8') }
-                if output['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
+            raise HttpHeaderError(f'unimplemented response Content-Type: {response.headers=!r}')
-                    raise ApiError(output['header']['functionExecutionStatus'])
-            return output
         return None
-
-class JsonHttpApiServer():
-    def __init__(self, api_func: JsonHttpApiFunction, call_handler = None):
-        """
-        Args:
-                api_func : API function definition (JsonHttpApiFunction)
-                call_handler : handler function to process the request. This function must accept the
-                               decoded request as a dictionary. The handler function must return a tuple consisting
-                               of the response in the form of a dictionary (may be empty), and a function execution
-                               status string ('Executed-Success', 'Executed-WithWarning', 'Failed' or 'Expired')
-        """
-        self.api_func = api_func
-        if call_handler:
-            self.call_handler = call_handler
-        else:
-            self.call_handler = self.default_handler
-
-    def default_handler(self, data: dict) -> (dict, str):
-        """default handler, used in case no call handler is provided."""
-        logger.error("no handler function for request: %s" % str(data))
-        return {}, 'Failed'
-
-    def call(self, request: Request) -> str:
-        """ Process an incoming request.
-        Args:
-                request : request object as received using twisted.web.server
-        Returns:
-                encoded JSON string (HTTP response code and headers are set by calling the appropriate methods on the
-                provided the request object)
-        """
-
-        # Make sure the request is done with the correct HTTP method
-        if (request.method.decode() != self.api_func.http_method):
-            raise ValueError('Wrong HTTP method %s!=%s' % (request.method.decode(), self.api_func.http_method))
-
-        # Decode the request
-        decoded_request = self.api_func.decode_server(json.loads(request.content.read()))
-
-        # Run call handler (see above)
-        data, fe_status = self.call_handler(decoded_request)
-
-        # In case a function execution status is returned, use it to generate and prepend the header field according to
-        # SGP.22, section 6.5.1.2 and 6.5.1.4 (the presence of the header filed is checked by the encode_server method)
-        if fe_status:
-            data = {'header' : {'functionExecutionStatus': {'status' : fe_status}}} | data
-
-        # Encode the message (the presence of mandatory fields is checked during encoding)
-        encoded = json.dumps(self.api_func.encode_server(data))
-
-        # Apply HTTP request headers according to SGP.22, section 6.5.1
-        res_headers = {
-            'Content-Type': 'application/json',
-            'X-Admin-Protocol': 'gsma/rsp/v2.5.0',
-        }
-        res_headers.update(self.api_func.extra_http_res_headers)
-        for header, value in res_headers.items():
-            request.setHeader(header, value)
-        request.setResponseCode(self.api_func.expected_http_status)
-
-        # Return the encoded result back to the caller for sending (using twisted/klein)
-        return encoded
-

pySim/esim/saip/__init__.py

@@ -34,7 +34,7 @@ from pySim import ts_102_222
 from pySim.utils import dec_imsi
 from pySim.ts_102_221 import FileDescriptor
 from pySim.filesystem import CardADF, Path
-from pySim.ts_31_102 import ADF_USIM
+from pySim.ts_31_102 import ADF_USIM, EF_UST, EF_SUCI_Calc_Info
 from pySim.ts_31_103 import ADF_ISIM
 from pySim.esim import compile_asn1_subdir
 from pySim.esim.saip import templates
@@ -1517,8 +1517,11 @@ class ProfileElementHeader(ProfileElement):
     def mandatory_service_add(self, service_name):
         self.decoded['eUICC-Mandatory-services'][service_name] = None
 
+    def mandatory_service_present(self, service_name):
+        return service_name in self.decoded['eUICC-Mandatory-services'].keys()
+
     def mandatory_service_remove(self, service_name):
-        if service_name in self.decoded['eUICC-Mandatory-services'].keys():
+        if self.mandatory_service_present(service_name):
             del self.decoded['eUICC-Mandatory-services'][service_name]
         else:
             raise ValueError("service not in eUICC-Mandatory-services list, cannot remove")
@@ -1726,12 +1729,61 @@ class ProfileElementSequence:
         if 'BT' in ftype_list:
             svc_set.add('ber-tlv')
         # FIXME:dfLinked files (scan all files, check for non-empty Fcp.linkPath presence of DFs)
-        # TODO: 5G related bits (derive from EF.UST or file presence?)
+
+        # 5G:
+        # - When SUCI is:
+        #   - enabled (EF.UST 124 = true)
+        #   AND
+        #   - calculated in the USIM (EF.UST 125 = true),
+        #   then eUICC-Mandatory-services needs 'get-identity'.
+        # - 'get-identity' implies that the eUICC must support ONE OF profile-A OR profile-B.
+        #   (One might assume from this that, when SUCI-CalcInfo for USIM in DF.SAIP contains both key types, then no
+        #   profile-A or B services need to be requested explicitly. However, the correct logic is:)
+        # - Iff the SUCI-CalcInfo for USIM (DF.SAIP) contains a key of profile-A ("identifier": 1),
+        #   then eUICC-Mandatory-services needs 'profile-a-x25519'.
+        # - Same: profile-B ("identifier": 2) needs 'profile-b-p256'.
+        # - (When SUCI is calculated in the UE, then the eUICC does not need to provide any of these services.)
+        suci_in_usim_enabled = False
+        try:
+            f_ust = self.get_pe_for_type("usim").files["ef-ust"]
+            ust = EF_UST().decode_bin(f_ust.body)
+            suci_in_usim_enabled = ust[124]['activated'] and ust[125]['activated']
+        except (KeyError, AttributeError):
+            pass
+        if suci_in_usim_enabled:
+            svc_set.add('get-identity')
+            # now check for profile-a and profile-b presence
+            suci_calcinfo_has_profile_a = False
+            suci_calcinfo_has_profile_b = False
+            try:
+                f_sucici = self.get_pe_for_type("df-saip").files["ef-suci-calc-info-usim"]
+                sucici = EF_SUCI_Calc_Info().decode_bin(f_sucici.body) or {}
+                for prot_scheme in sucici['prot_scheme_id_list']:
+                    if not isinstance(prot_scheme, dict):
+                        continue
+                    ps_id = prot_scheme["identifier"]
+                    if ps_id == 1:
+                        suci_calcinfo_has_profile_a = True
+                    elif ps_id == 2:
+                        suci_calcinfo_has_profile_b = True
+            except (KeyError, AttributeError):
+                pass
+            if suci_calcinfo_has_profile_a:
+                # The profile has a profile-A key, so require that
+                svc_set.add('profile-a-x25519')
+            if suci_calcinfo_has_profile_b:
+                # The profile has a profile-B key, so require that
+                svc_set.add('profile-b-p256')
+
         hdr_pe = self.get_pe_for_type('header')
         # patch in the 'manual' services from the existing list:
+        old_svc_set = set()
         for old_svc in hdr_pe.decoded['eUICC-Mandatory-services'].keys():
             if old_svc in manual_services:
-                svc_set.add(old_svc)
+                old_svc_set.add(old_svc)
+        logger.debug(f"{svc_set=}  +  {old_svc_set=}")
+        svc_set = svc_set.union(old_svc_set)
+        logger.debug(f"{svc_set=}")
         hdr_pe.decoded['eUICC-Mandatory-services'] = {x: None for x in svc_set}
 
     def rebuild_mandatory_gfstelist(self):

pySim/esim/saip/batch.py

@@ -19,10 +19,23 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import copy
-from typing import Generator
+import pprint
+import logging
+import traceback
+import inspect
+from typing import Generator, Union
 from pySim.esim.saip.personalization import ConfigurableParameter
 from pySim.esim.saip import param_source
-from pySim.esim.saip import ProfileElementSequence
+from pySim.esim.saip import ProfileElementSequence, ProfileElementSD
+from pySim.global_platform import KeyUsageQualifier
+from osmocom.utils import b2h
+
+logger = logging.getLogger(__name__)
+def _func_():
+    return inspect.currentframe().f_back.f_code.co_name
+
+# a list of ConfigurableParameter classes and/or ConfigurableParameter class instances
+ParamList = list[Union[type[ConfigurableParameter], ConfigurableParameter]]
 
 class BatchPersonalization:
     """Produce a series of eSIM profiles from predefined parameters.
@@ -115,6 +128,247 @@ class BatchPersonalization:
                     value = p.param_cls.validate_val(input_value)
                     p.param_cls.apply_val(pes, value)
                 except Exception as e:
+                    print(traceback.format_exc())
+                    logger.error('during %s: %r', _func_(), e)
                     raise ValueError(f'{p.param_cls.get_name()} fed by {p.src.name}: {e}') from e
 
+            pes.rebuild_mandatory_services()
+
             yield pes
+
+
+class UppAudit(dict):
+    """
+    Key-value pairs collected from a single UPP DER or PES.
+
+    UppAudit itself is a dict, callers may use the standard python dict API to access key-value pairs read from the UPP.
+    """
+
+    @classmethod
+    def from_der(cls, der: bytes, params: ParamList, der_size=False, additional_sd_keys=False):
+        """return a dict of parameter name and set of selected parameter values found in a DER encoded profile. Note:
+        some ConfigurableParameter implementations return more than one key-value pair, for example, Imsi returns
+        both 'IMSI' and 'IMSI-ACC' parameters.
+
+        e.g.
+            UppAudit.from_der(my_der, [Imsi, ])
+            --> {'IMSI': {'001010000000023'}, 'IMSI-ACC': {'5'}}
+
+        (where 'IMSI' == Imsi.name)
+
+        Read all parameters listed in params. params is a list of either ConfigurableParameter classes or
+        ConfigurableParameter class instances. This calls only classmethods, so each entry in params can either be the
+        class itself, or a class-instance of, a (non-abstract) ConfigurableParameter subclass.
+        For example, params = [Imsi, ] is equivalent to params = [Imsi(), ].
+
+        For der_size=True, also include a {'der_size':12345} entry.
+
+        For additional_sd_keys=True, output also all Security Domain KVN that there are *no* ConfigurableParameter
+        subclasses for. For example, SCP80 has reserved kvn 0x01..0x0f, but we offer only Scp80Kvn01, Scp80Kvn02,
+        Scp80Kvn03. So we would not show kvn 0x04..0x0f in an audit. additional_sd_keys=True includes audits of all SD
+        key KVN there may be in the UPP. This helps to spot SD keys that may already be present in a UPP template, with
+        unexpected / unusual kvn.
+        """
+
+        # make an instance of this class
+        upp_audit = cls()
+
+        if der_size:
+            upp_audit['der_size'] = set((len(der), ))
+
+        pes = ProfileElementSequence.from_der(der)
+        for param in params:
+            try:
+                for valdict in param.get_values_from_pes(pes):
+                    upp_audit.add_values(valdict)
+            except Exception as e:
+                raise ValueError(f'Error during audit for parameter {param}: {e}') from e
+
+        if not additional_sd_keys:
+            return upp_audit
+
+        # additional_sd_keys
+        for pe in pes.pe_list:
+            if pe.type != 'securityDomain':
+                continue
+            assert isinstance(pe, ProfileElementSD)
+
+            for key in pe.keys:
+                audit_key = f'SdKey_KVN{key.key_version_number:02x}_ID{key.key_identifier:02x}'
+                kuq_bin = KeyUsageQualifier.build(key.key_usage_qualifier).hex()
+                audit_val = f'{key.key_components=!r} key_usage_qualifier=0x{kuq_bin}={key.key_usage_qualifier!r}'
+                upp_audit.add_values({audit_key: audit_val})
+
+        return upp_audit
+
+    def get_single_val(self, key, allow_absent=False, absent_val=None):
+        """
+        Return the audit's value for the given audit key (like 'IMSI' or 'IMSI-ACC').
+        Any kind of value may occur multiple times in a profile. When all of these agree to the same unambiguous value,
+        return that value. When they do not agree, raise a ValueError.
+        """
+        # key should be a string, but if someone passes a ConfigurableParameter, just use its default name
+        if ConfigurableParameter.is_super_of(key):
+            key = key.get_name()
+
+        assert isinstance(key, str)
+        v = self.get(key)
+        if v is None and allow_absent:
+            return absent_val
+        if not isinstance(v, set):
+            raise ValueError(f'audit value should be a set(), got {v!r}')
+        if len(v) != 1:
+            raise ValueError(f'expected a single value for {key}, got {v!r}')
+        v = tuple(v)[0]
+        return v
+
+    @staticmethod
+    def audit_val_to_str(v):
+        """
+        Usually, we want to see a single value in an audit. Still, to be able to collect multiple ambiguous values,
+        audit values are always python sets. Turn it into a nice string representation: only the value when it is
+        unambiguous, otherwise a list of the ambiguous values.
+        A value may also be completely absent, then return 'not present'.
+        """
+        def try_single_val(w):
+            'change single-entry sets to just the single value'
+            if isinstance(w, set):
+                if len(w) == 1:
+                    return tuple(w)[0]
+                if len(w) == 0:
+                    return None
+            return w
+
+        v = try_single_val(v)
+        if isinstance(v, bytes):
+            v = b2h(v)
+        if v is None:
+            return 'not present'
+        return str(v)
+
+    def get_val_str(self, key):
+        """Return a string of the value stored for the given key"""
+        return UppAudit.audit_val_to_str(self.get(key))
+
+    def add_values(self, src:dict):
+        """Merge a plain dict of values into self, which is a dict of sets.
+        For example from
+            self == { 'a': {123} }
+        and
+            src == { 'a': 456, 'b': 789 }
+        then after this function call:
+            self == { 'a': {123, 456}, 'b': {789} }
+        """
+        assert isinstance(src, dict)
+        for key, srcval in src.items():
+            dstvalset = self.get(key)
+            if dstvalset is None:
+                dstvalset = set()
+                self[key] = dstvalset
+            dstvalset.add(srcval)
+
+    def __str__(self):
+        return '\n'.join(f'{key}: {self.get_val_str(key)}' for key in sorted(self.keys()))
+
+class BatchAudit(list):
+    """
+    Collect UppAudit instances for a batch of UPP, for example from a personalization.BatchPersonalization.
+    Produce an output CSV.
+
+    Usage example:
+
+        ba = BatchAudit(params=(personalization.Iccid, ))
+        for upp_der in upps:
+            ba.add_audit(upp_der)
+        print(ba.summarize())
+
+        with open('output.csv', 'wb') as csv_data:
+            csv_str = io.TextIOWrapper(csv_data, 'utf-8', newline='')
+            csv.writer(csv_str).writerows( ba.to_csv_rows() )
+            csv_str.flush()
+
+    BatchAudit itself is a list, callers may use the standard python list API to access the UppAudit instances.
+    """
+
+    def __init__(self, params: ParamList):
+        assert params
+        self.params = params
+
+    def add_audit(self, upp_der:bytes):
+        audit = UppAudit.from_der(upp_der, self.params)
+        self.append(audit)
+        return audit
+
+    def summarize(self):
+        batch_audit = UppAudit()
+
+        audits = self
+
+        if len(audits) > 2:
+            val_sep = ', ..., '
+        else:
+            val_sep = ', '
+
+        first_audit = None
+        last_audit = None
+        if len(audits) >= 1:
+            first_audit = audits[0]
+        if len(audits) >= 2:
+            last_audit = audits[-1]
+
+        if first_audit:
+            if last_audit:
+                for key in first_audit.keys():
+                    first_val = first_audit.get_val_str(key)
+                    last_val = last_audit.get_val_str(key)
+
+                    if first_val == last_val:
+                        val = first_val
+                    else:
+                        val_sep_with_newline = f"{val_sep.rstrip()}\n{' ' * (len(key) + 2)}"
+                        val = val_sep_with_newline.join((first_val, last_val))
+                    batch_audit[key] = val
+            else:
+                batch_audit.update(first_audit)
+
+        return batch_audit
+
+    def to_csv_rows(self, headers=True, sort_key=None, column_blacklist=None):
+        """generator that yields all audits' values as rows, useful feed to a csv.writer."""
+        columns = set()
+        for audit in self:
+            columns.update(audit.keys())
+
+        if column_blacklist:
+            columns.difference_update(set(column_blacklist))
+
+        columns = tuple(sorted(columns, key=sort_key))
+
+        if headers:
+            yield columns
+
+        for audit in self:
+            yield (audit.get_single_val(col, allow_absent=True, absent_val="") for col in columns)
+
+def esim_profile_introspect(upp):
+    pes = ProfileElementSequence.from_der(upp.read())
+    d = {}
+    d['upp'] = repr(pes)
+
+    def show_bytes_as_hexdump(item):
+        if isinstance(item, bytes):
+            return b2h(item)
+        if isinstance(item, list):
+            return list(show_bytes_as_hexdump(i) for i in item)
+        if isinstance(item, tuple):
+            return tuple(show_bytes_as_hexdump(i) for i in item)
+        if isinstance(item, dict):
+            d = {}
+            for k, v in item.items():
+                d[k] = show_bytes_as_hexdump(v)
+            return d
+        return item
+
+    l = list((pe.type, show_bytes_as_hexdump(pe.decoded)) for pe in pes)
+    d['pp'] = pprint.pformat(l, width=120)
+    return d

pySim/esim/saip/param_source.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import random
+import secrets
 import re
 from osmocom.utils import b2h
 
@@ -123,19 +123,30 @@ class DecimalRangeSource(InputExpandingParamSource):
     def val_to_digit(self, val:int):
         return "%0*d" % (self.num_digits, val)  # pylint: disable=consider-using-f-string
 
-class RandomDigitSource(DecimalRangeSource):
+class RandomSourceMixin:
+    random_impl = secrets.SystemRandom()
+
+class RandomDigitSource(DecimalRangeSource, RandomSourceMixin):
     """return a different sequence of random decimal digits each"""
     name = "random decimal digits"
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.used_keys = set()
+
     def get_next(self, csv_row:dict=None):
-        val = random.randint(self.first_value, self.last_value) # TODO secure random source?
+        # try to generate random digits that are always different from previously produced random digits
+        for _ in range(10):
+            val = self.random_impl.randint(self.first_value, self.last_value)
+            if val not in self.used_keys:
+                break
+        self.used_keys.add(val)
         return self.val_to_digit(val)
 
-class RandomHexDigitSource(InputExpandingParamSource):
+class RandomHexDigitSource(InputExpandingParamSource, RandomSourceMixin):
     """return a different sequence of random hexadecimal digits each"""
     name = "random hexadecimal digits"
     numeric_base = 16
-
     def __init__(self, input_str:str):
         super().__init__(input_str)
         input_str = self.input_str
@@ -147,9 +158,16 @@ class RandomHexDigitSource(InputExpandingParamSource):
         if (num_digits & 1) != 0:
             raise ValueError(f"hexadecimal value should have even number of digits, not {num_digits}")
         self.num_digits = num_digits
+        self.used_keys = set()
 
     def get_next(self, csv_row:dict=None):
-        val = random.randbytes(self.num_digits // 2) # TODO secure random source?
+        # try to generate random bytes that are always different from previously produced random bytes
+        for _ in range(10):
+            val = self.random_impl.randbytes(self.num_digits // 2)
+            if val not in self.used_keys:
+                break
+        self.used_keys.add(val)
+
         return b2h(val)
 
 class IncDigitSource(DecimalRangeSource):

pySim/euicc.py

@@ -226,9 +226,28 @@ class Icon(BER_TLV_IE, tag=0x94):
     _construct = GreedyBytes
 class ProfileClass(BER_TLV_IE, tag=0x95):
     _construct = Enum(Int8ub, test=0, provisioning=1, operational=2)
+class ProfilePolicyRules(BER_TLV_IE, tag=0x99):
+    _construct = GreedyBytes
+class NotificationConfigurationInfo(BER_TLV_IE, tag=0xb6):
+    _construct = GreedyBytes
+
+# ProfileOwner
+class ProfileOwnerPLMN(BER_TLV_IE, tag=0x80):
+    _construct = PlmnAdapter(Bytes(3))
+class ProfileOwnerGID1(BER_TLV_IE, tag=0x81):
+    _construct = GreedyBytes
+class ProfileOwnerGID2(BER_TLV_IE, tag=0x82):
+    _construct = GreedyBytes
+class ProfileOwner(BER_TLV_IE, tag=0xb7, nested=[ProfileOwnerPLMN, ProfileOwnerGID1, ProfileOwnerGID2]):
+    _construct = GreedyBytes
+
+class SMDPPProprietaryData(BER_TLV_IE, tag=0xb8):
+    _construct = GreedyBytes
+
 class ProfileInfo(BER_TLV_IE, tag=0xe3, nested=[Iccid, IsdpAid, ProfileState, ProfileNickname,
                                                 ServiceProviderName, ProfileName, IconType, Icon,
-                                                ProfileClass]): # FIXME: more IEs
+                                                ProfileClass, ProfilePolicyRules, NotificationConfigurationInfo,
+                                                ProfileOwner, SMDPPProprietaryData]):
     pass
 class ProfileInfoSeq(BER_TLV_IE, tag=0xa0, nested=[ProfileInfo]):
     pass
@@ -444,9 +463,28 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             d = rn.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['notification_sent_resp']))
 
-        def do_get_profiles_info(self, _opts):
+        get_profiles_info_parser = argparse.ArgumentParser()
+        get_profiles_info_parser.add_argument('--all', action='store_true', help='Retrieve all known tags of a profile')
+
+        @cmd2.with_argparser(get_profiles_info_parser)
+        def do_get_profiles_info(self, opts):
             """Perform an ES10c GetProfilesInfo function."""
-            pi = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, ProfileInfoListReq(), ProfileInfoListResp)
+            if opts.all:
+                tags = [nest.tag for nest in ProfileInfo.nested_collection_cls().nested]
+                u8tags = []
+                # TODO: rework TagList to support 2 byte tags to not filter it into u8 tags
+                for tag in tags:
+                    if tag <= 255:
+                        u8tags.append(tag)
+                    elif tag <= 65535:
+                        u8tags.append(tag >> 8)
+                        u8tags.append(tag & 0xff)
+                    # Ignoring 3 byte tags
+                req = ProfileInfoListReq(children=[TagList(decoded=u8tags)])
+            else:
+                req = ProfileInfoListReq()
+
+            pi = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, req, ProfileInfoListResp)
             d = pi.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['profile_info_list_resp']))
 

pySim/ts_31_102.py

@@ -327,7 +327,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         """conversion method to generate list of {hnet_pubkey_identifier, hnet_pubkey} dicts
         from flat [{hnet_pubkey_identifier: }, {net_pubkey: }, ...] list"""
         out = []
-        while len(l):
+        while l:
             a = l.pop(0)
             b = l.pop(0)
             z = {**a, **b}

pyproject.toml

@@ -4,3 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.pylint.main]
 ignored-classes = ["twisted.internet.reactor"]
+
+[tool.pylint.TYPECHECK]
+# SdKey subclasses are generated dynamically via SdKey.generate_sd_key_classes()
+generated-members = ["SdKey[A-Za-z0-9]+"]

tests/unittests/smdpp_data

@@ -0,0 +1 @@
+../../smdpp-data

tests/unittests/test_configurable_parameters.py

@@ -0,0 +1,749 @@
+#!/usr/bin/env python3
+
+# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: Neels Hofmeyr
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import enum
+import io
+import sys
+import unittest
+import json
+from importlib import resources
+from osmocom.utils import hexstr
+from pySim.esim.saip import ProfileElementSequence
+import pySim.esim.saip.personalization as p13n
+import smdpp_data.upp
+
+import xo
+update_expected_output = False
+
+def valstr(val):
+    if isinstance(val, io.BytesIO):
+        val = val.getvalue()
+    if isinstance(val, bytearray):
+        val = bytes(val)
+    return f'{val!r}'
+
+def valtypestr(val):
+    if isinstance(val, dict):
+        types = []
+        for v in val.values():
+            types.append(f'{type(v).__name__}')
+
+        val_type = '{' + ', '.join(types) + '}'
+    else:
+        val_type = f'{type(val).__name__}'
+    return f'{valstr(val)}:{val_type}'
+
+class ConfigurableParameterTest(unittest.TestCase):
+
+    def test_parameters(self):
+
+        upp_fnames = (
+                'SAIP2.1_gfmsuci.der',
+                'TS48v5_SAIP2.1B_NoBERTLV.der',
+                'TS48v5_SAIP2.3_NoBERTLV.der',
+               )
+
+        class Paramtest:
+            iff_present_default = False
+            def __init__(self, param_cls, val, expect_val, expect_clean_val=None, iff_present=None):
+                self.param_cls = param_cls
+                self.val = val
+                self.expect_clean_val = expect_clean_val
+                self.expect_val = expect_val
+                if iff_present is None:
+                    iff_present = Paramtest.iff_present_default
+                self.iff_present = iff_present
+
+        param_tests = [
+            Paramtest(param_cls=p13n.Imsi, val='123456',
+                      expect_clean_val=str('123456'),
+                      expect_val={'IMSI': hexstr('123456'),
+                                  'IMSI-ACC': '0040'}),
+            Paramtest(param_cls=p13n.Imsi, val=int(123456),
+                      expect_val={'IMSI': hexstr('123456'),
+                                  'IMSI-ACC': '0040'}),
+
+            Paramtest(param_cls=p13n.Imsi, val='123456789012345',
+                      expect_clean_val=str('123456789012345'),
+                      expect_val={'IMSI': hexstr('123456789012345'),
+                                  'IMSI-ACC': '0020'}),
+            Paramtest(param_cls=p13n.Imsi, val=int(123456789012345),
+                      expect_val={'IMSI': hexstr('123456789012345'),
+                                  'IMSI-ACC': '0020'}),
+
+            Paramtest(param_cls=p13n.Puk1,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+            Paramtest(param_cls=p13n.Puk1,
+                      val=int(12345678),
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+
+            Paramtest(param_cls=p13n.Puk2,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+
+            Paramtest(param_cls=p13n.Pin1,
+                      val='1234',
+                      expect_clean_val=b'1234\xff\xff\xff\xff',
+                      expect_val='1234'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val='123456',
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val=int(1234),
+                      expect_clean_val=b'1234\xff\xff\xff\xff',
+                      expect_val='1234'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val=int(123456),
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val=int(12345678),
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+
+            Paramtest(param_cls=p13n.Adm1,
+                      val='1234',
+                      expect_clean_val=b'1234\xff\xff\xff\xff',
+                      expect_val='1234'),
+            Paramtest(param_cls=p13n.Adm1,
+                      val='123456',
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+            Paramtest(param_cls=p13n.Adm1,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+            Paramtest(param_cls=p13n.Adm1,
+                      val=int(123456),
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val='Milenage',
+                      expect_clean_val=1,
+                      expect_val='Milenage'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val='TUAK',
+                      expect_clean_val=2,
+                      expect_val='TUAK'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val='usim-test',
+                      expect_clean_val=3,
+                      expect_val='usim-test'),
+
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val=1,
+                      expect_clean_val=1,
+                      expect_val='Milenage'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val=2,
+                      expect_clean_val=2,
+                      expect_val='TUAK'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val=3,
+                      expect_clean_val=3,
+                      expect_val='usim-test'),
+
+            Paramtest(param_cls=p13n.K,
+                      val='01020304050607080910111213141516',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=int(11020304050607080910111213141516),
+                      expect_clean_val=b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='11020304050607080910111213141516'),
+
+            Paramtest(param_cls=p13n.Opc,
+                      val='01020304050607080910111213141516',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.Opc,
+                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.Opc,
+                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.Opc,
+                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+
+            Paramtest(param_cls=p13n.SmspTpScAddr,
+                      val='+1234567',
+                      expect_clean_val=(True, '1234567'),
+                      expect_val='+1234567'),
+            Paramtest(param_cls=p13n.SmspTpScAddr,
+                      val=1234567,
+                      expect_clean_val=(False, '1234567'),
+                      expect_val='1234567'),
+
+            Paramtest(param_cls=p13n.TuakNumberOfKeccak,
+                      val='123',
+                      expect_clean_val=123,
+                      expect_val='123'),
+            Paramtest(param_cls=p13n.TuakNumberOfKeccak,
+                      val=123,
+                      expect_clean_val=123,
+                      expect_val='123'),
+
+            Paramtest(param_cls=p13n.MilenageRotationConstants,
+                      val='0a 0b 0c 01 02',
+                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_val='0a0b0c0102'),
+            Paramtest(param_cls=p13n.MilenageRotationConstants,
+                      val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_val='0a0b0c0102'),
+            Paramtest(param_cls=p13n.MilenageRotationConstants,
+                      val=bytearray(b'\x0a\x0b\x0c\x01\x02'),
+                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_val='0a0b0c0102'),
+
+            Paramtest(param_cls=p13n.MilenageXoringConstants,
+                      val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+                          ' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+                          ' cccccccccccccccccccccccccccccccc'
+                          ' 11111111111111111111111111111111'
+                          ' 22222222222222222222222222222222',
+                      expect_clean_val=b'\xaa' * 16
+                                       + b'\xbb' * 16
+                                       + b'\xcc' * 16
+                                       + b'\x11' * 16
+                                       + b'\x22' * 16,
+                      expect_val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+                                 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+                                 'cccccccccccccccccccccccccccccccc'
+                                 '11111111111111111111111111111111'
+                                 '22222222222222222222222222222222'),
+            Paramtest(param_cls=p13n.MilenageXoringConstants,
+                      val=b'\xaa' * 16
+                          + b'\xbb' * 16
+                          + b'\xcc' * 16
+                          + b'\x11' * 16
+                          + b'\x22' * 16,
+                      expect_clean_val=b'\xaa' * 16
+                                       + b'\xbb' * 16
+                                       + b'\xcc' * 16
+                                       + b'\x11' * 16
+                                       + b'\x22' * 16,
+                      expect_val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+                                 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+                                 'cccccccccccccccccccccccccccccccc'
+                                 '11111111111111111111111111111111'
+                                 '22222222222222222222222222222222'),
+
+
+            Paramtest(param_cls=p13n.MncLen,
+                      val='2',
+                      expect_clean_val=2,
+                      expect_val='2'),
+            Paramtest(param_cls=p13n.MncLen,
+                      val=3,
+                      expect_clean_val=3,
+                      expect_val='3'),
+
+            Paramtest(param_cls=p13n.EuiccMandatoryServiceGetIdentity,
+                      val='mandatory',
+                      expect_clean_val=True,
+                      expect_val='mandatory'),
+            Paramtest(param_cls=p13n.EuiccMandatoryServiceGetIdentity,
+                      val='optional',
+                      expect_clean_val=False,
+                      expect_val='optional'),
+
+            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileA,
+                      val='mandatory',
+                      expect_clean_val=True,
+                      expect_val='mandatory'),
+            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileA,
+                      val='optional',
+                      expect_clean_val=False,
+                      expect_val='optional'),
+
+            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileB,
+                      val='mandatory',
+                      expect_clean_val=True,
+                      expect_val='mandatory'),
+            Paramtest(param_cls=p13n.EuiccMandatoryServiceProfileB,
+                      val='optional',
+                      expect_clean_val=False,
+                      expect_val='optional'),
+        ]
+
+        Paramtest.iff_present_default = True
+
+        sucici = {
+             "prot_scheme_id_list": [
+                 {"priority": 0, "identifier": 2, "key_index": 1},
+                 {"priority": 1, "identifier": 1, "key_index": 2},
+                 ],
+             "hnet_pubkey_list": [
+                 {"hnet_pubkey_identifier": 27,
+                  "hnet_pubkey": "0472da71976234ce833a6907425867b82e074d44ef907dfb4b3e21c1c2256ebcd15a7ded52fcbb097a4ed250e036c7b9c8c7004c4eedc4f068cd7bf8d3f900e3b4"},
+                 {"hnet_pubkey_identifier": 30,
+                  "hnet_pubkey": "5a8d38864820197c3394b92613b20b91633cbd897119273bf8e4a6f4eec0a650"},
+                  ],
+        }
+
+        param_tests.extend([
+            Paramtest(param_cls=p13n.SuciActive, val='SUCI-on',
+                      expect_clean_val=True,
+                      expect_val={'5G-SUCI-active': 'SUCI-on'}),
+            Paramtest(param_cls=p13n.SuciActive, val='SUCI-off',
+                      expect_clean_val=False,
+                      expect_val={'5G-SUCI-active': 'SUCI-off'}),
+
+            Paramtest(param_cls=p13n.SuciInUsim, val='SUCI-in-UE',
+                      expect_clean_val=False,
+                      expect_val={'5G-SUCI-in-USIM': 'SUCI-in-UE'}),
+            Paramtest(param_cls=p13n.SuciInUsim, val='SUCI-in-USIM',
+                      expect_clean_val=True,
+                      expect_val={'5G-SUCI-in-USIM': 'SUCI-in-USIM'}),
+
+            Paramtest(param_cls=p13n.SuciRi, val='123',
+                      expect_clean_val='123',
+                      expect_val={'5G-SUCI-RI': '123'}),
+            Paramtest(param_cls=p13n.SuciRi, val='0',
+                      expect_clean_val='0',
+                      expect_val={'5G-SUCI-RI': '0'}),
+            Paramtest(param_cls=p13n.SuciRi, val='9999',
+                      expect_clean_val='9999',
+                      expect_val={'5G-SUCI-RI': '9999'}),
+
+            Paramtest(param_cls=p13n.SuciCalcInfoUe,
+                      val=json.dumps(sucici),
+                      expect_clean_val=sucici,
+                      expect_val={'5G-SUCI-CalcInfo-UE': json.dumps(sucici)}),
+
+            Paramtest(param_cls=p13n.SuciCalcInfoUsim,
+                      val=json.dumps(sucici),
+                      expect_clean_val=sucici,
+                      expect_val={'5G-SUCI-CalcInfo-USIM': json.dumps(sucici)}),
+
+            Paramtest(param_cls=p13n.GfmSuciRi, val='123',
+                      expect_clean_val='123',
+                      expect_val={'GFM-5G-SUCI-RI': '123'}),
+            Paramtest(param_cls=p13n.GfmSuciRi, val='0',
+                      expect_clean_val='0',
+                      expect_val={'GFM-5G-SUCI-RI': '0'}),
+            Paramtest(param_cls=p13n.GfmSuciRi, val='9999',
+                      expect_clean_val='9999',
+                      expect_val={'GFM-5G-SUCI-RI': '9999'}),
+
+            Paramtest(param_cls=p13n.GfmSuciCalcInfoUe,
+                      val=json.dumps(sucici),
+                      expect_clean_val=sucici,
+                      expect_val={'GFM-5G-SUCI-CalcInfo-UE': json.dumps(sucici)}),
+
+        ])
+
+        Paramtest.iff_present_default = False
+
+        for sdkey_cls in (
+            # thin out the number of tests, as a compromise between completeness and test runtime
+            p13n.SdKeyScp02Kvn20AesDek,
+           #p13n.SdKeyScp02Kvn20AesEnc,
+           #p13n.SdKeyScp02Kvn20AesMac,
+           #p13n.SdKeyScp02Kvn21AesDek,
+            p13n.SdKeyScp02Kvn21AesEnc,
+           #p13n.SdKeyScp02Kvn21AesMac,
+           #p13n.SdKeyScp02Kvn22AesDek,
+           #p13n.SdKeyScp02Kvn22AesEnc,
+            p13n.SdKeyScp02Kvn22AesMac,
+           #p13n.SdKeyScp02KvnffAesDek,
+           #p13n.SdKeyScp02KvnffAesEnc,
+           #p13n.SdKeyScp02KvnffAesMac,
+            p13n.SdKeyScp03Kvn30AesDek,
+           #p13n.SdKeyScp03Kvn30AesEnc,
+           #p13n.SdKeyScp03Kvn30AesMac,
+           #p13n.SdKeyScp03Kvn31AesDek,
+            p13n.SdKeyScp03Kvn31AesEnc,
+           #p13n.SdKeyScp03Kvn31AesMac,
+           #p13n.SdKeyScp03Kvn32AesDek,
+           #p13n.SdKeyScp03Kvn32AesEnc,
+            p13n.SdKeyScp03Kvn32AesMac,
+           #p13n.SdKeyScp80Kvn01AesDek,
+           #p13n.SdKeyScp80Kvn01AesEnc,
+           #p13n.SdKeyScp80Kvn01AesMac,
+            p13n.SdKeyScp80Kvn01DesDek,
+           #p13n.SdKeyScp80Kvn01DesEnc,
+           #p13n.SdKeyScp80Kvn01DesMac,
+           #p13n.SdKeyScp80Kvn02AesDek,
+            p13n.SdKeyScp80Kvn02AesEnc,
+           #p13n.SdKeyScp80Kvn02AesMac,
+           #p13n.SdKeyScp80Kvn02DesDek,
+           #p13n.SdKeyScp80Kvn02DesEnc,
+            p13n.SdKeyScp80Kvn02DesMac,
+           #p13n.SdKeyScp80Kvn03AesDek,
+           #p13n.SdKeyScp80Kvn03AesEnc,
+           #p13n.SdKeyScp80Kvn03AesMac,
+            p13n.SdKeyScp80Kvn03DesDek,
+           #p13n.SdKeyScp80Kvn03DesEnc,
+           #p13n.SdKeyScp80Kvn03DesMac,
+            p13n.SdKeyScp81Kvn40AesDek,
+           #p13n.SdKeyScp81Kvn40Tlspsk,
+           #p13n.SdKeyScp81Kvn41AesDek,
+            p13n.SdKeyScp81Kvn41Tlspsk,
+           #p13n.SdKeyScp81Kvn42AesDek,
+           #p13n.SdKeyScp81Kvn42Tlspsk,
+            ):
+
+            for key_len in sdkey_cls.allow_len:
+                val = '0102030405060708091011121314151617181920212223242526272829303132'
+                expect_clean_val = (b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'
+                                    b'\x17\x18\x19\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x30\x31\x32')
+                expect_val = '0102030405060708091011121314151617181920212223242526272829303132'
+
+                val = val[:key_len*2]
+                expect_clean_val = expect_clean_val[:key_len]
+                expect_val = val
+
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                # test bytes input
+                val = expect_clean_val
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                # test bytearray input
+                val = bytearray(expect_clean_val)
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                # test BytesIO input
+                val = io.BytesIO(expect_clean_val)
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                if key_len == 16:
+                    # test huge integer input.
+                    # needs to start with nonzero.. stupid
+                    val = 11020304050607080910111213141516
+                    expect_clean_val = (b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16')
+                    expect_val = '11020304050607080910111213141516'
+                    param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+        outputs = []
+
+        for upp_fname in upp_fnames:
+            test_idx = -1
+            try:
+
+                der = resources.read_binary(smdpp_data.upp, upp_fname)
+
+                for t in param_tests:
+                    test_idx += 1
+                    testlog = []
+                    testlog.append(f'{upp_fname} {t.param_cls.__name__}(val={valtypestr(t.val)})')
+
+                    param = None
+                    try:
+                        param = t.param_cls()
+                        param.input_value = t.val
+                        param.validate()
+                    except ValueError as e:
+                        raise ValueError(f'{" ".join(testlog)}: {e}') from e
+
+                    clean_val = param.value
+                    testlog.append(f'clean_val={valtypestr(clean_val)}')
+                    if t.expect_clean_val is not None and t.expect_clean_val != clean_val:
+                        raise ValueError(f'{" ".join(testlog)}: expected'
+                                         f' expect_clean_val={valtypestr(t.expect_clean_val)}')
+
+                    # on my laptop, deepcopy is about 30% slower than decoding the DER from scratch:
+                    # pes = copy.deepcopy(orig_pes)
+                    pes = ProfileElementSequence.from_der(der)
+
+                    found = list((t.param_cls.get_value_from_pes(pes) or {}).values())
+                    testlog.append(f"previous value: {found}")
+
+                    if t.iff_present and not found:
+                        testlog.append("skipping, param not in template.")
+                        output = "\nskip: " + "\n  ".join(testlog)
+                        outputs.append(output)
+                        print(output)
+                        continue
+
+                    try:
+                        param.apply(pes)
+                    except ValueError as e:
+                        raise ValueError(f'{" ".join(testlog)} apply_val(clean_val): {e}') from e
+
+                    changed_der = pes.to_der()
+
+                    pes2 = ProfileElementSequence.from_der(changed_der)
+
+                    read_back_val = t.param_cls.get_value_from_pes(pes2)
+
+                    # compose log string to show the precise type of dict values
+                    if isinstance(read_back_val, dict):
+                        types = set()
+                        for v in read_back_val.values():
+                            types.add(f'{type(v).__name__}')
+
+                        read_back_val_type = '{' + ', '.join(types) + '}'
+                    else:
+                        read_back_val_type = f'{type(read_back_val).__name__}'
+
+                    testlog.append(f'read_back_val={valtypestr(read_back_val)}')
+
+                    if isinstance(read_back_val, dict) and not t.param_cls.get_name() in read_back_val.keys():
+                        raise ValueError(f'{" ".join(testlog)}: expected to find name {t.param_cls.get_name()!r} in read_back_val')
+
+                    expect_val = t.expect_val
+                    if not isinstance(expect_val, dict):
+                        expect_val = { t.param_cls.get_name(): expect_val }
+                    if read_back_val != expect_val:
+                        raise ValueError(f'{" ".join(testlog)}: expected {expect_val=!r}:{type(t.expect_val).__name__}')
+
+                    output = "\nok: " + "\n  ".join(testlog)
+                    outputs.append(output)
+                    print(output)
+
+            except Exception as e:
+                raise RuntimeError(f'Error while testing UPP {upp_fname} {test_idx=}: {e}') from e
+
+        output = '\n'.join(outputs) + '\n'
+        xo_name = 'test_configurable_parameters'
+        if update_expected_output:
+            with resources.path(xo, xo_name) as xo_path:
+                with open(xo_path, 'w', encoding='utf-8') as f:
+                    f.write(output)
+        else:
+            xo_str = resources.read_text(xo, xo_name)
+            if xo_str != output:
+                at = 0
+                while at < len(output):
+                    if output[at] == xo_str[at]:
+                        at += 1
+                        continue
+                    break
+
+                raise RuntimeError(f'output differs from expected output at position {at}: "{output[at:at+20]}" != "{xo_str[at:at+20]}"')
+
+
+class TestValidateVal(unittest.TestCase):
+    """validate_val() tests for various ConfigurableParameter subclasses."""
+
+    def _ok(self, cls, val, expected=None):
+        result = cls.validate_val(val)
+        if expected is not None:
+            self.assertEqual(result, expected)
+        return result
+
+    def _err(self, cls, val):
+        with self.assertRaises(ValueError):
+            cls.validate_val(val)
+
+    # --- Iccid ---
+
+    def test_iccid_18digits_adds_luhn(self):
+        result = self._ok(p13n.Iccid, '998877665544332211')
+        self.assertIsInstance(result, str)
+        self.assertEqual(len(result), 19)
+        self.assertTrue(result.isdecimal())
+
+    def test_iccid_19digits_passthrough(self):
+        result = self._ok(p13n.Iccid, '9988776655443322110')
+        self.assertIsInstance(result, str)
+        self.assertEqual(len(result), 19)
+
+    def test_iccid_too_short(self):
+        self._err(p13n.Iccid, '12345678901234567')  # 17 digits
+
+    def test_iccid_too_long(self):
+        self._err(p13n.Iccid, '1' * 21)
+
+    def test_iccid_non_digits(self):
+        self._err(p13n.Iccid, '99887766554433221X')
+
+    # --- Imsi ---
+
+    def test_imsi_valid_short(self):
+        self._ok(p13n.Imsi, '001010', '001010')
+
+    def test_imsi_valid_long(self):
+        self._ok(p13n.Imsi, '001010123456789', '001010123456789')
+
+    def test_imsi_too_short(self):
+        self._err(p13n.Imsi, '12345')  # 5 digits, min is 6
+
+    def test_imsi_too_long(self):
+        self._err(p13n.Imsi, '1' * 16)
+
+    def test_imsi_non_digits(self):
+        self._err(p13n.Imsi, '00101A123456789')
+
+    # --- Pin1 ---
+
+    def test_pin1_4digits(self):
+        # DecimalHexParam encodes each digit as its ASCII byte, then rpad to 8 bytes with 0xff
+        self._ok(p13n.Pin1, '1234', b'1234\xff\xff\xff\xff')
+
+    def test_pin1_8digits(self):
+        self._ok(p13n.Pin1, '12345678', b'12345678')
+
+    def test_pin1_too_short(self):
+        self._err(p13n.Pin1, '123')
+
+    def test_pin1_too_long(self):
+        self._err(p13n.Pin1, '123456789')
+
+    def test_pin1_non_digits(self):
+        self._err(p13n.Pin1, '123A')
+
+    # --- Puk1 ---
+
+    def test_puk1_8digits(self):
+        self._ok(p13n.Puk1, '12345678', b'12345678')
+
+    def test_puk1_wrong_length(self):
+        self._err(p13n.Puk1, '1234567')   # 7 digits
+        self._err(p13n.Puk1, '123456789')  # 9 digits
+
+    def test_puk1_non_digits(self):
+        self._err(p13n.Puk1, '1234567X')
+
+    # --- K (BinaryParam) ---
+
+    def test_k_valid_hex_str(self):
+        self._ok(p13n.K, '000102030405060708090a0b0c0d0e0f',
+                 b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f')
+
+    def test_k_valid_bytes(self):
+        raw = bytes(range(16))
+        self._ok(p13n.K, raw, raw)
+
+    def test_k_wrong_length(self):
+        self._err(p13n.K, '00' * 15)  # 15 bytes, allow_len requires 16 or 32
+
+    def test_k_non_hex(self):
+        self._err(p13n.K, 'gg' * 16)
+
+    def test_k_odd_hex_digits(self):
+        self._err(p13n.K, '0' * 31)   # odd number of hex digits
+
+
+class TestEnumParam(unittest.TestCase):
+    """Tests for the EnumParam machinery, using AlgorithmID as the concrete subclass."""
+
+    # --- validate_val ---
+
+    def test_validate_by_name_exact(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val('Milenage'), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val('TUAK'), 2)
+        self.assertEqual(p13n.AlgorithmID.validate_val('usim-test'), 3)
+
+    def test_validate_by_int(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val(1), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val(2), 2)
+        self.assertEqual(p13n.AlgorithmID.validate_val(3), 3)
+
+    def test_validate_fuzzy_case(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val('milenage'), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val('MILENAGE'), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val('tuak'), 2)
+
+    def test_validate_fuzzy_hyphen_underscore(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val('usim-test'), 3)
+
+    def test_validate_invalid_name(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.validate_val('unknown')
+
+    def test_validate_invalid_int(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.validate_val(99)
+
+    def test_validate_returns_int(self):
+        result = p13n.AlgorithmID.validate_val('Milenage')
+        self.assertIsInstance(result, int)
+        self.assertNotIsInstance(result, enum.Enum)
+
+    # --- map_name_to_val ---
+
+    def test_map_name_exact(self):
+        self.assertEqual(p13n.AlgorithmID.map_name_to_val('Milenage'), 1)
+
+    def test_map_name_fuzzy(self):
+        self.assertEqual(p13n.AlgorithmID.map_name_to_val('milenage'), 1)
+        self.assertEqual(p13n.AlgorithmID.map_name_to_val('usim-test'), 3)
+
+    def test_map_name_strict_raises(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.map_name_to_val('unknown', strict=True)
+
+    def test_map_name_nonstrict_returns_none(self):
+        self.assertIsNone(p13n.AlgorithmID.map_name_to_val('unknown', strict=False))
+
+    # --- map_val_to_name ---
+
+    def test_map_val_known(self):
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(1), 'Milenage')
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(2), 'TUAK')
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(3), 'usim-test')
+
+    def test_map_val_unknown_nonstrict(self):
+        self.assertIsNone(p13n.AlgorithmID.map_val_to_name(99))
+
+    def test_map_val_unknown_strict(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.map_val_to_name(99, strict=True)
+
+    # --- name_normalize ---
+
+    def test_name_normalize(self):
+        self.assertEqual(p13n.AlgorithmID.name_normalize('Milenage'), 'Milenage')
+        self.assertEqual(p13n.AlgorithmID.name_normalize('milenage'), 'Milenage')
+        self.assertEqual(p13n.AlgorithmID.name_normalize('usim-test'), 'usim-test')
+
+    # --- clean_name_str ---
+
+    def test_clean_name_str(self):
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim-test'), 'usim-test')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim_test'), 'usim_test')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('Milenage'), 'milenage')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('foo bar!'), 'foobar')
+
+
+if __name__ == "__main__":
+    if '-u' in sys.argv:
+        update_expected_output = True
+        sys.argv.remove('-u')
+    unittest.main()

tests/unittests/test_esim_saip.py

@@ -21,7 +21,7 @@ import copy
 from osmocom.utils import h2b, b2h
 
 from pySim.esim.saip import *
-from pySim.esim.saip.personalization import *
+from pySim.esim.saip import personalization
 from pprint import pprint as pp
 
 
@@ -55,14 +55,56 @@ class SaipTest(unittest.TestCase):
     def test_personalization(self):
         """Test some of the personalization operations."""
         pes = copy.deepcopy(self.pes)
-        params = [Puk1('01234567'), Puk2(98765432), Pin1('1111'), Pin2(2222), Adm1('11111111'),
+        params = [personalization.Puk1('01234567'),
-                  K(h2b('000102030405060708090a0b0c0d0e0f')), Opc(h2b('101112131415161718191a1b1c1d1e1f'))]
+                  personalization.Puk2(98765432),
+                  personalization.Pin1('1111'),
+                  personalization.Pin2(2222),
+                  personalization.Adm1('11111111'),
+                  personalization.K(h2b('000102030405060708090a0b0c0d0e0f')),
+                  personalization.Opc(h2b('101112131415161718191a1b1c1d1e1f'))]
         for p in params:
             p.validate()
             p.apply(pes)
         # TODO: we don't actually test the results here, but we just verify there is no exception
         pes.to_der()
 
+    def test_personalization2(self):
+        """Test some of the personalization operations."""
+        cls = personalization.SdKeyScp80Kvn01DesEnc
+        pes = ProfileElementSequence.from_der(self.per_input)
+        prev_val = tuple(cls.get_values_from_pes(pes))
+        print(f'{prev_val=}')
+        self.assertTrue(prev_val)
+
+        set_val = '42342342342342342342342342342342'
+        param = cls(set_val)
+        param.validate()
+        param.apply(pes)
+
+        get_val1 = tuple(cls.get_values_from_pes(pes))
+        print(f'{get_val1=} {set_val=}')
+        self.assertEqual(get_val1, ({cls.name: set_val},))
+
+        get_val1b = tuple(cls.get_values_from_pes(pes))
+        print(f'{get_val1b=} {set_val=}')
+        self.assertEqual(get_val1b, ({cls.name: set_val},))
+
+        der = pes.to_der()
+
+        get_val1c = tuple(cls.get_values_from_pes(pes))
+        print(f'{get_val1c=} {set_val=}')
+        self.assertEqual(get_val1c, ({cls.name: set_val},))
+
+        # assertTrue to not dump the entire der.
+        # Expecting the modified DER to be different. If this assertion fails, then no change has happened in the output
+        # DER and the ConfigurableParameter subclass is buggy.
+        self.assertTrue(der != self.per_input)
+
+        pes2 = ProfileElementSequence.from_der(der)
+        get_val2 = tuple(cls.get_values_from_pes(pes2))
+        print(f'{get_val2=} {set_val=}')
+        self.assertEqual(get_val2, ({cls.name: set_val},))
+
     def test_constructor_encode(self):
         """Test that DER-encoding of PE created by "empty" constructor works without raising exception."""
         for cls in [ProfileElementMF, ProfileElementPuk, ProfileElementPin, ProfileElementTelecom,

tests/unittests/test_param_src.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+
+# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: Neels Hofmeyr
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import math
+from importlib import resources
+import unittest
+from pySim.esim.saip import param_source
+
+import xo
+update_expected_output = False
+
+class D:
+    mandatory = set()
+    optional = set()
+
+    def __init__(self, **kwargs):
+        if (set(kwargs.keys()) - set(self.optional)) != set(self.mandatory):
+            raise RuntimeError(f'{self.__class__.__name__}.__init__():'
+                               f' {set(kwargs.keys())=!r} - {self.optional=!r} != {self.mandatory=!r}')
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+        for k in self.optional:
+            if not hasattr(self, k):
+                setattr(self, k, None)
+
+decimals = '0123456789'
+hexadecimals = '0123456789abcdefABCDEF'
+
+class FakeRandom:
+    vals = b'\xab\xcfm\xf0\x98J_\xcf\x96\x87fp5l\xe7f\xd1\xd6\x97\xc1\xf9]\x8c\x86+\xdb\t^ke\xc1r'
+    i = 0
+
+    @classmethod
+    def next(cls):
+        cls.i = (cls.i + 1) % len(cls.vals)
+        return cls.vals[cls.i]
+
+    @staticmethod
+    def randint(a, b):
+        d = b - a
+        n_bytes = math.ceil(math.log(d, 2))
+        r = int.from_bytes( bytes(FakeRandom.next() for i in range(n_bytes)) )
+        return a + (r % (b - a))
+
+    @staticmethod
+    def randbytes(n):
+        return bytes(FakeRandom.next() for i in range(n))
+
+
+class ParamSourceTest(unittest.TestCase):
+
+    def test_param_source(self):
+
+        class Paramtest(D):
+            mandatory = (
+                    'param_source',
+                    'n',
+                    'expect',
+                    )
+            optional = (
+                    'expect_arg',
+                    'csv_rows',
+                    )
+            param_source: param_source.ParamSource
+            n: int
+            expect: object
+            expect_arg: object
+            csv_rows: object
+
+        def expect_const(t, vals):
+            return tuple(t.expect_arg) == tuple(vals)
+
+        def expect_random(t, vals):
+            chars = t.expect_arg.get('digits')
+            repetitions = (t.n - len(set(vals)))
+            if repetitions:
+                raise RuntimeError(f'expect_random: there are {repetitions} repetitions in the returned values: {vals}')
+            for val_i in range(len(vals)):
+                v = vals[val_i]
+                val_minlen = t.expect_arg.get('val_minlen')
+                val_maxlen = t.expect_arg.get('val_maxlen')
+                if len(v) < val_minlen or len(v) > val_maxlen:
+                    raise RuntimeError(f'expect_random: invalid length {len(v)} for value [{val_i}]: {v!r}, expecting'
+                                       f' {val_minlen}..{val_maxlen}')
+
+                if chars is not None and not all(c in chars for c in v):
+                    raise RuntimeError(f'expect_random: invalid char in value [{val_i}]: {v!r}')
+            return True
+
+        param_source_tests = [
+            Paramtest(param_source=param_source.ConstantSource.from_str('123'),
+                      n=3,
+                      expect=expect_const,
+                      expect_arg=('123', '123', '123')),
+            Paramtest(param_source=param_source.RandomDigitSource.from_str('12345'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': decimals,
+                                  'val_minlen': 5,
+                                  'val_maxlen': 5}),
+            Paramtest(param_source=param_source.RandomDigitSource.from_str('1..999'),
+                      n=10,
+                      expect=expect_random,
+                      expect_arg={'digits': decimals,
+                                  'val_minlen': 1,
+                                  'val_maxlen': 3}),
+            Paramtest(param_source=param_source.RandomDigitSource.from_str('001..999'),
+                      n=10,
+                      expect=expect_random,
+                      expect_arg={'digits': decimals,
+                                  'val_minlen': 3,
+                                  'val_maxlen': 3}),
+            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('12345678'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': hexadecimals,
+                                  'val_minlen': 8,
+                                  'val_maxlen': 8}),
+            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('0*8'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': hexadecimals,
+                                  'val_minlen': 8,
+                                  'val_maxlen': 8}),
+            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('00*4'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': hexadecimals,
+                                  'val_minlen': 8,
+                                  'val_maxlen': 8}),
+            Paramtest(param_source=param_source.IncDigitSource.from_str('10001'),
+                      n=3,
+                      expect=expect_const,
+                      expect_arg=('10001', '10002', '10003')),
+            Paramtest(param_source=param_source.CsvSource('column_name'),
+                      n=3,
+                      expect=expect_const,
+                      expect_arg=('first val', 'second val', 'third val'),
+                      csv_rows=(
+                          {'column_name': 'first val'},
+                          {'column_name': 'second val'},
+                          {'column_name': 'third val'},
+                      )),
+            ]
+
+        outputs = []
+
+        for t in param_source_tests:
+            try:
+                if hasattr(t.param_source, 'random_impl'):
+                    t.param_source.random_impl = FakeRandom
+
+                vals = []
+                for i in range(t.n):
+                    csv_row = None
+                    if t.csv_rows is not None:
+                        csv_row = t.csv_rows[i]
+                    vals.append( t.param_source.get_next(csv_row=csv_row) )
+                if not t.expect(t, vals):
+                    raise RuntimeError(f'invalid values returned: returned {vals}')
+                output = f'ok: {t.param_source.__class__.__name__} {vals=!r}'
+                outputs.append(output)
+                print(output)
+            except RuntimeError as e:
+                raise RuntimeError(f'{t.param_source.__class__.__name__} {t.n=} {t.expect.__name__}({t.expect_arg!r}): {e}') from e
+
+        output = '\n'.join(outputs) + '\n'
+        xo_name = 'test_param_src'
+        if update_expected_output:
+            with resources.path(xo, xo_name) as xo_path:
+                with open(xo_path, 'w', encoding='utf-8') as f:
+                    f.write(output)
+        else:
+            xo_str = resources.read_text(xo, xo_name)
+            if xo_str != output:
+                at = 0
+                while at < len(output):
+                    if output[at] == xo_str[at]:
+                        at += 1
+                        continue
+                    break
+
+                raise RuntimeError(f'output differs from expected output at position {at}: {xo_str[at:at+128]!r}')
+
+if __name__ == "__main__":
+    if '-u' in sys.argv:
+        update_expected_output = True
+        sys.argv.remove('-u')
+    unittest.main()

tests/unittests/xo/test_param_src

@@ -0,0 +1,9 @@
+ok: ConstantSource vals=['123', '123', '123']
+ok: RandomDigitSource vals=['13987', '49298', '55670']
+ok: RandomDigitSource vals=['650', '580', '49', '885', '497', '195', '320', '137', '245', '663']
+ok: RandomDigitSource vals=['638', '025', '232', '779', '826', '972', '650', '580', '049', '885']
+ok: RandomHexDigitSource vals=['6b65c172', 'abcf6df0', '984a5fcf']
+ok: RandomHexDigitSource vals=['96876670', '356ce766', 'd1d697c1']
+ok: RandomHexDigitSource vals=['f95d8c86', '2bdb095e', '6b65c172']
+ok: IncDigitSource vals=['10001', '10002', '10003']
+ok: CsvSource vals=['first val', 'second val', 'third val']