sysmocom_sjs1: CardModel for the sysmoUSIM-SJS1

SJS1 model lifted off the the legacy
SysmoUSIMSJS1.program(), ADM is ADM1 with CHV 0x0a,
already used by the generic UiccCardBase default.

Change-Id: I32b011c68de5f990e709718b313b6f28a31ca78b

.gitignore

@@ -3,6 +3,7 @@
 
 /docs/_*
 /docs/generated
+/docs/filesystem.rst
 /.cache
 /.local
 /build

README.md

@@ -97,7 +97,7 @@ Please install the following dependencies:
  - pyscard
  - pyserial
  - pytlv
- - pyyaml >= 5.1
+ - pyyaml >= 5.4
  - smpp.pdu (from `github.com/hologram-io/smpp.pdu`)
  - termcolor
 

contrib/csv-to-pgsql.py

@@ -285,10 +285,7 @@ if __name__ == '__main__':
     option_parser.add_argument("--admin", action='store_true', help="perform action as admin", default=False)
     opts = option_parser.parse_args()
 
-    PySimLogger.setup(print, {logging.WARN: "\033[33m"})
-    if (opts.verbose):
-        PySimLogger.set_verbose(True)
-        PySimLogger.set_level(logging.DEBUG)
+    PySimLogger.setup(print, {logging.WARN: "\033[33m"}, opts.verbose)
 
     # Open CSV file
     cr = open_csv(opts)

contrib/jenkins.sh

@@ -10,6 +10,11 @@
 
 export PYTHONUNBUFFERED=1
 
+setup_venv() {
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
+}
+
 if [ ! -d "./tests/" ] ; then
 	echo "###############################################"
 	echo "Please call from pySim-prog top directory"
@@ -23,8 +28,7 @@ fi
 
 case "$JOB_TYPE" in
 "test")
-	virtualenv -p python3 venv --system-site-packages
-	. venv/bin/activate
+	setup_venv
 
 	pip install -r requirements.txt
 	pip install pyshark
@@ -32,23 +36,27 @@ case "$JOB_TYPE" in
 	# Execute automatically discovered unit tests first
 	python -m unittest discover -v -s tests/unittests
 
-	# Run pySim-prog integration tests (requires physical cards)
-	cd tests/pySim-prog_test/
-        ./pySim-prog_test.sh
-	cd ../../
-
 	# Run pySim-trace test
 	tests/pySim-trace_test/pySim-trace_test.sh
+	;;
+"card-test") # tests requiring physical cards
+	setup_venv
 
-	# Run pySim-shell integration tests (requires physical cards)
+	pip install -r requirements.txt
+
+	# Run pySim-prog integration tests
+	cd tests/pySim-prog_test/
+	./pySim-prog_test.sh
+	cd ../../
+
+	# Run pySim-shell integration tests
 	python3 -m unittest discover -v -s ./tests/pySim-shell_test/
 
 	# Run pySim-smpp2sim test
 	tests/pySim-smpp2sim_test/pySim-smpp2sim_test.sh
 	;;
 "distcheck")
-	virtualenv -p python3 venv --system-site-packages
-	. venv/bin/activate
+	setup_venv
 
 	pip install .
 	pip install pyshark
@@ -61,8 +69,7 @@ case "$JOB_TYPE" in
 	# Print pylint version
 	pip3 freeze | grep pylint
 
-	virtualenv -p python3 venv --system-site-packages
-	. venv/bin/activate
+	setup_venv
 
 	pip install .
 
@@ -80,8 +87,7 @@ case "$JOB_TYPE" in
 		contrib/*.py
 	;;
 "docs")
-	virtualenv -p python3 venv --system-site-packages
-	. venv/bin/activate
+	setup_venv
 
 	pip install -r requirements.txt
 

contrib/scp81_admin_server.py

@@ -0,0 +1,524 @@
+#!/usr/bin/env python3
+"""TLS-PSK HTTP Remote Administration Server for SCP81 / RAM over HTTP"""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# The card (specifically a SD supporting SCP81) is the TLS client:
+# - it opens a TCP connection to this server,
+# - performs a TLS handshake authenticated with a PSK,
+# - and then drives the HTTP admin loop of to fetch remote APDU command strings
+# - and posts back their responses.
+# This program is the server side of that exchange, it:
+# - accepts the PSK-TLS connection,
+# - hands the card a queue of commands
+# - and logs the decoded responses.
+#
+# The two TS 102 226 annex B figure B.1 administration modes are supported over the
+# same session, selected with --mode:
+#   ram  GP Amendment B RAM:
+#           command is handled by (--targeted-application) a SD
+#   rfm  ETSI TS 102 226 RFM/RAM:
+#           command is routed to the Receiving/RFM Application specified by
+#           --targeted-application, for example UICC-filesystem/USIM-ADF RFM app.
+#
+# Remote APDU command/response bodies use the Expanded Remote Application
+# data format.
+#
+
+import ssl
+import socket
+import logging
+import argparse
+import threading
+from pathlib import Path
+from typing import List, Optional, Callable, Dict, Tuple
+
+from osmocom.utils import h2b, b2h
+
+from pySim.ota import encode_expanded_cmd, decode_expanded_resp
+
+logger = logging.getLogger(Path(__file__).stem)
+
+# Amendment B section 3.4
+ADMIN_PROTOCOL = 'globalplatform-remote-admin/1.0'
+CT_COMMAND = 'application/vnd.globalplatform.card-content-mgt;version=1.0'
+CT_RESPONSE = 'application/vnd.globalplatform.card-content-mgt-response;version=1.0'
+
+#  TS 102 226 annex B, figure B.1 RFM/RAM over HTTPS content types
+CT_RFM_COMMAND = 'application/vnd.etsi.scp.command-data;version=1.0'
+CT_RFM_RESPONSE = 'application/vnd.etsi.scp.response-data;version=1.0'
+
+MODE_CONTENT_TYPE = {
+    'ram': CT_COMMAND,          # GP Amendment B RAM: target = a Security Domain
+    'rfm': CT_RFM_COMMAND,      # ETSI TS 102 226 RFM/RAM: target = an application
+}
+
+# Amendment B Table 3-2. The 3DES and NULL suites are left out and can be enabled with
+# --ciphers / --seclevel.
+DEFAULT_CIPHERS = ':'.join([
+    'PSK-AES128-CBC-SHA256',    # TLS_PSK_WITH_AES_128_CBC_SHA256, TLS 1.2
+    'PSK-AES128-CBC-SHA',       # TLS_PSK_WITH_AES_128_CBC_SHA, TLS 1.0/1.1
+])
+
+TLS_VERSION_MAP = {
+    '1.0': ssl.TLSVersion.TLSv1,
+    '1.1': ssl.TLSVersion.TLSv1_1,
+    '1.2': ssl.TLSVersion.TLSv1_2,
+    '1.3': ssl.TLSVersion.TLSv1_3,
+}
+
+
+def format_aid(aid: str) -> str:
+    """AID -> //aid/<RID>/<PIX> for X-Admin-Targeted-Application from Amendment B section 3.4.2
+      First 5 bytes RID, the PIX the remainder, string in //aid/ notation is passed through."""
+    if aid.startswith('//aid/'):
+        return aid
+    aid = aid.replace(' ', '').lower()
+    if len(aid) < 10:
+        raise ValueError('AID %r is shorter than the 5 byte RID' % aid)
+    rid, pix = aid[:10], aid[10:]
+    return '//aid/%s/%s' % (rid, pix)
+
+
+def make_ssl_context(psk: bytes, identity: str, *,
+                     ciphers: str = DEFAULT_CIPHERS,
+                     min_tls: str = '1.2', max_tls: str = '1.3',
+                     seclevel: Optional[int] = None,
+                     identity_hint: Optional[str] = None,
+                     allow_any_identity: bool = False,
+                     extra_psks: Optional[Dict[str, bytes]] = None) -> ssl.SSLContext:
+    """PSK SSLContext, resolvesg the key from the client psk_identity
+
+    extra_psks can carry additional identity->key mappings.
+    allow_any_identity can be used for debugging
+    """
+    # the PSK callback must return immutable bytes, h2b() gives a bytearray
+    psk = bytes(psk)
+    keymap: Dict[str, bytes] = {identity: psk}
+    if extra_psks:
+        keymap.update({k: bytes(v) for k, v in extra_psks.items()})
+
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    ctx.minimum_version = TLS_VERSION_MAP[min_tls]
+    ctx.maximum_version = TLS_VERSION_MAP[max_tls]
+    cipher_str = ciphers
+    if seclevel is not None:
+        # @SECLEVEL=0 is to enable NULL/3DES/legacy PSK suites
+        cipher_str = '%s:@SECLEVEL=%d' % (ciphers, seclevel)
+    if cipher_str:
+        ctx.set_ciphers(cipher_str)
+
+    def psk_server_callback(client_identity: Optional[str]) -> bytes:
+        if allow_any_identity:
+            logger.info('PSK handshake: identity=%r (ACEPTING ANY!)', client_identity)
+            return psk
+        key = keymap.get(client_identity)
+        if key is None:
+            logger.warning('PSK handshake: unknown identity %r (known: %r) -> rejecting',
+                           client_identity, list(keymap.keys()))
+            return b''    # empty PSK aborts handshake
+        logger.info('PSK handshake: identity=%r resolved', client_identity)
+        return key
+
+    ctx.set_psk_server_callback(psk_server_callback, identity_hint=identity_hint)
+    return ctx
+
+
+class HttpRequest:
+    """A parsed HTTP request (request line + headers + body)."""
+    __slots__ = ('method', 'uri', 'version', 'headers', 'body')
+
+    def __init__(self, method: str, uri: str, version: str,
+                 headers: Dict[str, str], body: bytes):
+        self.method = method
+        self.uri = uri
+        self.version = version
+        self.headers = headers      # lower cased field names
+        self.body = body
+
+    def get(self, name: str, default=None) -> Optional[str]:
+        return self.headers.get(name.lower(), default)
+
+
+# http.client bound on a single HTTP line.
+MAX_LINE = 65536
+
+
+def _read_line(rfile) -> bytes:
+    """readline() with a bound. reaching the bound without a
+    terminator means the card is out of sync somehow, not that the line is long."""
+    line = rfile.readline(MAX_LINE)
+    if line and not line.endswith(b'\n'):
+        raise ValueError('HTTP line longer than %u bytes' % MAX_LINE)
+    return line
+
+
+def _read_chunked_body(rfile) -> bytes:
+    """Read a Transfer-Encoding: chunked body. Amendment B section 3.4.1 lets
+    the card send its response string with either a Content-Length or chunked"""
+    out = bytearray()
+    while True:
+        size_line = _read_line(rfile)
+        if not size_line:
+            break
+        size = int(size_line.split(b';', 1)[0].strip() or b'0', 16)
+        if size == 0:
+            # consume trailer headers up to the terminating blank line
+            while _read_line(rfile) not in (b'\r\n', b'\n', b''):
+                pass
+            break
+        chunk = rfile.read(size)
+        if len(chunk) != size:
+            raise ValueError('chunked body ended after %u of %u bytes' % (len(chunk), size))
+        out += chunk
+        _read_line(rfile)   # trailing CRLF after the chunk data
+    return bytes(out)
+
+
+def read_http_request(rfile, send: Optional[Callable[[bytes], None]] = None) -> Optional[HttpRequest]:
+    """Read one HTTP request from a buffered binary reader or None when closed"""
+    request_line = _read_line(rfile)
+    if not request_line:
+        return None
+    parts = request_line.rstrip(b'\r\n').decode('iso-8859-1').split(' ')
+    if len(parts) < 3:
+        raise ValueError('Malformed HTTP request line: %r' % request_line)
+    method, uri, version = parts[0], parts[1], parts[2]
+
+    headers: Dict[str, str] = {}
+    while True:
+        line = _read_line(rfile)
+        if line in (b'\r\n', b'\n', b''):
+            break
+        name, _, value = line.rstrip(b'\r\n').decode('iso-8859-1').partition(':')
+        headers[name.strip().lower()] = value.strip()
+
+    # Expect: 100-continue waits for the response before it sends the body,
+    # and RFC 2616 8.2.3 (Amendment B references RFC 2616 as [HTTP])
+    # requires the server to send it. Amendment B 3.4.1 does not mention this
+    # header, tho, might be useless.
+    if send and '100-continue' in headers.get('expect', '').lower():
+        logger.info('-> 100 Continue ')
+        send(b'HTTP/1.1 100 Continue\r\n\r\n')
+
+    body = b''
+    te = headers.get('transfer-encoding', '').lower()
+    if 'chunked' in te:
+        body = _read_chunked_body(rfile)
+    elif 'content-length' in headers:
+        n = int(headers['content-length'])
+        if n:
+            body = rfile.read(n)
+    return HttpRequest(method, uri, version, headers, body)
+
+
+def build_http_response(status_line: str, headers: List[Tuple[str, str]],
+                        body: bytes = b'') -> bytes:
+    """Serialise  HTTP response.  status_line 'HTTP/1.1 200 OK'."""
+    lines = [status_line]
+    lines += ['%s: %s' % (name, value) for name, value in headers]
+    head = ('\r\n'.join(lines) + '\r\n\r\n').encode('iso-8859-1')
+    return head + body
+
+
+def format_decoded_response(dec) -> str:
+    """hand over the data"""
+    bits = ['%u command(s) executed' % dec.number_of_commands]
+    for i, c in enumerate(dec.commands):
+        data = c.response_data or '-'
+        bits.append('  R-APDU[%u]: SW=%s data=%s' % (i, c.status_word, data))
+    if dec.get('truncated'):
+        bits.append('  TRUNCATED: an R-APDU returned SW 62F1, so the card cut the response data '
+                    'short and stopped executing the rest of the script ') # TS 102 226 5.2.1.1
+    if dec.bad_format is not None:
+        bits.append('  bad-format: %s' % dec.bad_format)
+    if dec.immediate_action_response is not None:
+        bits.append('  immediate-action-response: %s' % dec.immediate_action_response)
+    if dec.script_chaining_response is not None:
+        bits.append('  script-chaining-response: %s' % dec.script_chaining_response)
+    return '\n'.join(bits)
+
+
+class AdminSession:
+
+    def __init__(self, command_bodies: List[bytes],
+                 next_uri: Optional[str] = None,
+                 targeted_application: Optional[str] = None,
+                 on_response: Optional[Callable[[object], None]] = None,
+                 content_type: str = CT_COMMAND):
+        self.pending: List[bytes] = list(command_bodies)
+        self.next_uri = next_uri            # None -> echo the request URI
+        self.targeted_application = targeted_application
+        self.on_response = on_response
+        self.content_type = content_type    # Content-Type for the command body
+        self.responses: List[object] = []   # decoded Containers, in order
+
+    def record_response(self, dec) -> None:
+        self.responses.append(dec)
+        if self.on_response:
+            self.on_response(dec)
+
+
+def run_admin_loop(rfile, send: Callable[[bytes], None], session: AdminSession) -> AdminSession:
+    """Drive the admin loop for one connection.
+    Just keep answering the card POST requests with the next queued command
+    (200 OK + Expanded command body) until the queue is empty, end session with 204 No Content."""
+    while True:
+        req = read_http_request(rfile, send)
+        if req is None:
+            logger.info('connection closed by card')
+            return session
+
+        if req.method != 'POST':
+            logger.warning('unexpected method %s %s -> 405', req.method, req.uri)
+            send(build_http_response('HTTP/1.1 405 Method Not Allowed',
+                                     [('X-Admin-Protocol', ADMIN_PROTOCOL),
+                                      ('Connection', 'close')]))
+            return session
+
+        proto = req.get('x-admin-protocol')
+        if proto and proto != ADMIN_PROTOCOL:
+            logger.warning('card X-Admin-Protocol=%r (expected %r)', proto, ADMIN_PROTOCOL)
+        status = req.get('x-admin-script-status')
+        resume = req.get('x-admin-resume')
+        logger.info('POST %s from=%r status=%r resume=%r body=%uB',
+                    req.uri, req.get('x-admin-from'), status, resume, len(req.body))
+
+        # section 3.4.1:
+        # - body with "X-Admin-Script-Status: ok" carries the previous command
+        # response string (Expanded Remote response format);
+        # - other status values carry no body ().
+        if req.body:
+            # Expanded Remote response:
+            # - GP Amd B 'card-content-mgt-response'
+            # - ETSI 'scp.response-data'
+            logger.debug('  response Content-Type=%r raw body (%uB): %s',
+                         req.get('content-type'), len(req.body), b2h(req.body))
+            if status in (None, 'ok'):
+                try:
+                    dec = decode_expanded_resp(req.body)
+                    session.record_response(dec)
+                    logger.info('card response:\n%s', format_decoded_response(dec))
+                except Exception as e:
+                    logger.error('failed to decode response body %s: %s', b2h(req.body), e)
+            else:
+                logger.warning('body present with status=%r; ignoring', status) #  section 3.4.1
+        elif status and status != 'ok':
+            logger.info('card reported script-status=%r (no response body)', status)
+
+        if session.pending:
+            body = session.pending.pop(0)
+            next_uri = session.next_uri or req.uri
+            headers = [('X-Admin-Protocol', ADMIN_PROTOCOL),
+                       ('X-Admin-Next-URI', next_uri),
+                       ('Content-Type', session.content_type)]
+            if session.targeted_application:
+                headers.append(('X-Admin-Targeted-Application', session.targeted_application))
+            headers.append(('Content-Length', str(len(body))))
+            logger.info('-> 200 OK, next command (%uB): %s', len(body), b2h(body))
+            send(build_http_response('HTTP/1.1 200 OK', headers, body))
+        else:
+            # section 3.4.2: No more commands, end session
+            # No Content-Type or body for 204
+            logger.info('-> 204 No Content, ending administration session')
+            send(build_http_response('HTTP/1.1 204 No Content',
+                                     [('X-Admin-Protocol', ADMIN_PROTOCOL),
+                                      ('Connection', 'close')]))
+            return session
+
+
+class Scp81AdminServer:
+    """Threaded TLS-PSK server that runs the Amendment B admin loop against each
+    connecting card."""
+
+    def __init__(self, host: str, port: int, ssl_ctx: ssl.SSLContext,
+                 command_bodies: List[bytes],
+                 next_uri: Optional[str] = None,
+                 targeted_application: Optional[str] = None,
+                 on_response: Optional[Callable[[object], None]] = None,
+                 on_session_end: Optional[Callable[[AdminSession], None]] = None,
+                 content_type: str = CT_COMMAND):
+        self.host = host
+        self.port = port
+        self.ssl_ctx = ssl_ctx
+        self.command_bodies = command_bodies
+        self.next_uri = next_uri
+        self.targeted_application = targeted_application
+        self.content_type = content_type
+        self.on_response = on_response
+        self.on_session_end = on_session_end
+        self._sock: Optional[socket.socket] = None
+        self._stop = threading.Event()
+
+    def bind(self) -> int:
+        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self._sock.bind((self.host, self.port))
+        self._sock.listen(5)
+        self._sock.settimeout(0.5)
+        self.port = self._sock.getsockname()[1]
+        return self.port
+
+    def serve_forever(self) -> None:
+        if self._sock is None:
+            self.bind()
+        logger.info('SCP81 admin server listening on %s:%u (%u command(s) queued)',
+                    self.host, self.port, len(self.command_bodies))
+        while not self._stop.is_set():
+            try:
+                conn, addr = self._sock.accept()
+            except socket.timeout:
+                continue
+            except OSError:
+                break
+            threading.Thread(target=self._handle, args=(conn, addr), daemon=True).start()
+
+    def shutdown(self) -> None:
+        self._stop.set()
+        if self._sock is not None:
+            self._sock.close()
+
+    def _handle(self, conn: socket.socket, addr) -> None:
+        try:
+            tls = self.ssl_ctx.wrap_socket(conn, server_side=True)
+        except (ssl.SSLError, OSError) as e:
+            logger.warning('TLS-PSK handshake with %s failed: %s', addr, e)
+            try:
+                conn.close()
+            except OSError:
+                pass
+            return
+        logger.info('TLS-PSK established with %s: %s / %s', addr, tls.version(), tls.cipher())
+        session = AdminSession(self.command_bodies, next_uri=self.next_uri,
+                               targeted_application=self.targeted_application,
+                               on_response=self.on_response,
+                               content_type=self.content_type)
+        try:
+            rfile = tls.makefile('rb')
+            run_admin_loop(rfile, tls.sendall, session)
+        except (ssl.SSLError, OSError, ValueError) as e:
+            logger.warning('session with %s aborted: %s', addr, e)
+        finally:
+            try:
+                tls.close()
+            except OSError:
+                pass
+        logger.info('session with %s ended: %u response(s) collected', addr, len(session.responses))
+        if self.on_session_end:
+            self.on_session_end(session)
+
+
+def build_command_bodies(apdus: List[bytes], batch: bool = False,
+                         length_coding: str = 'definite') -> List[bytes]:
+    """wrpa apdus
+    each C-APDU -> one cmd message + one HTTP response per APDU
+    batch=True -> all C-APDUs in one Command Scripting template.
+    length_coding selects the definite or indefinite Command Scripting template."""
+    if not apdus:
+        return []
+    if batch:
+        return [encode_expanded_cmd(apdus, length_coding=length_coding)]
+    return [encode_expanded_cmd(a, length_coding=length_coding) for a in apdus]
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='TLS-PSK HTTP Remote Administration Server for SCP81 / RAM over HTTP')
+    parser.add_argument('--host', default='0.0.0.0', help='Host/IP to bind to (default: 0.0.0.0)')
+    parser.add_argument('--port', type=int, default=8443, help='TCP port to bind to (default: 8443)')
+    parser.add_argument('--psk', required=True,
+                        help='PSK TLS key, Amendment B key type 85 as hex')
+    parser.add_argument('--psk-identity', required=True,
+                        help='Expected PSK identity string presented by the card')
+    parser.add_argument('--psk-identity-hint', default=None,
+                        help='Optional PSK identity hint to send to the card (default: none)')
+    parser.add_argument('--allow-any-identity', action='store_true',
+                        help='DEBUG: Accept any psk_identity')
+    parser.add_argument('--ciphers', default=DEFAULT_CIPHERS,
+                        help='OpenSSL cipher string for TLS<=1.2')
+    parser.add_argument('--min-tls', default='1.2', choices=sorted(TLS_VERSION_MAP),
+                        help='Minimum TLS version (default: 1.2)')
+    parser.add_argument('--max-tls', default='1.3', choices=sorted(TLS_VERSION_MAP),
+                        help='Maximum TLS version (default: 1.3)')
+    parser.add_argument('--seclevel', type=int, default=None,
+                        help='OpenSSL @SECLEVEL to force (0 to enable NULL/3DES/legacy PSK)')
+    parser.add_argument('--uri', default=None,
+                        help='X-Admin-Next-URI to hand the card (default: request URI)')
+    parser.add_argument('--mode', choices=sorted(MODE_CONTENT_TYPE), default='ram',
+                        help='"ram" = GP Amendment B RAM to a SD (default), '
+                             '"rfm" = TS 102 226 RFM/RAM to the --targeted-application.')
+    parser.add_argument('--targeted-application', default=None,
+                        help='X-Admin-Targeted-Application AID (hex). '
+                             'Required by --mode rfm, optional for --mode ram')
+    parser.add_argument('--length-coding', choices=('definite', 'indefinite'), default='definite',
+                        help='Expanded format length coding "definite" "indefinite"')
+    parser.add_argument('--apdu', action='append', default=[], metavar='HEX',
+                        help='one of many C-APDU (hex) to send, executed in order')
+    parser.add_argument('--apdu-file', default=None,
+                        help='File with one C-APDU (hex) per line to push (# comments allowed)')
+    parser.add_argument('--batch', action='store_true',
+                        help='All C-APDUs in one large command message')
+    parser.add_argument('--raw-cmd', action='append', default=[], metavar='HEX',
+                        help='Debug, raw command')
+    parser.add_argument('-v', '--verbose', action='store_true', help='enable debug output')
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
+                        format='%(asctime)s %(levelname)s %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S')
+
+    if args.mode == 'rfm' and not args.targeted_application:
+        parser.error('--mode rfm requires --targeted-application <RFM Application AID>')
+    content_type = MODE_CONTENT_TYPE[args.mode]
+
+    apdus: List[bytes] = [h2b(a) for a in args.apdu]
+    if args.apdu_file:
+        for line in Path(args.apdu_file).read_text().splitlines():
+            line = line.split('#', 1)[0].strip()
+            if line:
+                apdus.append(h2b(line))
+    command_bodies = build_command_bodies(apdus, batch=args.batch,
+                                          length_coding=args.length_coding)
+    command_bodies += [h2b(r) for r in args.raw_cmd]
+    if not command_bodies:
+        logger.warning('no C-APDUs: the server will answer the first POST with 204...')
+
+    targeted = format_aid(args.targeted_application) if args.targeted_application else None
+    logger.info('mode=%s content-type=%s length-coding=%s targeted-application=%s',
+                args.mode, content_type, args.length_coding, targeted or '(none)')
+
+    ssl_ctx = make_ssl_context(h2b(args.psk), args.psk_identity,
+                               ciphers=args.ciphers,
+                               min_tls=args.min_tls, max_tls=args.max_tls,
+                               seclevel=args.seclevel,
+                               identity_hint=args.psk_identity_hint,
+                               allow_any_identity=args.allow_any_identity)
+
+    server = Scp81AdminServer(args.host, args.port, ssl_ctx, command_bodies,
+                              next_uri=args.uri, targeted_application=targeted,
+                              content_type=content_type)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info('shutting down')
+        server.shutdown()
+
+
+if __name__ == '__main__':
+    main()

contrib/scp81_trigger.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""scp81_trigger.py -- build the OTA packet that asks the card to open an SCP81 admin session."""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# Prints the apdu line for AdmSessTriggerParams TLV as the sms secured data, Expanded RFM mode,
+# to be fed into pysim_shell.py
+#
+# security params supplied either
+# - in the trigger
+# - from the cards data object,
+# trigger wins when both are present.
+# --no-sec omits them from the trigger so the stored ones are used.
+#
+# example params:
+#    --psk-id 'PSK Identity 123' --kvn 0x41 --kid-ref 5
+#    --ip 127.0.0.1 --port 8080 --buffer 512
+#    --host 172.96.0.1 --uri '/server/adminagent?cmd=1'
+
+import argparse
+import sys
+
+from osmocom.utils import b2h                                          # noqa: E402
+from pySim.cat import (sms_pp_download_envelope, BearerDescription,  # noqa: E402
+                       BufferSize, UiccTransportLevel, OtherAddress)
+from pySim.global_platform.http import (AdmSessTriggerParams, AdmSessionParams,  # noqa: E402
+                                        SecurityParams, HttpPostParams, RasConnectionParams,
+                                        AdminHostParam, AdminUriParam)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--psk-id", help="PSK identity for ClientHello (required, unless --no-sec)")
+    ap.add_argument("--kvn", type=lambda s: int(s, 0), help="key version of the PSK (required, unless --no-sec)")
+    ap.add_argument("--kid-ref", type=lambda s: int(s, 0), help="PSK key id (required, unless --no-sec)")
+    ap.add_argument("--host", help="HTTP Host header (required, unless --no-http)")
+    ap.add_argument("--uri", help="HTTP request URI (required, unless --no-http)")
+    ap.add_argument("--ip", help="administration server address, BIP (required, unless --no-conn)")
+    ap.add_argument("--port", type=int, help="administration server port (required, unless --no-conn)")
+    ap.add_argument("--buffer", type=int, help="BIP buffer size (required, unless --no-conn)")
+    ap.add_argument("--no-conn", action="store_true", help="omit the connection params (tag 0x84)")
+    ap.add_argument("--no-sec", action="store_true", help="omit the security params (tag 0x85)")
+    ap.add_argument("--no-http", action="store_true", help="omit the HTTP POST params (tag 0x89)")
+
+    args = ap.parse_args()
+
+    missing = []
+    if not args.no_conn:
+        missing += [n for n in ('ip', 'port', 'buffer') if getattr(args, n) is None]
+    if not args.no_sec:
+        missing += [n for n in ('psk_id', 'kvn', 'kid_ref') if getattr(args, n) is None]
+    if not args.no_http:
+        missing += [n for n in ('host', 'uri') if getattr(args, n) is None]
+    if missing:
+        ap.error("pass every value required: %s." % " ".join("--" + n.replace('_', '-') for n in missing))
+
+    session = []
+    if not args.no_conn:
+        session.append(RasConnectionParams(children=[
+            BearerDescription(decoded={'bearer_type': 'default', 'bearer_parameters': ''}),
+            BufferSize(decoded=args.buffer),
+            UiccTransportLevel(decoded={'protocol_type': 'tcp_uicc_client_remote',
+                                        'port_number': args.port}),
+            OtherAddress(decoded={'type_of_address': 'ipv4',
+                                  'address': bytes(int(b) for b in args.ip.split("."))})]))
+    if not args.no_sec:
+        session.append(SecurityParams(decoded={'psk_id': args.psk_id.encode(), 'kvn': args.kvn,
+                                               'kid': args.kid_ref, 'sha_type': None}))
+    if not args.no_http:
+        session.append(HttpPostParams(children=[AdminHostParam(decoded=args.host),
+                                                AdminUriParam(decoded=args.uri)]))
+    trig = AdmSessTriggerParams(children=[AdmSessionParams(children=session)]).to_tlv()
+
+    # stderr for logs, stdout for data
+    print("# trigger TLV   %d B  %s" % (len(trig), trig.hex()), file=sys.stderr)
+    print("# %-13s %d B  %s" % ("secured data", len(trig), trig.hex()), file=sys.stderr)
+    print(trig.hex())
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

contrib/smpp-ota-tool.py

@@ -24,7 +24,8 @@ import smpplib.gsm
 import smpplib.client
 import smpplib.consts
 import time
-from pySim.ota import OtaKeyset, OtaDialectSms, OtaAlgoCrypt, OtaAlgoAuth, CNTR_REQ, RC_CC_DS, POR_REQ
+from pySim.ota import OtaKeyset, OtaDialectSms, OtaAlgoCrypt, OtaAlgoAuth, OtaCheckError, CNTR_REQ, RC_CC_DS, POR_REQ
+from pySim.sms import ConcatenatedSmsReassembler
 from pySim.utils import b2h, h2b, is_hexstr
 from pathlib import Path
 
@@ -70,6 +71,8 @@ option_parser.add_argument("--por-req", choices=POR_REQ.decmapping.values(), def
 option_parser.add_argument('--src-addr', default='12', type=str, help='SMS source address (MSISDN)')
 option_parser.add_argument('--dest-addr', default='23', type=str, help='SMS destination address (MSISDN)')
 option_parser.add_argument('--timeout', default=10, type=int, help='Maximum response waiting time')
+option_parser.add_argument('--format', choices=['compact', 'expanded'], default='compact',
+                           help="Remote Application data format: 'compact' or 'expanded'")
 option_parser.add_argument('-a', '--apdu', action='append', required=True, type=is_hexstr, help='C-APDU to send')
 
 class SmppHandler:
@@ -77,7 +80,8 @@ class SmppHandler:
 
     def __init__(self, host: str, port: int,
                  system_id: str, password: str,
-                 ota_keyset: OtaKeyset, spi: dict, tar: bytes):
+                 ota_keyset: OtaKeyset, spi: dict, tar: bytes,
+                 remote_format: str = 'compact'):
         """
         Initialize connection to SMPP server and set static OTA SMS-TPDU ciphering parameters
         Args:
@@ -88,6 +92,7 @@ class SmppHandler:
                 ota_keyset: OTA keyset to be used for SMS-TPDU ciphering
                 spi: Security Parameter Indicator (SPI) to be used for SMS-TPDU ciphering
                 tar: Toolkit Application Reference (TAR) of the targeted card application
+                remote_format: Remote Application data format ('compact' or 'expanded', TS 102 226)
         """
 
         # Create and connect SMPP client
@@ -103,26 +108,58 @@ class SmppHandler:
         self.ota_keyset = ota_keyset
         self.tar = tar
         self.spi = spi
+        self.remote_format = remote_format
+        self.reassembler = ConcatenatedSmsReassembler()
 
     def __del__(self):
         if self.client:
             self.client.unbind()
             self.client.disconnect()
 
+    def _decode_resp(self, tpud: bytes) -> tuple:
+        """Decode a response SMS-TPDU into (response_packet, decoded).
+
+        Retry to decoding with ciphering disabled (in case the card has problems to decode the SMS-TDPU
+        we have sent, the response will contain an unencrypted error message)
+        """
+        try:
+            return self.ota_dialect.decode_resp(self.ota_keyset, self.spi, tpud,
+                                                remote_format=self.remote_format)
+        except (ValueError, OtaCheckError):
+            spi = self.spi.copy()
+            spi['por_shall_be_ciphered'] = False
+            spi['por_rc_cc_ds'] = 'no_rc_cc_ds'
+            return self.ota_dialect.decode_resp(self.ota_keyset, spi, tpud,
+                                                remote_format=self.remote_format)
+
     def message_received_handler(self, pdu):
-        if pdu.short_message:
-            logger.info("SMS-TPDU received: %s", b2h(pdu.short_message))
-            try:
-                dec = self.ota_dialect.decode_resp(self.ota_keyset, self.spi, pdu.short_message)
-            except ValueError:
-                # Retry to decoding with ciphering disabled (in case the card has problems to decode the SMS-TDPU
-                # we have sent, the response will contain an unencrypted error message)
-                spi = self.spi.copy()
-                spi['por_shall_be_ciphered'] = False
-                spi['por_rc_cc_ds'] = 'no_rc_cc_ds'
-                dec = self.ota_dialect.decode_resp(self.ota_keyset, spi, pdu.short_message)
-            logger.info("SMS-TPDU decoded: %s", dec)
-            self.response = dec
+        if not pdu.short_message:
+            return None
+        logger.info("SMS-TPDU received: %s", b2h(pdu.short_message))
+        tpud = self.reassembler.add(pdu.short_message)
+        if tpud is None:
+            logger.info("SMS-TPDU is part of concat message, waiting for more parts...")
+            return None
+        if tpud != pdu.short_message:
+            logger.info("SMS-TPDU reassembled: %s", b2h(tpud))
+        try:
+            res, decoded = self._decode_resp(tpud)
+        except Exception as e:
+            # for example ENVELOPE POR
+            logger.warning("Ignoring undecodable resp SMS-TPDU (%s: %s)", type(e).__name__, e)
+            return None
+        logger.info("SMS-TPDU decoded: %s", (res, decoded))
+        # large app response as reassembled SEND SHORT MESSAGE, but
+        # the ENVELOPE itself returns a POR without R-APDU.
+        # smpplib poll() drains all pending SMS in one call, so that PoR is processed
+        # right after the real response and would overwrite it,
+        # which leaves transceive_apdu with no last_response_data to return.
+        # Only allow a response that has no application data (decoded == None)
+        # if we do not already have a real one.
+        if decoded is None and self.response is not None and self.response[1] is not None:
+            logger.info("ignoring status response to keep earlier app response")
+            return None
+        self.response = (res, decoded)
         return None
 
     def message_sent_handler(self, pdu):
@@ -183,10 +220,14 @@ class SmppHandler:
                 tuple containing the last response data and the last status word as byte strings
         """
 
-        logger.info("C-APDU sending: %s...", b2h(apdu))
+        if isinstance(apdu, (list, tuple)):
+            logger.info("C-APDU(s) sending: %s...", [b2h(a) for a in apdu])
+        else:
+            logger.info("C-APDU sending: %s...", b2h(apdu))
 
         # translate to Secured OTA RFM
-        secured = self.ota_dialect.encode_cmd(self.ota_keyset, self.tar, self.spi, apdu=apdu)
+        secured = self.ota_dialect.encode_cmd(self.ota_keyset, self.tar, self.spi, apdu=apdu,
+                                              remote_format=self.remote_format)
         # add user data header
         tpdu = b'\x02\x70\x00' + secured
         # send via SMPP
@@ -200,6 +241,17 @@ class SmppHandler:
                 container_dict = dict(container)
                 resp = container_dict.get('last_response_data')
                 sw = container_dict.get('last_status_word')
+                # expanded format: decoded response carries
+                # per command R-APDU list; log each one.
+                for i, cmd in enumerate(container_dict.get('commands') or []):
+                    logger.info("R-APDU[%u] received: %s %s", i,
+                                cmd['response_data'], cmd['status_word'])
+                if container_dict.get('truncated'):
+                    logger.warning("Response was TRUNCATED (SW 62F1): the card cut the response "
+                                   "data short and did not execute the rest of the script")
+                if container_dict.get('bad_format') is not None:
+                    logger.warning("Response contains a Bad format TLV: %s",
+                                   container_dict['bad_format'])
         if resp is None:
             raise ValueError("Response does not contain any last_response_data, no R-APDU received!")
         if sw is None:
@@ -233,8 +285,14 @@ if __name__ == '__main__':
            'por_shall_be_ciphered': not opts.por_no_ciphering,
            'por_rc_cc_ds': opts.por_rc_cc_ds,
            'por': opts.por_req}
-    apdu = h2b("".join(opts.apdu))
+    if opts.format == 'expanded':
+        # TS 102 226 5.2.1.1: wrap each apdu in its own C-APDU TLV
+        apdu = [h2b(a) for a in opts.apdu]
+    else:
+        # compact: C-APDUs are concatenated as single command string
+        apdu = h2b("".join(opts.apdu))
 
-    smpp_handler = SmppHandler(opts.host, opts.port, opts.system_id, opts.password, ota_keyset, spi, h2b(opts.tar))
+    smpp_handler = SmppHandler(opts.host, opts.port, opts.system_id, opts.password, ota_keyset, spi,
+                               h2b(opts.tar), remote_format=opts.format)
     resp, sw = smpp_handler.transceive_apdu(apdu, opts.src_addr, opts.dest_addr, opts.timeout)
     print("%s %s" % (b2h(resp), b2h(sw)))

docs/card-key-provider.rst

@@ -305,16 +305,16 @@ the requested data.
 
 
 ADM PIN
-~~~~~~~
+^^^^^^^
 
 The `verify_adm` command will attempt to look up the `ADM1` column
 indexed by the ICCID of the SIM/UICC.
 
 
 SCP02 / SCP03
-~~~~~~~~~~~~~
+^^^^^^^^^^^^^
 
-SCP02 and SCP03 each use key triplets consisting if ENC, MAC and DEK
+SCP02 and SCP03 each use key triplets consisting of ENC, MAC and DEK
 keys.  For more details, see the applicable GlobalPlatform
 specifications.
 

docs/conf.py

@@ -13,6 +13,7 @@
 import os
 import sys
 sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath('.'))   # for local extensions (pysim_fs_sphinx, ...)
 
 
 # -- Project information -----------------------------------------------------
@@ -39,7 +40,8 @@ extensions = [
         "sphinx.ext.autodoc",
         "sphinxarg.ext",
         "sphinx.ext.autosectionlabel",
-        "sphinx.ext.napoleon"
+        "sphinx.ext.napoleon",
+        "pysim_fs_sphinx",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -64,3 +66,25 @@ html_theme = 'alabaster'
 html_static_path = ['_static']
 
 autoclass_content = 'both'
+
+# Mock optional server-side deps of es2p and http_json_api/es9p,
+# so that autodoc can import and document those modules.
+autodoc_mock_imports = ['klein', 'twisted']
+
+# Workaround for duplicate label warnings:
+# https://github.com/sphinx-doc/sphinx-argparse/issues/14
+#
+# sphinxarg.ext generates generic sub-headings ("Named arguments",
+# "Positional arguments", "Sub-commands", "General options", ...) for every
+# argparse command/tool.  These repeat across many files and trigger tons
+# of autosectionlabel duplicate-label warnings - suppress them.
+autosectionlabel_maxdepth = 3
+suppress_warnings = [
+    'autosectionlabel.filesystem',
+    'autosectionlabel.saip-tool',
+    'autosectionlabel.shell',
+    'autosectionlabel.smpp2sim',
+    'autosectionlabel.smpp-ota-tool',
+    'autosectionlabel.suci-keytool',
+    'autosectionlabel.trace',
+]

docs/index.rst

@@ -39,6 +39,7 @@ pySim consists of several parts:
    :caption: Contents:
 
    shell
+   filesystem
    trace
    legacy
    smpp2sim

docs/legacy.rst

@@ -205,7 +205,7 @@ Specifically, pySim-read will dump the following:
 
 * DF.GSM
 
- * EF,IMSI
+ * EF.IMSI
  * EF.GID1
  * EF.GID2
  * EF.SMSP

docs/put_key-tutorial.rst

@@ -0,0 +1,836 @@
+Guide: Managing GP Keys
+=======================
+
+Most of today's smartcards follow the GlobalPlatform Card Specification and the included Security Domain model.
+UICCs and eUCCCs are no exception here.
+
+The Security Domain acts as an on-card representative of a card authority or administrator. It is used to perform tasks
+like the installation of applications or the provisioning and rotation of secure channel keys. It also acts as a secure
+key storage and offers all kinds of cryptographic services to applications that are installed under a specific
+Security Domain (see also GlobalPlatform Card Specification, section 7).
+
+In this tutorial, we will show how to work with the key material (keysets) stored inside a Security Domain and how to
+rotate (replace) existing keys. We will also show how to provision new keys.
+
+.. warning:: Making changes to keysets requires extreme caution as misconfigured keysets may lock you out permanently.
+	     It's also strongly recommended to maintain at least one backup keyset that you can use as fallback in case
+	     the primary keyset becomes unusable for some reason.
+
+
+Selecting a Security Domain
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A typical smartcard, such as an UICC will have one primary Security Domain, called the Issuer Security Domain (ISD).
+When working with those cards, the ISD will show up in the UICC filesystem tree as `ADF.ISD` and can be selected like
+any other file.
+
+::
+
+   pySIM-shell (00:MF)> select ADF.ISD
+   {
+      "application_id": "a000000003000000",
+      "proprietary_data": {
+         "maximum_length_of_data_field_in_command_message": 255
+      }
+   }
+
+When working with eUICCs, multiple Security Domains are involved. The model is fundamentally different from the classic
+model with one primary Security Domain (ISD). In the case of eUICCs, an ISD-R (Issuer Security Domain - Root) and an
+ISD-P (Issuer Security Domain - Profile) exist (see also: GSMA SGP.02, section 2.2.1).
+
+The ISD-P is established by the ISD-R during the profile installation and serves as a secure container for an eSIM
+profile. Within the ISD-P the eSIM profile establishes a dedicated Security Domain called `MNO-SD` (see also GSMA
+SGP.02, section 2.2.4). This `MNO-SD` is comparable to the Issuer Security Domain (ISD) we find on UICCs. The AID of
+`MNO-SD` is either the default AID for the Issuer Security Domain (see also GlobalPlatform, section H.1.3) or a
+different value specified by the provider of the eSIM profile.
+
+Since the AID of the `MNO-SD` is not a fixed value, it is not known by `pySim-shell`. This means there will be no
+`ADF.ISD` file shown in the file system, but we can simply select the `ADF.ISD-R` first and then select the `MNO-SD`
+using a raw APDU. In the following example we assume that the default AID (``a000000151000000``) is used  The APDU
+would look like this: ``00a4040408`` + ``a000000151000000`` + ``00``
+
+::
+
+   pySIM-shell (00:MF)> select ADF.ISD-R
+   {
+       "application_id": "a0000005591010ffffffff8900000100",
+       "proprietary_data": {
+           "maximum_length_of_data_field_in_command_message": 255
+       },
+       "isdr_proprietary_application_template": {
+           "supported_version_number": "020300"
+       }
+   }
+   pySIM-shell (00:MF/ADF.ISD-R)> apdu 00a4040408a00000015100000000
+   SW: 9000, RESP: 6f108408a000000151000000a5049f6501ff
+
+After that, the prompt will still show the `ADF.ISD-R`, but we are actually in `ADF.ISD` and the standard GlobalPlatform
+operations like `establish_scpXX`, `get_data`, and `put_key` should work. By doing this, we simply have tricked
+`pySim-shell` into making the GlobalPlatform related commands available for some other Security Domain we are not
+interested in. With the raw APDU we then have swapped out the Security Domain under the hood. The same workaround can
+be applied to any Security Domain, provided that the AID is known to the user.
+
+
+Establishing a secure channel
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before we can make changes to the keysets in the currently selected Security Domain we must first establish a secure
+channel with that Security Domain. In the following examples we will use `SCP02` (see also GlobalPlatform Card
+Specification, section E.1.1) and `SCP03` (see also GlobalPlatform Card Specification – Amendment D) to establish the
+secure channel. `SCP02` is slightly older than `SCP03`. The main difference between the two is that `SCP02` uses 3DES
+while `SCP03` is based on AES.
+
+.. warning:: Secure channel protocols like `SCP02` and `SCP03` may manage an error counter to count failed login
+	     attempts. This means attempting to establish a secure channel with a wrong keyset multiple times may lock
+	     you out permanently. Double check the applied keyset before attempting to establish a secure channel.
+
+.. warning:: The key values used in the following examples are random key values used for illustration purposes only.
+	     Each UICC or eSIM profile is shipped with individual keys, which means that the keys used below will not
+	     work with your UICC or eSIM profile. You must replace the key values with the values you have received
+	     from your UICC vendor or eSIM profile provider.
+
+
+Example: `SCP02`
+----------------
+
+In the following example, we assume that we want to establish a secure channel with the ISD of a `sysmoUSIM-SJA5` UICC.
+Along with the card we have received the following keyset:
+
++---------+----------------------------------+
+| Keyname | Keyvalue                         |
++=========+==================================+
+| ENC/KIC | F09C43EE1A0391665CC9F05AF4E0BD10 |
++---------+----------------------------------+
+| MAC/KID | 01981F4A20999F62AF99988007BAF6CA |
++---------+----------------------------------+
+| DEK/KIK | 8F8AEE5CDCC5D361368BC45673D99195 |
++---------+----------------------------------+
+
+This keyset is tied to the key version number KVN 122 and is configured as a DES keyset. We can use this keyset to
+establish a secure channel using the SCP02 Secure Channel Protocol.
+
+::
+
+    pySIM-shell (00:MF/ADF.ISD)> establish_scp02 --key-enc F09C43EE1A0391665CC9F05AF4E0BD10 --key-mac 01981F4A20999F62AF99988007BAF6CA --key-dek 8F8AEE5CDCC5D361368BC45673D99195  --key-ver 112 --security-level 3
+    Successfully established a SCP02[03] secure channel
+
+
+Example: `SCP03`
+----------------
+
+The establishment of a secure channel via SCP03 works just the same. In the following example we will establish a
+secure channel to the `MNO-SD` of an eSIM profile. The SCP03 keyset we use is tied to KVN 48 and looks like this:
+
++---------+------------------------------------------------------------------+
+| Keyname | Keyvalue                                                         |
++=========+==================================================================+
+| ENC/KIC | 63af517c29ad6ac6fcadfe6ac8a3c8a041d8141c7eb845ef1cba6112a325e430 |
++---------+------------------------------------------------------------------+
+| MAC/KID | 54b9ad6713ae922f54014ed762132e7b59bdcd2a2a6beba98fb9afe6b4df27e1 |
++---------+------------------------------------------------------------------+
+| DEK/KIK | cbb933ba2389da93c86c112739cd96389139f16c6f80f7d16bf3593e407ca893 |
++---------+------------------------------------------------------------------+
+
+We assume that the `MNO-SD` is already selected (see above). We may now establish the SCP03 secure channel:
+
+::
+
+   pySIM-shell (00:MF/ADF.ISD-R)> establish_scp03 --key-enc 63af517c29ad6ac6fcadfe6ac8a3c8a041d8141c7eb845ef1cba6112a325e430 --key-mac 54b9ad6713ae922f54014ed762132e7b59bdcd2a2a6beba98fb9afe6b4df27e1 --key-dek cbb933ba2389da93c86c112739cd96389139f16c6f80f7d16bf3593e407ca893 --key-ver 48 --security-level 3
+   Successfully established a SCP03[03] secure channel
+
+
+
+Understanding Keysets
+~~~~~~~~~~~~~~~~~~~~~
+
+Before making any changes to keysets, it is recommended to check the status of the currently installed keysets. To do
+so, we use the `get_data` command to retrieve the `key_information`. This command does not require the establishment of
+a secure channel. We also cannot read back the key values themselves, but we get a summary of the installed keys
+together with their KVN numbers, IDs, algorithm and key length values.
+
+Example: `key_information` from a `sysmoISIM-SJA5`:
+
+::
+
+    pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> get_data key_information
+    {
+        "key_information": [
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 112,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 112,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 3,
+                    "key_version_number": 112,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 1,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 1,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 3,
+                    "key_version_number": 1,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 2,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 2,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 3,
+                    "key_version_number": 2,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 47,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 47,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 3,
+                    "key_version_number": 47,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+
+Example: `key_information` from a `sysmoEUICC1-C2T`:
+
+::
+
+    pySIM-shell (SCP03[03]:00:MF/ADF.ISD-R)> get_data key_information
+    {
+        "key_information": [
+            {
+                "key_information_data": {
+                    "key_identifier": 3,
+                    "key_version_number": 50,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 32
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 50,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 32
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 50,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 32
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 64,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 64,
+                    "key_types": [
+                        {
+                            "type": "tls_psk",
+                            "length": 16
+                        }
+                    ]
+                }
+            }
+        ]
+    }
+
+The output from those two examples above may seem lengthy, but in order to move on and to provision own keys
+successfully, it is important to understand each aspect of it.
+
+Key Version Number (KVN)
+------------------------
+
+Each key is associated with a Key Version Number (KVN). Multiple keys that share the same KVN belong to the same
+keyset. In the first example above we can see that four keysets with KVN numbers 112, 1, 2 and 47 are provisioned.
+In the second example we see two keysets. One with KVN 50 and one with KVN 64.
+
+The term "Key Version Number" is misleading as this number is not really a version number. It's actually a unique
+identifier for a specific keyset that also defines with which Secure Channel Protocol a key can be used. This means
+that the KVN is not just an arbitrary number. The following (incomplete) table gives a hint which KVN numbers may be
+used with which Secure Channel Protocol.
+
++-----------+-------------------------------------------------------+
+| KVN range | Secure Channel Protocol                               |
++===========+=======================================================+
+| 1-15      | reserved for `SCP80` (OTA SMS)                        |
++-----------+-------------------------------------------------------+
+| 17        | reserved for DAP specified in ETSI TS 102 226         |
++-----------+-------------------------------------------------------+
+| 32-47     | reserved for `SCP02`                                  |
++-----------+-------------------------------------------------------+
+| 48-63     | reserved for `SCP03`                                  |
++-----------+-------------------------------------------------------+
+| 64-79     | reserved for `SCP81` (GSMA SGP.02, section 2.2.5.1)   |
++-----------+-------------------------------------------------------+
+| 112       | Token key (RSA public or DES, also used with `SCP02`) |
++-----------+-------------------------------------------------------+
+| 113       | Receipt key (DES)                                     |
++-----------+-------------------------------------------------------+
+| 115       | DAP verification key (RS public or DES)               |
++-----------+-------------------------------------------------------+
+| 116       | reserved for CASD                                     |
++-----------+-------------------------------------------------------+
+| 117       | 16-byte DES key for Ciphered Load File Data Block     |
++-----------+-------------------------------------------------------+
+| 255       | reserved for ISD with SCP02 without SCP80 support     |
++-----------+-------------------------------------------------------+
+
+With that we can now understand that in the first example, the first and the last keyset is intended to be used with
+`SCP02` and that the second and the third keyset is intended to be used with `SCP80` (OTA SMS). In the second example we
+can see that the first keyset is intended to be used with `SCP03`, wheres the second should be usable with `SCP81`.
+
+
+Key Identifier
+--------------
+
+Each keyset consists of a number of keys, where each key has a different Key Identifier. The Key Identifier is usually
+an incrementing number that starts counting at 1. The Key Identifier is used to distinguish the keys within the keyset.
+The exact number of keys and their attributes depends on the secure channel protocol for which the keyset is intended
+for. Each secure channel protocol may have its specific requirements on how many keys of which which type, length or
+Key Identifier have to be present.
+
+However, almost all of the classic secure channel protocols (including  `SCP02`, `SCP03` and `SCP81`) make use of the
+following three-key scheme:
+
++----------------+---------+---------------------------------------+
+| Key Identifier | Keyname | Purpose                               |
++================+=========+=======================================+
+| 1              | ENC/KIC | encryption/decryption                 |
++----------------+---------+---------------------------------------+
+| 2              | MAC/KID | cryptographic checksumming/signing    |
++----------------+---------+---------------------------------------+
+| 3              | DEK/KIK | encryption/decryption of key material |
++----------------+---------+---------------------------------------+
+
+In this case, all three keys share the same length and are used with the same algorithm. The key length is often used
+to implicitly select sub-types of an algorithm. (e.g. a 16 byte key of type `aes` is associated with `AES128`, where a 32
+byte key would be associated with `AES256`).
+
+The second example shows that different schemes are possible. The `SCP80` keyset from the second example uses a scheme
+that works with two keys:
+
++----------------+---------+---------------------------------------+
+| Key Identifier | Keyname | Purpose                               |
++================+=========+=======================================+
+| 1              | TLS-PSK | pre-shared key used for TLS           |
++----------------+---------+---------------------------------------+
+| 2              | DEK/KIK | encryption/decryption of key material |
++----------------+---------+---------------------------------------+
+
+It should also be noted that the order in which keysets and keys appear is an implementation detail of the UICC/eUICC
+O/S. The order has no influence on how a keyset is interpreted. Only the Key Version Number (KVN) and the Key Identifier
+matter.
+
+
+Rotating a keyset
+~~~~~~~~~~~~~~~~~
+
+Rotating keys is one of the most basic tasks one might want to perform on an UICC/eUICC before using it productively. In
+the following example we will illustrate how key rotation can be done. When rotating keys, only the key itself may
+change. For example it is not possible to change the key length or the algorithm used (see also GlobalPlatform Card
+Specification, section 11.8.2.3.3). Any key of the current Security Domain can be rotated, this also includes the key
+that was used to establish the secure channel.
+
+In the following example we assume that the Security Domain is selected and a secure channel is already established. We
+intend to rotate the keyset with KVN 112. Since this keyset uses triple DES keys with a key length of 16, we must
+replace it with a keyset with keys of the same nature.
+
+The new keyset shall look like this:
+
++----------------+---------+----------------------------------+
+| Key Identifier | Keyname | Keyvalue                         |
++================+=========+==================================+
+| 1              | ENC/KIC | 542C37A6043679F2F9F71116418B1CD5 |
++----------------+---------+----------------------------------+
+| 2              | MAC/KID | 34F11BAC8E5390B57F4E601372339E3C |
++----------------+---------+----------------------------------+
+| 3              | DEK/KIK | 5524F4BECFE96FB63FC29D6BAAC6058B |
++----------------+---------+----------------------------------+
+
+When passing the keys to the `put_key` commandline, we set the Key Identifier of the first key using the `--key-id`
+parameter. This Key Identifier will be valid for the first key (KIC) we pass. For all consecutive keys, the Key
+Identifier will be incremented automatically (see also GlobalPlatform Card Specification, section 11.8.2.2). To Ensure
+that the new KIC, KID and KIK keys get the correct Key Identifiers, it is crucial to maintain order when passing the
+keys in the `--key-data` arguments. It is also important that each `--key-data` argument is preceded by a `--key-type`
+argument that sets the algorithm correctly (`des` in this case).
+
+Finally we have to target the keyset we want to rotate by its KVN. The `--old-key-version-nr` argument is set to 112
+as this identifies the keyset we want to rotate. The `--key-version-nr` is also set to 112 as we do not want
+KVN to be changed in this example. Changing the KVN while rotating a keyset is possible. In case the KVN has to change
+for some reason, the new KVN must be selected carefully to keep the key usable with the associated Secure Channel
+Protocol.
+
+The commandline that matches the keyset we had laid out above looks like this:
+::
+
+   pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> put_key --key-id 1 --key-type des --key-data 542C37A6043679F2F9F71116418B1CD5 --key-type des --key-data 34F11BAC8E5390B57F4E601372339E3C --key-type des --key-data 5524F4BECFE96FB63FC29D6BAAC6058B --old-key-version-nr 112 --key-version-nr 112
+
+After executing this put_key commandline, the keyset identified by KVN 122 is equipped with new keys. We can use
+`get_data key_information` to inspect the currently installed keysets. The output should appear unchanged as
+we only swapped out the keys. All other parameters, identifiers etc. should remain constant.
+
+.. warning:: It is technically possible to rotate a keyset in a `non atomic` way using one `put_key` commandline for
+	     each key. However, in case the targeted keyset is the one used to establish the current secure channel,
+	     this method should not be used since, depending on the UICC/eUICC model, half-written key material may
+	     interrupt the current secure channel.
+
+
+Removing a keyset
+~~~~~~~~~~~~~~~~~
+
+In some cases it is necessary to remove a keyset entirely. This can be done with the `delete_key` command. Here it is
+important to understand that `delete_key` only removes one specific key from a specific keyset. This means that you
+need to run a separate `delete_key` command for each key inside a keyset.
+
+In the following example we assume that the Security Domain is selected and a secure channel is already established. We
+intend to remove the keyset with KVN 112. This keyset consists of three keys.
+
+::
+
+   pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> delete_key --key-ver 112 --key-id 1
+   pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> delete_key --key-ver 112 --key-id 2
+   pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> delete_key --key-ver 112 --key-id 3
+
+To verify that the keyset has been deleted properly, we can use the `get_data key_information` command to inspect the
+current status of the installed keysets. We should see that the key with KVN 112 is no longer present.
+
+
+Adding a keyset
+~~~~~~~~~~~~~~~
+
+In the following we will discuss how to add an entirely new keyset. The procedure is almost identical with the key
+rotation procedure we have already discussed and it is assumed that all details about the key rotation are understood.
+In this section we will go into more detail and illustrate how to provision new 3DES, `AES128` and `AES256` keysets.
+
+It is important to keep in mind that storage space on smartcard is a precious resource. In many cases the amount of
+keysets that a Security Domain can store is limited. In some situations you may be forced to sacrifice one of your
+existing keysets in favor of a new keyset.
+
+The main difference between key rotation and the adding of new keys is that we do not simply replace an existing key.
+Instead an entirely new key is programmed into the Security Domain. Therefore the `put_key` commandline will have no
+`--old-key-version-nr` parameter. From the commandline perspective, this is already the only visible difference from a
+commandline that simply rotates a keyset. Since we are writing an entirely new keyset, we are free to chose the
+algorithm and the key length within the parameter range permitted by the targeted secure channel protocol. Otherwise
+the same rules apply.
+
+For reference, it should be mentioned that it is also possible to add or rotate keyset using multiple `put_key`
+commandlines. In this case one `put_key` commandline for each key is used. Each commandline will specify `--key-id` and
+`--key-version-nr` and one `--key-type` and `--key-data` tuple. However, when rotating or adding a keyset step-by-step,
+the whole process happens in a `non-atomic` way, which is less reliable. Therefore we will favor the `atomic method`
+
+In the following examples we assume that the Security Domain is selected and a secure channel is already established.
+
+
+Example: `3DES` key for `SCP02`
+-------------------------------
+
+Let's assume we want to provision a new 3DES keyset that we can use for SCP02. The keyset shall look like this:
+
++----------------+---------+----------------------------------+
+| Key Identifier | Keyname | Keyvalue                         |
++================+=========+==================================+
+| 1              | ENC/KIC | 542C37A6043679F2F9F71116418B1CD5 |
++----------------+---------+----------------------------------+
+| 2              | MAC/KID | 34F11BAC8E5390B57F4E601372339E3C |
++----------------+---------+----------------------------------+
+| 3              | DEK/KIK | 5524F4BECFE96FB63FC29D6BAAC6058B |
++----------------+---------+----------------------------------+
+
+The keyset shall be a associated with the KVN 46. We have made sure before that KVN 46 is still unused and that this
+KVN number is actually suitable for SCP02 keys. As we are using 3DES, it is obvious that we have to pass 3 keys with 16
+byte length.
+
+To program the key, we may use the following commandline. As we can see, this commandline is almost the exact same as
+the one from the key rotation example where we were rotating a 3DES key. The only difference is that we didn't specify
+an old KVN number and that we have chosen a different KVN.
+
+::
+
+   pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> put_key --key-id 1 --key-type des --key-data 542C37A6043679F2F9F71116418B1CD5 --key-type des --key-data 34F11BAC8E5390B57F4E601372339E3C --key-type des --key-data 5524F4BECFE96FB63FC29D6BAAC6058B --key-version-nr 46
+
+In case of success, the keyset should appear in the `key_information` among the other keysets that are already present.
+
+::
+
+    pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> get_data key_information
+    {
+        "key_information": [
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 46,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 46,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 3,
+                    "key_version_number": 46,
+                    "key_types": [
+                        {
+                            "type": "des",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            ...
+        ]
+    }
+
+
+Example: `AES128` key for `SCP80`
+---------------------------------
+
+In this example we intend to provision a new `AES128` keyset that we can use with SCP80 (OTA SMS). The keyset shall look
+like this:
+
++----------------+---------+----------------------------------+
+| Key Identifier | Keyname | Keyvalue                         |
++================+=========+==================================+
+| 1              | ENC/KIC | 542C37A6043679F2F9F71116418B1CD5 |
++----------------+---------+----------------------------------+
+| 2              | MAC/KID | 34F11BAC8E5390B57F4E601372339E3C |
++----------------+---------+----------------------------------+
+| 3              | DEK/KIK | 5524F4BECFE96FB63FC29D6BAAC6058B |
++----------------+---------+----------------------------------+
+
+In addition to that, we want to associate this key with KVN 3. We have inspected the currently installed keysets before
+and made sure that KVN 3 is still unused. We are also aware that for SCP80 we may only use KVN values from 1 to 15.
+
+For `AES128`, we specify the algorithm using the `--key-type aes` parameter. The selection between `AES128` and `AES256` is
+done implicitly using the key length. Since we want to use `AES128` in this case, all three keys have a length of 16 byte.
+
+::
+
+   pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> put_key --key-id 1 --key-type aes --key-data 542C37A6043679F2F9F71116418B1CD5 --key-type aes --key-data 34F11BAC8E5390B57F4E601372339E3C --key-type aes --key-data 5524F4BECFE96FB63FC29D6BAAC6058B --key-version-nr 3
+
+In case of success, the keyset should appear in the `key_information` among the other keysets that are already present.
+
+::
+
+    pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> get_data key_information
+    {
+        "key_information": [
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 3,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 3,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 3,
+                    "key_version_number": 3,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            ...
+        ]
+    }
+
+
+Example: `AES256` key for `SCP03`
+---------------------------------
+
+Let's assume we want to provision a new `AES256` keyset that we can use for SCP03. The keyset shall look like this:
+
++----------------+---------+------------------------------------------------------------------+
+| Key Identifier | Keyname | Keyvalue                                                         |
++================+=========+==================================================================+
+| 1              | ENC/KIC | 542C37A6043679F2F9F71116418B1CD5542C37A6043679F2F9F71116418B1CD5 |
++----------------+---------+------------------------------------------------------------------+
+| 2              | MAC/KID | 34F11BAC8E5390B57F4E601372339E3C34F11BAC8E5390B57F4E601372339E3C |
++----------------+---------+------------------------------------------------------------------+
+| 3              | DEK/KIK | 5524F4BECFE96FB63FC29D6BAAC6058B5524F4BECFE96FB63FC29D6BAAC6058B |
++----------------+---------+------------------------------------------------------------------+
+
+In addition to that, we assume that we want to associate this key with KVN 51. This KVN number falls in the range of
+48 - 63 and is therefore suitable for a key that shall be usable with SCP03. We also made sure before that KVN 51 is
+still unused.
+
+With that we can go ahead and make up the following commandline:
+::
+
+   pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> put_key --key-id 1 --key-type aes --key-data 542C37A6043679F2F9F71116418B1CD5542C37A6043679F2F9F71116418B1CD5 --key-type aes --key-data 34F11BAC8E5390B57F4E601372339E3C34F11BAC8E5390B57F4E601372339E3C --key-type aes --key-data 5524F4BECFE96FB63FC29D6BAAC6058B5524F4BECFE96FB63FC29D6BAAC6058B --key-version-nr 51
+
+In case of success, we should see the keyset in the `key_information`
+
+::
+
+    pySIM-shell (SCP02[03]:00:MF/ADF.ISD)> get_data key_information
+    {
+        "key_information": [
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 51,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 32
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 51,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 32
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 3,
+                    "key_version_number": 51,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 32
+                        }
+                    ]
+                }
+            },
+            ...
+        ]
+    }
+
+
+Example: `AES128` key for `SCP81`
+---------------------------------
+
+In this example we will show how to provision a new `AES128` keyset for `SCP81`. We will provision this keyset under
+KVN 64. The keyset we intend to apply shall look like this:
+
++----------------+---------+----------------------------------+
+| Key Identifier | Keyname | Keyvalue                         |
++================+=========+==================================+
+| 1              | TLS-PSK | 000102030405060708090a0b0c0d0e0f |
++----------------+---------+----------------------------------+
+| 2              | DEK/KIK | 000102030405060708090a0b0c0d0e0f |
++----------------+---------+----------------------------------+
+
+With that we can put together the following command line:
+
+::
+
+    put_key --key-id 1 --key-type tls_psk --key-data 000102030405060708090a0b0c0d0e0f --key-type aes --key-data 000102030405060708090a0b0c0d0e0f --key-version-nr 64
+
+In case of success, the keyset should appear in the `key_information` as follows:
+
+::
+
+    pySIM-shell (SCP03[03]:00:MF/ADF.ISD-R)> get_data key_information
+    {
+        "key_information": [
+            ...,
+            {
+                "key_information_data": {
+                    "key_identifier": 2,
+                    "key_version_number": 64,
+                    "key_types": [
+                        {
+                            "type": "aes",
+                            "length": 16
+                        }
+                    ]
+                }
+            },
+            {
+                "key_information_data": {
+                    "key_identifier": 1,
+                    "key_version_number": 64,
+                    "key_types": [
+                        {
+                            "type": "tls_psk",
+                            "length": 16
+                        }
+                    ]
+                }
+            }
+        ]
+    }

docs/pysim_fs_sphinx.py

@@ -0,0 +1,267 @@
+"""
+Sphinx extension: auto-generate docs/filesystem.rst from the pySim EF class hierarchy.
+
+Hooked into Sphinx's ``builder-inited`` event so the file is always regenerated
+from the live Python classes before Sphinx reads any source files.
+
+The table of root objects to document is in SECTIONS near the top of this file.
+EXCLUDED lists CardProfile/CardApplication subclasses intentionally omitted from
+SECTIONS, with reasons.  Both tables are read by tests/unittests/test_fs_coverage.py
+to ensure every class with EF/DF content is accounted for.
+"""
+
+import importlib
+import inspect
+import json
+import os
+import sys
+import textwrap
+
+# Ensure pySim is importable when this module is loaded as a Sphinx extension
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from pySim.filesystem import (CardApplication, CardDF, CardMF, CardEF, # noqa: E402
+                              TransparentEF, TransRecEF, LinFixedEF, CyclicEF, BerTlvEF)
+from pySim.profile import CardProfile # noqa: E402
+
+
+# Generic EF base classes whose docstrings describe the *type* of file
+# (Transparent, LinFixed, ...) rather than a specific file's content.
+# Suppress those boilerplate texts in the per-EF entries; they are only
+# useful once, at the top of the document or in a dedicated glossary.
+_EF_BASE_TYPES = frozenset([TransparentEF,
+                            TransRecEF,
+                            LinFixedEF,
+                            CyclicEF,
+                            BerTlvEF])
+
+
+# ---------------------------------------------------------------------------
+# Sections: (heading, module, class-name)
+# The class must be either a CardProfile (uses .files_in_mf) or a CardDF
+# subclass (uses .children).
+# ---------------------------------------------------------------------------
+SECTIONS = [
+    ('MF / TS 102 221 (UICC)',
+     'pySim.ts_102_221', 'CardProfileUICC'),
+    ('ADF.USIM / TS 31.102',
+     'pySim.ts_31_102',  'ADF_USIM'),
+    ('ADF.ISIM / TS 31.103',
+     'pySim.ts_31_103',  'ADF_ISIM'),
+    ('ADF.HPSIM / TS 31.104',
+     'pySim.ts_31_104',  'ADF_HPSIM'),
+    ('DF.GSM + DF.TELECOM / TS 51.011 (SIM)',
+     'pySim.ts_51_011',  'CardProfileSIM'),
+    ('CDMA / IS-820 (RUIM)',
+     'pySim.cdma_ruim',  'CardProfileRUIM'),
+    ('DF.EIRENE / GSM-R',
+     'pySim.gsm_r',      'DF_EIRENE'),
+    ('DF.SYSTEM / sysmocom SJA2+SJA5',
+     'pySim.sysmocom_sja2', 'DF_SYSTEM'),
+]
+
+# ---------------------------------------------------------------------------
+# Excluded: {(module, class-name)}
+# CardProfile and CardApplication subclasses that have EF/DF children but are
+# intentionally absent from SECTIONS.  Keeping this list explicit lets
+# test_fs_coverage.py detect newly added classes that the developer forgot to
+# add to either table.
+# ---------------------------------------------------------------------------
+EXCLUDED = {
+    # eUICC profiles inherit files_in_mf verbatim from CardProfileUICC; the
+    # eUICC-specific content lives in ISD-R / ISD-P applications, not in MF.
+    ('pySim.euicc', 'CardProfileEuiccSGP02'),
+    ('pySim.euicc', 'CardProfileEuiccSGP22'),
+    ('pySim.euicc', 'CardProfileEuiccSGP32'),
+    # CardApplication* classes are thin wrappers that embed an ADF_* instance.
+    # The ADF contents are already documented via the corresponding ADF_* entry
+    # in SECTIONS above.
+    ('pySim.ts_31_102', 'CardApplicationUSIM'),
+    ('pySim.ts_31_102', 'CardApplicationUSIMnonIMSI'),
+    ('pySim.ts_31_103', 'CardApplicationISIM'),
+    ('pySim.ts_31_104', 'CardApplicationHPSIM'),
+}
+
+# RST underline characters ordered by nesting depth
+_HEADING_CHARS = ['=', '=', '-', '~', '^', '"']
+# Level 0 uses '=' with overline (page title).
+# Level 1 uses '=' without overline (major sections).
+# Levels 2+ use the remaining characters for DFs.
+
+
+# ---------------------------------------------------------------------------
+# RST formatting helpers
+# ---------------------------------------------------------------------------
+
+def _heading(title: str, level: int) -> str:
+    """Return an RST heading string.  Level 0 gets an overline."""
+    char = _HEADING_CHARS[level]
+    rule = char * len(title)
+    if level == 0:
+        return f'{rule}\n{title}\n{rule}\n\n'
+    return f'{title}\n{rule}\n\n'
+
+
+def _json_default(obj):
+    """Fallback serialiser: bytes -> hex, anything else -> repr."""
+    if isinstance(obj, (bytes, bytearray)):
+        return obj.hex()
+    return repr(obj)
+
+
+def _examples_block(cls) -> str:
+    """Return RST code-block examples (one per vector), or '' if none exist.
+
+    Each example is rendered as a ``json5`` code-block with the hex-encoded
+    binary as a ``// comment`` on the first line, followed by the decoded JSON.
+    ``json5`` is used instead of ``json`` so that Pygments does not flag the
+    ``//`` comment as a syntax error.
+    """
+    vectors = []
+    for attr in ('_test_de_encode', '_test_decode'):
+        v = getattr(cls, attr, None)
+        if v:
+            vectors.extend(v)
+    if not vectors:
+        return ''
+
+    lines = ['**Examples**\n\n']
+
+    for t in vectors:
+        # 2-tuple: (encoded, decoded)
+        # 3-tuple: (encoded, record_nr, decoded)   — LinFixedEF / CyclicEF
+        if len(t) >= 3:
+            encoded, record_nr, decoded = t[0], t[1], t[2]
+            comment = f'record {record_nr}: {encoded.lower()}'
+        else:
+            encoded, decoded = t[0], t[1]
+            comment = f'file: {encoded.lower()}'
+
+        json_str = json.dumps(decoded, default=_json_default, indent=2)
+        json_indented = textwrap.indent(json_str, '   ')
+
+        lines.append('.. code-block:: json5\n\n')
+        lines.append(f'   // {comment}\n')
+        lines.append(json_indented + '\n')
+        lines.append('\n')
+
+    return ''.join(lines)
+
+
+def _document_ef(ef: CardEF) -> str:
+    """Return RST for a single EF.  Uses ``rubric`` to stay out of the TOC."""
+    cls = type(ef)
+
+    parts = [ef.fully_qualified_path_str()]
+    if ef.fid:
+        parts.append(f'({ef.fid.upper()})')
+    if ef.desc:
+        parts.append(f'\u2014 {ef.desc}')   # em-dash
+    title = ' '.join(parts)
+
+    lines = [f'.. rubric:: {title}\n\n']
+
+    # Only show a docstring if it is specific to this class.  EFs that are
+    # direct instances of a base type (TransparentEF, LinFixedEF, ...) carry
+    # only the generic "what is a TransparentEF" boilerplate; named subclasses
+    # without their own __doc__ have cls.__dict__['__doc__'] == None.  Either
+    # way, suppress the text here - it belongs at the document level, not
+    # repeated for every single EF entry.
+    doc = None if cls in _EF_BASE_TYPES else cls.__dict__.get('__doc__')
+    if doc:
+        lines.append(inspect.cleandoc(doc) + '\n\n')
+
+    examples = _examples_block(cls)
+    if examples:
+        lines.append(examples)
+
+    return ''.join(lines)
+
+
+def _document_df(df: CardDF, level: int) -> str:
+    """Return RST for a DF section and all its children, recursively."""
+    parts = [df.fully_qualified_path_str()]
+    if df.fid:
+        parts.append(f'({df.fid.upper()})')
+    if df.desc:
+        parts.append(f'\u2014 {df.desc}')   # em-dash
+    title = ' '.join(parts)
+
+    lines = [_heading(title, level)]
+
+    cls = type(df)
+    doc = None if cls in (CardDF, CardMF) else cls.__dict__.get('__doc__')
+    if doc:
+        lines.append(inspect.cleandoc(doc) + '\n\n')
+
+    for child in df.children.values():
+        if isinstance(child, CardDF):
+            lines.append(_document_df(child, level + 1))
+        elif isinstance(child, CardEF):
+            lines.append(_document_ef(child))
+
+    return ''.join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Top-level generator
+# ---------------------------------------------------------------------------
+
+def generate_filesystem_rst() -> str:
+    """Walk all registered sections and return the full RST document as a string."""
+    out = [
+        '.. This file is auto-generated by docs/pysim_fs_sphinx.py — do not edit.\n\n',
+        _heading('Card Filesystem Reference', 0),
+        'This page documents all Elementary Files (EFs) and Dedicated Files (DFs) '
+        'implemented in pySim, organised by their location in the card filesystem.\n\n',
+    ]
+
+    # Track already-documented classes so that DFs/EFs shared between profiles
+    # (e.g. DF.TELECOM / DF.GSM present in both CardProfileSIM and CardProfileRUIM)
+    # are only emitted once.
+    seen_types: set = set()
+
+    for section_title, module_path, class_name in SECTIONS:
+        module = importlib.import_module(module_path)
+        cls = getattr(module, class_name)
+        obj = cls()
+
+        if isinstance(obj, CardProfile):
+            files = obj.files_in_mf
+        elif isinstance(obj, CardApplication):
+            files = list(obj.adf.children.values())
+        elif isinstance(obj, CardDF):
+            files = list(obj.children.values())
+        else:
+            continue
+
+        # Filter out files whose class was already documented in an earlier section.
+        files = [f for f in files if type(f) not in seen_types]
+        if not files:
+            continue
+
+        out.append(_heading(section_title, 1))
+
+        for f in files:
+            seen_types.add(type(f))
+            if isinstance(f, CardDF):
+                out.append(_document_df(f, level=2))
+            elif isinstance(f, CardEF):
+                out.append(_document_ef(f))
+
+    return ''.join(out)
+
+
+# ---------------------------------------------------------------------------
+# Sphinx integration
+# ---------------------------------------------------------------------------
+
+def _on_builder_inited(app):
+    output_path = os.path.join(app.srcdir, 'filesystem.rst')
+    with open(output_path, 'w') as fh:
+        fh.write(generate_filesystem_rst())
+
+
+def setup(app):
+    app.connect('builder-inited', _on_builder_inited)
+    return {'version': '0.1', 'parallel_read_safe': True}

docs/saip-tool.rst

@@ -67,7 +67,7 @@ Inspecting applications
 
 To inspect the application PE contents of an existing profile package, sub-command `info` with parameter '--apps' can
 be used. This command lists out all application and their parameters in detail. This allows an application developer
-to check if the applet insertaion was carried out as expected.
+to check if the applet insertion was carried out as expected.
 
 Example: Listing applications and their parameters
 ::

docs/shell.rst

@@ -68,7 +68,7 @@ Usage Examples
 
    suci-tutorial
    cap-tutorial
-
+   put_key-tutorial
 
 Advanced Topics
 ---------------
@@ -602,8 +602,8 @@ This allows for easy interactive modification of records.
 If this command fails before the editor is spawned, it means that the current record contents is not decodable,
 and you should use the :ref:`update_record_decoded` or :ref:`update_record` command.
 
-If this command fails after making your modificatiosn in the editor, it means that the new file contents is not
-encodable; please check your input and/or us the raw :ref:`update_record` comamdn.
+If this command fails after making your modifications in the editor, it means that the new file contents is not
+encodable; please check your input and/or use the raw :ref:`update_record` command.
 
 
 decode_hex
@@ -708,8 +708,8 @@ This allows for easy interactive modification of file contents.
 If this command fails before the editor is spawned, it means that the current file contents is not decodable,
 and you should use the :ref:`update_binary_decoded` or :ref:`update_binary` command.
 
-If this command fails after making your modificatiosn in the editor, it means that the new file contents is not
-encodable; please check your input and/or us the raw :ref:`update_binary` comamdn.
+If this command fails after making your modifications in the editor, it means that the new file contents is not
+encodable; please check your input and/or use the raw :ref:`update_binary` command.
 
 
 decode_hex

docs/smpp-ota-tool.rst

@@ -170,6 +170,35 @@ ensures that a message can only be sent once.
 .. note:: The replay-protection-counter is implemented as a 5 byte integer value (see also ETSI TS 102 225, Table 3).
 	  When the counter has reached its maximum, it will not overflow nor can it be reset.
 
+Expanded remote application data format
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`smpp-ota-tool` uses the TS 102 226 section 5.1 compact remote application data format by default. This
+format concatenates C-APDUs into one command string and only the result of the LAST executed command is reported back.
+Retrieving the response data therefore requires a GET RESPONSE C-APDU, and only a single GET RESPONSE command may occur per script.
+
+The TS 102 226 section 5.2 expanded remote application data format removes these limitations: Each C-APDU is
+wrapped in its own C-APDU TLV inside a Command Scripting template, and the response is a Response Scripting template that contains one R-APDU TLV with the full response data and status word per executed command. To use it, pass
+``--format expanded``; every ``--apdu`` argument then becomes its own C-APDU TLV.
+
+.. note:: The expanded format does not use GET RESPONSE. To retrieve response data from a case 2 or case 4
+	  command, include an ``Le`` field in the C-APDU. i.e. ``Le='00'`` instructs the card to return all available
+	  response data in the R-APDU, with no 256-byte limit (TS 102 226, section 5.2.1.1). Without the ``Le``
+	  field no response data is returned, except a status word for the last command!.
+
+For example, a GP GET STATUS of all applications (``80F24002024F00``) returns a registry that can be much
+larger than 256 bytes. In the compact format the card would only answer with ``61xx`` procedure bytes. In the expanded
+format, appending ``Le='00'`` (i.e. ``80F24002024F0000``) makes the card return the whole registry in one exchange:
+
+::
+
+   $ PYTHONPATH=./ ./contrib/smpp-ota-tool.py --kic <KIC> --kid <KID> --kid-idx 1 --kic-idx 1 \
+       --algo-crypt triple_des_cbc2 --algo-auth triple_des_cbc2 --tar 000000 --cntr-req no_counter \
+       --format expanded --apdu 80F24002024F0000
+
+The response data (a concatenation of GlobalPlatform registry TLVs) can then be decoded with
+``pySim.global_platform.GpRegistryRelatedData.from_tlv()``.
+
 smpp-ota-tool syntax
 ~~~~~~~~~~~~~~~~~~~~
 

osmo-smdpp.py

@@ -136,6 +136,52 @@ from pySim.esim.x509_cert import CertAndPrivkey, CertificateSet, cert_get_subjec
 import logging # noqa: E402
 logger = logging.getLogger(__name__)
 
+
+def _disable_twisted_alpn_if_incompatible():
+    """Twisted <-> pyOpenSSL TLS compatibility guard applied at import.
+
+    Twisted TLSMemoryBIOFactory applies ALPN by setting the 'select' callback
+    on the SSL Context after it has already created a Connection from that
+    Context (_createConnection -> _applyProtocolNegotiation).
+    pyOpenSSL >= 25.0.0 makes a Context immutable once it has been used and
+    raises, which aborts every inbound TLS handshake, client sees unexpected-EOF
+    / decode_error that looks like a cert/cipher problem but is not.
+    pyOpenSSL < 25 does not import against recent cryptography, so downgrading
+    it is not a fix.
+
+    This server only speaks HTTP/1.1 anyway, so ALPN negotiation is not
+    needed.
+    """
+    def _major(v):
+        import re
+        m = re.match(r'\d+', (v or '').strip())
+        return int(m.group()) if m else 0
+
+    try:
+        import OpenSSL
+    except Exception:
+        return  # no pyOpenSSL ???
+    pyossl_ver = getattr(OpenSSL, '__version__', '0')
+    if _major(pyossl_ver) < 25:
+        return  # pre-25 pyOpenSSL allows mutating a used Context
+
+    try:
+        import twisted
+        from twisted.protocols import tls
+    except Exception:
+        return
+    factory = getattr(tls, 'TLSMemoryBIOFactory', None)
+    if factory is None or not hasattr(factory, '_applyProtocolNegotiation'):
+        return  # Twisted already fixed
+
+    factory._applyProtocolNegotiation = lambda self, connection: None
+    logger.warning("Disabled Twisted ALPN negotiation: Twisted %s + "
+                   "pyOpenSSL %s are incompatible for it",
+                   getattr(twisted, '__version__', '?'), pyossl_ver)
+
+
+_disable_twisted_alpn_if_incompatible()
+
 # HACK: make this configurable
 DATA_DIR = './smdpp-data'
 HOSTNAME = 'testsmdpplus1.example.com' # must match certificates!
@@ -479,7 +525,7 @@ class SmDppHttpServer:
         """See ES9+ InitiateAuthentication SGP.22 Section 5.6.1"""
         # Verify that the received address matches its own SM-DP+ address, where the comparison SHALL be
         # case-insensitive. Otherwise, the SM-DP+ SHALL return a status code "SM-DP+ Address - Refused".
-        if content['smdpAddress'] != self.server_hostname:
+        if content['smdpAddress'].lower() != self.server_hostname.lower():
            raise ApiError('8.8.1', '3.8', 'Invalid SM-DP+ Address')
 
         euiccChallenge = b64decode(content['euiccChallenge'])
@@ -640,7 +686,7 @@ class SmDppHttpServer:
                 # look up profile based on matchingID.  We simply check if a given file exists for now..
                 path = os.path.join(self.upp_dir, matchingId) + '.der'
                 # prevent directory traversal attack
-                if os.path.commonprefix((os.path.realpath(path),self.upp_dir)) != self.upp_dir:
+                if os.path.commonpath((os.path.realpath(path),self.upp_dir)) != self.upp_dir:
                     raise ApiError('8.2.6', '3.8', 'Refused')
                 if not os.path.isfile(path) or not os.access(path, os.R_OK):
                     raise ApiError('8.2.6', '3.8', 'Refused')
@@ -870,12 +916,17 @@ def main(argv):
                         action='store_true', default=False)
     parser.add_argument("-m", "--in-memory", help="Use ephermal in-memory session storage (for concurrent runs)",
                         action='store_true', default=False)
+    parser.add_argument("--smdp-address", default=HOSTNAME,
+                        help="ES9+ SM-DP+ address advertised, defaults to \"%(default)s\". "
+                             "Include the TLS port (e.g. %(default)s:8443) when binding a port other "
+                             "than 443, so it matches the address the LPA connects to. "
+                             "The TLS certificate identity is unaffected.")
     args = parser.parse_args()
 
     logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARNING)
 
     common_cert_path = os.path.join(DATA_DIR, args.certdir)
-    hs = SmDppHttpServer(server_hostname=HOSTNAME, ci_certs_path=os.path.join(common_cert_path, 'CertificateIssuer'), common_cert_path=common_cert_path, use_brainpool=args.brainpool)
+    hs = SmDppHttpServer(server_hostname=args.smdp_address, ci_certs_path=os.path.join(common_cert_path, 'CertificateIssuer'), common_cert_path=common_cert_path, use_brainpool=args.brainpool)
     if(args.nossl):
         hs.app.run(args.host, args.port)
     else:
@@ -904,7 +955,7 @@ def main(argv):
             with open(cert_pempath, 'wb') as pem_file:
                 pem_file.write(pem_cert)
 
-        SERVER_STRING = f'ssl:{args.port}:privateKey={cert_skpath}:certKey={cert_pempath}:dhParameters={dhparam_path}'
+        SERVER_STRING = f'ssl:{args.port}:privateKey={cert_skpath}:certKey={cert_pempath}:dhParameters={dhparam_path}:interface={args.host}'
         print(SERVER_STRING)
 
         hs.app.run(host=HOSTNAME, port=args.port, endpoint_description=SERVER_STRING)

pySim-prog.py

@@ -27,7 +27,6 @@
 import hashlib
 import argparse
 import os
-import random
 import re
 import sys
 import traceback
@@ -44,6 +43,11 @@ from pySim.legacy.ts_51_011 import EF
 from pySim.card_handler import *
 from pySim.utils import *
 
+from pathlib import Path
+import logging
+from pySim.log import PySimLogger
+
+log = PySimLogger.get(Path(__file__).stem)
 
 def parse_options():
 
@@ -185,6 +189,7 @@ def parse_options():
                       default=False, action="store_true")
     parser.add_argument("--card_handler", dest="card_handler_config", metavar="FILE",
                       help="Use automatic card handling machine")
+    parser.add_argument("--verbose", help="Enable verbose logging", action='store_true', default=False)
 
     options = parser.parse_args()
 
@@ -430,7 +435,7 @@ def gen_parameters(opts):
         if not re.match('^[0-9a-fA-F]{32}$', ki):
             raise ValueError('Ki needs to be 128 bits, in hex format')
     else:
-        ki = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
+        ki = os.urandom(16).hex()
 
     # OPC (random)
     if opts.opc is not None:
@@ -441,7 +446,7 @@ def gen_parameters(opts):
     elif opts.op is not None:
         opc = derive_milenage_opc(ki, opts.op)
     else:
-        opc = ''.join(['%02x' % random.randrange(0, 256) for i in range(16)])
+        opc = os.urandom(16).hex()
 
     pin_adm = sanitize_pin_adm(opts.pin_adm, opts.pin_adm_hex)
 
@@ -770,6 +775,9 @@ if __name__ == '__main__':
     # Parse options
     opts = parse_options()
 
+    # Setup logger
+    PySimLogger.setup(print, {logging.WARN: "\033[33m"}, opts.verbose)
+
     # Init card reader driver
     sl = init_reader(opts)
 
@@ -808,7 +816,7 @@ if __name__ == '__main__':
             print("")
             print("Card programming failed with an exception:")
             print("---------------------8<---------------------")
-            traceback.print_exc()
+            print(traceback.format_exc().rstrip())
             print("---------------------8<---------------------")
             print("")
             rc = -1

pySim-read.py

@@ -25,7 +25,6 @@
 import hashlib
 import argparse
 import os
-import random
 import re
 import sys
 
@@ -46,11 +45,17 @@ from pySim.utils import dec_imsi, dec_iccid
 from pySim.legacy.utils import format_xplmn_w_act, dec_st, dec_msisdn
 from pySim.ts_51_011 import EF_SMSP
 
+from pathlib import Path
+import logging
+from pySim.log import PySimLogger
+
+log = PySimLogger.get(Path(__file__).stem)
+
 option_parser = argparse.ArgumentParser(description='Legacy tool for reading some parts of a SIM card',
                                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+option_parser.add_argument("--verbose", help="Enable verbose logging", action='store_true', default=False)
 argparse_add_reader_args(option_parser)
 
-
 def select_app(adf: str, card: SimCard):
     """Select application by its AID"""
     sw = 0
@@ -75,6 +80,9 @@ if __name__ == '__main__':
     # Parse options
     opts = option_parser.parse_args()
 
+    # Setup logger
+    PySimLogger.setup(print, {logging.WARN: "\033[33m"}, opts.verbose)
+
     # Init card reader driver
     sl = init_reader(opts)
 

pySim-shell.py

@@ -24,21 +24,21 @@ import traceback
 import re
 import cmd2
 from packaging import version
-from cmd2 import style
-
 import logging
 from pySim.log import PySimLogger
 from osmocom.utils import auto_uint8
 
-# cmd2 >= 2.3.0 has deprecated the bg/fg in favor of Bg/Fg :(
-if version.parse(cmd2.__version__) < version.parse("2.3.0"):
-    from cmd2 import fg, bg # pylint: disable=no-name-in-module
-    RED = fg.red
-    YELLOW = fg.yellow
-    LIGHT_RED = fg.bright_red
-    LIGHT_GREEN = fg.bright_green
+# cmd2 >= 3.0 replaced Fg + style() with Color + stylize()
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
+    from cmd2 import Color, stylize # pylint: disable=no-name-in-module
+    RED = Color.RED
+    YELLOW = Color.YELLOW
+    LIGHT_RED = Color.BRIGHT_RED
+    LIGHT_GREEN = Color.BRIGHT_GREEN
+    def style(text, fg=None, bg=None, bold=False): # pylint: disable=function-redefined
+        return stylize(text, fg) if fg else text
 else:
-    from cmd2 import Fg, Bg # pylint: disable=no-name-in-module
+    from cmd2 import style, Fg # pylint: disable=no-name-in-module
     RED = Fg.RED
     YELLOW = Fg.YELLOW
     LIGHT_RED = Fg.LIGHT_RED
@@ -69,50 +69,26 @@ from pySim.ts_102_222 import Ts102222Commands
 from pySim.gsm_r import DF_EIRENE
 from pySim.cat import ProactiveCommand
 
-from pySim.card_key_provider import CardKeyProviderCsv, CardKeyProviderPgsql
-from pySim.card_key_provider import card_key_provider_register, card_key_provider_get_field, card_key_provider_get
+from pySim.card_key_provider import card_key_provider_argparse_add_args, card_key_provider_init
+from pySim.card_key_provider import card_key_provider_get_field, card_key_provider_get
 
 from pySim.app import init_card
 
 log = PySimLogger.get(Path(__file__).stem)
 
-class Cmd2Compat(cmd2.Cmd):
-    """Backwards-compatibility wrapper around cmd2.Cmd to support older and newer
-    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
-    def run_editor(self, file_path: Optional[str] = None) -> None:
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            return self._run_editor(file_path) # pylint: disable=no-member
-        else:
-            return super().run_editor(file_path) # pylint: disable=no-member
-
-class Settable2Compat(cmd2.Settable):
-    """Backwards-compatibility wrapper around cmd2.Settable to support older and newer
-    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
-    def __init__(self, name, val_type, description, settable_object, **kwargs):
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            super().__init__(name, val_type, description, **kwargs) # pylint: disable=no-value-for-parameter
-        else:
-            super().__init__(name, val_type, description, settable_object, **kwargs) # pylint: disable=too-many-function-args
-
-class PysimApp(Cmd2Compat):
+class PysimApp(cmd2.Cmd):
     CUSTOM_CATEGORY = 'pySim Commands'
     BANNER = """Welcome to pySim-shell!
 (C) 2021-2023 by Harald Welte, sysmocom - s.f.m.c. GmbH and contributors
 Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/shell.html """
 
     def __init__(self, verbose, card, rs, sl, ch, script=None):
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            kwargs = {'use_ipython': True}
-        else:
-            kwargs = {'include_ipy': True}
-
         self.verbose = verbose
-        self._onchange_verbose('verbose', False, self.verbose);
-
-        # pylint: disable=unexpected-keyword-arg
-        super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
-                         auto_load_commands=False, startup_script=script, **kwargs)
         PySimLogger.setup(self.poutput, {logging.WARN: YELLOW})
+        self._onchange_verbose('verbose', False, self.verbose)
+
+        super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
+                         auto_load_commands=False, startup_script=script, include_ipy=True)
         self.intro = style(self.BANNER, fg=RED)
         self.default_category = 'pySim-shell built-in commands'
         self.card = None
@@ -125,22 +101,27 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         self.numeric_path = False
         self.conserve_write = True
         self.json_pretty_print = True
-        self.apdu_trace = False
+        self.apdu_trace = getattr(sl, 'apdu_tracer', None) is not None
         self.apdu_strict = False
 
-        self.add_settable(Settable2Compat('numeric_path', bool, 'Print File IDs instead of names', self,
-                                          onchange_cb=self._onchange_numeric_path))
-        self.add_settable(Settable2Compat('conserve_write', bool, 'Read and compare before write', self,
-                                          onchange_cb=self._onchange_conserve_write))
-        self.add_settable(Settable2Compat('json_pretty_print', bool, 'Pretty-Print JSON output', self))
-        self.add_settable(Settable2Compat('apdu_trace', bool, 'Trace and display APDUs exchanged with card', self,
-                                          onchange_cb=self._onchange_apdu_trace))
-        self.add_settable(Settable2Compat('apdu_strict', bool,
-                                          'Enforce APDU responses according to ISO/IEC 7816-3, table 12', self,
-                                          onchange_cb=self._onchange_apdu_strict))
-        self.add_settable(Settable2Compat('verbose', bool,
-                                          'Enable/disable verbose logging', self,
-                                          onchange_cb=self._onchange_verbose))
+        self.add_settable(cmd2.Settable('numeric_path', bool,
+                                        'Print File IDs instead of names',
+                                        self, onchange_cb=self._onchange_numeric_path))
+        self.add_settable(cmd2.Settable('conserve_write', bool,
+                                        'Read and compare before write',
+                                        self, onchange_cb=self._onchange_conserve_write))
+        self.add_settable(cmd2.Settable('json_pretty_print', bool,
+                                        'Pretty-Print JSON output',
+                                        self))
+        self.add_settable(cmd2.Settable('apdu_trace', bool,
+                                        'Trace and display APDUs exchanged with card',
+                                        self, onchange_cb=self._onchange_apdu_trace))
+        self.add_settable(cmd2.Settable('apdu_strict', bool,
+                                        'Strictly apply APDU format according to ISO/IEC 7816-3, table 12',
+                                        self))
+        self.add_settable(cmd2.Settable('verbose', bool,
+                                        'Enable/disable verbose logging',
+                                        self, onchange_cb=self._onchange_verbose))
         self.equip(card, rs)
 
     def equip(self, card, rs):
@@ -218,13 +199,6 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
             else:
                 self.card._scc._tp.apdu_tracer = None
 
-    def _onchange_apdu_strict(self, param_name, old, new):
-        if self.card:
-            if new == True:
-                self.card._scc._tp.apdu_strict = True
-            else:
-                self.card._scc._tp.apdu_strict = False
-
     def _onchange_verbose(self, param_name, old, new):
         PySimLogger.set_verbose(new)
         if new == True:
@@ -236,8 +210,10 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         def __init__(self, cmd2_app):
             self.cmd2 = cmd2_app
 
-        def trace_response(self, cmd, sw, resp):
+        def trace_command(self, cmd):
             self.cmd2.poutput("-> %s %s" % (cmd[:10], cmd[10:]))
+
+        def trace_response(self, cmd, sw, resp):
             self.cmd2.poutput("<- %s: %s" % (sw, resp))
 
     def update_prompt(self):
@@ -281,7 +257,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
     apdu_cmd_parser.add_argument('--expect-sw', help='expect a specified status word', type=str, default=None)
     apdu_cmd_parser.add_argument('--expect-response-regex', help='match response against regex', type=str, default=None)
     apdu_cmd_parser.add_argument('--raw', help='Bypass the logical channel (and secure channel)', action='store_true')
-    apdu_cmd_parser.add_argument('APDU', type=is_hexstr, help='APDU as hex string')
+    apdu_cmd_parser.add_argument('APDU', type=is_hexstr, help='APDU as hex string (see also: ISO/IEC 7816-3, section 12.1')
 
     @cmd2.with_argparser(apdu_cmd_parser)
     def do_apdu(self, opts):
@@ -290,14 +266,23 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         tracked. Depending on the raw APDU sent, pySim-shell may not continue to work as expected if you e.g. select
         a different file."""
 
+        if not hasattr(self, 'apdu_strict_warning_displayed') and self.apdu_strict is False:
+            self.poutput("Warning: The default for the setable parameter `apdu_strict` will be changed from")
+            self.poutput("         `False` to `True` in future pySim-shell releases. In case you are using")
+            self.poutput("         the `apdu` command from a script that still mixes APDUs with TPDUs, consider")
+            self.poutput("         fixing or adding a `set apdu_strict false` line at the beginning.")
+            self.apdu_strict_warning_displayed = True;
+
         # When sending raw APDUs we access the scc object through _scc member of the card object. It should also be
         # noted that the apdu command plays an exceptional role since it is the only card accessing command that
         # can be executed without the presence of a runtime state (self.rs) object. However, this also means that
         # self.lchan is also not present (see method equip).
+        self.card._scc._tp.apdu_strict = self.apdu_strict
         if opts.raw or self.lchan is None:
             data, sw = self.card._scc.send_apdu(opts.APDU, apply_lchan = False)
         else:
             data, sw = self.lchan.scc.send_apdu(opts.APDU, apply_lchan = False)
+        self.card._scc._tp.apdu_strict = True
         if data:
             self.poutput("SW: %s, RESP: %s" % (sw, data))
         else:
@@ -366,7 +351,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
             self.poutput("")
             self.poutput("Card initialization (%s) failed with an exception:" % str(self.sl))
             self.poutput("---------------------8<---------------------")
-            traceback.print_exc()
+            self.poutput(traceback.format_exc().rstrip())
             self.poutput("---------------------8<---------------------")
             self.poutput("")
             return -1
@@ -480,7 +465,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
                 self.poutput("")
                 self.poutput("Card handling (%s) failed with an exception:" % str(self.sl))
                 self.poutput("---------------------8<---------------------")
-                traceback.print_exc()
+                self.poutput(traceback.format_exc().rstrip())
                 self.poutput("---------------------8<---------------------")
                 self.poutput("")
                 fail_count = fail_count + 1
@@ -1145,23 +1130,14 @@ global_group.add_argument("--skip-card-init", help="Skip all card/profile initia
 global_group.add_argument("--verbose", help="Enable verbose logging",
                           action='store_true', default=False)
 
-card_key_group = option_parser.add_argument_group('Card Key Provider Options')
-card_key_group.add_argument('--csv', metavar='FILE',
-                            default="~/.osmocom/pysim/card_data.csv",
-                            help='Read card data from CSV file')
-card_key_group.add_argument('--pgsql', metavar='FILE',
-                            default="~/.osmocom/pysim/card_data_pgsql.cfg",
-                            help='Read card data from PostgreSQL database (config file)')
-card_key_group.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
-                            help=argparse.SUPPRESS, dest='column_key')
-card_key_group.add_argument('--column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
-                            help='per-column AES transport key', dest='column_key')
-
 adm_group = global_group.add_mutually_exclusive_group()
-adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM1', dest='pin_adm', default=None,
+adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM', dest='pin_adm', default=None,
                        help='ADM PIN used for provisioning (overwrites default)')
-adm_group.add_argument('-A', '--pin-adm-hex', metavar='PIN_ADM1_HEX', dest='pin_adm_hex', default=None,
+adm_group.add_argument('-A', '--pin-adm-hex', metavar='PIN_ADM_HEX', dest='pin_adm_hex', default=None,
                        help='ADM PIN used for provisioning, as hex string (16 characters long)')
+global_group.add_argument('--pin-adm-type',
+                          choices=[x for x in pin_names.values() if x.startswith('ADM')],
+                          help='Override ADM number. Default is card-model-specific, usually 1')
 
 option_parser.add_argument('-e', '--execute-command', action='append', default=[],
                            help='A pySim-shell command that will be executed at startup')
@@ -1169,30 +1145,17 @@ option_parser.add_argument("command", nargs='?',
                            help="A pySim-shell command that would optionally be executed at startup")
 option_parser.add_argument('command_args', nargs=argparse.REMAINDER,
                            help="Optional Arguments for command")
+card_key_provider_argparse_add_args(option_parser)
 
 if __name__ == '__main__':
     startup_errors = False
     opts = option_parser.parse_args()
 
     # Ensure that we are able to print formatted warnings from the beginning.
-    PySimLogger.setup(print, {logging.WARN: YELLOW})
-    if opts.verbose:
-        PySimLogger.set_verbose(True)
-        PySimLogger.set_level(logging.DEBUG)
-    else:
-        PySimLogger.set_verbose(False)
-        PySimLogger.set_level(logging.INFO)
+    PySimLogger.setup(print, {logging.WARN: YELLOW}, opts.verbose)
 
-    # Register csv-file as card data provider, either from specified CSV
-    # or from CSV file in home directory
-    column_keys = {}
-    for par in opts.column_key:
-        name, key = par.split(':')
-        column_keys[name] = key
-    if os.path.isfile(os.path.expanduser(opts.csv)):
-        card_key_provider_register(CardKeyProviderCsv(os.path.expanduser(opts.csv), column_keys))
-    if os.path.isfile(os.path.expanduser(opts.pgsql)):
-        card_key_provider_register(CardKeyProviderPgsql(os.path.expanduser(opts.pgsql), column_keys))
+    # Init card key provider for automatic card key retrieval
+    card_key_provider_init(opts)
 
     # Init card reader driver
     sl = init_reader(opts, proactive_handler = Proact())
@@ -1213,7 +1176,7 @@ if __name__ == '__main__':
         startup_errors = True
         print("Card initialization (%s) failed with an exception:" % str(sl))
         print("---------------------8<---------------------")
-        traceback.print_exc()
+        print(traceback.format_exc().rstrip())
         print("---------------------8<---------------------")
         if not opts.noprompt:
             print("(you may still try to recover from this manually by using the 'equip' command.)")
@@ -1224,18 +1187,15 @@ if __name__ == '__main__':
 
     # If the user supplies an ADM PIN at via commandline args authenticate
     # immediately so that the user does not have to use the shell commands
-    pin_adm = sanitize_pin_adm(opts.pin_adm, opts.pin_adm_hex)
-    if pin_adm:
-        if not card:
-            print("Card error, cannot do ADM verification with supplied ADM pin now.")
-        try:
-            card._scc.verify_chv(card._adm_chv_num, h2b(pin_adm))
-        except Exception as e:
-            startup_errors = True
-            print("ADM verification (%s) failed with an exception:" % str(pin_adm))
-            print("---------------------8<---------------------")
-            print(e)
-            print("---------------------8<---------------------")
+    pin_adm_type = ""
+    if opts.pin_adm_type:
+        pin_adm_type = "--adm-type %s" % opts.pin_adm_type
+    if opts.pin_adm:
+        app.onecmd_plus_hooks("verify_adm %s %s" %
+                              (opts.pin_adm, pin_adm_type), add_to_history = False)
+    elif opts.pin_adm_hex:
+        app.onecmd_plus_hooks("verify_adm %s --pin-is-hex %s" %
+                              (opts.pin_adm_hex, pin_adm_type), add_to_history = False)
 
     # Run optional commands
     for c in opts.execute_command:

pySim-smpp2sim.py

@@ -30,10 +30,13 @@
 
 import argparse
 import logging
+import socket
+import threading
+import time
 import colorlog
 
 from twisted.protocols import basic
-from twisted.internet import defer, endpoints, protocol, reactor, task
+from twisted.internet import defer, endpoints, reactor, task
 from twisted.cred.portal import IRealm
 from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
 from twisted.cred.portal import Portal
@@ -47,13 +50,16 @@ from smpp.pdu import pdu_types, operations, pdu_encoding
 
 from pySim.sms import SMS_DELIVER, SMS_SUBMIT, AddressField
 
+from pySim.bip import Proact, terminal_profile
 from pySim.transport import LinkBase, ProactiveHandler, argparse_add_reader_args, init_reader, ApduTracer
 from pySim.commands import SimCardCommands
 from pySim.cards import UiccCardBase
 from pySim.exceptions import *
+from pySim.cat import sms_pp_download_envelope
 from pySim.cat import ProactiveCommand, SendShortMessage, SMS_TPDU, SMSPPDownload, BearerDescription
 from pySim.cat import DeviceIdentities, Address, OtherAddress, UiccTransportLevel, BufferSize
 from pySim.cat import ChannelStatus, ChannelData, ChannelDataLength
+from pySim.cat import EventList, EventDownload, Result
 from pySim.utils import b2h, h2b
 
 logger = logging.getLogger(__name__)
@@ -71,224 +77,6 @@ class MyApduTracer(ApduTracer):
         print("-> %s %s" % (cmd[:10], cmd[10:]))
         print("<- %s: %s" % (sw, resp))
 
-class TcpProtocol(protocol.Protocol):
-    def dataReceived(self, data):
-        pass
-
-    def connectionLost(self, reason):
-        pass
-
-
-def tcp_connected_callback(p: protocol.Protocol):
-    """called by twisted TCP client."""
-    logger.error("%s: connected!" % p)
-
-class ProactChannel:
-    """Representation of a single protective channel."""
-    def __init__(self, channels: 'ProactChannels', chan_nr: int):
-        self.channels = channels
-        self.chan_nr = chan_nr
-        self.ep = None
-
-    def close(self):
-        """Close the channel."""
-        if self.ep:
-            self.ep.disconnect()
-        self.channels.channel_delete(self.chan_nr)
-
-class ProactChannels:
-    """Wrapper class for maintaining state of proactive channels."""
-    def __init__(self):
-        self.channels = {}
-
-    def channel_create(self) -> ProactChannel:
-        """Create a new proactive channel, allocating its integer number."""
-        for i in range(1, 9):
-            if not i in self.channels:
-                self.channels[i] = ProactChannel(self, i)
-                return self.channels[i]
-        raise ValueError('Cannot allocate another channel: All channels active')
-
-    def channel_delete(self, chan_nr: int):
-        del self.channels[chan_nr]
-
-class Proact(ProactiveHandler):
-    #def __init__(self, smpp_factory):
-    #    self.smpp_factory = smpp_factory
-    def __init__(self):
-        self.channels = ProactChannels()
-
-    @staticmethod
-    def _find_first_element_of_type(instlist, cls):
-        for i in instlist:
-            if isinstance(i, cls):
-                return i
-        return None
-
-    """Call-back which the pySim transport core calls whenever it receives a
-    proactive command from the SIM."""
-    def handle_SendShortMessage(self, pcmd: ProactiveCommand):
-        # {'smspp_download': [{'device_identities': {'source_dev_id': 'network',
-        #                                            'dest_dev_id': 'uicc'}},
-        #                     {'address': {'ton_npi': {'ext': True,
-        #                                              'type_of_number': 'international',
-        #                                              'numbering_plan_id': 'isdn_e164'},
-        #                                  'call_number': '79'}},
-        #                     {'sms_tpdu': {'tpdu': '40048111227ff6407070611535004d02700000481516011212000001fe4c0943aea42e45021c078ae06c66afc09303608874b72f58bacadb0dcf665c29349c799fbb522e61709c9baf1890015e8e8e196e36153106c8b92f95153774'}}
-        #                    ]}
-        """Card requests sending a SMS. We need to pass it on to the ESME via SMPP."""
-        logger.info("SendShortMessage")
-        logger.info(pcmd)
-        # Relevant parts in pcmd: Address, SMS_TPDU
-        addr_ie = Proact._find_first_element_of_type(pcmd.children, Address)
-        sms_tpdu_ie = Proact._find_first_element_of_type(pcmd.children, SMS_TPDU)
-        raw_tpdu = sms_tpdu_ie.decoded['tpdu']
-        submit = SMS_SUBMIT.from_bytes(raw_tpdu)
-        submit.tp_da = AddressField(addr_ie.decoded['call_number'], addr_ie.decoded['ton_npi']['type_of_number'],
-                                    addr_ie.decoded['ton_npi']['numbering_plan_id'])
-        logger.info(submit)
-        self.send_sms_via_smpp(submit)
-
-    def handle_OpenChannel(self, pcmd: ProactiveCommand):
-        """Card requests opening a new channel via a UDP/TCP socket."""
-        # {'open_channel': [{'command_details': {'command_number': 1,
-        #                                        'type_of_command': 'open_channel',
-        #                                        'command_qualifier': 3}},
-        #                   {'device_identities': {'source_dev_id': 'uicc',
-        #                                          'dest_dev_id': 'terminal'}},
-        #                   {'bearer_description': {'bearer_type': 'default',
-        #                                           'bearer_parameters': ''}},
-        #                   {'buffer_size': 1024},
-        #                   {'uicc_transport_level': {'protocol_type': 'tcp_uicc_client_remote',
-        #                                             'port_number': 32768}},
-        #                   {'other_address': {'type_of_address': 'ipv4',
-        #                                      'address': '01020304'}}
-        #                  ]}
-        logger.info("OpenChannel")
-        logger.info(pcmd)
-        transp_lvl_ie = Proact._find_first_element_of_type(pcmd.children, UiccTransportLevel)
-        other_addr_ie = Proact._find_first_element_of_type(pcmd.children, OtherAddress)
-        bearer_desc_ie = Proact._find_first_element_of_type(pcmd.children, BearerDescription)
-        buffer_size_ie = Proact._find_first_element_of_type(pcmd.children, BufferSize)
-        if transp_lvl_ie.decoded['protocol_type'] != 'tcp_uicc_client_remote':
-            raise ValueError('Unsupported protocol_type')
-        if other_addr_ie.decoded.get('type_of_address', None) != 'ipv4':
-            raise ValueError('Unsupported type_of_address')
-        ipv4_bytes = h2b(other_addr_ie.decoded['address'])
-        ipv4_str = '%u.%u.%u.%u' % (ipv4_bytes[0], ipv4_bytes[1], ipv4_bytes[2], ipv4_bytes[3])
-        port_nr = transp_lvl_ie.decoded['port_number']
-        print("%s:%u" % (ipv4_str, port_nr))
-        channel = self.channels.channel_create()
-        channel.ep = endpoints.TCP4ClientEndpoint(reactor, ipv4_str, port_nr)
-        channel.prot = TcpProtocol()
-        d = endpoints.connectProtocol(channel.ep, channel.prot)
-        # FIXME: why is this never called despite the client showing the inbound connection?
-        d.addCallback(tcp_connected_callback)
-
-        # Terminal Response example: [
-        #  {'command_details': {'command_number': 1,
-        #                       'type_of_command': 'open_channel',
-        #                       'command_qualifier': 3}},
-        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
-        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
-        #  {'channel_status': '8100'},
-        #  {'bearer_description': {'bearer_type': 'default', 'bearer_parameters': ''}},
-        #  {'buffer_size': 1024}
-        # ]
-        return self.prepare_response(pcmd) + [ChannelStatus(decoded='8100'), bearer_desc_ie, buffer_size_ie]
-
-    def handle_CloseChannel(self, pcmd: ProactiveCommand):
-        """Close a channel."""
-        logger.info("CloseChannel")
-        logger.info(pcmd)
-
-    def handle_ReceiveData(self, pcmd: ProactiveCommand):
-        """Receive/read data from the socket."""
-        # {'receive_data': [{'command_details': {'command_number': 1,
-        #                                        'type_of_command': 'receive_data',
-        #                                        'command_qualifier': 0}},
-        #                   {'device_identities': {'source_dev_id': 'uicc',
-        #                                          'dest_dev_id': 'channel_1'}},
-        #                   {'channel_data_length': 9}
-        #                  ]}
-        logger.info("ReceiveData")
-        logger.info(pcmd)
-        # Terminal Response example: [
-        #  {'command_details': {'command_number': 1,
-        #                       'type_of_command': 'receive_data',
-        #                       'command_qualifier': 0}},
-        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
-        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
-        #  {'channel_data': '16030100040e000000'},
-        #  {'channel_data_length': 0}
-        # ]
-        return self.prepare_response(pcmd) + []
-
-    def handle_SendData(self, pcmd: ProactiveCommand):
-        """Send/write data received from the SIM to the socket."""
-        # {'send_data': [{'command_details': {'command_number': 1,
-        #                                     'type_of_command': 'send_data',
-        #                                     'command_qualifier': 1}},
-        #                {'device_identities': {'source_dev_id': 'uicc',
-        #                                       'dest_dev_id': 'channel_1'}},
-        #                {'channel_data': '160301003c010000380303d0f45e12b52ce5bb522750dd037738195334c87a46a847fe2b6886cada9ea6bf00000a00ae008c008b00b0002c010000050001000101'}
-        #               ]}
-        logger.info("SendData")
-        logger.info(pcmd)
-        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
-        chan_data_ie = Proact._find_first_element_of_type(pcmd.children, ChannelData)
-        chan_str = dev_id_ie.decoded['dest_dev_id']
-        chan_nr = 1 # FIXME
-        chan = self.channels.channels.get(chan_nr, None)
-        # FIXME chan.prot.transport.write(h2b(chan_data_ie.decoded))
-        # Terminal Response example: [
-        #  {'command_details': {'command_number': 1,
-        #                       'type_of_command': 'send_data',
-        #                       'command_qualifier': 1}},
-        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
-        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
-        #  {'channel_data_length': 255}
-        # ]
-        return self.prepare_response(pcmd) + [ChannelDataLength(decoded=255)]
-
-    def handle_SetUpEventList(self, pcmd: ProactiveCommand):
-        # {'set_up_event_list': [{'command_details': {'command_number': 1,
-        #                                             'type_of_command': 'set_up_event_list',
-        #                                             'command_qualifier': 0}},
-        #                        {'device_identities': {'source_dev_id': 'uicc',
-        #                                               'dest_dev_id': 'terminal'}},
-        #                        {'event_list': ['data_available', 'channel_status']}
-        #                       ]}
-        logger.info("SetUpEventList")
-        logger.info(pcmd)
-        # Terminal Response example: [
-        #  {'command_details': {'command_number': 1,
-        #                       'type_of_command': 'set_up_event_list',
-        #                       'command_qualifier': 0}},
-        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
-        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}}
-        # ]
-        return self.prepare_response(pcmd)
-
-    def getChannelStatus(self, pcmd: ProactiveCommand):
-        logger.info("GetChannelStatus")
-        logger.info(pcmd)
-        return self.prepare_response(pcmd) + []
-
-    def send_sms_via_smpp(self, submit: SMS_SUBMIT):
-        # while in a normal network the phone/ME would *submit* a message to the SMSC,
-        # we are actually emulating the SMSC itself, so we must *deliver* the message
-        # to the ESME
-        deliver = SMS_DELIVER.from_submit(submit)
-        deliver_smpp = deliver.to_smpp()
-
-        hackish_global_smpp.sendDataRequest(deliver_smpp)
-#       # obtain the connection/binding of system_id to be used for delivering MO-SMS to the ESME
-#       connection = smpp_server.getBoundConnections[system_id].getNextBindingForDelivery()
-#       connection.sendDataRequest(deliver_smpp)
-
-
-
 def dcs_is_8bit(dcs):
     if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
                                    pdu_types.DataCodingDefault.OCTET_UNSPECIFIED):
@@ -323,6 +111,11 @@ class MyServer:
         smppEndpoint = endpoints.TCP6ServerEndpoint(reactor, tcp_port, interface=bind_ip)
         smppEndpoint.listen(self.factory)
         self.tp = self.scc = self.card = None
+        # Serialise card/APDU access.
+        # - SMPP handler drives the card from reactor thread
+        # - BIP relay data-available path drives it from socket reader thread.
+        # The transport is not re-entrant, both must take this lock.
+        self._card_lock = threading.Lock()
 
     def connect_to_card(self, tp: LinkBase):
         self.tp = tp
@@ -333,8 +126,22 @@ class MyServer:
         self.scc.sel_ctrl = "0004"
         self.card.read_aids()
         self.card.select_adf_by_aid(adf='usim')
-        # FIXME: create a more realistic profile than ffffff
-        self.scc.terminal_profile('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff')
+        self.scc.terminal_profile(b2h(terminal_profile()))
+        # Connect the BIP relay inbound path to the card.
+        # relay socket receives data -> ME initiated ENVELOPE EVENT DOWNLOA
+        # -> triggers RECEIVE DATA proactive session.
+        # FIXME this cross-thread push to the card is exercised only with real hardware
+        # the card free tests cover socket relay + envelope construction, not delivery.
+        handler = getattr(tp, 'proactive_handler', None)
+        if isinstance(handler, Proact):
+            handler.data_available_sink = self._deliver_data_available
+
+    def _deliver_data_available(self, envelope_hex: str):
+        """push ME initiated ENVELOPE EVENT DOWNLOAD to the card"""
+        with self._card_lock:
+            logger.info("ENVELOPE(Data available): %s" % envelope_hex)
+            (data, sw) = self.scc.envelope(envelope_hex)
+            logger.info("SW %s: %s" % (sw, data))
 
     def _msgHandler(self, system_id, smpp, pdu):
         """Handler for incoming messages received via SMPP from ESME."""
@@ -362,14 +169,12 @@ class MyServer:
         tpdu = SMS_DELIVER.from_smpp_submit(pdu)
         logger.info(tpdu)
         # 2) wrap into the CAT ENVELOPE for SMS-PP-Download
-        tpdu_ie = SMS_TPDU(decoded={'tpdu': b2h(tpdu.to_bytes())})
-        addr_ie = Address(decoded={'ton_npi': {'ext':False, 'type_of_number':'unknown', 'numbering_plan_id':'unknown'}, 'call_number': '0123456'})
-        dev_ids = DeviceIdentities(decoded={'source_dev_id': 'network', 'dest_dev_id': 'uicc'})
-        sms_dl = SMSPPDownload(children=[dev_ids, addr_ie, tpdu_ie])
+        sms_dl = sms_pp_download_envelope(tpdu)
         # 3) send to the card
         envelope_hex = b2h(sms_dl.to_tlv())
         logger.info("ENVELOPE: %s" % envelope_hex)
-        (data, sw) = self.scc.envelope(envelope_hex)
+        with self._card_lock:
+            (data, sw) = self.scc.envelope(envelope_hex)
         logger.info("SW %s: %s" % (sw, data))
         if sw in ['9200', '9300']:
             # TODO send back RP-ERROR message with TP-FCS == 'SIM Application Toolkit Busy'
@@ -416,7 +221,8 @@ if __name__ == '__main__':
 
     opts = option_parser.parse_args()
 
-    tp = init_reader(opts, proactive_handler = Proact())
+    tp = init_reader(opts, proactive_handler = Proact(
+        sms_sink=lambda pdu: hackish_global_smpp.sendDataRequest(pdu)))
     if tp is None:
         exit(1)
     tp.connect()

pySim-trace.py

@@ -117,7 +117,7 @@ class Tracer:
             try:
                 apdu = self.source.read()
                 apdu_counter = apdu_counter + 1
-            except StopIteration:
+            except (StopIteration, KeyboardInterrupt):
                 print("%i APDUs parsed, stop iteration." % apdu_counter)
                 return 0
 

pySim/app.py

@@ -26,11 +26,15 @@ from pySim.cdma_ruim import CardProfileRUIM
 from pySim.ts_102_221 import CardProfileUICC
 from pySim.utils import all_subclasses
 from pySim.exceptions import SwMatchError
+from pySim.log import PySimLogger
 
-# we need to import this module so that the SysmocomSJA2 sub-class of
-# CardModel is created, which will add the ATR-based matching and
-# calling of SysmocomSJA2.add_files.  See  CardModel.apply_matching_models
+log = PySimLogger.get(__name__)
+
+# we need to import these modules so that the SysmocomSJA2 / SysmocomSJS1
+# sub-classes of CardModel are created, which will add the ATR-based matching
+# and calling of their add_files.  See CardModel.apply_matching_models
 import pySim.sysmocom_sja2
+import pySim.sysmocom_sjs1
 
 # we need to import these modules so that the various sub-classes of
 # CardProfile are created, which will be used in init_card() to iterate
@@ -54,7 +58,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
 
     # Wait up to three seconds for a card in reader and try to detect
     # the card type.
-    print("Waiting for card...")
+    log.info("Waiting for card...")
     sl.wait_for_card(3)
 
     # The user may opt to skip all card initialization. In this case only the
@@ -66,7 +70,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
     generic_card = False
     card = card_detect(scc)
     if card is None:
-        print("Warning: Could not detect card type - assuming a generic card type...")
+        log.warning("Could not detect card type - assuming a generic card type...")
         card = SimCardBase(scc)
         generic_card = True
 
@@ -76,7 +80,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
         # just means that pySim was unable to recognize the card profile. This
         # may happen in particular with unprovisioned cards that do not have
         # any files on them yet.
-        print("Unsupported card type!")
+        log.warning("Unsupported card type!")
         return None, card
 
     # ETSI TS 102 221, Table 9.3 specifies a default for the PIN key
@@ -87,7 +91,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
     if generic_card and isinstance(profile, CardProfileUICC):
         card._adm_chv_num = 0x0A
 
-    print("Info: Card is of type: %s" % str(profile))
+    log.info("Card is of type: %s", str(profile))
 
     # FIXME: this shouldn't really be here but somewhere else/more generic.
     # We cannot do it within pySim/profile.py as that would create circular

pySim/ara_m.py

@@ -72,10 +72,10 @@ class ApduArDO(BER_TLV_IE, tag=0xd0):
             if do[0] == 0x01:
                 self.decoded = {'generic_access_rule': 'always'}
                 return self.decoded
-            return ValueError('Invalid 1-byte generic APDU access rule')
+            raise ValueError('Invalid 1-byte generic APDU access rule')
         else:
             if len(do) % 8:
-                return ValueError('Invalid non-modulo-8 length of APDU filter: %d' % len(do))
+                raise ValueError('Invalid non-modulo-8 length of APDU filter: %d' % len(do))
             self.decoded = {'apdu_filter': []}
             offset = 0
             while offset < len(do):
@@ -90,19 +90,19 @@ class ApduArDO(BER_TLV_IE, tag=0xd0):
                 return b'\x00'
             if self.decoded['generic_access_rule'] == 'always':
                 return b'\x01'
-            return ValueError('Invalid 1-byte generic APDU access rule')
+            raise ValueError('Invalid 1-byte generic APDU access rule')
         else:
             if not 'apdu_filter' in self.decoded:
-                return ValueError('Invalid APDU AR DO')
+                raise ValueError('Invalid APDU AR DO')
             filters = self.decoded['apdu_filter']
             res = b''
             for f in filters:
                 if not 'header' in f or not 'mask' in f:
-                    return ValueError('APDU filter must contain header and mask')
+                    raise ValueError('APDU filter must contain header and mask')
                 header_b = h2b(f['header'])
                 mask_b = h2b(f['mask'])
                 if len(header_b) != 4 or len(mask_b) != 4:
-                    return ValueError('APDU filter header and mask must each be 4 bytes')
+                    raise ValueError('APDU filter header and mask must each be 4 bytes')
                 res += header_b + mask_b
             return res
 
@@ -269,7 +269,7 @@ class ADF_ARAM(CardADF):
             cmd_do_enc = cmd_do.to_ie()
             cmd_do_len = len(cmd_do_enc)
             if cmd_do_len > 255:
-                return ValueError('DO > 255 bytes not supported yet')
+                raise ValueError('DO > 255 bytes not supported yet')
         else:
             cmd_do_enc = b''
             cmd_do_len = 0
@@ -300,6 +300,51 @@ class ADF_ARAM(CardADF):
                                'major': v_major, 'minor': v_minor, 'patch': v_patch}}])
         return ADF_ARAM.xceive_apdu_tlv(scc, '80cadf21', cmd_do, ResponseAramConfigDO)
 
+    @staticmethod
+    def store_ref_ar_do(scc, aid:Hexstr, aid_empty:bool, device_app_id:Hexstr, pkg_ref:str,
+                        apdu_filter:Hexstr, apdu_never:bool, apdu_always:bool,
+                        nfc_always:bool, nfc_never:bool, android_permissions:Hexstr):
+        # REF
+        ref_do_content = []
+        if aid is not None:
+            ref_do_content += [{'aid_ref_do': aid}]
+        elif aid_empty:
+            ref_do_content += [{'aid_ref_empty_do': None}]
+        ref_do_content += [{'dev_app_id_ref_do': device_app_id}]
+        if pkg_ref:
+            ref_do_content += [{'pkg_ref_do': {'package_name_string': pkg_ref}}]
+        # AR
+        ar_do_content = []
+        if apdu_never:
+            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
+        elif apdu_always:
+            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
+        elif apdu_filter:
+            if len(apdu_filter) % 16:
+                raise ValueError(f'Invalid non-modulo-16 length of APDU filter: {len(apdu_filter)}')
+            offset = 0
+            apdu_filter_list = []
+            while offset < len(apdu_filter):
+                apdu_filter_list += [{'header': apdu_filter[offset:offset+8],
+                                      'mask': apdu_filter[offset+8:offset+16]}]
+                offset += 16 # Move offset to the beginning of the next apdu_filter object
+            ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter_list}}]
+        if nfc_never:
+            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
+        elif nfc_always:
+            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
+        if android_permissions:
+            ar_do_content += [{'perm_ar_do': {'permissions': android_permissions}}]
+        d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
+        csrado = CommandStoreRefArDO()
+        csrado.from_val_dict(d)
+        return ADF_ARAM.store_data(scc, csrado)
+
+    @staticmethod
+    def aram_delete_all(scc):
+        deldo = CommandDelete()
+        return ADF_ARAM.store_data(scc, deldo)
+
     @with_default_category('Application-Specific Commands')
     class AddlShellCommands(CommandSet):
         def do_aram_get_all(self, _opts):
@@ -334,58 +379,25 @@ class ADF_ARAM(CardADF):
         apdu_grp.add_argument(
             '--apdu-filter', help='APDU filter: multiple groups of 8 hex bytes (4 byte CLA/INS/P1/P2 followed by 4 byte mask)')
         nfc_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
-        nfc_grp.add_argument('--nfc-always', action='store_true',
-                             help='NFC event access is allowed')
         nfc_grp.add_argument('--nfc-never', action='store_true',
                              help='NFC event access is not allowed')
+        nfc_grp.add_argument('--nfc-always', action='store_true',
+                             help='NFC event access is allowed')
         store_ref_ar_do_parse.add_argument(
             '--android-permissions', help='Android UICC Carrier Privilege Permissions (8 hex bytes)')
 
         @cmd2.with_argparser(store_ref_ar_do_parse)
         def do_aram_store_ref_ar_do(self, opts):
             """Perform STORE DATA [Command-Store-REF-AR-DO] to store a (new) access rule."""
-            # REF
-            ref_do_content = []
-            if opts.aid is not None:
-                ref_do_content += [{'aid_ref_do': opts.aid}]
-            elif opts.aid_empty:
-                ref_do_content += [{'aid_ref_empty_do': None}]
-            ref_do_content += [{'dev_app_id_ref_do': opts.device_app_id}]
-            if opts.pkg_ref:
-                ref_do_content += [{'pkg_ref_do': {'package_name_string': opts.pkg_ref}}]
-            # AR
-            ar_do_content = []
-            if opts.apdu_never:
-                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
-            elif opts.apdu_always:
-                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
-            elif opts.apdu_filter:
-                if len(opts.apdu_filter) % 16:
-                    return ValueError('Invalid non-modulo-16 length of APDU filter: %d' % len(do))
-                offset = 0
-                apdu_filter = []
-                while offset < len(opts.apdu_filter):
-                    apdu_filter += [{'header': opts.apdu_filter[offset:offset+8],
-                                     'mask': opts.apdu_filter[offset+8:offset+16]}]
-                    offset += 16 # Move offset to the beginning of the next apdu_filter object
-                ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter}}]
-            if opts.nfc_always:
-                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
-            elif opts.nfc_never:
-                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
-            if opts.android_permissions:
-                ar_do_content += [{'perm_ar_do': {'permissions': opts.android_permissions}}]
-            d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
-            csrado = CommandStoreRefArDO()
-            csrado.from_val_dict(d)
-            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, csrado)
+            res_do = ADF_ARAM.store_ref_ar_do(self._cmd.lchan.scc, opts.aid, opts.aid_empty, opts.device_app_id,
+                                              opts.pkg_ref, opts.apdu_filter, opts.apdu_never, opts.apdu_always,
+                                              opts.nfc_always, opts.nfc_never, opts.android_permissions)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
         def do_aram_delete_all(self, _opts):
             """Perform STORE DATA [Command-Delete[all]] to delete all access rules."""
-            deldo = CommandDelete()
-            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, deldo)
+            res_do = ADF_ARAM.aram_delete_all(self._cmd.lchan.scc)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
@@ -394,7 +406,6 @@ class ADF_ARAM(CardADF):
             (Proprietary feature that is specific to sysmocom's fork of Bertrand Martel’s ARA-M implementation.)"""
             self._cmd.lchan.scc.send_apdu_checksw('80e2900001A1', '9000')
 
-
 # SEAC v1.1 Section 4.1.2.2 + 5.1.2.2
 sw_aram = {
     'ARA-M': {

pySim/bip.py

@@ -0,0 +1,627 @@
+# -*- coding: utf-8 -*-
+"""Bearer Independent Protocol relay"""
+
+#
+# (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# A ProactiveHandler with TCP sockets that backs the BIP channels,
+# so a card can run its own IP session (SCP81/HTTPS, CAT_TP, ...)
+#
+# Currently used by pySim-smpp2sim.py which connects the SMS path to its SMPP server.
+# Other drivers can pass their own sinks:
+#
+#     handler = Proact(data_available_sink=..., sms_sink=...)
+#     tp = init_reader(opts, proactive_handler=handler)
+
+
+import logging
+import socket
+import threading
+import time
+
+from osmocom.utils import b2h, h2b
+
+from pySim.transport import ProactiveHandler
+from pySim.sms import SMS_DELIVER, SMS_SUBMIT, AddressField
+from pySim.cat import (ProactiveCommand, SendShortMessage, SMS_TPDU, SMSPPDownload,
+                       BearerDescription, DeviceIdentities, Address, OtherAddress,
+                       UiccTransportLevel, BufferSize, ChannelStatus, ChannelData,
+                       ChannelDataLength, EventList, EventDownload, Result,
+                       CommandDetails, LocationInformation)
+
+logger = logging.getLogger(__name__)
+
+# PROVIDE LOCAL INFORMATION location, GERAN TS 31.111 8.19.1
+# - 3 byte PLMN of TS 24.008 10.5.1.3 -> 262-01
+# - 2 byte LAC and a 2 byte cid.
+DEFAULT_LOCATION = h2b('62f21000010001')
+
+
+def terminal_profile(num_channels: int = 7) -> bytes:
+    """TERMINAL PROFILE for what we implement, TS 102 223 5.2 and annex T.
+
+    Annex T table T.1 lists what a Connected Entity, a CAT client that is not the modem
+    which is pretty much what we are, may announce, and its inverse is what only a modem may announce.
+    """
+    if not 0 <= num_channels <= ProactChannels.MAX_CHANNELS:
+        raise ValueError('num_channels must be 0..%u' % ProactChannels.MAX_CHANNELS)
+    profile = bytearray(32)
+    # 1 (Download): b1 profile download, b2+b5 SMS-PP data download. Both of the latter, per the
+    # note in TS 31.111 5.2: "several bits may need to be set to 1 for the support of the same
+    # facility ... because of backward compatibility with SAT". The relay is OTA over SMS-PP.
+    profile[0] = 0x01 | 0x02 | 0x10
+    profile[1] = 0x01                    # 2 (Other): b1 command result
+    profile[2] = 0x80                    # 3: b8 REFRESH (empty result is a valid answer, 6.4.7)
+    profile[3] = 0x02                    # 4: b2 SEND SHORT MESSAGE (the OTA response path)
+    profile[4] = 0x01                    # 5: b1 SET UP EVENT LIST
+    profile[5] = 0x04 | 0x08             # 6: b3 Event Data available, b4 Event Channel status
+    # 12 (class "e"): b1..b5 OPEN CHANNEL, CLOSE CHANNEL, RECEIVE DATA, SEND DATA, GET CHANNEL
+    # STATUS.
+    profile[11] = 0x1f
+    # 13 (class "e" supported bearers): b2 GPRS, and b6..b8 the number of channels.
+    profile[12] = 0x02 | (num_channels << 5)
+    profile[13] = 0x40 | 0x20            # 14: b6 no display capability, b7 no keypad available
+    profile[16] = 0x01                   # 15: b1 TCP, UICC in client mode, remote connection
+    return bytes(profile)
+
+
+class ProactChannel:
+    """One BIP channel, TS 102 223 class "e", backed by a blocking TCP socket.
+
+    Created by ProactChannels.channel_create(). A reader thread fills the Rx buffer from the
+    socket, the Proact handlers drain it (RECEIVE DATA) and write to it (SEND DATA). Payload
+    is opaque, TLS or CAT_TP run on the card.
+
+    Args:
+        channels: the owning ProactChannels, notified of data arrival and of close()
+        chan_nr: channel number 1..7 as used in the Device identities
+    """
+    # Why blocking sockets and not Twisted endpoints, considering we have twisted?
+    # The proactive-command loop lives in a blocking while-loop,
+    # "pySim.transport.LinkBase.send_apdu_checksw" that runs on the Twisted reactor thread.
+    # A Twisted async TCP client only makes any progress when the reactor uhh... reacts, but
+    # the reactor is stuck in that loop for the whole proactive session -> the
+    # connectProtocol() Deferred never fires while we are handling OPEN/SEND/RECEIVE CHANNEL.
+    # Plain blocking sockets just work: connect() in handle_OpenChannel, send() in
+    # handle_SendData, recv() feeding a buffer for handle_ReceiveData. No need to make it
+    # harder than it has to be to handle the "massive" T0 bandwidth..
+    # how much we try to read off the socket per recv()
+    RECV_CHUNK = 4096
+
+    def __init__(self, channels: 'ProactChannels', chan_nr: int):
+        self.channels = channels
+        self.chan_nr = chan_nr
+        self.sock = None
+        # TS 102 223 says the terminal keeps an Rx buffer per channel; RECEIVE
+        # DATA drains it, and it is filled asynchronously as the peer sends.
+        self.rx_buf = bytearray()
+        self._rx_lock = threading.Lock()
+        self._reader = None
+        self._closing = False
+        self.peer_closed = False
+
+    def connect(self, host: str, port: int, timeout: float = 10.0):
+        """Open the blocking TCP socket and start the background Rx reader."""
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            s.settimeout(timeout)
+            s.connect((host, port))
+            # Back to blocking mode for the reader thread.
+            # CLOSE CHANNEL unblocks the pending recv() via shutdown().
+            s.settimeout(None)
+        except OSError:
+            s.close()
+            raise
+        self.sock = s
+        self._reader = threading.Thread(target=self._rx_loop,
+                                        name='bip-rx-%d' % self.chan_nr, daemon=True)
+        self._reader.start()
+
+    def _rx_loop(self):
+        """Continuously read from the socket into rx_buf, like a real ME.
+
+        TS 102 223 7.5.10.1 says the event is raised 'only if the targeted channel buffer is
+        empty when new data arrives in it', so the data available hook fires on the
+        empty->non-empty transition only. That is enough: every RECEIVE DATA response tells
+        the card how many bytes remain, so it keeps fetching until the buffer is empty, and
+        the next event restarts it when more data arrives."""
+        while not self._closing:
+            try:
+                data = self.sock.recv(self.RECV_CHUNK)
+            except (OSError, ValueError):
+                break
+            if not data:
+                self.peer_closed = True
+                break
+            with self._rx_lock:
+                was_empty = len(self.rx_buf) == 0
+                self.rx_buf.extend(data)
+            if was_empty and not self._closing:
+                self.channels.notify_data_available(self)
+
+    def send(self, data: bytes):
+        """Tx, write bytes to the socket == SEND DATA"""
+        self.sock.sendall(data)
+
+    def available_rx(self) -> int:
+        """Number of bytes waiting in the Rx buffer, what RECEIVE DATA can return right now."""
+        with self._rx_lock:
+            return len(self.rx_buf)
+
+    def take_rx(self, n: int):
+        """Take up to n bytes out of the Rx buffer. Returns (bytes, bytes still remaining)."""
+        with self._rx_lock:
+            chunk = bytes(self.rx_buf[:n])
+            del self.rx_buf[:n]
+            remaining = len(self.rx_buf)
+        return chunk, remaining
+
+    def wait_rx(self, timeout: float) -> int:
+        """wait up to timeout seconds until the rxbuf has data
+        returns the number of bytes available
+        Cards have a "data available" event, card free callers use
+        this to wait for the echoed bytes."""
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            avail = self.available_rx()
+            if avail or self.peer_closed:
+                return avail
+            time.sleep(0.005)
+        return self.available_rx()
+
+    def close(self):
+        """Close channel: stop reader, close socket, drop bookkeeping."""
+        self._closing = True
+        if self.sock is not None:
+            try:
+                self.sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+            try:
+                self.sock.close()
+            except OSError:
+                pass
+        # CLOSE CHANNEL synchronously handled inside the rx reader thread
+        # (data-available -> ENVELOPE -> FETCH -> handle_CloseChannel -> close),
+        # so close() can be called on the reader thread.
+        # Joining self raises "cannot join current thread" so better skip i..
+        # setting _closing + shutting down the socket already makes _rx_loop
+        # return on the next iteration anyway.
+        if self._reader is not None and self._reader is not threading.current_thread():
+            self._reader.join(timeout=1.0)
+        self.channels.channel_delete(self.chan_nr)
+
+class ProactChannels:
+    """The open BIP channels of one terminal, keyed by channel number.
+
+    Args:
+        on_data_available: callback(chan: ProactChannel), invoked from the channel's reader
+            thread when data arrives in an empty Rx buffer. Proact turns it into an
+            ENVELOPE EVENT DOWNLOAD (data available).
+    """
+
+    # TS 102 223 8.56 channel identifier in 3 bits as "1 to 7", 0 == no channel available
+    # TERMINAL PROFILE has to agree with byte 13 , "number of channels supported by terminal"
+    MAX_CHANNELS = 7
+
+    def __init__(self, on_data_available=None):
+        self.channels = {}
+        self._on_data_available = on_data_available
+
+    def channel_create(self) -> ProactChannel:
+        """Create a new proactive channel, allocating its integer number."""
+        for i in range(1, self.MAX_CHANNELS + 1):
+            if not i in self.channels:
+                self.channels[i] = ProactChannel(self, i)
+                return self.channels[i]
+        raise ValueError('Cannot allocate another channel: All channels active')
+
+    def channel_delete(self, chan_nr: int):
+        """Forget a channel, called by ProactChannel.close()."""
+        self.channels.pop(chan_nr, None)
+
+    def notify_data_available(self, chan: ProactChannel):
+        """Run the on_data_available callback for chan, if one was given."""
+        if self._on_data_available:
+            self._on_data_available(chan)
+
+class Proact(ProactiveHandler):
+    """ProactiveHandler that answers the BIP proactive commands with TCP sockets.
+
+    The transport calls the handle_* methods with the decoded proactive command and posts
+    the returned IE list as TERMINAL RESPONSE.
+
+    Args:
+        data_available_sink: callback(envelope_hex: str), called from a channel reader thread
+            with an encoded ENVELOPE EVENT DOWNLOAD (data available). The caller forwards it
+            to the card with the ENVELOPE command, the card then FETCHes RECEIVE DATA.
+            None: the event is only logged (card free / test mode).
+        sms_sink: callback(pdu), called with the SMPP deliver_sm of a SEND SHORT MESSAGE
+            the card issued; pySim-smpp2sim.py hands it to its SMPP server.
+            None: the SMS is logged and dropped.
+        location: Location information returned in PROVIDE LOCAL INFORMATION (location).
+    """
+    def __init__(self, data_available_sink=None, sms_sink=None, location: bytes = DEFAULT_LOCATION):
+        self.data_available_sink = data_available_sink
+        self.sms_sink = sms_sink
+        self.location = location
+        self.channels = ProactChannels(on_data_available=self._on_channel_data_available)
+
+    def handle_ProvideLocalInformation(self, pcmd: ProactiveCommand):
+        """only location
+
+        TS 102 223 6.8.7 says TERMINAL RESPONSE to PROVIDE LOCAL INFORMATION "shall"
+        contain the data object the command qualifier (6.6.15) asked for. At least answer '00',
+        location information, usually requested.
+
+        answering "terminal currently unable to process - no service", which is a handset
+        out of coverage makes SJA5 believe it and postpones the entire session!
+        it registers a location status event, starts a ten minute timer and waits for coverage."""
+        cmd_det_ie = Proact._find_first_element_of_type(pcmd.children, CommandDetails)
+        if cmd_det_ie is not None and cmd_det_ie.decoded['command_qualifier'] == 0x00:
+            return self.prepare_response(pcmd) + [LocationInformation(decoded=self.location)]
+        return self.prepare_response(pcmd)
+
+    def receive_fetch(self, pcmd: ProactiveCommand):
+        """Answer anything this handler has no specific handler for.
+
+        A card coming up will usually issue PROVIDE LOCAL INFORMATION,
+        POLL INTERVAL or TIMER MANAGEMENT before it gets anywhere near a BIP channel,
+        whatever the TERMINAL PROFILE announces.
+
+        Note that this is not the spec-correct answer. TS 102 223 6.8.7
+        says a successful TERMINAL RESPONSE to PROVIDE LOCAL INFORMATION "shall" carry the
+        requested Local information data object, and 6.8.13/6.8.14 says the same for TIMER
+        MANAGEMENT, this returns empty results for all of them, which works with real cards.
+
+        Always "performed_successfully", never "command_beyond_terminal_capability" because
+        answering that to PROVIDE LOCAL INFORMATION makes a card refuse to open the session.
+        """
+        logger.info("no handler for %s, answering performed_successfully",
+                    type(pcmd.decoded).__name__)
+        return self.prepare_response(pcmd, 'performed_successfully')
+
+    @staticmethod
+    def _find_first_element_of_type(instlist, cls):
+        for i in instlist:
+            if isinstance(i, cls):
+                return i
+        return None
+
+    @staticmethod
+    def _channel_nr_from_dev_ids(dev_id_ie: DeviceIdentities) -> int:
+        """Maps id like channel_1 -> channel number.
+        TS 102 223 Section 8.7 says low nibble is channel number,
+        channel-N = 0x21..0x27"""
+        dest = dev_id_ie.decoded['dest_dev_id']
+        return DeviceIdentities.DEV_IDS.inverse[dest] & 0x0f
+
+    def _channel_for(self, dev_id_ie: DeviceIdentities):
+        """Resolve the ProactChannel addressed by a command dev id, or None"""
+        return self.channels.channels.get(self._channel_nr_from_dev_ids(dev_id_ie), None)
+
+    @staticmethod
+    def _channel_status(chan_nr: int, established: bool = True) -> str:
+        """TS 102 223 Section 8.56 channel status value for the
+        default/network bearer:
+        - byte 3 low 3 bits = channel id
+        - bit 8 = link established
+        - byte 4 = 00 no further info"""
+        b3 = (0x80 if established else 0x00) | (chan_nr & 0x07)
+        return '%02x00' % b3
+
+    def _bip_response_head(self, pcmd: ProactiveCommand,
+                           general_result: str = 'performed_successfully',
+                           additional_information: str = ''):
+        """CommandDetails / DeviceIdentities / Result head part of a BIP TERMINAL
+        RESPONSE. Built on prepare_response() but with two changes:
+
+        - Device identities forced source=terminal, dest=UICC.
+          TS 102 223 6.8.2 mandates for every TERMINAL RESPONSE
+          prepare_response() inverts the commands device id, which is
+          right for a uicc->terminal command but would yield a wrong
+          channel_N->UICC for the channel addressed BIP commands.
+
+        - Result is recreated for non success cases. prepare_response()
+          hard codes empty "additional information", but for enum results
+          like BIP error -> AddlInfoBip the empty value cannot be encoded at
+          all, so we always ask prepare_response() for a success Result
+          and swap for a properly encoded one here."""
+        head = self.prepare_response(pcmd, 'performed_successfully')
+        for i, ie in enumerate(head):
+            if isinstance(ie, DeviceIdentities):
+                head[i] = DeviceIdentities(decoded={'source_dev_id': 'terminal',
+                                                    'dest_dev_id': 'uicc'})
+            elif isinstance(ie, Result) and general_result != 'performed_successfully':
+                res = Result()
+                res.from_dict({'result': {'general_result': general_result,
+                                          'additional_information': additional_information}})
+                head[i] = res
+        return head
+
+    def _build_data_available_envelope(self, chan: ProactChannel) -> bytes:
+        """TS 102 223 7.5.10.2 ENVELOPE EVENT DOWNLOAD
+        Event list, Device id terminal->UICC, Channel status,
+        Channel data length (bytes available or FF for > 255)."""
+        avail = min(chan.available_rx(), 0xff)
+        ed = EventDownload(children=[
+            EventList(decoded=['data_available']),
+            DeviceIdentities(decoded={'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}),
+            ChannelStatus(decoded=self._channel_status(chan.chan_nr)),
+            ChannelDataLength(decoded=avail),
+        ])
+        return ed.to_tlv()
+
+    def _on_channel_data_available(self, chan: ProactChannel):
+        """rx reader thread hook: socket data arrived while the channel buffer
+        was empty. card uses ENVELOPE EVENT DOWNLOAD + responds by FETCHing RECEIVE DATA
+        proactive command. Card free only builds and logs"""
+        envelope_hex = b2h(self._build_data_available_envelope(chan))
+        logger.info("channel %u: %u byte(s) available -> ENVELOPE(Data available) %s",
+                    chan.chan_nr, chan.available_rx(), envelope_hex)
+        if self.data_available_sink:
+            self.data_available_sink(envelope_hex)
+
+    # handle_*: called by the transport with the decoded proactive command, the returned IE
+    # list becomes the TERMINAL RESPONSE.
+    def handle_SendShortMessage(self, pcmd: ProactiveCommand):
+        # {'smspp_download': [{'device_identities': {'source_dev_id': 'network',
+        #                                            'dest_dev_id': 'uicc'}},
+        #                     {'address': {'ton_npi': {'ext': True,
+        #                                              'type_of_number': 'international',
+        #                                              'numbering_plan_id': 'isdn_e164'},
+        #                                  'call_number': '79'}},
+        #                     {'sms_tpdu': {'tpdu': '40048111227ff6407070611535004d02700000481516011212000001fe4c0943aea42e45021c078ae06c66afc09303608874b72f58bacadb0dcf665c29349c799fbb522e61709c9baf1890015e8e8e196e36153106c8b92f95153774'}}
+        #                    ]}
+        """SEND SHORT MESSAGE: hand the MO-SMS to sms_sink, answer with success so the card
+        continues with the next part of a multi part response."""
+        logger.info("SendShortMessage")
+        logger.info(pcmd)
+        # Relevant parts in pcmd: Address, SMS_TPDU
+        addr_ie = Proact._find_first_element_of_type(pcmd.children, Address)
+        sms_tpdu_ie = Proact._find_first_element_of_type(pcmd.children, SMS_TPDU)
+        raw_tpdu = sms_tpdu_ie.decoded['tpdu']
+        submit = SMS_SUBMIT.from_bytes(raw_tpdu)
+        submit.tp_da = AddressField(addr_ie.decoded['call_number'], addr_ie.decoded['ton_npi']['type_of_number'],
+                                    addr_ie.decoded['ton_npi']['numbering_plan_id'])
+        logger.info(submit)
+        self.send_sms_via_smpp(submit)
+        # Return a successful TERMINAL RESPONSE.
+        # This is important:
+        # - without it the transport cannot complete the proactive command
+        # - for a multi part OTA response, the card would never be asked to give us
+        # the remaining SMS chunks.
+        # 'pcmd' is a decoded SendShortMessage IE, which contains CommandDetails and
+        # DeviceIdentities that prepare_response() echoes/inverts.
+        return self.prepare_response(pcmd)
+
+    def handle_OpenChannel(self, pcmd: ProactiveCommand):
+        """OPEN CHANNEL: connect a TCP socket to the given address and port, allocate a
+        channel number and report it in the Channel status of the response."""
+        # {'open_channel': [{'command_details': {'command_number': 1,
+        #                                        'type_of_command': 'open_channel',
+        #                                        'command_qualifier': 3}},
+        #                   {'device_identities': {'source_dev_id': 'uicc',
+        #                                          'dest_dev_id': 'terminal'}},
+        #                   {'bearer_description': {'bearer_type': 'default',
+        #                                           'bearer_parameters': ''}},
+        #                   {'buffer_size': 1024},
+        #                   {'uicc_transport_level': {'protocol_type': 'tcp_uicc_client_remote',
+        #                                             'port_number': 32768}},
+        #                   {'other_address': {'type_of_address': 'ipv4',
+        #                                      'address': '01020304'}}
+        #                  ]}
+        logger.info("OpenChannel")
+        logger.info(pcmd)
+        transp_lvl_ie = Proact._find_first_element_of_type(pcmd.children, UiccTransportLevel)
+        other_addr_ie = Proact._find_first_element_of_type(pcmd.children, OtherAddress)
+        bearer_desc_ie = Proact._find_first_element_of_type(pcmd.children, BearerDescription)
+        buffer_size_ie = Proact._find_first_element_of_type(pcmd.children, BufferSize)
+
+        def refuse(additional_information: str, chan_nr: int = 0):
+            """TERMINAL RESPONSE refusing the OPEN CHANNEL
+
+            - always a BIP error, only the cause byte of TS 102 223 8.12.11 differs
+            - chan_nr 0 -> "no channel available" in the Channel status, 8.56
+            - 6.8.18, 6.8.20, 6.8.21 want chan status, Bearer desc and buf size
+              in a successful or unsuccessful response
+            """
+            ies = [ChannelStatus(decoded=self._channel_status(chan_nr, established=False))]
+            ies += [ie for ie in (bearer_desc_ie, buffer_size_ie) if ie is not None]
+            return self._bip_response_head(pcmd, 'bearer_independent_protocol_error',
+                                           additional_information) + ies
+
+        # UICC/terminal interface transport level is Optional, TS 102 223 6.6.27.x. Absent means
+        # the CAT application runs its own network and transport layer, which we do not do.
+        if transp_lvl_ie is None or transp_lvl_ie.decoded['protocol_type'] != 'tcp_uicc_client_remote':
+            logger.warning("OpenChannel: unsupported UICC/terminal interface transport level (%s) "
+                           "-> refusing", transp_lvl_ie.decoded if transp_lvl_ie else '(absent)')
+            return refuse('requested_uicc_if_transp_level_not_available')
+        if other_addr_ie is None or other_addr_ie.decoded.get('type_of_address', None) != 'ipv4':
+            # No cause byte fits a wrong address family. '06' is about the transport level data
+            # object, and 8.12.11 leaves '14' ("IPv4 only allowed") reserved by 3GPP, so '00'.
+            logger.warning("OpenChannel: unsupported data destination address (%s) -> refusing",
+                           other_addr_ie.decoded if other_addr_ie else '(absent)')
+            return refuse('no_specific_cause')
+        addr_bytes = h2b(other_addr_ie.decoded['address']) if isinstance(
+                other_addr_ie.decoded['address'], str) else other_addr_ie.decoded['address']
+        ipv4_str = '%u.%u.%u.%u' % (addr_bytes[0], addr_bytes[1], addr_bytes[2], addr_bytes[3])
+        port_nr = transp_lvl_ie.decoded['port_number']
+        logger.info("OpenChannel: connecting to %s:%u", ipv4_str, port_nr)
+        try:
+            channel = self.channels.channel_create()
+        except ValueError:
+            # TS 102 223 6.4.27.2 and 6.4.27.3: no channel left -> BIP error
+            logger.warning("OpenChannel: all %u channels are in use -> refusing",
+                           len(self.channels.channels))
+            return refuse('no_channel_availabile')
+        # yes, blocking connect()
+        try:
+            channel.connect(ipv4_str, port_nr)
+        except OSError as e:
+            logger.warning("OpenChannel: connect to %s:%u failed: %s", ipv4_str, port_nr, e)
+            self.channels.channel_delete(channel.chan_nr)
+            # TS 102 223 6.4.30 is the only clause naming a cause for a link that could not be
+            # established: BIP error, channel closed. 6.4.27.4 lists no error cases at all.
+            return refuse('channel_closed', channel.chan_nr)
+
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'open_channel',
+        #                       'command_qualifier': 3}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_status': '8100'},
+        #  {'bearer_description': {'bearer_type': 'default', 'bearer_parameters': ''}},
+        #  {'buffer_size': 1024}
+        # ]
+        return self._bip_response_head(pcmd) + [
+                ChannelStatus(decoded=self._channel_status(channel.chan_nr)),
+                bearer_desc_ie, buffer_size_ie]
+
+    def handle_CloseChannel(self, pcmd: ProactiveCommand):
+        """CLOSE CHANNEL: close the socket of the addressed channel and free its number."""
+        logger.info("CloseChannel")
+        logger.info(pcmd)
+        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
+        chan = self._channel_for(dev_id_ie)
+        if chan is None:
+            # channel closed / invalid
+            return self._bip_response_head(pcmd, 'bearer_independent_protocol_error',
+                                           'channel_id_not_valid')
+        chan.close()
+        return self._bip_response_head(pcmd)
+
+    def handle_ReceiveData(self, pcmd: ProactiveCommand):
+        """RECEIVE DATA: the card fetches up to Channel data length bytes from the Rx buffer
+        of the addressed channel, the response also carries how many bytes remain."""
+        # {'receive_data': [{'command_details': {'command_number': 1,
+        #                                        'type_of_command': 'receive_data',
+        #                                        'command_qualifier': 0}},
+        #                   {'device_identities': {'source_dev_id': 'uicc',
+        #                                          'dest_dev_id': 'channel_1'}},
+        #                   {'channel_data_length': 9}
+        #                  ]}
+        logger.info("ReceiveData")
+        logger.info(pcmd)
+        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
+        req_len_ie = Proact._find_first_element_of_type(pcmd.children, ChannelDataLength)
+        chan = self._channel_for(dev_id_ie)
+        if chan is None:
+            return self._bip_response_head(pcmd, 'bearer_independent_protocol_error',
+                                           'channel_id_not_valid')
+        # TS 102 223 8.54: RECEIVE DATA contains the requested count the card wants
+        requested = req_len_ie.decoded if req_len_ie is not None else chan.available_rx()
+        data, remaining = chan.take_rx(requested)
+        # TS 102 223 6.4.29:
+        # - return data available in the Rx buffer + num bytes still remaining (FF if > 255)
+        # - if fewer than requested available terminal must NOT wait, report and returns what we have
+        general_result = 'performed_successfully'
+        if len(data) < requested:
+            general_result = 'performed_with_missing_information'
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'receive_data',
+        #                       'command_qualifier': 0}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_data': '16030100040e000000'},
+        #  {'channel_data_length': 0}
+        # ]
+        return self._bip_response_head(pcmd, general_result) + [
+                ChannelData(decoded=b2h(data)),
+                ChannelDataLength(decoded=min(remaining, 0xff))]
+
+    def handle_SendData(self, pcmd: ProactiveCommand):
+        """SEND DATA: write the Channel data of the command to the socket of the addressed
+        channel."""
+        # {'send_data': [{'command_details': {'command_number': 1,
+        #                                     'type_of_command': 'send_data',
+        #                                     'command_qualifier': 1}},
+        #                {'device_identities': {'source_dev_id': 'uicc',
+        #                                       'dest_dev_id': 'channel_1'}},
+        #                {'channel_data': '160301003c010000380303d0f45e12b52ce5bb522750dd037738195334c87a46a847fe2b6886cada9ea6bf00000a00ae008c008b00b0002c010000050001000101'}
+        #               ]}
+        logger.info("SendData")
+        logger.info(pcmd)
+        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
+        chan_data_ie = Proact._find_first_element_of_type(pcmd.children, ChannelData)
+        chan = self._channel_for(dev_id_ie)
+        if chan is None:
+            return self._bip_response_head(pcmd, 'bearer_independent_protocol_error',
+                                           'channel_id_not_valid')
+        # lets accept hexstrings as well
+        payload = chan_data_ie.decoded
+        if isinstance(payload, str):
+            payload = h2b(payload)
+        # command_qualifier bit 1 selects 'send immediately' / Tx-buffer store and forward
+        # For TCP stream all we have is a socket and TCP takes care of segmentation,
+        # so just send.
+        chan.send(payload)
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'send_data',
+        #                       'command_qualifier': 1}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_data_length': 255}
+        # ]
+        # TS 102 223 6.4.30 / 8.54 Channel data length = free space tx buf; FF == > 255 available
+        return self._bip_response_head(pcmd) + [ChannelDataLength(decoded=255)]
+
+    def handle_SetUpEventList(self, pcmd: ProactiveCommand):
+        """SET UP EVENT LIST: acknowledged, data available and channel status are always on."""
+        # {'set_up_event_list': [{'command_details': {'command_number': 1,
+        #                                             'type_of_command': 'set_up_event_list',
+        #                                             'command_qualifier': 0}},
+        #                        {'device_identities': {'source_dev_id': 'uicc',
+        #                                               'dest_dev_id': 'terminal'}},
+        #                        {'event_list': ['data_available', 'channel_status']}
+        #                       ]}
+        logger.info("SetUpEventList")
+        logger.info(pcmd)
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'set_up_event_list',
+        #                       'command_qualifier': 0}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}}
+        # ]
+        return self.prepare_response(pcmd)
+
+    def getChannelStatus(self, pcmd: ProactiveCommand):
+        logger.info("GetChannelStatus")
+        logger.info(pcmd)
+        return self.prepare_response(pcmd) + []
+
+    def send_sms_via_smpp(self, submit: SMS_SUBMIT):
+        # while in a normal network the phone/ME would *submit* a message to the SMSC,
+        # we are actually emulating the SMSC itself, so we must *deliver* the message
+        # to the ESME
+        deliver = SMS_DELIVER.from_submit(submit)
+        deliver_smpp = deliver.to_smpp()
+
+        if self.sms_sink is None:
+            logger.info('no sms_sink: dropping MO-SMS %s', deliver_smpp)
+            return
+        self.sms_sink(deliver_smpp)
+#       # obtain the connection/binding of system_id to be used for delivering MO-SMS to the ESME
+#       connection = smpp_server.getBoundConnections[system_id].getNextBindingForDelivery()
+#       connection.sendDataRequest(deliver_smpp)
+
+
+

pySim/card_key_provider.py

@@ -33,10 +33,12 @@ from Cryptodome.Cipher import AES
 from osmocom.utils import h2b, b2h
 from pySim.log import PySimLogger
 
+import os
 import abc
 import csv
 import logging
 import yaml
+import argparse
 
 log = PySimLogger.get(__name__)
 
@@ -130,6 +132,31 @@ class CardKeyFieldCryptor:
         cipher = AES.new(h2b(self.transport_keys[field_name.upper()]), AES.MODE_CBC, self.__IV)
         return b2h(cipher.encrypt(h2b(plaintext_val)))
 
+    @staticmethod
+    def argparse_add_args(arg_parser: argparse.ArgumentParser):
+        arg_parser.add_argument('--column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                                help='per-column AES transport key', dest='column_key')
+        # Depprecated argument, replaced by --column-key (see above)
+        arg_parser.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                                help=argparse.SUPPRESS, dest='column_key')
+
+    @staticmethod
+    def transport_keys_from_opts(opts: argparse.Namespace) -> dict:
+        """
+        Transport keys are passed via the commandline using the '--column-key' option. Each column requires a
+        dedicated transport key. This method can be used to extract the column keys parameters from the commandline
+        options into a dict that can be directly passed to the construtor with the transport_keys argument.
+
+        Args:
+                opts: parsed commandline options (Namespace)
+        """
+
+        transport_keys = {}
+        for par in opts.column_key:
+            name, key = par.split(':')
+            transport_keys[name] = key
+        return transport_keys
+
 class CardKeyProvider(abc.ABC):
     """Base class, not containing any concrete implementation."""
 
@@ -148,24 +175,33 @@ class CardKeyProvider(abc.ABC):
                 fond None shall be returned.
         """
 
+    @staticmethod
+    def argparse_add_args(arg_parser: argparse.ArgumentParser):
+        """
+        Add the commandline arguments relevant for this card key provider.
+
+        Args:
+                arg_parser : argument parser group
+        """
+
     def __str__(self):
         return type(self).__name__
 
 class CardKeyProviderCsv(CardKeyProvider):
     """Card key provider implementation that allows to query against a specified CSV file."""
 
-    def __init__(self, csv_filename: str, transport_keys: dict):
+    def __init__(self, csv_filename: str, field_cryptor: CardKeyFieldCryptor):
         """
         Args:
                 csv_filename : file name (path) of CSV file containing card-individual key/data
-                transport_keys : (see class CardKeyFieldCryptor)
+                field_cryptor : (see class CardKeyFieldCryptor)
         """
         log.info("Using CSV file as card key data source: %s" % csv_filename)
         self.csv_file = open(csv_filename, 'r')
         if not self.csv_file:
             raise RuntimeError("Could not open CSV file '%s'" % csv_filename)
         self.csv_filename = csv_filename
-        self.crypt = CardKeyFieldCryptor(transport_keys)
+        self.crypt = field_cryptor
 
     def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         self.csv_file.seek(0)
@@ -188,14 +224,20 @@ class CardKeyProviderCsv(CardKeyProvider):
             return None
         return return_dict
 
+    @staticmethod
+    def argparse_add_args(arg_parser: argparse.ArgumentParser):
+        arg_parser.add_argument('--csv', metavar='FILE',
+                                default="~/.osmocom/pysim/card_data.csv",
+                                help='Read card data from CSV file')
+
 class CardKeyProviderPgsql(CardKeyProvider):
     """Card key provider implementation that allows to query against a specified PostgreSQL database table."""
 
-    def __init__(self, config_filename: str, transport_keys: dict):
+    def __init__(self, config_filename: str, field_cryptor: CardKeyFieldCryptor):
         """
         Args:
                 config_filename : file name (path) of CSV file containing card-individual key/data
-                transport_keys : (see class CardKeyFieldCryptor)
+                field_cryptor : (see class CardKeyFieldCryptor)
         """
         import psycopg2
         log.info("Using SQL database as card key data source: %s" % config_filename)
@@ -212,7 +254,7 @@ class CardKeyProviderPgsql(CardKeyProvider):
                                          host=config.get('host'))
             self.tables = config.get('table_names')
             log.info("Card key database tables: %s" % str(self.tables))
-            self.crypt = CardKeyFieldCryptor(transport_keys)
+            self.crypt = field_cryptor
 
     def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         import psycopg2
@@ -252,6 +294,11 @@ class CardKeyProviderPgsql(CardKeyProvider):
             result[k] = self.crypt.decrypt_field(k, result.get(k))
         return result
 
+    @staticmethod
+    def argparse_add_args(arg_parser: argparse.ArgumentParser):
+        arg_parser.add_argument('--pgsql', metavar='FILE',
+                                default="~/.osmocom/pysim/card_data_pgsql.cfg",
+                                help='Read card data from PostgreSQL database (config file)')
 
 def card_key_provider_register(provider: CardKeyProvider, provider_list=card_key_providers):
     """Register a new card key provider.
@@ -305,3 +352,19 @@ def card_key_provider_get_field(field: str, key: str, value: str, provider_list=
     fields = [field]
     result = card_key_provider_get(fields, key, value, card_key_providers)
     return result.get(field.upper())
+
+def card_key_provider_argparse_add_args(arg_parser: argparse.ArgumentParser):
+    """Add card key provider commandline options to the given argument parser"""
+    card_key_group = arg_parser.add_argument_group('Card Key Provider Options')
+    CardKeyProviderCsv.argparse_add_args(card_key_group)
+    CardKeyProviderPgsql.argparse_add_args(card_key_group)
+    CardKeyFieldCryptor.argparse_add_args(card_key_group)
+
+def card_key_provider_init(opts: argparse.Namespace):
+    """Initialize card key provider depending on the user provided commandline options"""
+    transport_keys = CardKeyFieldCryptor.transport_keys_from_opts(opts)
+    card_key_field_cryptor = CardKeyFieldCryptor(transport_keys)
+    if os.path.isfile(os.path.expanduser(opts.csv)):
+        card_key_provider_register(CardKeyProviderCsv(os.path.expanduser(opts.csv), card_key_field_cryptor))
+    if os.path.isfile(os.path.expanduser(opts.pgsql)):
+        card_key_provider_register(CardKeyProviderPgsql(os.path.expanduser(opts.pgsql), card_key_field_cryptor))

pySim/cat.py

@@ -22,7 +22,7 @@ from typing import List
 from bidict import bidict
 from construct import Int8ub, Int16ub, Byte, BitsInteger
 from construct import Struct, Enum, BitStruct, this
-from construct import Switch, GreedyRange, FlagsEnum
+from construct import Switch, GreedyRange, FlagsEnum, Adapter
 from osmocom.tlv import TLV_IE, COMPR_TLV_IE, BER_TLV_IE, TLV_IE_Collection
 from osmocom.construct import PlmnAdapter, BcdAdapter, GsmStringAdapter, TonNpi, GsmString, Bytes, GreedyBytes
 from osmocom.utils import b2h, h2b
@@ -318,11 +318,58 @@ class FileList(COMPR_TLV_IE, tag=0x92):
 
 # TS 102 223 Section 8.19
 class LocationInformation(COMPR_TLV_IE, tag=0x93):
-    pass
+    # 8.19: coding is per access technology, and the lengths differ (TS 131.111 8.19.1-.4: GERAN 7,
+    # UTRAN/E-UTRAN 9, NG-RAN 11) with nothing in the IE to say which -> keep the value opaque.
+    _construct = GreedyBytes
 
-# TS 102 223 Section 8.20
+class MobileIdentityAdapter(Adapter):
+    """TS 124.008 section 10.5.1.4 figure 10.5.4 + table 10.5.4
+
+    NOT a plain BCD string:
+    - bits 1-3 type of identity + odd/even bit 4
+    - digit 1 in bits 5-8, following octets contain 2 digits, low nibble first
+    - if even length: high nibble of last octet 1111
+    So IMEI IE of 8 bytes is 15 digits + framing nibble."""
+
+    # Table 10.5.4 bits 321
+    TYPE_IMSI = 1
+    TYPE_IMEI = 2
+    TYPE_IMEISV = 3
+
+    def __init__(self, subcon, type_of_identity: int):
+        super().__init__(subcon)
+        self.type_of_identity = type_of_identity
+
+    def _decode(self, obj, context, path):
+        data = bytes(obj)
+        if not data:
+            return ''
+        # TS 24.008 figure 10.5.4: octet 3 holds type of identity (b1-3), odd/even (b4) and
+        # digit 1 in its high nibble, the remaining digits follow BCD swapped from octet 4
+        odd = bool(data[0] & 0x08)          # bit 4: 1 = odd number of digits
+        digits = '%x' % (data[0] >> 4)      # bits 5-8: digit 1
+        for octet in data[1:]:
+            digits += '%x%x' % (octet & 0x0f, octet >> 4)
+        if not odd:
+            digits = digits[:-1]            # drop the 1111 end mark
+        return digits
+
+    def _encode(self, obj, context, path):
+        digits = str(obj)
+        odd = len(digits) % 2
+        first = (int(digits[0], 16) << 4) | (0x08 if odd else 0x00) | self.type_of_identity
+        rest = digits[1:] if odd else digits[1:] + 'f'
+        return bytes([first]) + bytes((int(rest[i+1], 16) << 4) | int(rest[i], 16)
+                                      for i in range(0, len(rest), 2))
+
+# TS 102 223 Section 8.20, len is fixed at 8: "The IMEI is coded [..] as the
+# value part of the Mobile Identity IE as specified in TS 124 008", and the
+# IMEI itself is the 15 digits of TS 123 003.
 class IMEI(COMPR_TLV_IE, tag=0x94):
-    _construct = BcdAdapter(GreedyBytes)
+    _test_de_encode = [
+        ( '94081a32547698103254', '123456789012345' ),
+    ]
+    _construct = MobileIdentityAdapter(GreedyBytes, MobileIdentityAdapter.TYPE_IMEI)
 
 # TS 102 223 Section 8.21
 class HelpRequest(COMPR_TLV_IE, tag=0x95):
@@ -536,9 +583,9 @@ class Aid(COMPR_TLV_IE, tag=0xAF):
 
 # TS 102 223 Section 8.61
 class AccessTechnology(COMPR_TLV_IE, tag=0xBF):
-    SingleAccessTech = Enum(Int8ub, gsm=0, tia_eia_533=1, tia_eia_136_270=2, utran=3, tetra=4,
-                                    tia_eia_95_b=5, cdma1000_1x=6, cdma2000_hrpd=7, eutran=8,
-                                    ehrpd=9, nr=0x0a)
+    SingleAccessTech = Enum(Int8ub, gsm=0, tia_eia_553=1, tia_eia_136_270=2, utran=3, tetra=4,
+                                    tia_eia_95_b=5, cdma2000_1x=6, cdma2000_hrpd=7, eutran=8,
+                                    ehrpd=9, nr=0x0a, satellite_nr=0x0b, satellite_eutran=0x0c)
     _construct = GreedyRange(SingleAccessTech)
 
 # TS 102 223 Section 8.63
@@ -596,6 +643,14 @@ class UtranEutranMeasurementQualifier(COMPR_TLV_IE, tag=0xE9):
                                 eutran_inter_rat_utran=0x08,
                                 eutran_inter_rat_nr=0x09)
 
+# TS 102 223 Section 8.74, length is not fixed, because IMEISV is 16 digits per TS 123.003
+# -> even count needs the '1111' end mark and is 9 bytes long
+class IMEISV(COMPR_TLV_IE, tag=0xE2):
+    _test_de_encode = [
+        ( 'e2091332547698103254f6', '1234567890123456' ),
+    ]
+    _construct = MobileIdentityAdapter(GreedyBytes, MobileIdentityAdapter.TYPE_IMEISV)
+
 # TS 102 223 Section 8.75
 class NetworkSearchMode(COMPR_TLV_IE, tag=0xE5):
     _construct = Enum(Int8ub, manual=0, automatic=1)
@@ -729,8 +784,12 @@ class DnsServerAddress(COMPR_TLV_IE, tag=0xC0):
 
 # TS 102 223 Section 8.105
 class SupportedRadioAccessTechnologies(COMPR_TLV_IE, tag=0xB4):
+    # 2 bytes/entry:
+    # - technology of 8.61
+    # - state byte b1 is 0 disabled/1 enabled
+    # - b2-b8 RFU.
     AccessTechTuple = Struct('technology'/AccessTechnology.SingleAccessTech,
-                             'state'/FlagsEnum(Int8ub, enabled=0))
+                             'state'/FlagsEnum(Int8ub, enabled=1))
     _construct = GreedyRange(AccessTechTuple)
 
 # TS 102 223 Section 8.107
@@ -763,6 +822,22 @@ class SMSPPDownload(BER_TLV_IE, tag=0xD1,
                     nested=[DeviceIdentities, Address, SMS_TPDU]):
     pass
 
+
+def sms_pp_download_envelope(tpdu, call_number: str = '0123456') -> SMSPPDownload:
+    """TS 31.111 Section 7.1.1.2 wrap of a SMS-DELIVER TPDU in the ENVELOPE (SMS-PP Download)
+     call_number :
+     SMSC address to report, defined in TS 31.111 7.1.1.2 as
+    "the RP_Originating_Address of the Service Centre (TS-Service-Centre-Address, 3GPP TS 24.011)"
+    its presence is Conditional, and the note there says the UICC should be fine
+    if its missing, so for remote management its presence should suffice (?).
+    """
+    return SMSPPDownload(children=[
+        DeviceIdentities(decoded={'source_dev_id': 'network', 'dest_dev_id': 'uicc'}),
+        Address(decoded={'ton_npi': {'ext': False, 'type_of_number': 'unknown',
+                                     'numbering_plan_id': 'unknown'},
+                         'call_number': call_number}),
+        SMS_TPDU(decoded={'tpdu': b2h(tpdu.to_bytes())})])
+
 # TS 101 220 Table 7.17 + 31.111 7.1.1.3
 class SMSCBDownload(BER_TLV_IE, tag=0xD2,
                     nested=[DeviceIdentities, CBSPage]):

pySim/cdma_ruim.py

@@ -131,7 +131,7 @@ class EF_AD(TransparentEF):
                  desc='Administrative Data', size=(3, None), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct(
-            # Byte 1: Display Condition
+            # Byte 1: MS operation mode
             'ms_operation_mode'/Enum(Byte, self.OP_MODE),
             # Bytes 2-3: Additional information
             'additional_info'/Bytes(2),

pySim/esim/http_json_api.py

@@ -19,7 +19,7 @@ import abc
 import requests
 import logging
 import json
-from typing import Optional
+from typing import Optional, Tuple
 import base64
 from twisted.web.server import Request
 
@@ -180,7 +180,7 @@ class JsonHttpApiFunction(abc.ABC):
     # receives from the a requesting client. The same applies vice versa to class variables that have an "output_"
     # prefix.
 
-    # path of the API function (e.g. '/gsma/rsp2/es2plus/confirmOrder')
+    # path of the API function (e.g. '/gsma/rsp2/es2plus/confirmOrder', see also method rewrite_url).
     path = None
 
     # dictionary of input parameters. key is parameter name, value is ApiParam class
@@ -336,6 +336,22 @@ class JsonHttpApiFunction(abc.ABC):
                 output[p] = p_class.decode(v)
         return output
 
+    def rewrite_url(self, data: dict, url: str) -> Tuple[dict, str]:
+        """
+        Rewrite a static URL using information passed in the data dict. This method may be overloaded by a derived
+        class to allow fully dynamic URLs. The input parameters required for the URL rewriting may be passed using
+        data parameter. In case those parameters are additional parameters that are not intended to be passed to
+        the encode_client method later, they must be removed explcitly.
+
+        Args:
+                data: (see JsonHttpApiClient and JsonHttpApiServer)
+                url: statically generated URL string (see comment in JsonHttpApiClient)
+        """
+
+        # This implementation is a placeholder in which we do not perform any URL rewriting. We just pass through data
+        # and url unmodified.
+        return data, url
+
 class JsonHttpApiClient():
     def __init__(self, api_func: JsonHttpApiFunction, url_prefix: str, func_req_id: Optional[str],
                  session: requests.Session):
@@ -352,8 +368,16 @@ class JsonHttpApiClient():
         self.session = session
 
     def call(self, data: dict, func_call_id: Optional[str] = None, timeout=10) -> Optional[dict]:
-        """Make an API call to the HTTP API endpoint represented by this object. Input data is passed in `data` as
-        json-serializable dict. Output data is returned as json-deserialized dict."""
+        """
+        Make an API call to the HTTP API endpoint represented by this object. Input data is passed in `data` as
+        json-serializable fields. `data` may also contain additional parameters required for URL rewriting (see
+        rewrite_url in class JsonHttpApiFunction). Output data is returned as json-deserialized dict.
+
+        Args:
+                data: Input data required to perform the request.
+                func_call_id: Function Call Identifier, if present a header field is generated automatically.
+                timeout: Maximum amount of time to wait for the request to complete.
+        """
 
         # In case a function caller ID is supplied, use it together with the stored function requestor ID to generate
         # and prepend the header field according to SGP.22, section 6.5.1.1 and 6.5.1.3. (the presence of the header
@@ -362,6 +386,11 @@ class JsonHttpApiClient():
             data = {'header' : {'functionRequesterIdentifier': self.func_req_id,
                                 'functionCallIdentifier': func_call_id}} | data
 
+        # The URL used for the HTTP request (see below) normally consists of the initially given url_prefix
+        # concatenated with the path defined by the JsonHttpApiFunction definition. This static URL path may be
+        # rewritten by rewrite_url method defined in the JsonHttpApiFunction.
+        data, url = self.api_func.rewrite_url(data, self.url_prefix + self.api_func.path)
+
         # Encode the message (the presence of mandatory fields is checked during encoding)
         encoded = json.dumps(self.api_func.encode_client(data))
 
@@ -373,7 +402,6 @@ class JsonHttpApiClient():
         req_headers.update(self.api_func.extra_http_req_headers)
 
         # Perform HTTP request
-        url = self.url_prefix + self.api_func.path
         logger.debug("HTTP REQ %s - hdr: %s '%s'" % (url, req_headers, encoded))
         response = self.session.request(self.api_func.http_method, url, data=encoded, headers=req_headers, timeout=timeout)
         logger.debug("HTTP RSP-STS: [%u] hdr: %s" % (response.status_code, response.headers))

pySim/esim/saip/__init__.py

@@ -34,7 +34,7 @@ from pySim import ts_102_222
 from pySim.utils import dec_imsi
 from pySim.ts_102_221 import FileDescriptor
 from pySim.filesystem import CardADF, Path
-from pySim.ts_31_102 import ADF_USIM
+from pySim.ts_31_102 import ADF_USIM, EF_UST, EF_SUCI_Calc_Info
 from pySim.ts_31_103 import ADF_ISIM
 from pySim.esim import compile_asn1_subdir
 from pySim.esim.saip import templates
@@ -441,7 +441,7 @@ class File:
             elif k == 'fillFileContent':
                 stream.write(v)
             else:
-                return ValueError("Unknown key '%s' in tuple list" % k)
+                raise ValueError("Unknown key '%s' in tuple list" % k)
         return stream.getvalue()
 
     def file_content_to_tuples(self, optimize:bool = False) -> List[Tuple]:
@@ -1079,6 +1079,13 @@ class SecurityDomainKey:
                 'keyVersionNumber': bytes([self.key_version_number]),
                 'keyComponents': [k.to_saip_dict() for k in self.key_components]}
 
+    def get_key_component(self, key_type):
+        for kc in self.key_components:
+            if kc.key_type == key_type:
+                return kc.key_data
+        return None
+
+
 class ProfileElementSD(ProfileElement):
     """Class representing a securityDomain ProfileElement."""
     type = 'securityDomain'
@@ -1719,7 +1726,52 @@ class ProfileElementSequence:
         if 'BT' in ftype_list:
             svc_set.add('ber-tlv')
         # FIXME:dfLinked files (scan all files, check for non-empty Fcp.linkPath presence of DFs)
-        # TODO: 5G related bits (derive from EF.UST or file presence?)
+
+        # 5G:
+        # - When SUCI is:
+        #   - enabled (EF.UST 124 = true)
+        #   AND
+        #   - calculated in the USIM (EF.UST 125 = true),
+        #   then eUICC-Mandatory-services needs 'get-identity'.
+        # - 'get-identity' implies that the eUICC must support ONE OF profile-A OR profile-B.
+        #   (One might assume from this that, when SUCI-CalcInfo for USIM in DF.SAIP contains both key types, then no
+        #   profile-A or B services need to be requested explicitly. However, the correct logic is:)
+        # - Iff the SUCI-CalcInfo for USIM (DF.SAIP) contains a key of profile-A ("identifier": 1),
+        #   then eUICC-Mandatory-services needs 'profile-a-x25519'.
+        # - Same: profile-B ("identifier": 2) needs 'profile-b-p256'.
+        # - (When SUCI is calculated in the UE, then the eUICC does not need to provide any of these services.)
+        suci_in_usim_enabled = False
+        try:
+            f_ust = self.get_pe_for_type("usim").files["ef-ust"]
+            ust = EF_UST().decode_bin(f_ust.body)
+            suci_in_usim_enabled = ust[124]['activated'] and ust[125]['activated']
+        except (KeyError, AttributeError):
+            pass
+        if suci_in_usim_enabled:
+            svc_set.add('get-identity')
+            # now check for profile-a and profile-b presence
+            suci_calcinfo_has_profile_a = False
+            suci_calcinfo_has_profile_b = False
+            try:
+                f_sucici = self.get_pe_for_type("df-saip").files["ef-suci-calc-info-usim"]
+                sucici = EF_SUCI_Calc_Info().decode_bin(f_sucici.body) or {}
+                for prot_scheme in sucici['prot_scheme_id_list']:
+                    if not isinstance(prot_scheme, dict):
+                        continue
+                    ps_id = prot_scheme["identifier"]
+                    if ps_id == 1:
+                        suci_calcinfo_has_profile_a = True
+                    elif ps_id == 2:
+                        suci_calcinfo_has_profile_b = True
+            except (KeyError, AttributeError):
+                pass
+            if suci_calcinfo_has_profile_a:
+                # The profile has a profile-A key, so require that
+                svc_set.add('profile-a-x25519')
+            if suci_calcinfo_has_profile_b:
+                # The profile has a profile-B key, so require that
+                svc_set.add('profile-b-p256')
+
         hdr_pe = self.get_pe_for_type('header')
         # patch in the 'manual' services from the existing list:
         for old_svc in hdr_pe.decoded['eUICC-Mandatory-services'].keys():

pySim/esim/saip/batch.py

@@ -0,0 +1,362 @@
+"""Implementation of Personalization of eSIM profiles in SimAlliance/TCA Interoperable Profile:
+   Run a batch of N personalizations"""
+
+# (C) 2025-2026 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: nhofmeyr@sysmocom.de
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import copy
+import pprint
+from typing import Generator, Union
+from pySim.esim.saip.personalization import ConfigurableParameter
+from pySim.esim.saip import param_source
+from pySim.esim.saip import ProfileElementSequence, ProfileElementSD
+from pySim.global_platform import KeyUsageQualifier
+from osmocom.utils import b2h
+
+# a list of ConfigurableParameter classes and/or ConfigurableParameter class instances
+ParamList = list[Union[type[ConfigurableParameter], ConfigurableParameter]]
+
+class BatchPersonalization:
+    """Produce a series of eSIM profiles from predefined parameters.
+    Personalization parameters are derived from pysim.esim.saip.param_source.ParamSource.
+
+    Usage example:
+
+    der_input = open('some_file', 'rb').read()
+    pes = ProfileElementSequence.from_der(der_input)
+    p = BatchPersonalization(
+        n=10,
+        src_pes=pes,
+        csv_rows=get_csv_reader())
+
+    p.add_param_and_src(
+        personalization.Iccid(),
+        param_source.IncDigitSource(
+            num_digits=18,
+            first_value=123456789012340001,
+            last_value=123456789012340010))
+
+    # add more parameters here, using ConfigurableParameter and ParamSource subclass instances to define the profile
+    # ...
+
+    # generate all 10 profiles (from n=10 above)
+    for result_pes in p.generate_profiles():
+        upp = result_pes.to_der()
+        store_upp(upp)
+    """
+
+    class ParamAndSrc:
+        """tie a ConfigurableParameter to a source of actual values"""
+        def __init__(self, param: ConfigurableParameter, src: param_source.ParamSource):
+            if isinstance(param, type):
+                self.param_cls = param
+            else:
+                self.param_cls = param.__class__
+            self.src = src
+
+    def __init__(self,
+                 n: int,
+                 src_pes: ProfileElementSequence,
+                 params: list[ParamAndSrc]=None,
+                 csv_rows: Generator=None,
+                ):
+        """
+        n: number of eSIM profiles to generate.
+        src_pes: a decoded eSIM profile as ProfileElementSequence, to serve as template. This is not modified, only
+                 copied.
+        params: list of ParamAndSrc instances, defining a ConfigurableParameter and corresponding ParamSource to fill in
+                profile values.
+        csv_rows: A generator (e.g. iter(list_of_rows)) producing all CSV rows one at a time, starting with a row
+                  containing the column headers. This is compatible with the python csv.reader. Each row gets passed to
+                  ParamSource.get_next(), such that ParamSource implementations can access the row items. See
+                  param_source.CsvSource.
+        """
+        self.n = n
+        self.params = params or []
+        self.src_pes = src_pes
+        self.csv_rows = csv_rows
+
+    def add_param_and_src(self, param:ConfigurableParameter, src:param_source.ParamSource):
+        self.params.append(BatchPersonalization.ParamAndSrc(param, src))
+
+    def generate_profiles(self):
+        # get first row of CSV: column names
+        csv_columns = None
+        if self.csv_rows:
+            try:
+                csv_columns = next(self.csv_rows)
+            except StopIteration as e:
+                raise ValueError('the input CSV file appears to be empty') from e
+
+        for i in range(self.n):
+            csv_row = None
+            if self.csv_rows and csv_columns:
+                try:
+                    csv_row_list = next(self.csv_rows)
+                except StopIteration as e:
+                    raise ValueError(f'not enough rows in the input CSV for eSIM nr {i+1} of {self.n}') from e
+
+                csv_row = dict(zip(csv_columns, csv_row_list))
+
+            pes = copy.deepcopy(self.src_pes)
+
+            for p in self.params:
+                try:
+                    input_value = p.src.get_next(csv_row=csv_row)
+                    assert input_value is not None
+                    value = p.param_cls.validate_val(input_value)
+                    p.param_cls.apply_val(pes, value)
+                except Exception as e:
+                    raise ValueError(f'{p.param_cls.get_name()} fed by {p.src.name}: {e}') from e
+
+            pes.rebuild_mandatory_services()
+
+            yield pes
+
+
+class UppAudit(dict):
+    """
+    Key-value pairs collected from a single UPP DER or PES.
+
+    UppAudit itself is a dict, callers may use the standard python dict API to access key-value pairs read from the UPP.
+    """
+
+    @classmethod
+    def from_der(cls, der: bytes, params: ParamList, der_size=False, additional_sd_keys=False):
+        """return a dict of parameter name and set of selected parameter values found in a DER encoded profile. Note:
+        some ConfigurableParameter implementations return more than one key-value pair, for example, Imsi returns
+        both 'IMSI' and 'IMSI-ACC' parameters.
+
+        e.g.
+            UppAudit.from_der(my_der, [Imsi, ])
+            --> {'IMSI': {'001010000000023'}, 'IMSI-ACC': {'5'}}
+
+        (where 'IMSI' == Imsi.name)
+
+        Read all parameters listed in params. params is a list of either ConfigurableParameter classes or
+        ConfigurableParameter class instances. This calls only classmethods, so each entry in params can either be the
+        class itself, or a class-instance of, a (non-abstract) ConfigurableParameter subclass.
+        For example, params = [Imsi, ] is equivalent to params = [Imsi(), ].
+
+        For der_size=True, also include a {'der_size':12345} entry.
+
+        For additional_sd_keys=True, output also all Security Domain KVN that there are *no* ConfigurableParameter
+        subclasses for. For example, SCP80 has reserved kvn 0x01..0x0f, but we offer only Scp80Kvn01, Scp80Kvn02,
+        Scp80Kvn03. So we would not show kvn 0x04..0x0f in an audit. additional_sd_keys=True includes audits of all SD
+        key KVN there may be in the UPP. This helps to spot SD keys that may already be present in a UPP template, with
+        unexpected / unusual kvn.
+        """
+
+        # make an instance of this class
+        upp_audit = cls()
+
+        if der_size:
+            upp_audit['der_size'] = set((len(der), ))
+
+        pes = ProfileElementSequence.from_der(der)
+        for param in params:
+            try:
+                for valdict in param.get_values_from_pes(pes):
+                    upp_audit.add_values(valdict)
+            except Exception as e:
+                raise ValueError(f'Error during audit for parameter {param}: {e}') from e
+
+        if not additional_sd_keys:
+            return upp_audit
+
+        # additional_sd_keys
+        for pe in pes.pe_list:
+            if pe.type != 'securityDomain':
+                continue
+            assert isinstance(pe, ProfileElementSD)
+
+            for key in pe.keys:
+                audit_key = f'SdKey_KVN{key.key_version_number:02x}_ID{key.key_identifier:02x}'
+                kuq_bin = KeyUsageQualifier.build(key.key_usage_qualifier).hex()
+                audit_val = f'{key.key_components=!r} key_usage_qualifier=0x{kuq_bin}={key.key_usage_qualifier!r}'
+                upp_audit.add_values({audit_key: audit_val})
+
+        return upp_audit
+
+    def get_single_val(self, key, allow_absent=False, absent_val=None):
+        """
+        Return the audit's value for the given audit key (like 'IMSI' or 'IMSI-ACC').
+        Any kind of value may occur multiple times in a profile. When all of these agree to the same unambiguous value,
+        return that value. When they do not agree, raise a ValueError.
+        """
+        # key should be a string, but if someone passes a ConfigurableParameter, just use its default name
+        if ConfigurableParameter.is_super_of(key):
+            key = key.get_name()
+
+        assert isinstance(key, str)
+        v = self.get(key)
+        if v is None and allow_absent:
+            return absent_val
+        if not isinstance(v, set):
+            raise ValueError(f'audit value should be a set(), got {v!r}')
+        if len(v) != 1:
+            raise ValueError(f'expected a single value for {key}, got {v!r}')
+        v = tuple(v)[0]
+        return v
+
+    @staticmethod
+    def audit_val_to_str(v):
+        """
+        Usually, we want to see a single value in an audit. Still, to be able to collect multiple ambiguous values,
+        audit values are always python sets. Turn it into a nice string representation: only the value when it is
+        unambiguous, otherwise a list of the ambiguous values.
+        A value may also be completely absent, then return 'not present'.
+        """
+        def try_single_val(w):
+            'change single-entry sets to just the single value'
+            if isinstance(w, set):
+                if len(w) == 1:
+                    return tuple(w)[0]
+                if len(w) == 0:
+                    return None
+            return w
+
+        v = try_single_val(v)
+        if isinstance(v, bytes):
+            v = b2h(v)
+        if v is None:
+            return 'not present'
+        return str(v)
+
+    def get_val_str(self, key):
+        """Return a string of the value stored for the given key"""
+        return UppAudit.audit_val_to_str(self.get(key))
+
+    def add_values(self, src:dict):
+        """Merge a plain dict of values into self, which is a dict of sets.
+        For example from
+            self == { 'a': {123} }
+        and
+            src == { 'a': 456, 'b': 789 }
+        then after this function call:
+            self == { 'a': {123, 456}, 'b': {789} }
+        """
+        assert isinstance(src, dict)
+        for key, srcval in src.items():
+            dstvalset = self.get(key)
+            if dstvalset is None:
+                dstvalset = set()
+                self[key] = dstvalset
+            dstvalset.add(srcval)
+
+    def __str__(self):
+        return '\n'.join(f'{key}: {self.get_val_str(key)}' for key in sorted(self.keys()))
+
+class BatchAudit(list):
+    """
+    Collect UppAudit instances for a batch of UPP, for example from a personalization.BatchPersonalization.
+    Produce an output CSV.
+
+    Usage example:
+
+        ba = BatchAudit(params=(personalization.Iccid, ))
+        for upp_der in upps:
+            ba.add_audit(upp_der)
+        print(ba.summarize())
+
+        with open('output.csv', 'wb') as csv_data:
+            csv_str = io.TextIOWrapper(csv_data, 'utf-8', newline='')
+            csv.writer(csv_str).writerows( ba.to_csv_rows() )
+            csv_str.flush()
+
+    BatchAudit itself is a list, callers may use the standard python list API to access the UppAudit instances.
+    """
+
+    def __init__(self, params: ParamList):
+        assert params
+        self.params = params
+
+    def add_audit(self, upp_der:bytes):
+        audit = UppAudit.from_der(upp_der, self.params)
+        self.append(audit)
+        return audit
+
+    def summarize(self):
+        batch_audit = UppAudit()
+
+        audits = self
+
+        if len(audits) > 2:
+            val_sep = ', ..., '
+        else:
+            val_sep = ', '
+
+        first_audit = None
+        last_audit = None
+        if len(audits) >= 1:
+            first_audit = audits[0]
+        if len(audits) >= 2:
+            last_audit = audits[-1]
+
+        if first_audit:
+            if last_audit:
+                for key in first_audit.keys():
+                    first_val = first_audit.get_val_str(key)
+                    last_val = last_audit.get_val_str(key)
+
+                    if first_val == last_val:
+                        val = first_val
+                    else:
+                        val_sep_with_newline = f"{val_sep.rstrip()}\n{' ' * (len(key) + 2)}"
+                        val = val_sep_with_newline.join((first_val, last_val))
+                    batch_audit[key] = val
+            else:
+                batch_audit.update(first_audit)
+
+        return batch_audit
+
+    def to_csv_rows(self, headers=True, sort_key=None):
+        """generator that yields all audits' values as rows, useful feed to a csv.writer."""
+        columns = set()
+        for audit in self:
+            columns.update(audit.keys())
+
+        columns = tuple(sorted(columns, key=sort_key))
+
+        if headers:
+            yield columns
+
+        for audit in self:
+            yield (audit.get_single_val(col, allow_absent=True, absent_val="") for col in columns)
+
+def esim_profile_introspect(upp):
+    pes = ProfileElementSequence.from_der(upp.read())
+    d = {}
+    d['upp'] = repr(pes)
+
+    def show_bytes_as_hexdump(item):
+        if isinstance(item, bytes):
+            return b2h(item)
+        if isinstance(item, list):
+            return list(show_bytes_as_hexdump(i) for i in item)
+        if isinstance(item, tuple):
+            return tuple(show_bytes_as_hexdump(i) for i in item)
+        if isinstance(item, dict):
+            d = {}
+            for k, v in item.items():
+                d[k] = show_bytes_as_hexdump(v)
+            return d
+        return item
+
+    l = list((pe.type, show_bytes_as_hexdump(pe.decoded)) for pe in pes)
+    d['pp'] = pprint.pformat(l, width=120)
+    return d

pySim/esim/saip/param_source.py

@@ -0,0 +1,221 @@
+# Implementation of SimAlliance/TCA Interoperable Profile handling: parameter sources for batch personalization.
+#
+# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: nhofmeyr@sysmocom.de
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import secrets
+import re
+from osmocom.utils import b2h
+
+class ParamSourceExn(Exception):
+    pass
+
+class ParamSourceExhaustedExn(ParamSourceExn):
+    pass
+
+class ParamSourceUndefinedExn(ParamSourceExn):
+    pass
+
+class ParamSource:
+    """abstract parameter source. For usage, see personalization.BatchPersonalization."""
+
+    # This name should be short but descriptive, useful for a user interface, like 'random decimal digits'.
+    name = "none"
+    numeric_base = None # or 10 or 16
+
+    def __init__(self, input_str:str):
+        """Subclasses should call super().__init__(input_str) before evaluating self.input_str. Each subclass __init__()
+        may in turn manipulate self.input_str to apply expansions or decodings."""
+        self.input_str = input_str
+
+    def get_next(self, csv_row:dict=None):
+        """Subclasses implement this: return the next value from the parameter source.
+           When there are no more values from the source, raise a ParamSourceExhaustedExn.
+           This default implementation is an empty source."""
+        raise ParamSourceExhaustedExn()
+
+    @classmethod
+    def from_str(cls, input_str:str):
+        """compatibility with earlier version of ParamSource. Just use the constructor."""
+        return cls(input_str)
+
+class ConstantSource(ParamSource):
+    """one value for all"""
+    name = "constant"
+
+    def get_next(self, csv_row:dict=None):
+        return self.input_str
+
+class InputExpandingParamSource(ParamSource):
+
+    def __init__(self, input_str:str):
+        super().__init__(input_str)
+        self.input_str = self.expand_input_str(self.input_str)
+
+    @classmethod
+    def expand_input_str(cls, input_str:str):
+        # user convenience syntax '0*32' becomes '00000000000000000000000000000000'
+        if "*" not in input_str:
+            return input_str
+        # re:                "XX            *        123" with optional spaces
+        tokens = re.split(r"([^ \t]+)[ \t]*\*[ \t]*([0-9]+)", input_str)
+        if len(tokens) < 3:
+            return input_str
+        parts = []
+        for unchanged, snippet, repeat_str in zip(tokens[0::3], tokens[1::3], tokens[2::3]):
+            parts.append(unchanged)
+            repeat = int(repeat_str)
+            parts.append(snippet * repeat)
+
+        return "".join(parts)
+
+class DecimalRangeSource(InputExpandingParamSource):
+    """abstract: decimal numbers with a value range"""
+
+    numeric_base = 10
+
+    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
+        """Constructor to set up values from a (user entered) string: DecimalRangeSource(input_str).
+        Constructor to set up values directly: DecimalRangeSource(num_digits=3, first_value=123, last_value=456)
+
+        num_digits produces leading zeros when first_value..last_value are shorter.
+        """
+        assert ((input_str is not None and (num_digits, first_value, last_value) == (None, None, None))
+                or (input_str is None and None not in (num_digits, first_value, last_value)))
+
+        if input_str is not None:
+            super().__init__(input_str)
+
+            input_str = self.input_str
+
+            if ".." in input_str:
+                first_str, last_str = input_str.split('..')
+                first_str = first_str.strip()
+                last_str = last_str.strip()
+            else:
+                first_str = input_str.strip()
+                last_str = None
+
+            num_digits = len(first_str)
+            first_value = int(first_str)
+            last_value = int(last_str if last_str is not None else "9" * num_digits)
+
+        assert num_digits > 0
+        assert first_value <= last_value
+        self.num_digits = num_digits
+        self.first_value = first_value
+        self.last_value = last_value
+
+    def val_to_digit(self, val:int):
+        return "%0*d" % (self.num_digits, val)  # pylint: disable=consider-using-f-string
+
+class RandomSourceMixin:
+    random_impl = secrets.SystemRandom()
+
+class RandomDigitSource(DecimalRangeSource, RandomSourceMixin):
+    """return a different sequence of random decimal digits each"""
+    name = "random decimal digits"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.used_keys = set()
+
+    def get_next(self, csv_row:dict=None):
+        # try to generate random digits that are always different from previously produced random digits
+        for _ in range(10):
+            val = self.random_impl.randint(self.first_value, self.last_value)
+            if val not in self.used_keys:
+                break
+        self.used_keys.add(val)
+        return self.val_to_digit(val)
+
+class RandomHexDigitSource(InputExpandingParamSource, RandomSourceMixin):
+    """return a different sequence of random hexadecimal digits each"""
+    name = "random hexadecimal digits"
+    numeric_base = 16
+    def __init__(self, input_str:str):
+        super().__init__(input_str)
+        input_str = self.input_str
+
+        num_digits = len(input_str.strip())
+        if num_digits < 1:
+            raise ValueError("zero number of digits")
+        # hex digits always come in two
+        if (num_digits & 1) != 0:
+            raise ValueError(f"hexadecimal value should have even number of digits, not {num_digits}")
+        self.num_digits = num_digits
+        self.used_keys = set()
+
+    def get_next(self, csv_row:dict=None):
+        # try to generate random bytes that are always different from previously produced random bytes
+        for _ in range(10):
+            val = self.random_impl.randbytes(self.num_digits // 2)
+            if val not in self.used_keys:
+                break
+        self.used_keys.add(val)
+
+        return b2h(val)
+
+class IncDigitSource(DecimalRangeSource):
+    """incrementing sequence of digits"""
+    name = "incrementing decimal digits"
+
+    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
+        """input_str: the range of values to iterate. Format: 'FIRST..LAST' (e.g. '0001..9999') or
+        just 'FIRST' (iterates to the maximum value for the given digit width). Leading zeros in
+        FIRST determine the digit width and are preserved in returned values."""
+        super().__init__(input_str, num_digits, first_value, last_value)
+        self.next_val = None
+        self.reset()
+
+    def reset(self):
+        """Restart from the first value of the defined range passed to __init__()."""
+        self.next_val = self.first_value
+
+    def get_next(self, csv_row:dict=None):
+        val = self.next_val
+        if val is None:
+            raise ParamSourceExhaustedExn()
+
+        returnval = self.val_to_digit(val)
+
+        val += 1
+        if val > self.last_value:
+            self.next_val = None
+        else:
+            self.next_val = val
+
+        return returnval
+
+class CsvSource(ParamSource):
+    """apply a column from a CSV row, as passed in to ParamSource.get_next(csv_row)"""
+    name = "from CSV"
+
+    def __init__(self, input_str:str):
+        """input_str: the CSV column name to read values from.
+        The caller passes the current CSV row to get_next(), from which CsvSource picks the column matching
+        this name."""
+        super().__init__(input_str)
+        self.csv_column = self.input_str
+
+    def get_next(self, csv_row:dict=None):
+        val = None
+        if csv_row:
+            val = csv_row.get(self.csv_column)
+        if val is None:
+            raise ParamSourceUndefinedExn(f"no value for CSV column {self.csv_column!r}")
+        return val

pySim/esim/saip/personalization.py

@@ -16,13 +16,24 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import abc
+import enum
 import io
-from typing import List, Tuple
+import re
+from typing import List, Tuple, Generator, Optional
 
+from construct.core import StreamError
 from osmocom.tlv import camel_to_snake
-from pySim.utils import enc_iccid, enc_imsi, h2b, rpad, sanitize_iccid
-from pySim.esim.saip import ProfileElement, ProfileElementSequence
+from osmocom.utils import hexstr
+from pySim.utils import enc_iccid, dec_iccid, enc_imsi, dec_imsi, h2b, b2h, rpad, sanitize_iccid
+from pySim.ts_31_102 import EF_AD
 from pySim.ts_51_011 import EF_SMSP
+from pySim.esim.saip import param_source
+from pySim.esim.saip import ProfileElement, ProfileElementSD, ProfileElementSequence
+from pySim.esim.saip import SecurityDomainKey, SecurityDomainKeyComponent
+from pySim.global_platform import KeyUsageQualifier, KeyType
+
+def unrpad(s: hexstr, c='f') -> hexstr:
+    return hexstr(s.rstrip(c))
 
 def remove_unwanted_tuples_from_list(l: List[Tuple], unwanted_keys: List[str]) -> List[Tuple]:
     """In a list of tuples, remove all tuples whose first part equals 'unwanted_key'."""
@@ -43,7 +54,6 @@ class ClassVarMeta(abc.ABCMeta):
         x = super().__new__(metacls, name, bases, namespace)
         for k, v in kwargs.items():
             setattr(x, k, v)
-        setattr(x, 'name', camel_to_snake(name))
         return x
 
 class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
@@ -63,6 +73,7 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
     min_len: minimum length of an input str; min_len = 4
     max_len: maximum length of an input str; max_len = 8
     allow_len: permit only specific lengths; allow_len = (8, 16, 32)
+    numeric_base: indicate hex / decimal, if any; numeric_base = None; numeric_base = 10; numeric_base = 16
 
     Subclasses may change the meaning of these by overriding validate_val(), for example that the length counts
     resulting bytes instead of a hexstring length. Most subclasses will be covered by the default validate_val().
@@ -117,6 +128,8 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
     max_len = None
     allow_len = None # a list of specific lengths
     example_input = None
+    default_source = None # a param_source.ParamSource subclass
+    numeric_base = None # or 10 or 16
 
     def __init__(self, input_value=None):
         self.input_value = input_value # the raw input value as given by caller
@@ -178,19 +191,28 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
             if cls.allow_chars is not None:
                 if any(c not in cls.allow_chars for c in val):
                     raise ValueError(f"invalid characters in input value {val!r}, valid chars are {cls.allow_chars}")
+        elif isinstance(val, io.BytesIO):
+            val = val.getvalue()
+
+        if hasattr(val, '__len__'):
+            val_len = len(val)
+        else:
+            # e.g. int length
+            val_len = len(str(val))
+
         if cls.allow_len is not None:
             l = cls.allow_len
             # cls.allow_len could be one int, or a tuple of ints. Wrap a single int also in a tuple.
             if not isinstance(l, (tuple, list)):
                 l = (l,)
-            if len(val) not in l:
-                raise ValueError(f'length must be one of {cls.allow_len}, not {len(val)}: {val!r}')
+            if val_len not in l:
+                raise ValueError(f'length must be one of {cls.allow_len}, not {val_len}: {val!r}')
         if cls.min_len is not None:
-            if len(val) < cls.min_len:
-                raise ValueError(f'length must be at least {cls.min_len}, not {len(val)}: {val!r}')
+            if val_len < cls.min_len:
+                raise ValueError(f'length must be at least {cls.min_len}, not {val_len}: {val!r}')
         if cls.max_len is not None:
-            if len(val) > cls.max_len:
-                raise ValueError(f'length must be at most {cls.max_len}, not {len(val)}: {val!r}')
+            if val_len > cls.max_len:
+                raise ValueError(f'length must be at most {cls.max_len}, not {val_len}: {val!r}')
         return val
 
     @classmethod
@@ -199,6 +221,49 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
         Write the given val in the right format in all the right places in pes."""
         pass
 
+    @classmethod
+    def get_value_from_pes(cls, pes: ProfileElementSequence):
+        """Same as get_values_from_pes() but expecting a single value.
+           get_values_from_pes() may return values like this:
+             [{ 'AlgorithmID': 'Milenage' }, { 'AlgorithmID': 'Milenage' }]
+           This ensures that all these entries are identical and would return only
+              { 'AlgorithmID': 'Milenage' }.
+
+           This is relevant for any profile element that may appear multiple times in the same PES (only a few),
+           where each occurrence should reflect the same value (all currently known parameters).
+        """
+
+        val = None
+        for v in cls.get_values_from_pes(pes):
+            if val is None:
+                val = v
+            elif val != v:
+                raise ValueError(f'get_value_from_pes(): got distinct values: {val!r} != {v!r}')
+        return val
+
+    @classmethod
+    @abc.abstractmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence) -> Generator:
+        """This is what subclasses implement: yield all values from a decoded profile package.
+           Find all values in the pes, and yield them decoded to a valid cls.input_value format.
+           Should be a generator function, i.e. use 'yield' instead of 'return'.
+
+           Yielded value must be a dict(). Usually, an implementation will return only one key, like
+
+              { "ICCID": "1234567890123456789" }
+
+           Some implementations have more than one value to return, like
+
+              { "IMSI": "00101012345678", "IMSI-ACC" : "5" }
+
+           Implementation example:
+
+             for pe in pes:
+                if my_condition(pe):
+                    yield { cls.name: b2h(my_bin_value_from(pe)) }
+           """
+        pass
+
     @classmethod
     def get_len_range(cls):
         """considering all of min_len, max_len and allow_len, get a tuple of the resulting (min, max) of permitted
@@ -219,6 +284,20 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
             return (None, None)
         return (min(vals), max(vals))
 
+    @classmethod
+    def get_typical_input_len(cls):
+        '''return a good length to use as the visible width of a user interface input field.
+           May be overridden by subclasses.
+           This default implementation returns the maximum allowed value length -- a good fit for most subclasses.
+           '''
+        return cls.get_len_range()[1] or 16
+
+    @classmethod
+    def is_super_of(cls, other_class):
+        try:
+            return issubclass(other_class, cls)
+        except TypeError:
+            return False
 
 class DecimalParam(ConfigurableParameter):
     """Decimal digits. The input value may be a string of decimal digits like '012345', or an int. The output of
@@ -226,6 +305,7 @@ class DecimalParam(ConfigurableParameter):
     """
     allow_types = (str, int)
     allow_chars = '0123456789'
+    numeric_base = 10
 
     @classmethod
     def validate_val(cls, val):
@@ -249,6 +329,7 @@ class DecimalHexParam(DecimalParam):
     @classmethod
     def validate_val(cls, val):
         val = super().validate_val(val)
+        assert isinstance(val, str)
         val = ''.join('%02x' % ord(x) for x in val)
         if cls.rpad is not None:
             c = cls.rpad_char
@@ -256,9 +337,21 @@ class DecimalHexParam(DecimalParam):
         # a DecimalHexParam subclass expects the apply_val() input to be a bytes instance ready for the pes
         return h2b(val)
 
+    @classmethod
+    def decimal_hex_to_str(cls, val):
+        """useful for get_values_from_pes() implementations of subclasses"""
+        if isinstance(val, bytes):
+            val = b2h(val)
+        assert isinstance(val, hexstr)
+        if cls.rpad is not None:
+            c = cls.rpad_char or 'f'
+            val = unrpad(val, c)
+        return val.to_bytes().decode('ascii')
+
 class IntegerParam(ConfigurableParameter):
     allow_types = (str, int)
     allow_chars = '0123456789'
+    numeric_base = 10
 
     # two integers, if the resulting int should be range limited
     min_val = None
@@ -279,14 +372,28 @@ class IntegerParam(ConfigurableParameter):
             raise ValueError(f'Value {val} is out of range, must be [{cls.min_val}..{cls.max_val}]')
         return val
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for valdict in super().get_values_from_pes(pes):
+            for key, val in valdict.items():
+                if isinstance(val, int):
+                    valdict[key] = str(val)
+            yield valdict
+
 class BinaryParam(ConfigurableParameter):
-    allow_types = (str, io.BytesIO, bytes, bytearray)
+    allow_types = (str, io.BytesIO, bytes, bytearray, int)
     allow_chars = '0123456789abcdefABCDEF'
     strip_chars = ' \t\r\n'
+    numeric_base = 16
+    default_source = param_source.RandomHexDigitSource
 
     @classmethod
     def validate_val(cls, val):
         # take care that min_len and max_len are applied to the binary length by converting to bytes first
+        if isinstance(val, int):
+            min_len, _max_len = cls.get_len_range()
+            val = '%0*d' % (min_len, val)
+
         if isinstance(val, str):
             if cls.strip_chars is not None:
                 val = ''.join(c for c in val if c not in cls.strip_chars)
@@ -301,6 +408,82 @@ class BinaryParam(ConfigurableParameter):
         val = super().validate_val(val)
         return bytes(val)
 
+    @classmethod
+    def get_typical_input_len(cls):
+        # override to return twice the length, because of hex digits.
+        min_len, max_len = cls.get_len_range()
+        if max_len is None:
+            return None
+        # two hex characters per value octet.
+        # (maybe *3 to also allow for spaces?)
+        return max_len * 2
+
+
+class EnumParam(ConfigurableParameter):
+    """ConfigurableParameter for named integer enumeration values.
+
+    Subclasses must define a nested enum.IntEnum named 'Values' listing all valid names and their
+    integer codes.  apply_val() and get_values_from_pes() are not implemented here and this must
+    be inherited from another mixin."""
+
+    class Values(enum.IntEnum):
+        pass  # subclasses override this
+
+    @classmethod
+    def validate_val(cls, val) -> int:
+        if isinstance(val, int):
+            try:
+                return int(cls.Values(val))
+            except ValueError:
+                pass
+        elif isinstance(val, str):
+            member = cls.map_name_to_val(val, strict=False)
+            if member is not None:
+                return member
+
+        valid = ', '.join(m.name for m in cls.Values)
+        raise ValueError(f"{cls.get_name()}: invalid argument: {val!r}. Valid arguments are: {valid}")
+
+    @classmethod
+    def map_name_to_val(cls, name: str, strict=True) -> int:
+        """Return the integer value for a given enum member name. Performs an exact match first,
+        then falls back to fuzzy matching (case-insensitive, punctuation-insensitive)."""
+        try:
+            return int(cls.Values[name])
+        except KeyError:
+            pass
+
+        clean = cls.clean_name_str(name)
+        for member in cls.Values:
+            if cls.clean_name_str(member.name) == clean:
+                return int(member)
+
+        if strict:
+            valid = ', '.join(m.name for m in cls.Values)
+            raise ValueError(f"{cls.get_name()}: {name!r} is not a known value. Known values are: {valid}")
+        return None
+
+    @classmethod
+    def map_val_to_name(cls, val, strict=False) -> str:
+        """Return the enum member name for a given integer value."""
+        try:
+            return cls.Values(val).name
+        except ValueError:
+            if strict:
+                raise ValueError(f"{cls.get_name()}: {val!r} ({type(val).__name__}) is not a known value.")
+            return None
+
+    @classmethod
+    def name_normalize(cls, name: str) -> str:
+        """Map a (possibly fuzzy) name to its canonical enum member name."""
+        return cls.Values(cls.map_name_to_val(name)).name
+
+    @classmethod
+    def clean_name_str(cls, val: str) -> str:
+        """Strip punctuation and case for fuzzy name comparison.
+        Treats hyphens and underscores as equivalent (both removed)."""
+        return re.sub('[^0-9A-Za-z]', '', val).lower()
+
 
 class Iccid(DecimalParam):
     """ICCID Parameter. Input: string of decimal digits.
@@ -309,6 +492,7 @@ class Iccid(DecimalParam):
     min_len = 18
     max_len = 20
     example_input = '998877665544332211'
+    default_source = param_source.IncDigitSource
 
     @classmethod
     def validate_val(cls, val):
@@ -322,6 +506,17 @@ class Iccid(DecimalParam):
         # patch MF/EF.ICCID
         file_replace_content(pes.get_pe_for_type('mf').decoded['ef-iccid'], h2b(enc_iccid(val)))
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        padded = b2h(pes.get_pe_for_type('header').decoded['iccid'])
+        iccid = unrpad(padded)
+        yield { cls.name: iccid }
+
+        for pe in pes.get_pes_for_type('mf'):
+            iccid_f = pe.files.get('ef-iccid', None)
+            if iccid_f is not None:
+                yield { cls.name: dec_iccid(b2h(iccid_f.body)) }
+
 class Imsi(DecimalParam):
     """Configurable IMSI. Expects value to be a string of digits. Automatically sets the ACC to
     the last digit of the IMSI."""
@@ -330,6 +525,7 @@ class Imsi(DecimalParam):
     min_len = 6
     max_len = 15
     example_input = '00101' + ('0' * 10)
+    default_source = param_source.IncDigitSource
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -342,6 +538,18 @@ class Imsi(DecimalParam):
             file_replace_content(pe.decoded['ef-acc'], acc.to_bytes(2, 'big'))
         # TODO: DF.GSM_ACCESS if not linked?
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('usim'):
+            imsi_f = pe.files.get('ef-imsi', None)
+            acc_f = pe.files.get('ef-acc', None)
+            y = {}
+            if imsi_f:
+                y[cls.name] = dec_imsi(b2h(imsi_f.body))
+            if acc_f:
+                y[cls.name + '-ACC'] = b2h(acc_f.body)
+            yield y
+
 class SmspTpScAddr(ConfigurableParameter):
     """Configurable SMSC (SMS Service Centre) TP-SC-ADDR. Expects to be a phone number in national or
     international format (designated by a leading +). Automatically sets the NPI to E.164 and the TON based on
@@ -350,25 +558,45 @@ class SmspTpScAddr(ConfigurableParameter):
     name = 'SMSP-TP-SC-ADDR'
     allow_chars = '+0123456789'
     strip_chars = ' \t\r\n'
+    numeric_base = 10
     max_len = 21 # '+' and 20 digits
     min_len = 1
     example_input = '+49301234567'
+    default_source = param_source.ConstantSource
 
-    @classmethod
-    def validate_val(cls, val):
-        val = super().validate_val(val)
-        addr_str = str(val)
+    @staticmethod
+    def str_to_tuple(addr_str):
         if addr_str[0] == '+':
             digits = addr_str[1:]
             international = True
         else:
             digits = addr_str
             international = False
+        return (international, digits)
+
+    @staticmethod
+    def tuple_to_str(addr_tuple):
+        international, digits = addr_tuple
+        if international:
+            ret = '+'
+        else:
+            ret = ''
+        ret += digits
+        return ret
+
+    @classmethod
+    def validate_val(cls, val):
+        val = super().validate_val(val)
+
+        addr_tuple = cls.str_to_tuple(str(val))
+
+        international, digits = addr_tuple
         if len(digits) > 20:
             raise ValueError(f'TP-SC-ADDR must not exceed 20 digits: {digits!r}')
         if not digits.isdecimal():
             raise ValueError(f'TP-SC-ADDR must only contain decimal digits: {digits!r}')
-        return (international, digits)
+
+        return addr_tuple
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -392,98 +620,318 @@ class SmspTpScAddr(ConfigurableParameter):
             ef_smsp_dec['tp_sc_addr']['ton_npi']['type_of_number'] = 'international' if international else 'unknown'
             # ensure the parameter_indicators.tp_sc_addr is True
             ef_smsp_dec['parameter_indicators']['tp_sc_addr'] = True
-            # re-encode into the File body
-            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1)
+
+            # alpha_id padding: to make room for a human readable SMSC name that can be provisioned to the profile later
+            # on, alpha_id needs to be empty but padded 0xff to some length.
+            # - alpha_id is optional, setting alpha_id = '' ensures the IE is present.
+            # - the length of the file is 28+Y where Y is the length of the alpha_id -- here the intended length of our padding
+            #   (see 3GPP TS 31.102 4.2.27 EF.SMSP). So if we want a maximum length of alpha_id = 14, we set the total
+            #   file size to 28+14 = 42.
+            # - this file size has to go in two places: encode_record_bin() needs to know the length to encode the right
+            #   length of fillFileContent.
+            # - the f_smsp needs to show the right file size in the PES, as in
+            #      'ef-smsp': [('fileDescriptor', {'efFileSize': '2a', ...
+            #   (where 2a == 42)
+            # - To generate the right amount of fillFileContent, pass total_len=42 to encode_record_bin().
+            # - To show the right size in the PES, set f_smsp.rec_len = 42
+            ef_smsp_dec['alpha_id'] = ''
+            f_smsp.rec_len = 42
+
+            # re-encode into the File body.
+            #
             #print("SMSP  (new): %s" % f_smsp.body)
             # re-generate the pe.decoded member from the File instance
+            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1, total_len=f_smsp.rec_len)
             pe.file2pe(f_smsp)
 
-class SdKey(BinaryParam, metaclass=ClassVarMeta):
-    """Configurable Security Domain (SD) Key.  Value is presented as bytes."""
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('usim'):
+            f_smsp = pe.files['ef-smsp']
+            ef_smsp = EF_SMSP()
+            ef_smsp_dec = ef_smsp.decode_record_bin(f_smsp.body, 1)
+
+            tp_sc_addr = ef_smsp_dec.get('tp_sc_addr', None)
+
+            digits = tp_sc_addr.get('call_number', None)
+
+            ton_npi = tp_sc_addr.get('ton_npi', None)
+            international = ton_npi.get('type_of_number', None)
+            international = (international == 'international')
+
+            yield { cls.name: cls.tuple_to_str((international, digits)) }
+
+
+class MncLen(EnumParam):
+    """MNC length.  Sets only the MNC length field in EF.AD (Administrative Data).
+    Accepted values: integer 2 or 3, digit strings '2' or '3', or enum names 'MNC2'/'MNC3'.
+    """
+    name = 'MNC-LEN'
+    example_input = '2'
+    default_source = param_source.ConstantSource
+
+    class Values(enum.IntEnum):
+        MNC2 = 2
+        MNC3 = 3
+
+    @classmethod
+    def validate_val(cls, val):
+        if isinstance(val, str) and val.isdigit():
+            val = int(val)
+        return super().validate_val(val)
+
+    @classmethod
+    def _get_f_ad(cls, pe: ProfileElement):
+        if not hasattr(pe, 'files'):
+            return None
+        f_ad = pe.files.get('ef-ad', None)
+        if f_ad and f_ad.body:
+            return f_ad
+        return None
+
+    @classmethod
+    def _decode_f_ad(cls, f_ad):
+        try:
+            ef_ad_dec = EF_AD().decode_bin(f_ad.body)
+        except StreamError:
+            return None
+        if 'mnc_len' not in ef_ad_dec:
+            return None
+        return ef_ad_dec
+
+    @classmethod
+    def apply_val(cls, pes: ProfileElementSequence, val: int):
+        for pe in pes.get_pes_for_type('usim'):
+            f_ad = cls._get_f_ad(pe)
+            if f_ad is None:
+                continue
+            # decode existing values
+            ef_ad_dec = cls._decode_f_ad(f_ad)
+            if ef_ad_dec is None:
+                continue
+            # change mnc_len
+            ef_ad_dec['mnc_len'] = val
+            # re-encode into the File body
+            f_ad.body = EF_AD().encode_bin(ef_ad_dec)
+            pe.file2pe(f_ad)
+
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('usim'):
+            f_ad = cls._get_f_ad(pe)
+            if f_ad is None:
+                continue
+            ef_ad_dec = cls._decode_f_ad(f_ad)
+            if ef_ad_dec is None:
+                continue
+            mnc_len = ef_ad_dec.get('mnc_len')
+            yield { cls.name: str(mnc_len) }
+
+
+class SdKey(BinaryParam):
+    """Configurable Security Domain (SD) Key.  Value is presented as bytes.
+       Non-abstract implementations are generated in SdKey.generate_sd_key_classes"""
     # these will be set by subclasses
     key_type = None
-    key_id = None
     kvn = None
+    key_id = None
     key_usage_qual = None
 
     @classmethod
-    def _apply_sd(cls, pe: ProfileElement, value):
-        assert pe.type == 'securityDomain'
-        for key in pe.decoded['keyList']:
-            if key['keyIdentifier'][0] == cls.key_id and key['keyVersionNumber'][0] == cls.kvn:
-                assert len(key['keyComponents']) == 1
-                key['keyComponents'][0]['keyData'] = value
-                return
-        # Could not find matching key to patch, create a new one
-        key = {
-            'keyUsageQualifier': bytes([cls.key_usage_qual]),
-            'keyIdentifier': bytes([cls.key_id]),
-            'keyVersionNumber': bytes([cls.kvn]),
-            'keyComponents': [
-                { 'keyType': bytes([cls.key_type]), 'keyData': value },
-            ]
-        }
-        pe.decoded['keyList'].append(key)
+    def apply_val(cls, pes: ProfileElementSequence, val):
+        set_components = [ SecurityDomainKeyComponent(cls.key_type, val) ]
+
+        for pe in pes.pe_list:
+            if pe.type != 'securityDomain':
+                continue
+            assert isinstance(pe, ProfileElementSD)
+
+            key = pe.find_key(key_version_number=cls.kvn, key_id=cls.key_id)
+            if not key:
+                # Could not find matching key to patch, create a new one
+                key = SecurityDomainKey(
+                        key_version_number=cls.kvn,
+                        key_id=cls.key_id,
+                        key_usage_qualifier=cls.key_usage_qual,
+                        key_components=set_components,
+                        )
+                pe.add_key(key)
+            else:
+                # A key of this KVN and ID already exists in the profile.
+
+                # Keep the key_usage_qualifier as it was in the profile, so skip this here:
+                # key.key_usage_qualifier = cls.key_usage_qual
+
+                key.key_components = set_components
 
     @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, value):
-        for pe in pes.get_pes_for_type('securityDomain'):
-            cls._apply_sd(pe, value)
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.pe_list:
+            if pe.type != 'securityDomain':
+                continue
+            assert isinstance(pe, ProfileElementSD)
 
-class SdKeyScp80_01(SdKey, kvn=0x01, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp80_01Kic(SdKeyScp80_01, key_id=0x01, key_usage_qual=0x18): # FIXME: ordering?
-    pass
-class SdKeyScp80_01Kid(SdKeyScp80_01, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp80_01Kik(SdKeyScp80_01, key_id=0x03, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp81_01(SdKey, kvn=0x81): # FIXME
-    pass
-class SdKeyScp81_01Psk(SdKeyScp81_01, key_id=0x01, key_type=0x85, key_usage_qual=0x3C):
-    pass
-class SdKeyScp81_01Dek(SdKeyScp81_01, key_id=0x02, key_type=0x88, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp02_20(SdKey, kvn=0x20, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp02_20Enc(SdKeyScp02_20, key_id=0x01, key_usage_qual=0x18):
-    pass
-class SdKeyScp02_20Mac(SdKeyScp02_20, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp02_20Dek(SdKeyScp02_20, key_id=0x03, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp03_30(SdKey, kvn=0x30, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp03_30Enc(SdKeyScp03_30, key_id=0x01, key_usage_qual=0x18):
-    pass
-class SdKeyScp03_30Mac(SdKeyScp03_30, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp03_30Dek(SdKeyScp03_30, key_id=0x03, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp03_31(SdKey, kvn=0x31, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp03_31Enc(SdKeyScp03_31, key_id=0x01, key_usage_qual=0x18):
-    pass
-class SdKeyScp03_31Mac(SdKeyScp03_31, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp03_31Dek(SdKeyScp03_31, key_id=0x03, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp03_32(SdKey, kvn=0x32, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp03_32Enc(SdKeyScp03_32, key_id=0x01, key_usage_qual=0x18):
-    pass
-class SdKeyScp03_32Mac(SdKeyScp03_32, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp03_32Dek(SdKeyScp03_32, key_id=0x03, key_usage_qual=0x48):
-    pass
+            key = pe.find_key(key_version_number=cls.kvn, key_id=cls.key_id)
+            if not key:
+                continue
+            kc = key.get_key_component(cls.key_type)
+            if kc:
+                yield { cls.name: b2h(kc) }
 
 
+    LEN_128 = (16,)
+    LEN_128_192_256 = (16, 24, 32)
+    LEN_128_256 = (16, 32)
 
+    DES =    ('DES', dict(key_type=KeyType.des, allow_len=LEN_128) )
+    AES =    ('AES', dict(key_type=KeyType.aes, allow_len=LEN_128_192_256) )
+
+    ENC =    ('ENC', dict(key_id=0x01, key_usage_qual=0x18) )
+    MAC =    ('MAC', dict(key_id=0x02, key_usage_qual=0x14) )
+    DEK =    ('DEK', dict(key_id=0x03, key_usage_qual=0x48) )
+
+    TLSPSK_PSK = ('TLSPSK', dict(key_type=KeyType.tls_psk, key_id=0x01, key_usage_qual=0x3c, allow_len=LEN_128_192_256) )
+    TLSPSK_DEK = ('DEK',    dict(key_id=0x02, key_usage_qual=0x48) )
+
+    # THIS IS THE LIST that controls which SdKeyXxx subclasses exist:
+    SD_KEY_DEFS = (
+        # name    KVN                     x variants            x variants
+        ('SCP02', (0x20, 0x21, 0x22, 0xff), (AES, ),              (ENC, MAC, DEK) ),
+        ('SCP03', (0x30, 0x31, 0x32),       (AES, ),              (ENC, MAC, DEK) ),
+        ('SCP80', (0x01, 0x02, 0x03),       (DES, AES),           (ENC, MAC, DEK) ),
+
+        # key_id=1
+        ('SCP81', (0x40, 0x41, 0x42),       (TLSPSK_PSK, ),                       ),
+        # key_id=2
+        ('SCP81', (0x40, 0x41, 0x42),       (DES, AES),           (TLSPSK_DEK, )  ),
+    )
+
+    all_implementations = None
+
+    @classmethod
+    def generate_sd_key_classes(cls, sd_key_defs=SD_KEY_DEFS):
+        '''This generates python classes to be exported in this module, as subclasses of class SdKey.
+
+        We create SdKey subclasses dynamically from a list.
+        You can list all of them via:
+          from pySim.esim.saip.personalization import SdKey
+          SdKey.all_implementations
+        or
+          print('\n'.join(sorted(f'{x.__name__}\t{x.name}' for x in SdKey.all_implementations)))
+
+        at time of writing this comment, this prints:
+
+        SdKeyScp02Kvn20AesDek	SCP02-KVN20-AES-DEK
+        SdKeyScp02Kvn20AesEnc	SCP02-KVN20-AES-ENC
+        SdKeyScp02Kvn20AesMac	SCP02-KVN20-AES-MAC
+        SdKeyScp02Kvn21AesDek	SCP02-KVN21-AES-DEK
+        SdKeyScp02Kvn21AesEnc	SCP02-KVN21-AES-ENC
+        SdKeyScp02Kvn21AesMac	SCP02-KVN21-AES-MAC
+        SdKeyScp02Kvn22AesDek	SCP02-KVN22-AES-DEK
+        SdKeyScp02Kvn22AesEnc	SCP02-KVN22-AES-ENC
+        SdKeyScp02Kvn22AesMac	SCP02-KVN22-AES-MAC
+        SdKeyScp02KvnffAesDek	SCP02-KVNff-AES-DEK
+        SdKeyScp02KvnffAesEnc	SCP02-KVNff-AES-ENC
+        SdKeyScp02KvnffAesMac	SCP02-KVNff-AES-MAC
+        SdKeyScp03Kvn30AesDek	SCP03-KVN30-AES-DEK
+        SdKeyScp03Kvn30AesEnc	SCP03-KVN30-AES-ENC
+        SdKeyScp03Kvn30AesMac	SCP03-KVN30-AES-MAC
+        SdKeyScp03Kvn31AesDek	SCP03-KVN31-AES-DEK
+        SdKeyScp03Kvn31AesEnc	SCP03-KVN31-AES-ENC
+        SdKeyScp03Kvn31AesMac	SCP03-KVN31-AES-MAC
+        SdKeyScp03Kvn32AesDek	SCP03-KVN32-AES-DEK
+        SdKeyScp03Kvn32AesEnc	SCP03-KVN32-AES-ENC
+        SdKeyScp03Kvn32AesMac	SCP03-KVN32-AES-MAC
+        SdKeyScp80Kvn01AesDek	SCP80-KVN01-AES-DEK
+        SdKeyScp80Kvn01AesEnc	SCP80-KVN01-AES-ENC
+        SdKeyScp80Kvn01AesMac	SCP80-KVN01-AES-MAC
+        SdKeyScp80Kvn01DesDek	SCP80-KVN01-DES-DEK
+        SdKeyScp80Kvn01DesEnc	SCP80-KVN01-DES-ENC
+        SdKeyScp80Kvn01DesMac	SCP80-KVN01-DES-MAC
+        SdKeyScp80Kvn02AesDek	SCP80-KVN02-AES-DEK
+        SdKeyScp80Kvn02AesEnc	SCP80-KVN02-AES-ENC
+        SdKeyScp80Kvn02AesMac	SCP80-KVN02-AES-MAC
+        SdKeyScp80Kvn02DesDek	SCP80-KVN02-DES-DEK
+        SdKeyScp80Kvn02DesEnc	SCP80-KVN02-DES-ENC
+        SdKeyScp80Kvn02DesMac	SCP80-KVN02-DES-MAC
+        SdKeyScp80Kvn03AesDek	SCP80-KVN03-AES-DEK
+        SdKeyScp80Kvn03AesEnc	SCP80-KVN03-AES-ENC
+        SdKeyScp80Kvn03AesMac	SCP80-KVN03-AES-MAC
+        SdKeyScp80Kvn03DesDek	SCP80-KVN03-DES-DEK
+        SdKeyScp80Kvn03DesEnc	SCP80-KVN03-DES-ENC
+        SdKeyScp80Kvn03DesMac	SCP80-KVN03-DES-MAC
+        SdKeyScp81Kvn40AesDek	SCP81-KVN40-AES-DEK
+        SdKeyScp81Kvn40DesDek	SCP81-KVN40-DES-DEK
+        SdKeyScp81Kvn40Tlspsk	SCP81-KVN40-TLSPSK
+        SdKeyScp81Kvn41AesDek	SCP81-KVN41-AES-DEK
+        SdKeyScp81Kvn41DesDek	SCP81-KVN41-DES-DEK
+        SdKeyScp81Kvn41Tlspsk	SCP81-KVN41-TLSPSK
+        SdKeyScp81Kvn42AesDek	SCP81-KVN42-AES-DEK
+        SdKeyScp81Kvn42DesDek	SCP81-KVN42-DES-DEK
+        SdKeyScp81Kvn42Tlspsk	SCP81-KVN42-TLSPSK
+        '''
+
+        SdKey.all_implementations = []
+
+        def camel(s):
+            return s[:1].upper() + s[1:].lower()
+
+        def do_variants(name, kvn, remaining_variants, labels=[], attrs={}):
+            'recurse to unfold as many variants as there may be'
+            if remaining_variants:
+                # not a leaf node, collect more labels and attrs
+                variants = remaining_variants[0]
+                remaining_variants = remaining_variants[1:]
+
+                for label, valdict in variants:
+                    # pass copies to recursion
+                    inner_labels = list(labels)
+                    inner_attrs = dict(attrs)
+
+                    inner_labels.append(label)
+                    inner_attrs.update(valdict)
+                    do_variants(name, kvn, remaining_variants,
+                                labels=inner_labels,
+                                attrs=inner_attrs)
+                return
+
+            # leaf node. create a new class with all the accumulated vals
+            parts = [name, f'KVN{kvn:02x}',] + labels
+            cls_label = '-'.join(p for p in parts if p)
+
+            parts = ['Sd', 'Key', name, f'Kvn{kvn:02x}'] + labels
+            clsname = ''.join(camel(p) for p in parts)
+
+            max_key_len = attrs.get('allow_len')[-1]
+
+            attrs.update({
+                'name' : cls_label,
+                'kvn': kvn,
+                'example_input': f'00*{max_key_len}',
+                })
+
+            # below line is like
+            # class SdKeyScpNNKvnXXYyyZzz(SdKey):
+            #     <set attrs>
+            cls_def = type(clsname, (cls,), attrs)
+
+            # for some unknown reason, subclassing from abc.ABC makes cls_def.__module__ == 'abc',
+            # but we don't want 'abc.SdKeyScp03Kvn32AesEnc'.
+            # Make sure it is 'pySim.esim.saip.personalization.SdKeyScp03Kvn32AesEnc'
+            cls_def.__module__ = __name__
+
+            globals()[clsname] = cls_def
+            SdKey.all_implementations.append(cls_def)
+
+
+        for items in sd_key_defs:
+            name, kvns = items[:2]
+            variants = items[2:]
+            for kvn in kvns:
+                do_variants(name, kvn, variants)
+
+# this creates all of the classes named like SdKeyScp02Kvn20AesDek to be published in this python module:
+SdKey.generate_sd_key_classes()
 
 def obtain_all_pe_from_pelist(l: List[ProfileElement], wanted_type: str) -> ProfileElement:
     return (pe for pe in l if pe.type == wanted_type)
@@ -502,7 +950,8 @@ class Puk(DecimalHexParam):
     allow_len = 8
     rpad = 16
     keyReference = None
-    example_input = '0' * allow_len
+    example_input = f'0*{allow_len}'
+    default_source = param_source.RandomDigitSource
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -516,6 +965,14 @@ class Puk(DecimalHexParam):
         raise ValueError("input template UPP has unexpected structure:"
                          f" cannot find pukCode with keyReference={cls.keyReference}")
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        mf_pes = pes.pes_by_naa['mf'][0]
+        for pukCodes in obtain_all_pe_from_pelist(mf_pes, 'pukCodes'):
+            for pukCode in pukCodes.decoded['pukCodes']:
+                if pukCode['keyReference'] == cls.keyReference:
+                    yield { cls.name: cls.decimal_hex_to_str(pukCode['pukValue']) }
+
 class Puk1(Puk):
     name = 'PUK1'
     keyReference = 0x01
@@ -529,7 +986,8 @@ class Pin(DecimalHexParam):
     rpad = 16
     min_len = 4
     max_len = 8
-    example_input = '0' * max_len
+    example_input = f'0*{max_len}'
+    default_source = param_source.RandomDigitSource
     keyReference = None
 
     @staticmethod
@@ -551,9 +1009,24 @@ class Pin(DecimalHexParam):
             raise ValueError('input template UPP has unexpected structure:'
                              + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference}')
 
+    @classmethod
+    def _read_all_pinvalues_from_pe(cls, pe: ProfileElement):
+        "This is a separate function because subclasses may feed different pe arguments."
+        for pinCodes in obtain_all_pe_from_pelist(pe, 'pinCodes'):
+            if pinCodes.decoded['pinCodes'][0] != 'pinconfig':
+                continue
+
+            for pinCode in pinCodes.decoded['pinCodes'][1]:
+                if pinCode['keyReference'] == cls.keyReference:
+                    yield { cls.name: cls.decimal_hex_to_str(pinCode['pinValue']) }
+
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        yield from cls._read_all_pinvalues_from_pe(pes.pes_by_naa['mf'][0])
+
 class Pin1(Pin):
     name = 'PIN1'
-    example_input = '0' * 4  # PIN are usually 4 digits
+    example_input = '0*4'  # PIN are usually 4 digits
     keyReference = 0x01
 
 class Pin2(Pin1):
@@ -572,6 +1045,14 @@ class Pin2(Pin1):
                     raise ValueError('input template UPP has unexpected structure:'
                             + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference} in {naa=}')
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for naa in pes.pes_by_naa:
+            if naa not in ['usim','isim','csim','telecom']:
+                continue
+            for pe in pes.pes_by_naa[naa]:
+                yield from cls._read_all_pinvalues_from_pe(pe)
+
 class Adm1(Pin):
     name = 'ADM1'
     keyReference = 0x0A
@@ -596,26 +1077,59 @@ class AlgoConfig(ConfigurableParameter):
             raise ValueError('input template UPP has unexpected structure:'
                              f' {cls.__name__} cannot find algoParameter with key={cls.algo_config_key}')
 
-class AlgorithmID(DecimalParam, AlgoConfig):
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('akaParameter'):
+            algoConfiguration = pe.decoded['algoConfiguration']
+            if len(algoConfiguration) < 2:
+                continue
+            if algoConfiguration[0] != 'algoParameter':
+                continue
+            if not algoConfiguration[1]:
+                continue
+            val = algoConfiguration[1].get(cls.algo_config_key, None)
+            if val is None:
+                continue
+            if isinstance(val, bytes):
+                val = b2h(val)
+            # if it is an int (algorithmID), just pass thru as int
+            yield { cls.name: val }
+
+class AlgorithmID(EnumParam, AlgoConfig):
+    """use validate_val() from EnumParam, and apply_val() from AlgoConfig.
+    In get_values_from_pes(), return enum value names, not raw values."""
+    name = "Algorithm"
     algo_config_key = 'algorithmID'
-    allow_len = 1
-    example_input = 1  # Milenage
+    example_input = "Milenage"
+    default_source = param_source.ConstantSource
+
+    # as in pySim/esim/asn1/saip/PE_Definitions-3.3.1.asn
+    class Values(enum.IntEnum):
+        Milenage  = 1
+        TUAK      = 2
+        usim_test = 3  # input 'usim-test' also accepted via fuzzy matching
+
+    # EnumParam.validate_val() returns the int values from Values
 
     @classmethod
-    def validate_val(cls, val):
-        val = super().validate_val(val)
-        val = int(val)
-        valid = (1, 2, 3)
-        if val not in valid:
-            raise ValueError(f'Invalid algorithmID {val!r}, must be one of {valid}')
-        return val
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        # return enum names, not raw values.
+        # use of super(): this intends to call AlgoConfig.get_values_from_pes() so that the cls argument is this cls
+        # here (AlgorithmID); i.e. AlgoConfig.get_values_from_pes(pes) doesn't work, because AlgoConfig needs to look up
+        # cls.algo_config_key.
+        for d in super(cls, cls).get_values_from_pes(pes):
+            if cls.name in d:
+                # convert int to value string
+                val = d[cls.name]
+                d[cls.name] = cls.map_val_to_name(val, strict=True)
+            yield d
 
 class K(BinaryParam, AlgoConfig):
     """use validate_val() from BinaryParam, and apply_val() from AlgoConfig"""
     name = 'K'
     algo_config_key = 'key'
     allow_len = (128 // 8, 256 // 8) # length in bytes (from BinaryParam); TUAK also allows 256 bit
-    example_input = '00' * allow_len[0]
+    example_input = f'00*{allow_len[0]}'
 
 class Opc(K):
     name = 'OPc'
@@ -629,6 +1143,7 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):
     algo_config_key = 'rotationConstants'
     allow_len = 5 # length in bytes (from BinaryParam)
     example_input = '40 00 20 40 60'
+    default_source = param_source.ConstantSource
 
     @classmethod
     def validate_val(cls, val):
@@ -641,7 +1156,7 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):
 
 class MilenageXoringConstants(BinaryParam, AlgoConfig):
     """XOR-ing constants c1,c2,c3,c4,c5 of Milenage, 128bit each. See 3GPP TS 35.206 Sections 2.3 + 5.3.
-    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concetenation
+    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concatenation
     of::
 
      00000000000000000000000000000000
@@ -659,6 +1174,7 @@ class MilenageXoringConstants(BinaryParam, AlgoConfig):
                      ' 00000000000000000000000000000002'
                      ' 00000000000000000000000000000004'
                      ' 00000000000000000000000000000008')
+    default_source = param_source.ConstantSource
 
 class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
     """Number of iterations of Keccak-f[1600] permutation as recomended by Section 7.2 of 3GPP TS 35.231"""
@@ -667,3 +1183,4 @@ class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
     min_val = 1
     max_val = 255
     example_input = '1'
+    default_source = param_source.ConstantSource

pySim/euicc.py

@@ -226,9 +226,28 @@ class Icon(BER_TLV_IE, tag=0x94):
     _construct = GreedyBytes
 class ProfileClass(BER_TLV_IE, tag=0x95):
     _construct = Enum(Int8ub, test=0, provisioning=1, operational=2)
+class ProfilePolicyRules(BER_TLV_IE, tag=0x99):
+    _construct = GreedyBytes
+class NotificationConfigurationInfo(BER_TLV_IE, tag=0xb6):
+    _construct = GreedyBytes
+
+# ProfileOwner
+class ProfileOwnerPLMN(BER_TLV_IE, tag=0x80):
+    _construct = PlmnAdapter(Bytes(3))
+class ProfileOwnerGID1(BER_TLV_IE, tag=0x81):
+    _construct = GreedyBytes
+class ProfileOwnerGID2(BER_TLV_IE, tag=0x82):
+    _construct = GreedyBytes
+class ProfileOwner(BER_TLV_IE, tag=0xb7, nested=[ProfileOwnerPLMN, ProfileOwnerGID1, ProfileOwnerGID2]):
+    _construct = GreedyBytes
+
+class SMDPPProprietaryData(BER_TLV_IE, tag=0xb8):
+    _construct = GreedyBytes
+
 class ProfileInfo(BER_TLV_IE, tag=0xe3, nested=[Iccid, IsdpAid, ProfileState, ProfileNickname,
                                                 ServiceProviderName, ProfileName, IconType, Icon,
-                                                ProfileClass]): # FIXME: more IEs
+                                                ProfileClass, ProfilePolicyRules, NotificationConfigurationInfo,
+                                                ProfileOwner, SMDPPProprietaryData]):
     pass
 class ProfileInfoSeq(BER_TLV_IE, tag=0xa0, nested=[ProfileInfo]):
     pass
@@ -444,9 +463,28 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             d = rn.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['notification_sent_resp']))
 
-        def do_get_profiles_info(self, _opts):
+        get_profiles_info_parser = argparse.ArgumentParser()
+        get_profiles_info_parser.add_argument('--all', action='store_true', help='Retrieve all known tags of a profile')
+
+        @cmd2.with_argparser(get_profiles_info_parser)
+        def do_get_profiles_info(self, opts):
             """Perform an ES10c GetProfilesInfo function."""
-            pi = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, ProfileInfoListReq(), ProfileInfoListResp)
+            if opts.all:
+                tags = [nest.tag for nest in ProfileInfo.nested_collection_cls().nested]
+                u8tags = []
+                # TODO: rework TagList to support 2 byte tags to not filter it into u8 tags
+                for tag in tags:
+                    if tag <= 255:
+                        u8tags.append(tag)
+                    elif tag <= 65535:
+                        u8tags.append(tag >> 8)
+                        u8tags.append(tag & 0xff)
+                    # Ignoring 3 byte tags
+                req = ProfileInfoListReq(children=[TagList(decoded=u8tags)])
+            else:
+                req = ProfileInfoListReq()
+
+            pi = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, req, ProfileInfoListResp)
             d = pi.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['profile_info_list_resp']))
 

pySim/exceptions.py

@@ -38,15 +38,16 @@ class SwMatchError(Exception):
     """Raised when an operation specifies an expected SW but the actual SW from
        the card doesn't match."""
 
-    def __init__(self, sw_actual: str, sw_expected: str, rs=None):
+    def __init__(self, sw_actual: str, sw_expected, rs=None):
         """
         Args:
                 sw_actual : the SW we actually received from the card (4 hex digits)
-                sw_expected : the SW we expected to receive from the card (4 hex digits)
+                sw_expected : the SW we expected to receive from the card (4 hex digits),
+                              or a list of acceptable ones
                 rs : interpreter class to convert SW to string
         """
         self.sw_actual = sw_actual
-        self.sw_expected = sw_expected
+        self.sw_expected = '/'.join(sw_expected) if isinstance(sw_expected, (list, tuple)) else sw_expected
         self.rs = rs
 
     @property

pySim/filesystem.py

@@ -30,6 +30,7 @@ import tempfile
 import json
 import abc
 import inspect
+import os
 
 import cmd2
 from cmd2 import CommandSet, with_default_category
@@ -43,6 +44,7 @@ from pySim.utils import sw_match, decomposeATR
 from pySim.jsonpath import js_path_modify
 from pySim.commands import SimCardCommands
 from pySim.exceptions import SwMatchError
+from pySim.log import PySimLogger
 
 # int: a single service is associated with this file
 # list: any of the listed services requires this file
@@ -51,6 +53,8 @@ CardFileService = Union[int, List[int], Tuple[int, ...]]
 
 Size = Tuple[int, Optional[int]]
 
+log = PySimLogger.get(__name__)
+
 class CardFile:
     """Base class for all objects in the smart card filesystem.
     Serve as a common ancestor to all other file types; rarely used directly.
@@ -552,6 +556,85 @@ class CardADF(CardDF):
         return lchan.selected_file.application.export(as_json, lchan)
 
 
+class JsonEditor:
+    """Context manager for editing a JSON-encoded EF value in an external editor.
+
+    Writes the current JSON value (plus encode/decode examples as //-comments)
+    to a temporary file, opens the user's editor, then reads the result back
+    (stripping comment lines) and returns it as the context variable::
+
+        with JsonEditor(self._cmd, orig_json, ef) as edited_json:
+            if edited_json != orig_json:
+                ...write back...
+    """
+    def __init__(self, cmd, orig_json, ef):
+        self._cmd = cmd
+        self._orig_json = orig_json
+        self._ef = ef
+        self._file = None
+
+    @staticmethod
+    def _strip_comments(text: str) -> str:
+        """Strip //-comment lines from text before JSON parsing."""
+        # TODO: also strip inline comments?
+        return '\n'.join(line for line in text.splitlines() if not line.lstrip().startswith('//'))
+
+    def _append_examples_as_comments(self, text_file) -> None:
+        """Append encode/decode test vectors as //-comment lines to an open file.
+        The examples are taken from _test_de_encode and _test_decode class
+        attributes (same source as the auto-generated filesystem documentation).
+        The comment block is intentionally ignored on read-back by _strip_comments."""
+        vectors = []
+        for attr in ('_test_de_encode', '_test_decode'):
+            v = getattr(type(self._ef), attr, None)
+            if v:
+                vectors.extend(v)
+        if not vectors:
+            return
+        ef = self._ef
+        parts = [ef.fully_qualified_path_str()]
+        if ef.fid:
+            parts.append(f'({ef.fid.upper()})')
+        if ef.desc:
+            parts.append(f'- {ef.desc}')
+        text_file.write(f'\n\n// {" ".join(parts)}\n')
+        text_file.write('// Examples (ignored on save):\n')
+        for t in vectors:
+            if len(t) >= 3:
+                encoded, record_nr, decoded = t[0], t[1], t[2]
+                text_file.write(f'// record {record_nr}: {encoded}\n')
+            else:
+                encoded, decoded = t[0], t[1]
+                text_file.write(f'// file: {encoded}\n')
+            for line in json.dumps(decoded, indent=4, cls=JsonEncoder).splitlines():
+                text_file.write(f'// {line}\n')
+
+    def __enter__(self) -> object:
+        """Write JSON + examples to a temp file, run the editor, return parsed result.
+
+        On JSONDecodeError the user is offered the option to re-open the file
+        and fix the mistake interactively.  The temp file is removed by __exit__()
+        on success, or when the user declines to retry."""
+        self._file = tempfile.NamedTemporaryFile(prefix='pysim_', suffix='.json',
+                                                 mode='w', delete=False)
+        json.dump(self._orig_json, self._file, indent=4, cls=JsonEncoder)
+        self._append_examples_as_comments(self._file)
+        self._file.close()
+        while True:
+            self._cmd.run_editor(self._file.name)
+            try:
+                with open(self._file.name, 'r') as f:
+                    return json.loads(self._strip_comments(f.read()))
+            except json.JSONDecodeError as e:
+                self._cmd.perror(f'Invalid JSON: {e}')
+                answer = self._cmd.read_input('Re-open file for editing? [y]es/[n]o: ')
+                if answer not in ('y', 'yes'):
+                    return self._orig_json
+
+    def __exit__(self, *args):
+        os.unlink(self._file.name)
+
+
 class CardEF(CardFile):
     """EF (Entry File) in the smart card filesystem"""
 
@@ -657,15 +740,8 @@ class TransparentEF(CardEF):
         def do_edit_binary_decoded(self, _opts):
             """Edit the JSON representation of the EF contents in an editor."""
             (orig_json, _sw) = self._cmd.lchan.read_binary_dec()
-            with tempfile.TemporaryDirectory(prefix='pysim_') as dirname:
-                filename = '%s/file' % dirname
-                # write existing data as JSON to file
-                with open(filename, 'w') as text_file:
-                    json.dump(orig_json, text_file, indent=4, cls=JsonEncoder)
-                # run a text editor
-                self._cmd.run_editor(filename)
-                with open(filename, 'r') as text_file:
-                    edited_json = json.load(text_file)
+            ef = self._cmd.lchan.selected_file
+            with JsonEditor(self._cmd, orig_json, ef) as edited_json:
                 if edited_json == orig_json:
                     self._cmd.poutput("Data not modified, skipping write")
                 else:
@@ -959,15 +1035,8 @@ class LinFixedEF(CardEF):
         def do_edit_record_decoded(self, opts):
             """Edit the JSON representation of one record in an editor."""
             (orig_json, _sw) = self._cmd.lchan.read_record_dec(opts.RECORD_NR)
-            with tempfile.TemporaryDirectory(prefix='pysim_') as dirname:
-                filename = '%s/file' % dirname
-                # write existing data as JSON to file
-                with open(filename, 'w') as text_file:
-                    json.dump(orig_json, text_file, indent=4, cls=JsonEncoder)
-                # run a text editor
-                self._cmd.run_editor(filename)
-                with open(filename, 'r') as text_file:
-                    edited_json = json.load(text_file)
+            ef = self._cmd.lchan.selected_file
+            with JsonEditor(self._cmd, orig_json, ef) as edited_json:
                 if edited_json == orig_json:
                     self._cmd.poutput("Data not modified, skipping write")
                 else:
@@ -1543,14 +1612,14 @@ class CardModel(abc.ABC):
         card_atr = scc.get_atr()
         for atr in cls._atrs:
             if atr == card_atr:
-                print("Detected CardModel:", cls.__name__)
+                log.info("Detected CardModel: %s", cls.__name__)
                 return True
         # if nothing found try to just compare the Historical Bytes of the ATR
         card_atr_hb = decomposeATR(card_atr)['hb']
         for atr in cls._atrs:
             atr_hb = decomposeATR(atr)['hb']
             if atr_hb == card_atr_hb:
-                print("Detected CardModel:", cls.__name__)
+                log.info("Detected CardModel: %s", cls.__name__)
                 return True
         return False
 

pySim/global_platform/__init__.py

@@ -18,10 +18,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
 import io
+import hashlib
 from copy import deepcopy
 from typing import Optional, List, Dict, Tuple
 from construct import Optional as COptional
 from construct import Struct, GreedyRange, FlagsEnum, Int16ub, Int24ub, Padding, Bit, Const
+from construct import Construct, stream_read, stream_write
 from Cryptodome.Random import get_random_bytes
 from Cryptodome.Cipher import DES, DES3, AES
 from osmocom.utils import *
@@ -35,6 +37,9 @@ from pySim.filesystem import *
 from pySim.profile import CardProfile
 from pySim.ota import SimFileAccessAndToolkitAppSpecParams
 from pySim.javacard import CapFile
+from pySim.log import PySimLogger
+
+log = PySimLogger.get(__name__)
 
 # GPCS Table 11-48 Load Parameter Tags
 class NonVolatileCodeMinMemoryReq(BER_TLV_IE, tag=0xC6):
@@ -148,6 +153,24 @@ sw_table = {
     },
 }
 
+class PutKeyLength(Construct):
+    """A length field of a PUT KEY data field, GP CardSpec v2.3.1 11.8.2.3.1
+    - all lengths ASN.1 BER-TLV (ITU-T X.690 Section 8.1.3)
+    - except that the length 128 may also be coded on one byte as '80' for backwards compatibility
+    80 does not introduce the indefinite form here which is unused in GP as far as i know.
+    That legacy form is accepted when parsing, but never generated, which agrees with the spec"""
+    def _parse(self, stream, context, path):
+        first = stream_read(stream, 1, path)[0]
+        if first <= 0x80:
+            return first
+        return int.from_bytes(stream_read(stream, first & 0x7f, path), 'big')
+
+    def _build(self, obj, stream, context, path):
+        data = bertlv_encode_len(obj)
+        stream_write(stream, data, len(data), path)
+        return obj
+
+
 # GlobalPlatform 2.1.1 Section 9.1.6
 KeyType = Enum(Byte,    des=0x80,
                         tls_psk=0x85,                           # v2.3.1 Section 11.1.8
@@ -276,7 +299,7 @@ class ListOfSupportedOptions(BER_TLV_IE, tag=0x81):
 class SupportedKeysForScp03(BER_TLV_IE, tag=0x82):
     _construct = FlagsEnum(Byte, aes128=0x01, aes192=0x02, aes256=0x04)
 class SupportedTlsCipherSuitesForScp81(BER_TLV_IE, tag=0x83):
-    _consuruct = GreedyRange(Int16ub)
+    _construct = GreedyRange(Int16ub)
 class ScpInformation(BER_TLV_IE, tag=0xa0, nested=[ScpType, ListOfSupportedOptions, SupportedKeysForScp03,
                                                    SupportedTlsCipherSuitesForScp81]):
     pass
@@ -319,7 +342,7 @@ class CurrentSecurityLevel(BER_TLV_IE, tag=0xd3):
 # GlobalPlatform v2.3.1 Section 11.3.3.1.3
 class ApplicationAID(BER_TLV_IE, tag=0x4f):
     _construct = GreedyBytes
-class ApplicationTemplate(BER_TLV_IE, tag=0x61, ntested=[ApplicationAID]):
+class ApplicationTemplate(BER_TLV_IE, tag=0x61, nested=[ApplicationAID]):
     pass
 class ListOfApplications(BER_TLV_IE, tag=0x2f00, nested=[ApplicationTemplate]):
     pass
@@ -512,6 +535,63 @@ class GpRegistryRelatedData(BER_TLV_IE, tag=0xe3, nested=[ApplicationAID, LifeCy
                                                           ExecutableModuleAID, AssociatedSecurityDomainAID]):
     pass
 
+# GP CS v2.3.1 Table 11-36/11-37 possible data objects requested/returned from GET STATUS for each registry entry.
+# Applications and Executable Load Files have _different_ sets, so a tag list requesting them has
+# to match the subset because 11.4.2.3 warns that asking for a data object an entry does not have
+# "may" be answered with an error status.
+GetStatusTagListIEs = {
+    # Table 11-36 GP Application Data
+    'isd':               [ApplicationAID, LifeCycleState, Privileges, ImplicitSelectionParameter,
+                          ExecutableLoadFileAID, AssociatedSecurityDomainAID],
+    'applications':      [ApplicationAID, LifeCycleState, Privileges, ImplicitSelectionParameter,
+                          ExecutableLoadFileAID, AssociatedSecurityDomainAID],
+    # Table 11-37 GP Executable Load File Data. 84 only for the subset that asks for the modules (Note 2)!
+    'files':             [ApplicationAID, LifeCycleState, ExecutableLoadFileVersionNumber,
+                          AssociatedSecurityDomainAID],
+    'files_and_modules': [ApplicationAID, LifeCycleState, ExecutableLoadFileVersionNumber,
+                          ExecutableModuleAID, AssociatedSecurityDomainAID],
+}
+
+def get_status_tag_list(subset: str) -> bytes:
+    """Encode the GET STATUS tag list for the given status subset"""
+    tags = b''.join([bertlv_encode_tag(ie.tag) for ie in GetStatusTagListIEs[subset]])
+    return b'\x5c' + bertlv_encode_len(len(tags)) + tags
+
+# GP CS v2.3.1 Appendix H.2 / Table H-1
+# oid prefix {iso(1) member-body(2) country-USA(840) globalPlatform(114283)} + card management type 2
+# afterwards GP version.
+OID_GP_CARD_MGMT_TYPE = h2b('2a864886fc6b02')
+
+def _find_tlv_value(decoded, key: str):
+    """depth first search for the nested decoded TLV_IE dict/list"""
+    if isinstance(decoded, dict):
+        for k, v in decoded.items():
+            if k == key:
+                return v
+            found = _find_tlv_value(v, key)
+            if found is not None:
+                return found
+    elif isinstance(decoded, list):
+        for item in decoded:
+            found = _find_tlv_value(item, key)
+            if found is not None:
+                return found
+    return None
+
+def decode_gp_version(card_data: bytes) -> Optional[Tuple[int, ...]]:
+    """GP version from Card Data returned by GET DATA, like (2, 1, 1) or (2, 2).
+    None if cm type OID is absent/unknown"""
+    cd = CardData()
+    cd.from_tlv(card_data)
+    ctv = _find_tlv_value(cd.to_dict(), 'card_management_type_and_version')
+    oid = _find_tlv_value(ctv, 'object_identifier') if ctv is not None else None
+    if oid is None:
+        return None
+    oid = h2b(oid) if isinstance(oid, str) else bytes(oid)
+    if not oid.startswith(OID_GP_CARD_MGMT_TYPE):
+        return None
+    return tuple(oid[len(OID_GP_CARD_MGMT_TYPE):])
+
 # Application Dedicated File of a Security Domain
 class ADF_SD(CardADF):
     StoreData = BitStruct('last_block'/Flag,
@@ -562,14 +642,14 @@ class ADF_SD(CardADF):
 
         @cmd2.with_argparser(store_data_parser)
         def do_store_data(self, opts):
-            """Perform the GlobalPlatform GET DATA command in order to store some card-specific data.
-            See GlobalPlatform CardSpecification v2.3Section 11.11 for details."""
+            """Perform the GlobalPlatform STORE DATA command in order to store some card-specific data.
+            See GlobalPlatform CardSpecification v2.3 Section 11.11 for details."""
             response_permitted = opts.response == 'may_be_returned'
             self.store_data(h2b(opts.DATA), opts.data_structure, opts.encryption, response_permitted)
 
         def store_data(self, data: bytes, structure:str = 'none', encryption:str = 'none', response_permitted: bool = False) -> bytes:
-            """Perform the GlobalPlatform GET DATA command in order to store some card-specific data.
-            See GlobalPlatform CardSpecification v2.3Section 11.11 for details."""
+            """Perform the GlobalPlatform STORE DATA command in order to store some card-specific data.
+            See GlobalPlatform CardSpecification v2.3 Section 11.11 for details."""
             max_cmd_len = self._cmd.lchan.scc.max_cmd_len
             # Table 11-89 of GP Card Specification v2.3
             remainder = data
@@ -585,7 +665,7 @@ class ADF_SD(CardADF):
                 data, _sw = self._cmd.lchan.scc.send_apdu_checksw(hdr + b2h(chunk) + "00")
                 block_nr += 1
                 response += data
-            return data
+            return h2b(response)
 
         put_key_parser = argparse.ArgumentParser()
         put_key_parser.add_argument('--old-key-version-nr', type=auto_uint8, default=0, help='Old Key Version Number')
@@ -602,8 +682,8 @@ class ADF_SD(CardADF):
             See GlobalPlatform CardSpecification v2.3 Section 11.8 for details.
 
             The KCV (Key Check Values) can either be explicitly specified using `--key-check`, or will
-            otherwise be automatically generated for DES and AES keys.  You can suppress the latter using
-            `--suppress-key-check`.
+            otherwise be automatically generated for DES, AES and TLS-PSK keys.  You can suppress the
+            latter using `--suppress-key-check`.
 
             Example (SCP80 KIC/KID/KIK):
                 put_key --key-version-nr 1 --key-id 0x01    --key-type aes --key-data 000102030405060708090a0b0c0d0e0f
@@ -620,33 +700,81 @@ class ADF_SD(CardADF):
             kdb = []
             for i in range(0, len(opts.key_type)):
                 if opts.key_check and len(opts.key_check) > i:
-                    kcv = opts.key_check[i]
+                    kcv = h2b(opts.key_check[i])
                 elif opts.suppress_key_check:
-                    kcv = ''
+                    kcv = b''
                 else:
-                    kcv_bin = compute_kcv(opts.key_type[i], h2b(opts.key_data[i])) or b''
-                    kcv = b2h(kcv_bin)
-                if self._cmd.lchan.scc.scp:
-                    # encrypted key data with DEK of current SCP
-                    kcb = b2h(self._cmd.lchan.scc.scp.encrypt_key(h2b(opts.key_data[i])))
-                else:
-                    # (for example) during personalization, DEK might not be required)
-                    kcb = opts.key_data[i]
-                kdb.append({'key_type': opts.key_type[i], 'kcb': kcb, 'kcv': kcv})
+                    kcv = compute_kcv(opts.key_type[i], h2b(opts.key_data[i])) or b''
+                kdb.append({'key_type': opts.key_type[i], 'clear_key': h2b(opts.key_data[i]), 'kcv': kcv})
             p2 = opts.key_id
             if len(opts.key_type) > 1:
                 p2 |= 0x80
             self.put_key(opts.old_key_version_nr, opts.key_version_nr, p2, kdb)
 
-        # Table 11-68: Key Data Field - Format 1 (Basic Format)
-        KeyDataBasic = GreedyRange(Struct('key_type'/KeyType,
-                                          'kcb'/Prefixed(Int8ub, GreedyBytes),
-                                          'kcv'/Prefixed(Int8ub, GreedyBytes)))
+        # Table 11-68: Key Data Field - Format 1 (Basic Format).  The key component block length is
+        # BER-TLV coded (Section 11.8.2.3.1), the key check value length is always '00' - '7F'.
+        KeyDataBasic = Struct('key_type'/KeyType,
+                              'kcb'/Prefixed(PutKeyLength(), GreedyBytes),
+                              'kcv'/Prefixed(Int8ub, GreedyBytes))
 
-        def put_key(self, old_kvn:int, kvn: int, kid: int, key_dict: dict) -> bytes:
+        @classmethod
+        def encode_key_data_basic(cls, key_type: str, kcb: bytes, kcv: bytes) -> bytes:
+            """Generic Basic key data field, GP CardSpec v2.3 Table 11-68):
+                tag || L1 || <maybe L2> KCB || <1-byte length> KCV"""
+            return cls.KeyDataBasic.build({'key_type': key_type, 'kcb': kcb, 'kcv': kcv})
+
+        @classmethod
+        def encode_key_data_psk(cls, clear_key: bytes, ciphered_key: bytes, kcv: bytes) -> bytes:
+            """Single PSK TLS '85' key data field per GP Amendment B 1.2, 3.9.1 / Table 3-13:
+                85 | L1 | <L2>  <ciphered PSK key> | <KCV length> | <KCV>
+            - framing is like Basic Format, but the kcb is always GP CardSpec Table 11-70
+            so always with the length of the clear text key value, even without padding!
+            - 'ciphered_key' is DEK(block-padded clear key), no additional length prefix."""
+            kcb = bertlv_encode_len(len(clear_key)) + ciphered_key
+            return cls.encode_key_data_basic('tls_psk', kcb, kcv)
+
+        @classmethod
+        def build_put_key_data(cls, kvn: int, keys: List[dict], scp) -> bytes:
+            """Assemble the PUT KEY data field, mixed PSK + DES DEK is supported:
+            - new KVN followed by one key data field per key.
+            - tls_psk keys per GP Amendment B
+            - other key types generic Basic format
+            Param 'keys' is a dict:
+            - 'key_type' (str)
+            - 'clear_key' (bytes)
+            - 'kcv' (bytes / empty).
+            'scp' may be None (e.g. during personalization, when the DEK may not be required)."""
+            key_data = kvn.to_bytes(1, 'big')
+            for k in keys:
+                clear = k['clear_key']
+                if k['key_type'] == 'tls_psk':
+                    # len always part of the data see CardSpec Table 11-70 vs Table 11-71
+                    if scp:
+                        ciphered = scp.dek_encrypt(scp.pad_to_blocksize(clear))
+                    else:
+                        ciphered = clear
+                    key_data += cls.encode_key_data_psk(clear, ciphered, k['kcv'])
+                else:
+                    if scp:
+                        ciphered = scp.encrypt_key(clear)
+                    else:
+                        # (for example) during personalization, DEK might not be required
+                        ciphered = clear
+                    key_data += cls.encode_key_data_basic(k['key_type'], ciphered, k['kcv'])
+            return key_data
+
+        def put_key(self, old_kvn:int, kvn: int, kid: int, keys: List[dict]) -> bytes:
             """Perform the GlobalPlatform PUT KEY command in order to store a new key on the card.
             See GlobalPlatform CardSpecification v2.3 Section 11.8 for details."""
-            key_data = kvn.to_bytes(1, 'big') + build_construct(ADF_SD.AddlShellCommands.KeyDataBasic, key_dict)
+            key_data = self.build_put_key_data(kvn, keys, self._cmd.lchan.scc.scp)
+            # Lc of Table 11-64 is a single byte, while LOAD or STORE DATA splits we can't:
+            # 11.8.2.3.3 splits a key at component boundaries -> not helping here
+            max_cmd_len = self._cmd.lchan.scc.max_cmd_len
+            if len(key_data) > max_cmd_len:
+                raise ValueError('key data field of %u bytes exceeds the maximum command length of %u '
+                                 '(limited by the overhead of the current secure channel); use fewer '
+                                 'keys per command, a single key component that large needs STORE DATA' %
+                                 (len(key_data), max_cmd_len))
             hdr = "80D8%02x%02x%02x" % (old_kvn, kid, len(key_data))
             data, _sw = self._cmd.lchan.scc.send_apdu_checksw(hdr + b2h(key_data) + "00")
             return data
@@ -665,26 +793,78 @@ class ADF_SD(CardADF):
             for grd in grd_list:
                 self._cmd.poutput_json(grd.to_dict())
 
+        def gp_version(self) -> Optional[Tuple[int, ...]]:
+            """GP version the selected SD reports in its Card Recognition
+            Data, e.g. (2, 1, 1). Card Recognition Data "shall be present" v2.1.1/v2.3.1 section 7.4.1.3,
+            so this must succeed no matter the GP version. None if card did not answer GET DATA / OID unknown.
+            Cached, it cannot change during a session."""
+            if not hasattr(self, '_gp_version'):
+                self._gp_version = None
+                try:
+                    data, _sw = self._cmd.lchan.scc.get_data(cla=0x80, tag=CardData.tag)
+                    self._gp_version = decode_gp_version(h2b(data))
+                except (SwMatchError, ValueError) as e:
+                    log.warning("Could not determine GlobalPlatform version: %s", e)
+            return self._gp_version
+
         def get_status(self, subset:str, aid_search_qualifier:Hexstr = '') -> List[GpRegistryRelatedData]:
-            subset_hex = b2h(build_construct(StatusSubset, subset))
             aid = ApplicationAID(decoded=aid_search_qualifier)
-            cmd_data = aid.to_tlv() + h2b('5c054f9f70c5cc')
-            p2 = 0x02 # TLV format according to Table 11-36
+            # GPC CardSpec v2.3.1 Table 11-35 says only the AID search tag is mandatory, tag list is
+            # Optional and not present in the older v2.1.1, where section 9.4.2.3 defines the data
+            # field as the search qualifier.
+            # Cards like the sja5 implementing that old GP version reject anything else with 6A80
+            # from v2.1.1 Table 9-26 so only send a tag list to a card that announces v2.2 or later.
+            #
+            # Not sending one is not a problem on older cards, the tag list only gives us data beyond
+            # what 11.4.3.1 gives us anyway, for example the associated SD AID which matters on an eUICC
+            # where entries belong to different SD.
+            version = self.gp_version()
+            log.debug("Card Recognition Data reports GlobalPlatform %s",
+                      '.'.join(str(v) for v in version) if version else 'unknown')
+            if version is not None and version >= (2, 2):
+                try:
+                    return self._get_status(subset, aid.to_tlv() + get_status_tag_list(subset))
+                except SwMatchError as e:
+                    # Retry if v2.2 or later but rejected the tag list anyway.
+                    # 6A80 and 6A88 are the error conditions GET STATUS defines in table 11-39.
+                    # Retrying beats not ending up with a list again...
+                    if e.sw_actual not in ('6a80', '6a88'):
+                        raise
+                    log.warning("Card reports GlobalPlatform %s but answered %s to the GET STATUS tag list; "
+                                "retrying with the default search",
+                                '.'.join(str(v) for v in version), e.sw_actual)
+            return self._get_status(subset, aid.to_tlv(), empty_on_6a88=True)
+
+        def _get_status(self, subset:str, cmd_data:bytes,
+                        empty_on_6a88: bool = False) -> List[GpRegistryRelatedData]:
+            subset_hex = b2h(build_construct(StatusSubset, subset))
+            p2 = 0x02 # GPC v2.3.1 11.4.2.2 table 11-34, b2: response data structure per table 11-36
             grd_list = []
             while True:
                 hdr = "80F2%s%02x%02x" % (subset_hex, p2, len(cmd_data))
                 data, sw = self._cmd.lchan.scc.send_apdu(hdr + b2h(cmd_data) + "00")
+                if sw == '6a88':
+                    # Table 11-39 "Referenced data not found". After collecting all pages this can
+                    # only mean "nothing more matches" -> listing is complete. On the first page
+                    # it is ambiguous, empty result or bad command data field, so leave that to get_status()
+                    # which knows if a tag list was sent.
+                    if grd_list or empty_on_6a88:
+                        return grd_list
+                    raise SwMatchError(sw, ['9000', '6310'])
+                if sw not in ['9000', '6310']:
+                    # Never return a silently truncated registry
+                    raise SwMatchError(sw, ['9000', '6310'])
                 remainder = h2b(data)
                 while len(remainder):
                     # tlv sequence, each element is one GpRegistryRelatedData()
                     grd = GpRegistryRelatedData()
                     _dec, remainder = grd.from_tlv(remainder)
                     grd_list.append(grd)
-                if sw != '6310':
+                if sw == '9000':
                     return grd_list
-                else:
-                    p2 |= 0x01
-            return grd_list
+                # 6310 = more data available, table 11-38: reissue as get next occurrence(s), b1 of
+                # table 11-34. Keeps b2 unchanged.
+                p2 |= 0x01
 
         set_status_parser = argparse.ArgumentParser()
         set_status_parser.add_argument('scope', choices=list(SetStatusScope.ksymapping.values()),
@@ -826,23 +1006,32 @@ class ADF_SD(CardADF):
         load_parser_from_grp.add_argument('--from-hex', type=is_hexstr, help='load from hex string')
         load_parser_from_grp.add_argument('--from-file', type=argparse.FileType('rb', 0), help='load from binary file')
         load_parser_from_grp.add_argument('--from-cap-file', type=argparse.FileType('rb', 0), help='load from JAVA-card CAP file')
+        load_parser.add_argument('--chunk-len', type=auto_uint8, default=None,
+                                 help='Block size for the LOAD command; default: as large as the current secure channel overhead permits, at most 240')
 
         @cmd2.with_argparser(load_parser)
         def do_load(self, opts):
             """Perform a GlobalPlatform LOAD command. (We currently only support loading without DAP and
             without ciphering.)"""
             if opts.from_hex is not None:
-                self.load(h2b(opts.from_hex))
+                self.load(h2b(opts.from_hex), opts.chunk_len)
             elif opts.from_file is not None:
-                self.load(opts.from_file.read())
+                self.load(opts.from_file.read(), opts.chunk_len)
             elif opts.from_cap_file is not None:
                 cap = CapFile(opts.from_cap_file)
-                self.load(cap.get_loadfile())
+                self.load(cap.get_loadfile(), opts.chunk_len)
             else:
                 raise ValueError('load source not specified!')
 
-        def load(self, contents:bytes, chunk_len:int = 240):
-            # TODO:tune chunk_len based on the overhead of the used SCP?
+        def load(self, contents:bytes, chunk_len:Optional[int] = None):
+            # scc.max_cmd_len knows the overhead the currently active SCP
+            # 240 is the old default, keep it for now.
+            max_chunk_len = self._cmd.lchan.scc.max_cmd_len
+            if chunk_len is None:
+                chunk_len = min(240, max_chunk_len)
+            elif not 1 <= chunk_len <= max_chunk_len:
+                raise ValueError('chunk_len must be in range 1..%u (limited by the overhead of the current secure channel)' %
+                                 max_chunk_len)
             # build TLV according to GPC_SPE_034 section 11.6.2.3 / Table 11-58 for unencrypted case
             remainder = b'\xC4' + bertlv_encode_len(len(contents)) + contents
             # transfer this in various chunks to the card
@@ -859,22 +1048,30 @@ class ADF_SD(CardADF):
                 _rsp_hex, _sw = self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)
             self._cmd.poutput("Loaded a total of %u bytes in %u blocks. Don't forget install_for_install (and make selectable) now!" % (total_size, block_nr))
 
-        install_cap_parser = argparse.ArgumentParser()
+        install_cap_parser = argparse.ArgumentParser(usage='%(prog)s FILE [--install-parameters | --install-parameters-*]')
         install_cap_parser.add_argument('cap_file', type=str, metavar='FILE',
                                         help='JAVA-CARD CAP file to install')
-        install_cap_parser_inst_prm_g = install_cap_parser.add_mutually_exclusive_group()
-        install_cap_parser_inst_prm_g.add_argument('--install-parameters', type=is_hexstr, default=None,
-                                                   help='install Parameters (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
-        install_cap_parser_inst_prm_g_grp = install_cap_parser_inst_prm_g.add_argument_group()
-        install_cap_parser_inst_prm_g_grp.add_argument('--install-parameters-volatile-memory-quota',
-                                                       type=int, default=None,
-                                                       help='volatile memory quota (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
-        install_cap_parser_inst_prm_g_grp.add_argument('--install-parameters-non-volatile-memory-quota',
-                                                       type=int, default=None,
-                                                       help='non volatile memory quota (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
-        install_cap_parser_inst_prm_g_grp.add_argument('--install-parameters-stk',
-                                                       type=is_hexstr, default=None,
-                                                       help='Load Parameters (ETSI TS 102 226, section 8.2.1.3.2.1)')
+        # Ideally, the parser should enforce that:
+        #  * either the `--install-parameters` is given alone,
+        #  * or distinct `--install-parameters-*` are optionally given instead.
+        # We tried to achieve this using mutually exclusive groups (add_mutually_exclusive_group).
+        # However, group nesting was never supported, often failed to work correctly, and was unintentionally
+        # exposed through inheritance.  It has been deprecated since version 3.11, removed in version 3.14.
+        # Hence, we have to implement the enforcement manually.
+        install_cap_parser_inst_prm_grp = install_cap_parser.add_argument_group('Install Parameters')
+        install_cap_parser_inst_prm_grp.add_argument('--install-parameters', type=is_hexstr, default=None,
+                                                     help='install Parameters (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        install_cap_parser_inst_prm_grp.add_argument('--install-parameters-volatile-memory-quota',
+                                                     type=int, default=None,
+                                                     help='volatile memory quota (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        install_cap_parser_inst_prm_grp.add_argument('--install-parameters-non-volatile-memory-quota',
+                                                     type=int, default=None,
+                                                     help='non volatile memory quota (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        install_cap_parser_inst_prm_grp.add_argument('--install-parameters-stk',
+                                                     type=is_hexstr, default=None,
+                                                     help='Load Parameters (ETSI TS 102 226, section 8.2.1.3.2.1)')
+        install_cap_parser.add_argument('--chunk-len', type=auto_uint8, default=None,
+                                        help='Block size for the LOAD command; default: as large as the current secure channel overhead permits, at most 240')
 
         @cmd2.with_argparser(install_cap_parser)
         def do_install_cap(self, opts):
@@ -888,9 +1085,17 @@ class ADF_SD(CardADF):
             load_file_aid = cap.get_loadfile_aid()
             module_aid = cap.get_applet_aid()
             application_aid = module_aid
-            if opts.install_parameters:
+            if opts.install_parameters is not None:
+                # `--install-parameters` and `--install-parameters-*` are mutually exclusive
+                # make sure that none of `--install-parameters-*` is given; abort otherwise
+                if any(p is not None for p in [opts.install_parameters_non_volatile_memory_quota,
+                                               opts.install_parameters_volatile_memory_quota,
+                                               opts.install_parameters_stk]):
+                    self.install_cap_parser.error('arguments --install-parameters-* are '
+                                                  'not allowed with --install-parameters')
                 install_parameters = opts.install_parameters;
             else:
+                # `--install-parameters-*` are all optional
                 install_parameters = gen_install_parameters(opts.install_parameters_non_volatile_memory_quota,
                                                             opts.install_parameters_volatile_memory_quota,
                                                             opts.install_parameters_stk)
@@ -905,7 +1110,7 @@ class ADF_SD(CardADF):
             self._cmd.poutput("step #1: install for load...")
             self.do_install_for_load("--load-file-aid %s --security-domain-aid %s" % (load_file_aid, security_domain_aid))
             self._cmd.poutput("step #2: load...")
-            self.load(load_file)
+            self.load(load_file, opts.chunk_len)
             self._cmd.poutput("step #3: install_for_install (and make selectable)...")
             self.do_install_for_install("--load-file-aid %s --module-aid %s --application-aid %s --install-parameters %s --make-selectable" %
                                         (load_file_aid, module_aid, application_aid, install_parameters))
@@ -1051,10 +1256,16 @@ def compute_kcv_aes(key:bytes) -> bytes:
     cipher = AES.new(key, AES.MODE_ECB)
     return cipher.encrypt(plaintext)
 
+def compute_kcv_psk(key:bytes) -> bytes:
+    # GP Amendment B v1.2, 3.9.1 / Table 3-13
+    # KCV of a PSK TLS key is the 3 highest-order bytes of the SHA-1 digest of the clear key value.
+    return hashlib.sha1(key).digest()
+
 # dict is keyed by the string name of the KeyType enum above in this file
 KCV_CALCULATOR = {
         'aes': compute_kcv_aes,
         'des': compute_kcv_des,
+        'tls_psk': compute_kcv_psk,
     }
 
 def compute_kcv(key_type: str, key: bytes) -> Optional[bytes]:

pySim/global_platform/install_param.py

@@ -17,6 +17,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from typing import Optional
+
 from osmocom.construct import *
 from osmocom.utils import *
 from osmocom.tlv import *
@@ -46,7 +48,9 @@ class InstallParams(TLV_IE_Collection, nested=[AppSpecificParams, SystemSpecific
     # GPD_SPE_013, table 11-49
     pass
 
-def gen_install_parameters(non_volatile_memory_quota:int, volatile_memory_quota:int, stk_parameter:str):
+def gen_install_parameters(non_volatile_memory_quota: Optional[int] = None,
+                           volatile_memory_quota: Optional[int] = None,
+                           stk_parameter: Optional[str] = None):
 
     # GPD_SPE_013, table 11-49
 
@@ -54,19 +58,17 @@ def gen_install_parameters(non_volatile_memory_quota:int, volatile_memory_quota:
     install_params = InstallParams()
     install_params_dict = [{'app_specific_params': None}]
 
-    #Conditional
-    if non_volatile_memory_quota and volatile_memory_quota and stk_parameter:
-        system_specific_params = []
-        #Optional
-        if non_volatile_memory_quota:
-            system_specific_params += [{'non_volatile_memory_quota': non_volatile_memory_quota}]
-        #Optional
-        if volatile_memory_quota:
-            system_specific_params += [{'volatile_memory_quota': volatile_memory_quota}]
-        #Optional
-        if stk_parameter:
-            system_specific_params += [{'stk_parameter': stk_parameter}]
-        install_params_dict += [{'system_specific_params': system_specific_params}]
+    # Collect system specific parameters (optional)
+    system_specific_params = []
+    if non_volatile_memory_quota is not None:
+        system_specific_params.append({'non_volatile_memory_quota': non_volatile_memory_quota})
+    if volatile_memory_quota is not None:
+        system_specific_params.append({'volatile_memory_quota': volatile_memory_quota})
+    if stk_parameter is not None:
+        system_specific_params.append({'stk_parameter': stk_parameter})
+    # Add system specific parameters to the install parameters, if any
+    if system_specific_params:
+        install_params_dict.append({'system_specific_params': system_specific_params})
 
     install_params.from_dict(install_params_dict)
     return b2h(install_params.to_bytes())

pySim/global_platform/scp.py

@@ -27,9 +27,9 @@ from osmocom.utils import b2h
 from osmocom.tlv import bertlv_parse_len, bertlv_encode_len
 from pySim.utils import parse_command_apdu
 from pySim.secure_channel import SecureChannel
+from pySim.log import PySimLogger
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
+log = PySimLogger.get(__name__)
 
 def scp02_key_derivation(constant: bytes, counter: int, base_key: bytes) -> bytes:
     assert len(constant) == 2
@@ -75,7 +75,7 @@ class Scp02SessionKeys:
             h = e.encrypt(strxor(h, bytes(padded_data[8*i:8*(i+1)])))
         h = d.decrypt(h)
         h = e.encrypt(h)
-        logger.debug("mac_1des(%s,icv=%s) -> %s", b2h(data), b2h(icv), b2h(h))
+        log.debug("mac_1des(%s,icv=%s) -> %s", b2h(data), b2h(icv), b2h(h))
         if self.des_icv_enc:
             self.icv = self.des_icv_enc.encrypt(h)
         else:
@@ -89,7 +89,7 @@ class Scp02SessionKeys:
         h = b'\x00' * 8
         for i in range(q):
             h = e.encrypt(strxor(h, bytes(padded_data[8*i:8*(i+1)])))
-        logger.debug("mac_3des(%s) -> %s", b2h(data), b2h(h))
+        log.debug("mac_3des(%s) -> %s", b2h(data), b2h(h))
         return h
 
     def __init__(self, counter: int, card_keys: 'GpCardKeyset', icv_encrypt=True):
@@ -182,6 +182,29 @@ class SCP(SecureChannel, abc.ABC):
         """Should we perform R-ENC?"""
         return self.security_level & 0x20
 
+    @property
+    @abc.abstractmethod
+    def mac_len(self) -> int:
+        """Length of the appended C-MAC, to be provided by derived class."""
+
+    @property
+    def overhead(self) -> int:
+        """Worst-case len that wrapping a command APDU adds to its data field at the
+        current sec level is (255 - overhead), C-MAC + C-DECRYPTION encryption padding."""
+        if not self.do_cmac:
+            return 0
+        if not self.do_cenc:
+            return self.mac_len
+        # see Secure Channel Protocol '03' Card Specification v2.3 - Amendment D v1.1.2
+        # which defers to GPCS v2.3 Section B.2 which then defers to
+        # NIST SP 800-38B for encryption and points out that
+        # the padding is, as expected, just the usual padding from NIST SP 800-38A
+        # C-DECRYPTION pads with ('80'+['00'...] at least 1 byte) up to
+        # the cipher block size + C-MAC on top -> largest usable data field
+        # is one byte less than the largest block-size multiple within 255 - mac_len.
+        bs = self.sk.blocksize
+        return 255 - ((255 - self.mac_len) // bs * bs - 1)
+
     def __str__(self) -> str:
         return "%s[%02x]" % (self.__class__.__name__, self.security_level)
 
@@ -215,11 +238,20 @@ class SCP(SecureChannel, abc.ABC):
     def gen_ext_auth_apdu(self, security_level: int = 0x01) -> bytes:
         pass
 
+    def pad_to_blocksize(self, data: bytes) -> bytes:
+        """Right pad the data with zero bytes to a multiple of the DEK cipher block size."""
+        if len(data) % self.sk.blocksize:
+            # not '+=' which would mutate the callers bytearray in place..
+            data = data + b'\x00' * (self.sk.blocksize - len(data) % self.sk.blocksize)
+        return data
+
     def encrypt_key(self, key: bytes) -> bytes:
         """Encrypt a key with the DEK."""
-        num_pad = len(key) % self.sk.blocksize
-        if num_pad:
-            return bertlv_encode_len(len(key)) + self.dek_encrypt(key + b'\x00'*num_pad)
+        if len(key) % self.sk.blocksize:
+            # The kcv is right padded before encryption and the kcb
+            # is formatted as described in Table 11-70: preceded by the actual length of the
+            # clear text kcv.
+            return bertlv_encode_len(len(key)) + self.dek_encrypt(self.pad_to_blocksize(key))
         return self.dek_encrypt(key)
 
     def decrypt_key(self, encrypted_key:bytes) -> bytes:
@@ -232,9 +264,8 @@ class SCP(SecureChannel, abc.ABC):
             # Block provides the actual length of the key component value, which allows recovering the
             # clear-text key component value after decryption of the encrypted key component value and removal
             # of padding bytes.
-            decrypted = self.dek_decrypt(encrypted_key)
-            key_len, remainder = bertlv_parse_len(decrypted)
-            return remainder[:key_len]
+            key_len, remainder = bertlv_parse_len(encrypted_key)
+            return self.dek_decrypt(remainder)[:key_len]
         else:
             # If the length of the Key Component Block is a multiple of the block size of the encryption
             # algorithm (i.e.  8 bytes for DES, 16 bytes for AES), then it shall be assumed that no padding
@@ -260,10 +291,8 @@ class SCP02(SCP):
     # Key Version Number 0x70 is a non-spec special-case of sysmoISIM-SJA2/SJA5 and possibly more sysmocom products
     # Key Version Number 0x01 is a non-spec special-case of sysmoUSIM-SJS1
     kvn_ranges = [[0x01, 0x01], [0x20, 0x2f], [0x70, 0x70]]
-
-    def __init__(self, *args, **kwargs):
-        self.overhead = 8
-        super().__init__(*args, **kwargs)
+    # C-MAC (Single DES + final 3DES, B.1.2.2) is always one full DES block
+    mac_len = 8
 
     def dek_encrypt(self, plaintext:bytes) -> bytes:
         # See also GPC section B.1.1.2, E.4.7, and E.4.1
@@ -276,10 +305,10 @@ class SCP02(SCP):
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self, card_challenge: bytes, host_challenge: bytes):
-        logger.debug("host_challenge(%s), card_challenge(%s)", b2h(host_challenge), b2h(card_challenge))
+        log.debug("host_challenge(%s), card_challenge(%s)", b2h(host_challenge), b2h(card_challenge))
         self.host_cryptogram = self.sk.calc_mac_3des(self.sk.counter.to_bytes(2, 'big') + card_challenge + host_challenge)
         self.card_cryptogram = self.sk.calc_mac_3des(self.host_challenge + self.sk.counter.to_bytes(2, 'big') + card_challenge)
-        logger.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
+        log.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
 
     def gen_init_update_apdu(self, host_challenge: bytes = b'\x00'*8) -> bytes:
         """Generate INITIALIZE UPDATE APDU."""
@@ -291,7 +320,7 @@ class SCP02(SCP):
         resp = self.constr_iur.parse(resp_bin)
         self.card_challenge = resp['card_challenge']
         self.sk = Scp02SessionKeys(resp['seq_counter'], self.card_keys)
-        logger.debug(self.sk)
+        log.debug(self.sk)
         self._compute_cryptograms(self.card_challenge, self.host_challenge)
         if self.card_cryptogram != resp['card_cryptogram']:
             raise ValueError("card cryptogram doesn't match")
@@ -311,7 +340,7 @@ class SCP02(SCP):
 
     def _wrap_cmd_apdu(self, apdu: bytes, *args, **kwargs) -> bytes:
         """Wrap Command APDU for SCP02: calculate MAC and encrypt."""
-        logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
+        log.debug("wrap_cmd_apdu(%s)", b2h(apdu))
 
         if not self.do_cmac:
             return apdu
@@ -338,10 +367,16 @@ class SCP02(SCP):
             # CMAC on modified APDU
             mlc = lc + 8
             clac = cla | CLA_SM
+        if mlc >= 256:
+            raise ValueError('Modified Lc (%u) would exceed maximum when appending 8 bytes of mac' % mlc)
         mac = self.sk.calc_mac_1des(bytes([clac]) + apdu[1:4] + bytes([mlc]) + data)
         if self.do_cenc:
+            padded_data = pad80(data, 8)
+            if len(padded_data) + 8 >= 256:
+                raise ValueError('Modified Lc (%u) would exceed maximum when appending padding and mac' %
+                                 (len(padded_data) + 8))
             k = DES3.new(self.sk.enc, DES.MODE_CBC, b'\x00'*8)
-            data = k.encrypt(pad80(data, 8))
+            data = k.encrypt(padded_data)
             lc = len(data)
 
         lc += 8
@@ -378,7 +413,7 @@ def scp03_key_derivation(constant: bytes, context: bytes, base_key: bytes, l: Op
     if l is None:
         l = len(base_key) * 8
 
-    logger.debug("scp03_kdf(constant=%s, context=%s, base_key=%s, l=%u)", b2h(constant), b2h(context), b2h(base_key), l)
+    log.debug("scp03_kdf(constant=%s, context=%s, base_key=%s, l=%u)", b2h(constant), b2h(context), b2h(base_key), l)
     output_len = l // 8
     # SCP03 Section 4.1.5 defines a different parameter order than NIST SP 800-108, so we cannot use the
     # existing Cryptodome.Protocol.KDF.SP800_108_Counter function :(
@@ -438,7 +473,7 @@ class Scp03SessionKeys:
         """Obtain the ICV value computed as described in 6.2.6.
         This method has two modes:
             * is_response=False for computing the ICV for C-ENC. Will pre-increment the counter.
-            * is_response=False for computing the ICV for R-DEC."""
+            * is_response=True for computing the ICV for R-DEC."""
         if not is_response:
             self.block_nr += 1
         # The binary value of this number SHALL be left padded with zeroes to form a full block.
@@ -451,7 +486,7 @@ class Scp03SessionKeys:
         # This block SHALL be encrypted with S-ENC to produce the ICV for command encryption.
         cipher = AES.new(self.s_enc, AES.MODE_CBC, iv)
         icv = cipher.encrypt(data)
-        logger.debug("_get_icv(data=%s, is_resp=%s) -> icv=%s", b2h(data), is_response, b2h(icv))
+        log.debug("_get_icv(data=%s, is_resp=%s) -> icv=%s", b2h(data), is_response, b2h(icv))
         return icv
 
     # TODO: Resolve duplication with pySim.esim.bsp.BspAlgoCryptAES128 which provides pad80-wrapping
@@ -477,9 +512,13 @@ class SCP03(SCP):
 
     def __init__(self, *args, **kwargs):
         self.s_mode = kwargs.pop('s_mode', 8)
-        self.overhead = self.s_mode
         super().__init__(*args, **kwargs)
 
+    @property
+    def mac_len(self) -> int:
+        # C-MAC truncated to 8 in S8 or 16 bytes in S16 mode
+        return self.s_mode
+
     def dek_encrypt(self, plaintext:bytes) -> bytes:
         cipher = AES.new(self.card_keys.dek, AES.MODE_CBC, b'\x00'*16)
         return cipher.encrypt(plaintext)
@@ -489,12 +528,12 @@ class SCP03(SCP):
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self):
-        logger.debug("host_challenge(%s), card_challenge(%s)", b2h(self.host_challenge), b2h(self.card_challenge))
+        log.debug("host_challenge(%s), card_challenge(%s)", b2h(self.host_challenge), b2h(self.card_challenge))
         # Card + Host Authentication Cryptogram: Section 6.2.2.2 + 6.2.2.3
         context = self.host_challenge + self.card_challenge
         self.card_cryptogram = scp03_key_derivation(self.sk.DERIV_CONST_AUTH_CGRAM_CARD, context, self.sk.s_mac, l=self.s_mode*8)
         self.host_cryptogram = scp03_key_derivation(self.sk.DERIV_CONST_AUTH_CGRAM_HOST, context, self.sk.s_mac, l=self.s_mode*8)
-        logger.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
+        log.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
 
     def gen_init_update_apdu(self, host_challenge: Optional[bytes] = None) -> bytes:
         """Generate INITIALIZE UPDATE APDU."""
@@ -514,7 +553,7 @@ class SCP03(SCP):
         self.i_param = resp['i_param']
         # derive session keys and compute cryptograms
         self.sk = Scp03SessionKeys(self.card_keys, self.host_challenge, self.card_challenge)
-        logger.debug(self.sk)
+        log.debug(self.sk)
         self._compute_cryptograms()
         # verify computed cryptogram matches received cryptogram
         if self.card_cryptogram != resp['card_cryptogram']:
@@ -529,7 +568,7 @@ class SCP03(SCP):
 
     def _wrap_cmd_apdu(self, apdu: bytes, skip_cenc: bool = False) -> bytes:
         """Wrap Command APDU for SCP03: calculate MAC and encrypt."""
-        logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
+        log.debug("wrap_cmd_apdu(%s)", b2h(apdu))
 
         if not self.do_cmac:
             return apdu
@@ -584,7 +623,7 @@ class SCP03(SCP):
         # status word: in this case only the status word shall be returned in the response. All status words
         # except '9000' and warning status words (i.e. '62xx' and '63xx') shall be interpreted as error status
         # words.
-        logger.debug("unwrap_rsp_apdu(sw=%s, rsp_apdu=%s)", sw, rsp_apdu)
+        log.debug("unwrap_rsp_apdu(sw=%s, rsp_apdu=%s)", sw, rsp_apdu)
         if not self.do_rmac:
             assert not self.do_renc
             return rsp_apdu
@@ -600,9 +639,9 @@ class SCP03(SCP):
         if self.do_renc:
             # decrypt response data
             decrypted = self.sk._decrypt(response_data)
-            logger.debug("decrypted: %s", b2h(decrypted))
+            log.debug("decrypted: %s", b2h(decrypted))
             # remove padding
             response_data = unpad80(decrypted)
-            logger.debug("response_data: %s", b2h(response_data))
+            log.debug("response_data: %s", b2h(response_data))
 
         return response_data

pySim/global_platform/uicc.py

@@ -91,6 +91,7 @@ class UiccSdInstallParams(TLV_IE_Collection, nested=[UiccScp, AcceptExtradAppsAn
 
 # Key Usage:
 # KVN 0x01 .. 0x0F reserved for SCP80
+# KVN 0x81 .. 0x8f reserved for SCP81
 # KVN 0x11 reserved for DAP specified in ETSI TS 102 226
 # KVN 0x20 .. 0x2F reserved for SCP02
 #   KID 0x01 = ENC; 0x02 = MAC; 0x03 = DEK

pySim/legacy/cards.py

@@ -152,7 +152,8 @@ class SimCard(SimCardBase):
         return sw
 
     def update_smsp(self, smsp):
-        data, sw = self._scc.update_record(EF['SMSP'], 1, rpad(smsp, 84))
+        print("using update_smsp")
+        data, sw = self._scc.update_record(EF['SMSP'], 1, smsp, leftpad=True)
         return sw
 
     def update_ad(self, mnc=None, opmode=None, ofm=None, path=EF['AD']):

pySim/log.py

@@ -24,7 +24,16 @@
 #
 
 import logging
-from cmd2 import style
+import enum
+import cmd2
+from packaging import version
+
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
+    from cmd2 import stylize as _stylize # pylint: disable=no-name-in-module
+    def _style(text, fg=None): # pylint: disable=function-redefined
+        return _stylize(text, fg) if fg else text
+else: # cmd2>=2.6.2
+    from cmd2 import style as _style # pylint: disable=no-name-in-module
 
 class _PySimLogHandler(logging.Handler):
     def __init__(self, log_callback):
@@ -44,7 +53,7 @@ class PySimLogger:
     """
 
     LOG_FMTSTR = "%(levelname)s: %(message)s"
-    LOG_FMTSTR_VERBOSE = "%(module)s.%(lineno)d -- " + LOG_FMTSTR
+    LOG_FMTSTR_VERBOSE = "%(name)s.%(lineno)d -- " + LOG_FMTSTR
     __formatter = logging.Formatter(LOG_FMTSTR)
     __formatter_verbose = logging.Formatter(LOG_FMTSTR_VERBOSE)
 
@@ -63,7 +72,7 @@ class PySimLogger:
         raise RuntimeError('static class, do not instantiate')
 
     @staticmethod
-    def setup(print_callback = None, colors:dict = {}):
+    def setup(print_callback = None, colors:dict = {}, verbose_debug:bool = False):
         """
         Set a print callback function and color scheme. This function call is optional. In case this method is not
         called, default settings apply.
@@ -72,10 +81,20 @@ class PySimLogger:
                              have the following format: print_callback(message:str)
             colors : An optional dict through which certain log levels can be assigned a color.
                      (e.g. {logging.WARN: YELLOW})
+            verbose_debug: Enable verbose logging and set the loglevel DEBUG when set to true. Otherwise the
+                           non-verbose logging is used and the loglevel is set to INFO. This setting can be changed
+                           using the set_verbose and set_level methods at any time.
         """
         PySimLogger.print_callback = print_callback
         PySimLogger.colors = colors
 
+        if (verbose_debug):
+            PySimLogger.set_verbose(True)
+            PySimLogger.set_level(logging.DEBUG)
+        else:
+            PySimLogger.set_verbose(False)
+            PySimLogger.set_level(logging.INFO)
+
     @staticmethod
     def set_verbose(verbose:bool = False):
         """
@@ -108,10 +127,10 @@ class PySimLogger:
                 formatted_message = logging.Formatter.format(PySimLogger.__formatter, record)
             color = PySimLogger.colors.get(record.levelno)
             if color:
-                if isinstance(color, str):
+                if isinstance(color, str) and not isinstance(color, enum.Enum):
                     PySimLogger.print_callback(color + formatted_message + "\033[0m")
                 else:
-                    PySimLogger.print_callback(style(formatted_message, fg = color))
+                    PySimLogger.print_callback(_style(formatted_message, fg = color))
             else:
                 PySimLogger.print_callback(formatted_message)
 

pySim/ota.py

@@ -18,10 +18,12 @@
 import zlib
 import abc
 import struct
-from typing import Optional, Tuple
-from construct import Enum, Int8ub, Int16ub, Struct, BitsInteger, BitStruct
+from typing import Optional, Tuple, List, Union
+from construct import ConstructError, Enum, Int8ub, Int16ub, Struct, BitsInteger, BitStruct
 from construct import Flag, Padding, Switch, this, PrefixedArray, GreedyRange
+from construct import Const, Prefixed, Select, Construct, SizeofError, stream_read, stream_write
 from osmocom.construct import *
+from osmocom.tlv import bertlv_encode_len
 from osmocom.utils import b2h
 
 from pySim.sms import UserDataHeader
@@ -56,6 +58,217 @@ CompactRemoteResp = Struct('number_of_commands'/Int8ub,
                            'last_status_word'/HexAdapter(Bytes(2)),
                            'last_response_data'/HexAdapter(GreedyBytes))
 
+######################################################################
+# Expanded Remote Application data format, ETSI TS 102 226 V19.0.0 (2025-11) Section 5.2
+#   5.2.1   Expanded Remote command structure
+#   5.2.1.1 C-APDU TLV
+#   5.2.1.2 Immediate Action TLV
+#   5.2.1.3 Error Action TLV
+#   5.2.1.4 Script Chaining TLV
+#   5.2.2   Expanded Remote response structure (tables 5.10 .. 5.16)
+#
+# definite length coding and indefinite length coding are supported.
+#
+# BER-TLV tag values from ETSI TS 101 220 V19.0.0 tables 7.18, 7.19, 7.20
+# C-APDU / R-APDU ETSI TS 102 223 Section 8.35 + 8.36
+# inside these the CR flag of the tag is 0 (TS 101 220 tables 7.19/7.20),
+# so tag bytes are 22 and 23 and not A2/A3.
+#
+# This layer sits above the TS 102 225 security layer.
+######################################################################
+
+class BerTlvLength(Construct):
+    """A definite-length BER-TLV length field used by the "expanded remote
+    application data format" from ISO/IEC 8825-1 referenced by TS 102 226 5.2
+
+    - short form (0..127 -> single octet)
+    - long form (128.. -> 0x8N followed by N length octets)
+    Indefinite length coding (first octet 0x80, TS 102 226 tables 5.2a/5.10a)
+    is omitted here because it is only recommended for HTTPS/CoAP transport, not SMS."""
+    def _parse(self, stream, context, path):
+        first = stream_read(stream, 1, path)[0]
+        if first < 0x80:
+            return first
+        num_octets = first & 0x7f
+        if num_octets == 0:
+            raise NotImplementedError('indefinite coding is not supported')
+        return int.from_bytes(stream_read(stream, num_octets, path), 'big')
+
+    def _build(self, obj, stream, context, path):
+        encoded = bertlv_encode_len(obj)
+        stream_write(stream, encoded, len(encoded), path)
+        return obj
+
+    def _sizeof(self, context, path):
+        raise SizeofError('BER-TLV length has a variable size?!')
+
+BerTlvLen = BerTlvLength()
+
+class _RApduValueAdapter(Adapter):
+    """Split/join value of R-APDU COMPREHENSION-TLV TS 102 223 8.36
+      [R-APDU data (x-2 bytes)] SW1 SW2."""
+    def _decode(self, obj, context, path):
+        raw = bytes(obj)
+        return Container(response_data=b2h(raw[:-2]), status_word=b2h(raw[-2:]))
+
+    def _encode(self, obj, context, path):
+        return h2b(obj['response_data']) + h2b(obj['status_word'])
+
+#### Command Scripting template TS 102 226 tables 5.2 / 5.2a, TS 101 220 tables 7.18/7.19
+#
+# The two TS 101 220 table 7.18 length codings use different template tags:
+# - definite tag AA
+# - indefinite AE
+# In both codings the inner Command TLVs use definite length coding, only the
+# surrounding template differs.
+
+# TS 102 223 8.35
+ExpandedC_APDU = Struct('_tag'/Const(b'\x22'),
+                        'c_apdu'/Prefixed(BerTlvLen, HexAdapter(GreedyBytes)))
+
+# shared by both length codings.
+ExpandedCmdItems = GreedyRange(ExpandedC_APDU)
+
+# TS 102 226 table 5.2: Command Scripting template, definite length coding only
+ExpandedCmd = Struct('_tag'/Const(b'\xaa'),
+                     'commands'/Prefixed(BerTlvLen, ExpandedCmdItems))
+
+# TS 102 226 table 5.2a: indefinite length coding, 'AE 80 <C-APDU TLVs> 00 00'. GreedyRange
+# stops at the first octet that is not a C-APDU tag, which is the end-of-contents marker.
+ExpandedCmdIndef = Struct('_tag'/Const(b'\xae'), '_indef'/Const(b'\x80'),
+                          'commands'/ExpandedCmdItems, '_eoc'/Const(b'\x00\x00'))
+
+#### Response Scripting template TS 102 226 5.2.2, tables 5.10-5.16, TS 101 220 table 7.20
+
+# TS 102 223 8.36
+ExpandedR_APDU = Struct('_tag'/Const(b'\x23'),
+                        'r_apdu'/Prefixed(BerTlvLen, _RApduValueAdapter(GreedyBytes)))
+
+# TS 102 226 table 5.11
+# Value is an integer per ISO/IEC 8825-1, likely just one octet.
+ExpandedNumExecuted = Struct('_tag'/Const(b'\x80'),
+                             'number_of_commands'/Prefixed(BerTlvLen, GreedyInteger()))
+
+# TS 102 226 table 5.12
+ExpandedBadFormat = Struct('_tag'/Const(b'\x90'),
+                           'bad_format'/Prefixed(BerTlvLen,
+                               Enum(Int8ub, unknown_tag=1, wrong_length=2, length_not_found=3)))
+
+# TS 102 226 table 5.14
+ExpandedImmediateActionResp = Struct('_tag'/Const(b'\x81'),
+                                     'immediate_action_response'/Prefixed(BerTlvLen,
+                                         Enum(Int8ub, suspension_error=1)))
+
+# TS 102 226 table 5.16
+ExpandedScriptChainingResp = Struct('_tag'/Const(b'\x83'),
+                                    'script_chaining_response'/Prefixed(BerTlvLen,
+                                        Enum(Int8ub, no_previous_script=1,
+                                             not_supported=2, unable_to_process=3)))
+
+# response TLVs shared by the def and indef Response Scripting templates
+ExpandedRespItems = GreedyRange(Select(ExpandedR_APDU,
+                                       ExpandedBadFormat,
+                                       ExpandedImmediateActionResp,
+                                       ExpandedScriptChainingResp))
+
+# - starts with the "Number of executed command TLV objects" (table 5.10/5.13/5.15)
+# - followed by a sequence of R-APDU TLVs
+# - and/or one of the error # response TLVs
+ExpandedRemoteResp = Struct('_tag'/Const(b'\xab'),
+                            'body'/Prefixed(BerTlvLen, Struct(
+                                'num_executed'/ExpandedNumExecuted,
+                                'responses'/ExpandedRespItems)))
+
+# TS 102 226 table 5.10a: indefinite length coding, no "number of executed" TLV
+ExpandedRemoteRespIndef = Struct('_tag'/Const(b'\xaf'), '_indef'/Const(b'\x80'),
+                                 'responses'/ExpandedRespItems, '_eoc'/Const(b'\x00\x00'))
+
+
+def encode_expanded_cmd(apdus: Union[bytes, List[bytes]],
+                        length_coding: str = 'definite') -> bytes:
+    """builds the Command Scripting template, TS 102 226 5.2.1
+
+    Args:
+        apdus: single C-APDU bytes or list of C-APDUs bytes. Each
+               C-APDU is wrapped into a C-APDU TLV- This function does not add
+               or modify Le.
+        length_coding: 'definite' (the default, tag 'AA', table 5.2) or
+               'indefinite' (tag 'AE', table 5.2a: 'AE 80 <cmd TLVs> 00 00').
+               Inner C-APDU TLVs use definite length coding in both cases.
+    Returns:
+        encoded Command Scripting template as bytes
+    """
+    if isinstance(apdus, (bytes, bytearray)):
+        apdus = [apdus]
+    commands = [{'c_apdu': b2h(a)} for a in apdus]
+    if length_coding == 'definite':
+        return ExpandedCmd.build({'commands': commands})
+    if length_coding == 'indefinite':
+        return ExpandedCmdIndef.build({'commands': commands})
+    raise ValueError("Invalid length_coding: %r" % length_coding)
+
+
+def decode_expanded_resp(data: bytes) -> Container:
+    """Decode a Response Scripting template, TS 102 226 5.2.2 def and indef length
+    coding
+
+    returned Container has:
+        number_of_commands   -- "number of executed command TLV objects" table 5.11
+                                 for definite coding. indefinite coding does not have
+                                 this TLV, so report the number of returned R-APDUs instead.
+        commands             -- list of Containers, one per R-APDU TLV, each
+                                with 'response_data' and 'status_word' hexstr
+        last_response_data   -- response_data of the last R-APDU or ''
+        last_status_word     -- status_word of the last R-APDU or None
+        truncated            -- True if any R-APDU has SW 62F1.
+                                5.2.1.1 states card sets that status when it had to truncate
+                                C-APDU response data, and "this shall terminate the
+                                processing of the command list".
+                                so the response is short AND the remaining commands never ran.
+        bad_format           -- error type of a trailing Bad format TLV if present
+        immediate_action_response -- Immediate Action Response TLV, if there was a suspension error
+        script_chaining_response  -- Script Chaining Response TLV, if there was a chaining error
+
+    The 'last_response_data'/'last_status_word'/'number_of_commands' keys are compatible with
+    CompactRemoteResp so existing callers keep working."""
+    if isinstance(data, str):
+        data = h2b(data)
+    try:
+        if data[:1] == b'\xaf':
+            responses = ExpandedRemoteRespIndef.parse(data)['responses']
+            num_executed = None
+        else:
+            parsed = ExpandedRemoteResp.parse(data)
+            responses = parsed['body']['responses']
+            num_executed = parsed['body']['num_executed']['number_of_commands']
+    except ConstructError as e:
+        raise ValueError('malformed Response Scripting template: %s' % e) from e
+
+    commands = []
+    bad_format = None
+    immediate_action_response = None
+    script_chaining_response = None
+    for item in responses:
+        if 'r_apdu' in item:
+            commands.append(Container(response_data=item['r_apdu']['response_data'],
+                                      status_word=item['r_apdu']['status_word']))
+        elif 'bad_format' in item:
+            bad_format = item['bad_format']
+        elif 'immediate_action_response' in item:
+            immediate_action_response = item['immediate_action_response']
+        elif 'script_chaining_response' in item:
+            script_chaining_response = item['script_chaining_response']
+    # TS 102 226 5.2.1.1: 62F1 means response of a C-APDU was truncated, processing terminated
+    truncated = any(c['status_word'].lower() == '62f1' for c in commands)
+    return Container(number_of_commands=num_executed if num_executed is not None else len(commands),
+                     commands=commands,
+                     last_response_data=commands[-1]['response_data'] if commands else '',
+                     last_status_word=commands[-1]['status_word'] if commands else None,
+                     truncated=truncated,
+                     bad_format=bad_format,
+                     immediate_action_response=immediate_action_response,
+                     script_chaining_response=script_chaining_response)
+
 RC_CC_DS = Enum(BitsInteger(2), no_rc_cc_ds=0, rc=1, cc=2, ds=3)
 CNTR_REQ = Enum(BitsInteger(2), no_counter=0, counter_no_replay_or_seq=1, counter_must_be_higher=2, counter_must_be_lower=3)
 POR_REQ = Enum(BitsInteger(2), no_por=0, por_required=1, por_only_when_error=2)
@@ -149,13 +362,23 @@ class OtaDialect(abc.ABC):
         raise ValueError("Invalid rc_cc_ds: %s" % spi['rc_cc_ds'])
 
     @abc.abstractmethod
-    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict, apdu: bytes) -> bytes:
+    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict,
+                   apdu: Union[bytes, List[bytes]], remote_format: str = 'compact') -> bytes:
+        """Encode a command for a format.
+
+        remote_format:
+        'compact' TS 102 226 5.1, DEFAULT assumes apdus are opaque already-concatenated command strings
+        'expanded' TS 102 226 5.2 wraps a single C-APDU or list of C-APDUs in a Command Scripting template."""
         pass
 
     @abc.abstractmethod
-    def decode_resp(self, otak: OtaKeyset, spi: dict, apdu: bytes) -> (object, Optional["CompactRemoteResp"]):
-        """Decode a response into a response packet and, if indicted (by a
-        response status of `"por_ok"`) a decoded response.
+    def decode_resp(self, otak: OtaKeyset, spi: dict, apdu: bytes,
+                    remote_format: str = 'compact') -> (object, Optional[object]):
+        """Decode response into response packet + a decoded response if por_ok.
+
+        remote_format:
+        'compact' -> DEFAULT TS 102 226 5.1.2 CompactRemoteResp2
+        'expanded' -> container returned by decode_expanded_resp(), TS 102 226 5.2.2
 
         The response packet's common characteristics are not fully determined,
         and (so far) completely proprietary per dialect."""
@@ -221,12 +444,12 @@ class OtaAlgoCrypt(OtaAlgo, abc.ABC):
         for subc in cls.__subclasses__():
             if subc.enum_name == otak.algo_crypt:
                 return subc(otak)
-        raise ValueError('No implementation for crypt algorithm %s' % otak.algo_auth)
+        raise ValueError('No implementation for crypt algorithm %s' % otak.algo_crypt)
 
 class OtaAlgoAuth(OtaAlgo, abc.ABC):
     def __init__(self, otak: OtaKeyset):
         if self.enum_name != otak.algo_auth:
-            raise ValueError('Cannot use algorithm %s with key for %s' % (self.enum_name, otak.algo_crypt))
+            raise ValueError('Cannot use algorithm %s with key for %s' % (self.enum_name, otak.algo_auth))
         super().__init__(otak)
 
     def sign(self, data:bytes) -> bytes:
@@ -335,7 +558,16 @@ class OtaDialectSms(OtaDialect):
                                'secured_data'/GreedyBytes)
     hdr_construct = Struct('chl'/Int8ub, 'spi'/SPI, 'kic'/KIC, 'kid'/KID_CC, 'tar'/Bytes(3))
 
-    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict, apdu: bytes) -> bytes:
+    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict,
+                   apdu: Union[bytes, List[bytes]], remote_format: str = 'compact') -> bytes:
+        # as above:
+        # expanded format is a Command Scripting template wrapping the C-APDU(s)
+        # compact format passes already concatenated command string
+        if remote_format == 'expanded':
+            apdu = encode_expanded_cmd(apdu)
+        elif remote_format != 'compact':
+            raise ValueError("Invalid remote_format: %s" % remote_format)
+
         # length of signature in octets
         len_sig = self._compute_sig_len(spi)
         pad_cnt = 0
@@ -446,7 +678,10 @@ class OtaDialectSms(OtaDialect):
         return hdr_dec['tar'], spi, apdu
 
 
-    def decode_resp(self, otak: OtaKeyset, spi: dict, data: bytes) -> ("OtaDialectSms.SmsResponsePacket", Optional["CompactRemoteResp"]):
+    def decode_resp(self, otak: OtaKeyset, spi: dict, data: bytes,
+                    remote_format: str = 'compact') -> ("OtaDialectSms.SmsResponsePacket", Optional[object]):
+        if remote_format not in ('compact', 'expanded'):
+            raise ValueError("Invalid remote_format: %s ?!" % remote_format)
         if isinstance(data, str):
             data = h2b(data)
         # plain-text POR:   027100000e0ab000110000000000000001612f
@@ -492,9 +727,11 @@ class OtaDialectSms(OtaDialect):
         else:
             raise OtaCheckError('Unknown por_rc_cc_ds: %s' % spi['por_rc_cc_ds'])
 
-        # TODO: ExpandedRemoteResponse according to TS 102 226 5.2.2
         if res.response_status == 'por_ok' and len(res['secured_data']):
-            dec = CompactRemoteResp.parse(res['secured_data'])
+            if remote_format == 'expanded':
+                dec = decode_expanded_resp(res['secured_data'])
+            else:
+                dec = CompactRemoteResp.parse(res['secured_data'])
         else:
             dec = None
         return (res, dec)

pySim/sms.py

@@ -19,6 +19,7 @@
 
 import typing
 import abc
+import logging
 from bidict import bidict
 from construct import Int8ub, Byte, Bit, Flag, BitsInteger
 from construct import Struct, Enum, Tell, BitStruct, this, Padding
@@ -28,6 +29,8 @@ from osmocom.utils import Hexstr, h2b, b2h
 
 from smpp.pdu import pdu_types, operations
 
+logger = logging.getLogger(__name__)
+
 BytesOrHex = typing.Union[Hexstr, bytes]
 
 class UserDataHeader:
@@ -60,6 +63,109 @@ class UserDataHeader:
         return self._construct.build({'ies':self.ies, 'data':b''})
 
 
+class ConcatenatedSmsReassembler:
+    """3GPP TS 23.040 section 9.2.3.24 concat multi part reassembly
+
+    A large user-data payload (e.g. a big OTA response packet) is split by the
+    sending entity into several SMS,
+    each carries a
+    - "concat short messages" IE in its UDH that identifies the set (ref num),
+    - total number of parts
+    - this parts seqno.
+    supports both:
+    IEI 0x00, section 9.2.3.24.1 8-bit ref form
+    IEI 0x08, section 9.2.3.24.8 the 16-bit ref form
+
+    Feed each received TP-User-Data (UDH + payload) to add() which
+    returns the reassembled TP-User-Data once all parts of the set have arrived,
+    or None as long as parts are still missing.
+
+    A non-concatenated SMS is returned unchanged,
+    just like one where the concat IE holds a reserved value:
+    TS 23.040 9.2.3.24.1 says
+    - both a total of zero
+    - a sequence number that is zero or greater than the total
+    that "the receiving entity shall ignore the whole IE",
+    we treat the message as a single, non-concatenated one and warn, not
+    as an error, so the caller does not die.
+
+    The reassembled TP-User-Data is built with a UDH that contains
+    the non-concat IEs seen in the parts, for example the the OTA "response packet"
+    indicator IE 0x71, followed by the concatenated payloads in sequence order,
+    so exactly the single-SMS form the sender would have produced for a payload that fits
+    into one SMS.
+    This allows convenient decoding by the normal single part path."""
+
+    # IEI: Concatenated short messages, 8-bit reference number
+    # (see 3GPP TS 23.040 section 9.2.3.24 and section 9.2.3.24.1)
+    CONCAT_8BIT = 0x00
+    # IEI: Concatenated short message, 16-bit reference number
+    # (see 3GPP TS 23.040 section 9.2.3.24 and section 9.2.3.24.8)
+    CONCAT_16BIT = 0x08
+
+    def __init__(self, max_sets: int = 8):
+        # keyed by (iei, ref, total): {'parts': {seq: payload}, 'header_ies'}, insertion ordered
+        self.sets = {}
+        self.max_sets = max_sets    # incomplete sets kept, oldest is dropped beyond that
+
+    @classmethod
+    def _parse_concat_ie(cls, ies) -> typing.Optional[typing.Tuple[int, int, int, int]]:
+        """Return (iei, ref, total, seq) of the concat IE, or None"""
+        for ie in ies:
+            if ie['iei'] == cls.CONCAT_8BIT and ie['length'] == 3:
+                v = ie['value']
+                return cls.CONCAT_8BIT, v[0], v[1], v[2]
+            if ie['iei'] == cls.CONCAT_16BIT and ie['length'] == 4:
+                v = ie['value']
+                return cls.CONCAT_16BIT, int.from_bytes(v[0:2], 'big'), v[2], v[3]
+        return None
+
+    def add(self, tpud: BytesOrHex) -> typing.Optional[bytes]:
+        """Add one TP-User-Data.
+        Returns
+        - the reassembled TP-User-Data if set is complete or sms not multipart,
+        - else None"""
+        if isinstance(tpud, str):
+            tpud = h2b(tpud)
+        udh, payload = UserDataHeader.from_bytes(tpud)
+        concat = self._parse_concat_ie(udh.ies)
+        if concat is None:
+            return tpud
+        iei, ref, total, seq = concat
+        if total < 1 or seq < 1 or seq > total:
+            # TS 23.040 9.2.3.24.1 / 9.2.3.24.8, total zero or seqno zero / > total:
+            # Ignoring the IE means the message has no valid concat IE, which is a single part message.
+            # Better warn and hand it back rather than raise, so we don't kill the callers receive loop/session
+            logger.warning('Ignoring reserved concat IE (ref=%u total=%u seq=%u), treating the '
+                           'message as non-concat', ref, total, seq)
+            return tpud
+        # TS 23.040 9.2.3.24.1 Total is constant in a set, refno only unique per IE form -> both set identity
+        # - full count = seqno 1..total is present
+        # - part disagreeing on the total ends up as set that cannot complete like set with missing parts
+        key = (iei, ref, total)
+        if key not in self.sets and len(self.sets) >= self.max_sets:
+            del self.sets[next(iter(self.sets))]
+        s = self.sets.setdefault(key, {'parts': {}, 'header_ies': []})
+        s['parts'][seq] = payload
+        # - remember the non concat IEs (OTA 0x71 indicator for example)
+        # - keep first seen occurrence of each IEI,
+        # so app IE present only in the first segment is preserved independent of arrival order
+        seen = {ie['iei'] for ie in s['header_ies']}
+        for ie in udh.ies:
+            if ie['iei'] in (self.CONCAT_8BIT, self.CONCAT_16BIT):
+                continue
+            if ie['iei'] not in seen:
+                s['header_ies'].append(ie)
+                seen.add(ie['iei'])
+        if len(s['parts']) < total:
+            return None
+        # all parts present -> reassemble in seq order
+        del self.sets[(iei, ref, total)]
+        body = b''.join(s['parts'][i] for i in range(1, total + 1))
+        header = UserDataHeader(s['header_ies']).to_bytes()
+        return header + body
+
+
 def smpp_dcs_is_8bit(dcs: pdu_types.DataCoding) -> bool:
     """Determine if the given SMPP data coding scheme is 8-bit or not."""
     if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
@@ -140,8 +246,8 @@ class AddressField:
     def to_bytes(self) -> bytes:
         """Encode the AddressField into the binary representation as used in T-PDU."""
         num_digits = len(self.digits)
-        if num_digits % 2:
-            self.digits += 'f'
+        # don't store the filler nibble or get_bytes() encodes it as digit and ends up too large
+        digits = self.digits + 'f' if num_digits % 2 else self.digits
         d = {
             'addr_len': num_digits,
             'type_of_addr': {
@@ -149,7 +255,7 @@ class AddressField:
                 'type_of_number': self.ton,
                 'numbering_plan_id': self.npi,
                 },
-            'digits': self.digits,
+            'digits': digits,
             }
         return self._construct.build(d)
 
@@ -169,8 +275,14 @@ class SMS_TPDU(abc.ABC):
 
 class SMS_DELIVER(SMS_TPDU):
     """Representation of a SMS-DELIVER T-PDU. This is the Network to MS/UE (downlink) direction."""
-    flags_construct = BitStruct('tp_rp'/Flag, 'tp_udhi'/Flag, 'tp_rp'/Flag, 'tp_sri'/Flag,
-                                Padding(1), 'tp_mms'/Flag, 'tp_mti'/BitsInteger(2))
+    flags_construct = BitStruct('tp_rp'/Flag,
+                                'tp_udhi'/Flag,
+                                'tp_sri'/Flag,
+                                Padding(1),
+                                'tp_lp'/Flag,
+                                'tp_mms'/Flag,
+                                'tp_mti'/BitsInteger(2))
+
     def __init__(self, **kwargs):
         kwargs['tp_mti'] = 0
         super().__init__(**kwargs)

pySim/sysmocom_sjs1.py

@@ -0,0 +1,61 @@
+# coding=utf-8
+"""Utilities / Functions related to sysmocom sysmoUSIM-SJS1 cards
+
+(C) 2026 by sysmocom - s.f.m.c. GmbH
+All Rights Reserved
+
+Author: Eric Wild <ewild@sysmocom.de>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from construct import Struct, Bytes, Flag
+from osmocom.utils import *
+from osmocom.construct import *
+
+from pySim.filesystem import *
+from pySim.runtime import RuntimeState
+
+
+class EF_Ki(TransparentEF):
+    _test_de_encode = [
+        ('000102030405060708090a0b0c0d0e0f',
+         {'key': h2b('000102030405060708090a0b0c0d0e0f')}),
+    ]
+
+    def __init__(self, fid='00ff', name='EF.Ki'):
+        super().__init__(fid, name=name, desc='K/Ki authentication key', size=(16, 16))
+        self._construct = Struct('key'/Bytes(16))
+
+
+class EF_OPc(TransparentEF):
+    _test_de_encode = [
+        ('016ca53d7a0a804561646816d7b0c702fb',
+         {'use_opc_instead_of_op': True, 'op_opc': h2b('6ca53d7a0a804561646816d7b0c702fb')}),
+    ]
+
+    def __init__(self, fid='00f7', name='EF.OPc'):
+        super().__init__(fid, name=name, desc='OP/OPc for milenage', size=(17, 17))
+        self._construct = Struct('use_opc_instead_of_op'/Flag, 'op_opc'/Bytes(16))
+
+
+class SysmoUSIMSJS1(CardModel):
+    _atrs = ["3b9f96801fc78031a073be21136743200718000001a5"]
+
+    @classmethod
+    def add_files(cls, rs: RuntimeState):
+        """Add sysmoUSIM-SJS1 specific files to given RuntimeState."""
+        # the key material lives in DF.GSM shared with ADF.USIM
+        if '7f20' in rs.mf.children:
+            rs.mf.children['7f20'].add_files([EF_Ki(), EF_OPc()])

pySim/transport/__init__.py

@@ -45,8 +45,10 @@ class ApduTracer:
 
 class StdoutApduTracer(ApduTracer):
     """Minimalistic APDU tracer, printing commands to stdout."""
-    def trace_response(self, cmd, sw, resp):
+    def trace_command(self, cmd):
         log.info("-> %s %s", cmd[:10], cmd[10:])
+
+    def trace_response(self, cmd, sw, resp):
         log.info("<- %s: %s", sw, resp)
 
     def trace_reset(self):
@@ -70,10 +72,26 @@ class ProactiveHandler(abc.ABC):
         raise NotImplementedError('No handler method for %s' % pcmd.decoded)
 
     def prepare_response(self, pcmd: ProactiveCommand, general_result: str = 'performed_successfully'):
+        # TERMINAL RESPONSE per ETSI TS 102 223 section 6.8: Command details (6.8.1) echoed from the
+        # command, Device identities (6.8.2) with source and destination swapped, Result (6.8.3).
+        # pcmd can be
+        # - decoded proactive command IE (.children contains CommandDetails/DeviceIdentities)
+        # - ProactiveCommand collection wrapper (empty .children).
+        # Normalise to the children obj, so both work:
+        # - handler that passes its decoded command
+        # - fallback path that passes collection
+        children = list(getattr(pcmd, 'children', None) or [])
+        if not any(isinstance(c, CommandDetails) for c in children):
+            decoded = getattr(pcmd, 'decoded', None)
+            if decoded is not None and decoded is not pcmd:
+                children = list(getattr(decoded, 'children', None) or [])
         # The Command Details are echoed from the command that has been processed.
-        (command_details,) = [c for c in pcmd.children if isinstance(c, CommandDetails)]
+        command_details = next((c for c in children if isinstance(c, CommandDetails)), None)
         # invert the device identities
-        (command_dev_ids,) = [c for c in pcmd.children if isinstance(c, DeviceIdentities)]
+        command_dev_ids = next((c for c in children if isinstance(c, DeviceIdentities)), None)
+        if command_details is None or command_dev_ids is None:
+            raise ValueError('failed to prepare TERMINAL RESPONSE: proactive command has no '
+                             'CommandDetails/DeviceIdentities (%r)' % (pcmd,))
         rsp_dev_ids = DeviceIdentities()
         rsp_dev_ids.from_dict({'device_identities': {
                                     'dest_dev_id': command_dev_ids.decoded['source_dev_id'],
@@ -90,7 +108,7 @@ class LinkBase(abc.ABC):
         self.sw_interpreter = sw_interpreter
         self.apdu_tracer = apdu_tracer
         self.proactive_handler = proactive_handler
-        self.apdu_strict = False
+        self.apdu_strict = True
 
     @abc.abstractmethod
     def __str__(self) -> str:
@@ -301,24 +319,67 @@ class LinkBaseTpdu(LinkBase):
 
         prev_tpdu = tpdu
         data, sw = self.send_tpdu(tpdu)
+        log.debug("T0: case #%u TPDU: %s => %s %s", case, tpdu, data or "(no data)", sw or "(no status word)")
+        if sw is None:
+            raise ValueError("no status word received")
 
-        # When we have sent the first APDU, the SW may indicate that there are response bytes
-        # available. There are two SWs commonly used for this 9fxx (sim) and 61xx (usim), where
-        # xx is the number of response bytes available.
-        # See also:
-        if sw is not None:
-            while (sw[0:2] in ['9f', '61', '62', '63']):
-                # SW1=9F: 3GPP TS 51.011 9.4.1, Responses to commands which are correctly executed
-                # SW1=61: ISO/IEC 7816-4, Table 5 — General meaning of the interindustry values of SW1-SW2
-                # SW1=62: ETSI TS 102 221 7.3.1.1.4 Clause 4b): 62xx, 63xx, 9xxx != 9000
-                tpdu_gr = tpdu[0:2] + 'c00000' + sw[2:4]
+        # After sending the APDU/TPDU the UICC/eUICC or SIM may response with a status word that indicates that further
+        # TPDUs have to be sent in order to complete the task.
+        if case == 4 or self.apdu_strict == False:
+            # In case the APDU is a case #4 APDU, the UICC/eUICC/SIM may indicate that there is response data
+            # available which has to be retrieved using a GET RESPONSE command TPDU.
+            #
+            # ETSI TS 102 221, section 7.3.1.1.4 is very cleare about the fact that the GET RESPONSE mechanism
+            # shall only apply on case #4 APDUs but unfortunately it is impossible to distinguish between case #3
+            # and case #4 when the APDU format is not strictly followed. In order to be able to detect case #4
+            # correctly the Le byte (usually 0x00) must be present, is often forgotten. To avoid problems with
+            # legacy scripts that use raw APDU strings, we will still loosely apply GET RESPONSE based on what
+            # the status word indicates. Unless the user explicitly enables the strict mode (set apdu_strict true)
+            #
+            # The dummy GET RESPONSE of clause 4b (see below) is one shot: it turns a warning SW into the 61xx
+            # that announces the response length. It is only ever a valid reaction to the SW returned for the
+            # _command_ TPDU. Once a response has been fetched there is nothing left to announce, so a warning
+            # SW is the final result of the command and has to be passed on to the caller unmodified.
+            #
+            # This matters because the 62xx/63xx range is not exclusive to ETSI TS 102 221.
+            # GPC v2.3.1 section 11.4.3.2 table 11-38 GP GET STATUS (80 F2) answers
+            # 6310 "more data available", meaning "reissue with P2 bit 1 set" as per section 11.4.2.2 table 11-34
+            # rather than "response data is waiting". Trying a random GET RESPONSE at that point
+            # makes the card answer 6982 and tears down the whole SCP session and following commands fail with 6985.
+            dummy_gr_allowed = not data
+            while True:
+                if sw in ['9000', '9100']:
+                    # A status word of 9000 (or 9100 in case there is pending data from a proactive SIM command)
+                    # indicates that either no response data was returnd or all response data has been retrieved
+                    # successfully. We may discontinue the processing at this point.
+                    break;
+                if sw[0:2] in ['61', '9f']:
+                    # A status word of 61xx or 9fxx indicates that there is (still) response data available. We
+                    # send a GET RESPONSE command with the length value indicated in the second byte of the status
+                    # word. (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4a and 3GPP TS 51.011 9.4.1 and
+                    # ISO/IEC 7816-4, Table 5)
+                    le_gr = sw[2:4]
+                elif sw[0:2] in ['62', '63'] and dummy_gr_allowed:
+                    # There are corner cases (status word is 62xx or 63xx) where the UICC/eUICC/SIM asks us
+                    # to send a dummy GET RESPONSE command. We send a GET RESPONSE command with a length of 0.
+                    # (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4b and ETSI TS 151 011, section 9.4.1)
+                    le_gr = '00'
+                else:
+                    # A status word other then the ones covered by the above logic may indicate an error. In this
+                    # case we will discontinue the processing as well.
+                    # (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4c)
+                    break
+                tpdu_gr = tpdu[0:2] + 'c00000' + le_gr
                 prev_tpdu = tpdu_gr
-                d, sw = self.send_tpdu(tpdu_gr)
-                data += d
-            if sw[0:2] == '6c':
-                # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
-                tpdu_gr = prev_tpdu[0:8] + sw[2:4]
-                data, sw = self.send_tpdu(tpdu_gr)
+                data_gr, sw = self.send_tpdu(tpdu_gr)
+                log.debug("T0: GET RESPONSE TPDU: %s => %s %s", tpdu_gr, data_gr or "(no data)", sw or "(no status word)")
+                data += data_gr
+                dummy_gr_allowed = False
+        if sw[0:2] == '6c':
+            # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
+            tpdu_gr = prev_tpdu[0:8] + sw[2:4]
+            data, sw = self.send_tpdu(tpdu_gr)
+            log.debug("T0: repated case #%u TPDU: %s => %s %s", case, tpdu_gr, data or "(no data)", sw or "(no status word)")
 
         return data, sw
 

pySim/transport/pcsc.py

@@ -26,6 +26,7 @@ from smartcard.CardRequest import CardRequest
 from smartcard.Exceptions import NoCardException, CardRequestTimeoutException, CardConnectionException
 from smartcard.System import readers
 from smartcard.ExclusiveConnectCardConnection import ExclusiveConnectCardConnection
+from smartcard.ATR import ATR
 
 from osmocom.utils import h2i, i2h, Hexstr
 
@@ -80,23 +81,25 @@ class PcscSimLink(LinkBaseTpdu):
 
     def connect(self):
         try:
-            # To avoid leakage of resources, make sure the reader
-            # is disconnected
+            # To avoid leakage of resources, make sure the reader is disconnected
             self.disconnect()
 
             # Make card connection and select a suitable communication protocol
+            # (Even though pyscard provides an automatic protocol selection, we will make an independent decision
+            #  based on the ATR. There are two reasons for that:
+            #  1) In case a card supports T=0 and T=1, we perfer to use T=0.
+            #  2) The automatic protocol selection may be unreliabe on some platforms
+            #     see also: https://osmocom.org/issues/6952)
             self._con.connect()
-            supported_protocols = self._con.getProtocol();
-            self.disconnect()
-            if (supported_protocols & CardConnection.T0_protocol):
-                protocol =  CardConnection.T0_protocol
+            atr = ATR(self._con.getATR())
+            if atr.isT0Supported():
+                self._con.setProtocol(CardConnection.T0_protocol)
                 self.set_tpdu_format(0)
-            elif (supported_protocols & CardConnection.T1_protocol):
-                protocol = CardConnection.T1_protocol
+            elif atr.isT1Supported():
+                self._con.setProtocol(CardConnection.T1_protocol)
                 self.set_tpdu_format(1)
             else:
                 raise ReaderError('Unsupported card protocol')
-            self._con.connect(protocol)
         except CardConnectionException as exc:
             raise ProtocolError() from exc
         except NoCardException as exc:

pySim/ts_102_221.py

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 from bidict import bidict
+import copy
 
 from construct import Select, Const, Bit, Struct, Int16ub, FlagsEnum, GreedyString, ValidationError
 from construct import Optional as COptional, Computed
@@ -335,6 +336,8 @@ class TerminalCapability(BER_TLV_IE, tag=0xa9, nested=[TerminalPowerSupply, Exte
 
 # ETSI TS 102 221 Section 9.2.7 + ISO7816-4 9.3.3/9.3.4
 class _AM_DO_DF(DataObject):
+    """ISO7816-4:2005 5.4.3.1 Table 16"""
+
     def __init__(self):
         super().__init__('access_mode', 'Access Mode', tag=0x80)
 
@@ -381,7 +384,7 @@ class _AM_DO_DF(DataObject):
 
 
 class _AM_DO_EF(DataObject):
-    """ISO7816-4 9.3.2 Table 18 + 9.3.3.1 Table 31"""
+    """ISO7816-4:2005 5.4.3.1 Table 17"""
 
     def __init__(self):
         super().__init__('access_mode', 'Access Mode', tag=0x80)
@@ -429,7 +432,7 @@ class _AM_DO_EF(DataObject):
 
 
 class _AM_DO_CHDR(DataObject):
-    """Command Header Access Mode DO according to ISO 7816-4 Table 32."""
+    """Command Header Access Mode DO according to ISO 7816-4:2005 5.4.3.2 Table 22."""
 
     def __init__(self, tag):
         super().__init__('command_header', 'Command Header Description', tag=tag)
@@ -543,8 +546,9 @@ class CRT_DO(DataObject):
         pin = pin_names.inverse[self.decoded]
         return b'\x83\x01' + pin.to_bytes(1, 'big') + b'\x95\x01\x08'
 
-# ISO7816-4 9.3.3 Table 33
 class SecCondByte_DO(DataObject):
+    """ISO7816-4:2005 5.4.3.1 Table 20"""
+
     def __init__(self, tag=0x9d):
         super().__init__('security_condition_byte', tag=tag)
 
@@ -732,36 +736,57 @@ class EF_ARR(LinFixedEF):
                 raise ValueError
         return by_mode
 
+    @staticmethod
+    def __get_do_sequence(decode_for_df : bool = False):
+        if decode_for_df:
+            return DataObjectSequence('arr', sequence=[AM_DO_DF, SC_DO])
+        else:
+            return DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+
     def _decode_record_bin(self, raw_bin_data, **kwargs):
-        # we can only guess if we should decode for EF or DF here :(
-        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        # we can only guess if we should decode for EF or DF here, but our caller may
+        # be able to pass us a hint:
+        arr_seq = self.__get_do_sequence(kwargs.get('decode_for_df', False))
         dec = arr_seq.decode_multi(raw_bin_data)
         # we cannot pass the result through flatten() here, as we don't have a related
         # 'un-flattening' decoder, and hence would be unable to encode :(
         return dec[0]
 
     def _encode_record_bin(self, in_json, **kwargs):
-        # we can only guess if we should decode for EF or DF here :(
-        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        # we can only guess if we should decode for EF or DF here, but our caller may
+        # be able to pass us a hint:
+        arr_seq = self.__get_do_sequence(kwargs.get('encode_for_df', False))
         return arr_seq.encode_multi(in_json)
 
     @with_default_category('File-Specific Commands')
     class AddlShellCommands(CommandSet):
-        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
+        read_arr_argparser = copy.deepcopy(LinFixedEF.ShellCommands.read_rec_dec_parser)
+        read_arr_argparser.add_argument('--decode-for-df', action='store_true',
+                                        help='Decode EF.ARR record as if used by a DF (default: EF)')
+
+        @cmd2.with_argparser(read_arr_argparser)
         def do_read_arr_record(self, opts):
             """Read one EF.ARR record in flattened, human-friendly form."""
-            (data, _sw) = self._cmd.lchan.read_record_dec(opts.RECORD_NR)
+            (hexdata, _sw) = self._cmd.lchan.read_record(opts.RECORD_NR)
+            data = self._cmd.lchan.selected_file._decode_record_bin(h2b(hexdata),
+                                                                    decode_for_df = opts.decode_for_df)
             data = self._cmd.lchan.selected_file.flatten(data)
             self._cmd.poutput_json(data, opts.oneline)
 
-        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
+        read_arrs_argparser = copy.deepcopy(LinFixedEF.ShellCommands.read_recs_dec_parser)
+        read_arrs_argparser.add_argument('--decode-for-df', action='store_true',
+                                        help='Decode EF.ARR records as if used by a DF (default: EF)')
+
+        @cmd2.with_argparser(read_arrs_argparser)
         def do_read_arr_records(self, opts):
             """Read + decode all EF.ARR records in flattened, human-friendly form."""
             num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
             # collect all results in list so they are rendered as JSON list when printing
             data_list = []
             for recnr in range(1, 1 + num_of_rec):
-                (data, _sw) = self._cmd.lchan.read_record_dec(recnr)
+                (hexdata, _sw) = self._cmd.lchan.read_record(recnr)
+                data = self._cmd.lchan.selected_file._decode_record_bin(h2b(hexdata),
+                                                                        decode_for_df = opts.decode_for_df)
                 data = self._cmd.lchan.selected_file.flatten(data)
                 data_list.append(data)
             self._cmd.poutput_json(data_list, opts.oneline)

pySim/ts_31_102.py

@@ -285,6 +285,14 @@ class EF_SUCI_Calc_Info(TransparentEF):
                                 {"hnet_pubkey_identifier": 11, "hnet_pubkey":
                                  h2b("d1bc365f4997d17ce4374e72181431cbfeba9e1b98d7618f79d48561b144672a")}]} ),
     ]
+    _test_decode = [
+        ( 'A000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
+          {"prot_scheme_id_list": [],
+           "hnet_pubkey_list": []} ),
+        ( 'A000A100FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
+          {"prot_scheme_id_list": [],
+           "hnet_pubkey_list": []} ),
+    ]
     # 3GPP TS 31.102 Section 4.4.11.8
     class ProtSchemeIdList(BER_TLV_IE, tag=0xa0):
         # FIXME: 3GPP TS 24.501 Protection Scheme Identifier
@@ -327,7 +335,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         """conversion method to generate list of {hnet_pubkey_identifier, hnet_pubkey} dicts
         from flat [{hnet_pubkey_identifier: }, {net_pubkey: }, ...] list"""
         out = []
-        while len(l):
+        while l:
             a = l.pop(0)
             b = l.pop(0)
             z = {**a, **b}
@@ -389,7 +397,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         # remaining data holds Home Network Public Key Data Object
         hpkl = EF_SUCI_Calc_Info.HnetPubkeyList()
         hpkl.from_tlv(in_bytes[pos:])
-        hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'])
+        hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'] or [])
 
         return {
             'prot_scheme_id_list': prot_scheme_id_list,
@@ -1058,7 +1066,7 @@ class EF_OCSGL(LinFixedEF):
 # TS 31.102 Section 4.4.11.2 (Rel 15)
 class EF_5GS3GPPLOCI(TransparentEF):
     def __init__(self, fid='4f01', sfid=0x01, name='EF.5GS3GPPLOCI', size=(20, 20),
-                 desc='5S 3GP location information', **kwargs):
+                 desc='5GS 3GPP location information', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         upd_status_constr = Enum(
             Byte, updated=0, not_updated=1, roaming_not_allowed=2)
@@ -1326,7 +1334,7 @@ class EF_5G_PROSE_UIR(TransparentEF):
         pass
     class FiveGDdnmfCtfAddrForUploading(BER_TLV_IE, tag=0x97):
         pass
-    class ProSeConfigDataForUeToNetworkRelayUE(BER_TLV_IE, tag=0xa0,
+    class ProSeConfigDataForUsageInfoReporting(BER_TLV_IE, tag=0xa0,
                                                nested=[EF_5G_PROSE_DD.ValidityTimer,
                                                        CollectionPeriod, ReportingWindow,
                                                        ReportingIndicators,
@@ -1336,7 +1344,7 @@ class EF_5G_PROSE_UIR(TransparentEF):
                  desc='5G ProSe configuration data for usage information reporting', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
         # contains TLV structure despite being TransparentEF, not BER-TLV ?!?
-        self._tlv = EF_5G_PROSE_UIR.ProSeConfigDataForUeToNetworkRelayUE
+        self._tlv = EF_5G_PROSE_UIR.ProSeConfigDataForUsageInfoReporting
 
 # TS 31.102 Section 4.4.13.8 (Rel 18)
 class EF_5G_PROSE_U2URU(TransparentEF):

pySim/ts_51_011.py

@@ -251,6 +251,16 @@ class EF_SMSP(LinFixedEF):
                                                       "numbering_plan_id": "isdn_e164" },
                             "call_number": "4915790109999" },
             "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
+        ( 'e1ffffffffffffffffffffffff0891945197109099f9ffffff0000a9',
+          { "alpha_id": "", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
+                                                          "tp_pid": True, "tp_dcs": True, "tp_vp": True },
+            "tp_dest_addr": { "length": 255, "ton_npi": { "ext": True, "type_of_number": "reserved_for_extension",
+                                                          "numbering_plan_id": "reserved_for_extension" },
+                              "call_number": "" },
+            "tp_sc_addr": { "length": 8, "ton_npi": { "ext": True, "type_of_number": "international",
+                                                      "numbering_plan_id": "isdn_e164" },
+                            "call_number": "4915790109999" },
+            "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
         ( '454e6574776f726b73fffffffffffffff1ffffffffffffffffffffffffffffffffffffffffffffffff0000a7',
           { "alpha_id": "ENetworks", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
                                                                "tp_pid": True, "tp_dcs": True, "tp_vp": False },
@@ -261,6 +271,26 @@ class EF_SMSP(LinFixedEF):
                                                         "numbering_plan_id": "reserved_for_extension" },
                             "call_number": "" },
             "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 1440 } ),
+        ( 'fffffffffffffffffffffffffffffffffffffffffffffffffdffffffffffffffffffffffff07919403214365f7ffffffffffffff',
+          { "alpha_id": "", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
+                                                               "tp_pid": False, "tp_dcs": False, "tp_vp": False },
+            "tp_dest_addr": { "length": 255, "ton_npi": { "ext": True, "type_of_number": "reserved_for_extension",
+                                                          "numbering_plan_id": "reserved_for_extension" },
+                              "call_number": "" },
+            "tp_sc_addr": { "length": 7, "ton_npi": { "ext": True, "type_of_number": "international",
+                                                        "numbering_plan_id": "isdn_e164" },
+                            "call_number": "49301234567" },
+            "tp_pid": b"\xff", "tp_dcs": b"\xff", "tp_vp_minutes": 635040 } ),
+        ( 'fffffffffffffffffffffffffffffffffffffffffffffffffc0b919403214365f7ffffffff07919403214365f7ffffffffffffff',
+          { "alpha_id": "", "parameter_indicators": { "tp_dest_addr": True, "tp_sc_addr": True,
+                                                               "tp_pid": False, "tp_dcs": False, "tp_vp": False },
+            "tp_dest_addr": { "length": 11, "ton_npi": { "ext": True, "type_of_number": "international",
+                                                          "numbering_plan_id": "isdn_e164" },
+                              "call_number": "49301234567" },
+            "tp_sc_addr": { "length": 7, "ton_npi": { "ext": True, "type_of_number": "international",
+                                                        "numbering_plan_id": "isdn_e164" },
+                            "call_number": "49301234567" },
+            "tp_pid": b"\xff", "tp_dcs": b"\xff", "tp_vp_minutes": 635040 } ),
     ]
     _test_no_pad = True
     class ValidityPeriodAdapter(Adapter):
@@ -289,17 +319,30 @@ class EF_SMSP(LinFixedEF):
 
     @staticmethod
     def sc_addr_len(ctx):
-        """Compute the length field for an address field (like TP-DestAddr or TP-ScAddr)."""
+        """Compute the length field for an address field (see also: 3GPP TS 24.011, section 8.2.5.2)."""
         if not hasattr(ctx, 'call_number') or len(ctx.call_number) == 0:
             return 0xff
         else:
+            # octets required for the call_number + one octet for ton_npi
             return bytes_for_nibbles(len(ctx.call_number)) + 1
 
+    @staticmethod
+    def dest_addr_len(ctx):
+        """Compute the length field for an address field (see also: 3GPP TS 23.040, section 9.1.2.5)."""
+        if not hasattr(ctx, 'call_number') or len(ctx.call_number) == 0:
+            return 0xff
+        else:
+            # number of call_number digits
+            return len(ctx.call_number)
+
     def __init__(self, fid='6f42', sfid=None, name='EF.SMSP', desc='Short message service parameters', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(28, None), **kwargs)
         ScAddr = Struct('length'/Rebuild(Int8ub, lambda ctx: EF_SMSP.sc_addr_len(ctx)),
                         'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
-        self._construct = Struct('alpha_id'/COptional(GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28)))),
+        DestAddr = Struct('length'/Rebuild(Int8ub, lambda ctx: EF_SMSP.dest_addr_len(ctx)),
+                        'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
+        # (see comment below)
+        self._construct = Struct('alpha_id'/GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28))),
                                  'parameter_indicators'/InvertAdapter(BitStruct(
                                                                         Const(7, BitsInteger(3)),
                                                                         'tp_vp'/Flag,
@@ -307,13 +350,31 @@ class EF_SMSP(LinFixedEF):
                                                                         'tp_pid'/Flag,
                                                                         'tp_sc_addr'/Flag,
                                                                         'tp_dest_addr'/Flag)),
-                                 'tp_dest_addr'/ScAddr,
+                                 'tp_dest_addr'/DestAddr,
                                  'tp_sc_addr'/ScAddr,
-
                                  'tp_pid'/Bytes(1),
                                  'tp_dcs'/Bytes(1),
                                  'tp_vp_minutes'/EF_SMSP.ValidityPeriodAdapter(Byte))
 
+    # Ensure 'alpha_id' is always present
+    def encode_record_hex(self, abstract_data: dict, record_nr: int, total_len: int = None) -> str:
+        # Problem: TS 51.011 Section 10.5.6 describes the 'alpha_id' field as optional. However, this is only true
+        # at the time when the record length of the file is set up in the file system. A card manufacturer may decide
+        # to remove the field by setting the record length to 28. Likewise, the card manaufacturer may also decide to
+        # set the field to a distinct length by setting the record length to a value greater than 28 (e.g. 14 bytes
+        # 'alpha_id' + 28 bytes). Due to the fixed nature of the record length, this eventually means that in practice
+        # 'alpha_id' is a mandatory field with a fixed length.
+        #
+        # Due to the problematic specification of 'alpha_id' as a pseudo-optional field at the beginning of a
+        # fixed-size memory, the construct definition in self._construct has been incorrectly implemented and the field
+        # has been marked as COptional. We may correct the problem by removing COptional. But to maintain compatibility,
+        # we then have to ensure that in case the field is not provided (None), it is set to an empty string ('').
+        #
+        # See also ts_31_102.py, class EF_OCI for a correct example.
+        if abstract_data['alpha_id'] is None:
+            abstract_data['alpha_id'] = ''
+        return super().encode_record_hex(abstract_data, record_nr, total_len)
+
 # TS 51.011 Section 10.5.7
 class EF_SMSS(TransparentEF):
     class MemCapAdapter(Adapter):
@@ -389,7 +450,7 @@ class DF_TELECOM(CardDF):
 # TS 51.011 Section 10.3.1
 class EF_LP(TransRecEF):
     _test_de_encode = [
-            ( "24", "24"),
+            ( "24", ["24"] ),
         ]
     def __init__(self, fid='6f05', sfid=None, name='EF.LP', size=(1, None), rec_len=1,
                  desc='Language Preference'):
@@ -446,8 +507,8 @@ class EF_IMSI(TransparentEF):
 # TS 51.011 Section 10.3.4
 class EF_PLMNsel(TransRecEF):
     _test_de_encode = [
-            ( "22F860",  { "mcc": "228", "mnc": "06" } ),
-            ( "330420",  { "mcc": "334", "mnc": "020" } ),
+            ( "22F860",  [{ "mcc": "228", "mnc": "06" }] ),
+            ( "330420",  [{ "mcc": "334", "mnc": "020" }] ),
         ]
     def __init__(self, fid='6f30', sfid=None, name='EF.PLMNsel', desc='PLMN selector',
                  size=(24, None), rec_len=3, **kwargs):
@@ -661,7 +722,7 @@ class EF_AD(TransparentEF):
 # TS 51.011 Section 10.3.20 / 10.3.22
 class EF_VGCS(TransRecEF):
     _test_de_encode = [
-            ( "92f9ffff", "299" ),
+            ( "92f9ffff", ["299"] ),
         ]
     def __init__(self, fid='6fb1', sfid=None, name='EF.VGCS', size=(4, 200), rec_len=4,
                  desc='Voice Group Call Service', **kwargs):
@@ -797,9 +858,9 @@ class EF_LOCIGPRS(TransparentEF):
 # TS 51.011 Section 10.3.35..37
 class EF_xPLMNwAcT(TransRecEF):
     _test_de_encode = [
-        ( '62F2104000', { "mcc": "262", "mnc": "01", "act": [ "E-UTRAN NB-S1", "E-UTRAN WB-S1" ] } ),
-        ( '62F2108000', { "mcc": "262", "mnc": "01", "act": [ "UTRAN" ] } ),
-        ( '62F220488C', { "mcc": "262", "mnc": "02", "act": ['E-UTRAN NB-S1', 'E-UTRAN WB-S1', 'EC-GSM-IoT', 'GSM', 'NG-RAN'] } ),
+        ( '62F2104000', [{ "mcc": "262", "mnc": "01", "act": [ "E-UTRAN NB-S1", "E-UTRAN WB-S1" ] }] ),
+        ( '62F2108000', [{ "mcc": "262", "mnc": "01", "act": [ "UTRAN" ] }] ),
+        ( '62F220488C', [{ "mcc": "262", "mnc": "02", "act": ['E-UTRAN NB-S1', 'E-UTRAN WB-S1', 'EC-GSM-IoT', 'GSM', 'NG-RAN'] }] ),
     ]
     def __init__(self, fid='1234', sfid=None, name=None, desc=None, size=(40, None), rec_len=5, **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
@@ -1034,9 +1095,10 @@ class EF_ICCID(TransparentEF):
 # TS 102 221 Section 13.3 / TS 31.101 Section 13 / TS 51.011 Section 10.1.2
 class EF_PL(TransRecEF):
     _test_de_encode = [
-        ( '6465', "de" ),
-        ( '656e', "en" ),
-        ( 'ffff', None ),
+        ( '6465', ["de"] ),
+        ( '656e', ["en"] ),
+        ( 'ffff', [None] ),
+        ( '656e64657275ffffffff', ["en", "de", "ru", None, None] ),
     ]
 
     def __init__(self, fid='2f05', sfid=0x05, name='EF.PL', desc='Preferred Languages'):
@@ -1117,8 +1179,8 @@ class DF_GSM(CardDF):
             EF_MBI(),
             EF_MWIS(),
             EF_CFIS(),
-            EF_EXT('6fc8', None, 'EF.EXT6', desc='Externsion6 (MBDN)'),
-            EF_EXT('6fcc', None, 'EF.EXT7', desc='Externsion7 (CFIS)'),
+            EF_EXT('6fc8', None, 'EF.EXT6', desc='Extension6 (MBDN)'),
+            EF_EXT('6fcc', None, 'EF.EXT7', desc='Extension7 (CFIS)'),
             EF_SPDI(),
             EF_MMSN(),
             EF_EXT('6fcf', None, 'EF.EXT8', desc='Extension8 (MMSN)'),
@@ -1201,9 +1263,11 @@ class CardProfileSIM(CardProfile):
 
     @staticmethod
     def decode_select_response(resp_hex: str) -> object:
-        # we try to build something that resembles a dict resulting from the TLV decoder
-        # of TS 102.221 (FcpTemplate), so that higher-level code only has to deal with one
-        # format of SELECT response
+        """
+        Decode the select response to a dict representation, similar to the one of TS 102.221 (see ts_102_221.py,
+        class FcpTemplate), so that higher-level code only has to deal with one respresentation. See also
+        3GPP TS 51.011, section 9.2.1
+        """
         resp_bin = h2b(resp_hex)
         struct_of_file_map = {
             0: 'transparent',
@@ -1241,13 +1305,24 @@ class CardProfileSIM(CardProfile):
                 record_len = resp_bin[14]
                 ret['file_descriptor']['record_len'] = record_len
                 ret['file_descriptor']['num_of_rec'] = ret['file_size'] // record_len
-            ret['access_conditions'] = b2h(resp_bin[8:10])
-            if resp_bin[11] & 0x01 == 0:
+            ret['access_conditions'] = b2h(resp_bin[8:11])
+
+            # Life cycle status integer, see also ETSI TS 102 221, table 11.7b
+            lcsi = resp_bin[11]
+            if lcsi == 0x00:
+                ret['life_cycle_status_int'] = 'no_information'
+            elif lcsi == 0x01:
+                ret['life_cycle_status_int'] = 'creation'
+            elif lcsi == 0x03:
+                ret['life_cycle_status_int'] = 'initialization'
+            elif lcsi & 0xFD == 0x05:
                 ret['life_cycle_status_int'] = 'operational_activated'
-            elif resp_bin[11] & 0x04:
+            elif lcsi & 0xFD == 0x04:
                 ret['life_cycle_status_int'] = 'operational_deactivated'
+            elif lcsi & 0xFC == 0x0C:
+                ret['life_cycle_status_int'] = 'termination'
             else:
-                ret['life_cycle_status_int'] = 'terminated'
+                ret['life_cycle_status_int'] = lcsi
         return ret
 
     @classmethod

pySim/utils.py

@@ -139,7 +139,6 @@ def enc_plmn(mcc: Hexstr, mnc: Hexstr) -> Hexstr:
 
 def dec_plmn(threehexbytes: Hexstr) -> dict:
     res = {'mcc': "0", 'mnc': "0"}
-    dec_mcc_from_plmn_str(threehexbytes)
     res['mcc'] = dec_mcc_from_plmn_str(threehexbytes)
     res['mnc'] = dec_mnc_from_plmn_str(threehexbytes)
     return res
@@ -911,7 +910,8 @@ class DataObjectCollection:
     def encode(self, decoded) -> bytes:
         res = bytearray()
         for i in decoded:
-            obj = self.members_by_name(i[0])
+            name = i[0]
+            obj = self.members_by_name[name]
             res.append(obj.to_tlv())
         return res
 

pyproject.toml

@@ -4,3 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.pylint.main]
 ignored-classes = ["twisted.internet.reactor"]
+
+[tool.pylint.TYPECHECK]
+# SdKey subclasses are generated dynamically via SdKey.generate_sd_key_classes()
+generated-members = ["SdKey[A-Za-z0-9]+"]

requirements.txt

@@ -1,12 +1,12 @@
 pyscard
 pyserial
 pytlv
-cmd2>=2.6.2,<3.0
+cmd2>=2.6.2,<4.0
 jsonpath-ng
 construct>=2.10.70
 bidict
 pyosmocom>=0.0.12
-pyyaml>=5.1
+pyyaml>=5.4
 termcolor
 colorlog
 pycryptodomex

setup.py

@@ -21,12 +21,12 @@ setup(
         "pyscard",
         "pyserial",
         "pytlv",
-        "cmd2 >= 1.5.0, < 3.0",
+        "cmd2 >= 2.6.2, < 4.0",
         "jsonpath-ng",
         "construct >= 2.10.70",
         "bidict",
         "pyosmocom >= 0.0.12",
-        "pyyaml >= 5.1",
+        "pyyaml >= 5.4",
         "termcolor",
         "colorlog",
         "pycryptodomex",

tests/pySim-prog_test/Fairwaves-SIM.ok

@@ -1,11 +1,11 @@
-Using PC/SC reader interface
+INFO: Using PC/SC reader interface
 Reading ...
 Autodetected card type: Fairwaves-SIM
 ICCID: 8988219000000117833
 IMSI: 001010000000111
 GID1: ffffffffffffffff
 GID2: ffffffffffffffff
-SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
+SMSP: ffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
 SMSC: 0015555
 SPN: Fairwaves
 Show in HPLMN: False

tests/pySim-prog_test/Wavemobile-SIM.ok

@@ -1,11 +1,11 @@
-Using PC/SC reader interface
+INFO: Using PC/SC reader interface
 Reading ...
 Autodetected card type: Wavemobile-SIM
 ICCID: 89445310150011013678
 IMSI: 001010000000102
 GID1: Can't read file -- SW match failed! Expected 9000 and got 6a82.
 GID2: Can't read file -- SW match failed! Expected 9000 and got 6a82.
-SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
+SMSP: ffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
 SMSC: 0015555
 SPN: wavemobile
 Show in HPLMN: False

tests/pySim-prog_test/fakemagicsim.ok

@@ -1,4 +1,4 @@
-Using PC/SC reader interface
+INFO: Using PC/SC reader interface
 Reading ...
 Autodetected card type: fakemagicsim
 ICCID: 1122334455667788990

tests/pySim-prog_test/sysmoISIM-SJA2.ok

@@ -1,4 +1,4 @@
-Using PC/SC reader interface
+INFO: Using PC/SC reader interface
 Reading ...
 Autodetected card type: sysmoISIM-SJA2
 ICCID: 8988211000000467343

tests/pySim-prog_test/sysmoISIM-SJA5.ok

@@ -1,4 +1,4 @@
-Using PC/SC reader interface
+INFO: Using PC/SC reader interface
 Reading ...
 Autodetected card type: sysmoISIM-SJA5
 ICCID: 8949440000001155314

tests/pySim-prog_test/sysmoUSIM-SJS1.ok

@@ -1,4 +1,4 @@
-Using PC/SC reader interface
+INFO: Using PC/SC reader interface
 Reading ...
 Autodetected card type: sysmoUSIM-SJS1
 ICCID: 8988211320300000028

tests/pySim-prog_test/sysmosim-gr1.ok

@@ -1,4 +1,4 @@
-Using PC/SC reader interface
+INFO: Using PC/SC reader interface
 Reading ...
 Autodetected card type: sysmosim-gr1
 ICCID: 2222334455667788990

tests/pySim-shell_test/apdu/test_apdu.script

@@ -7,10 +7,24 @@ set apdu_strict true
 # No command data field, No response data field present
 apdu 00700001 --expect-sw 9000 --expect-response-regex '^$'
 
+# Case #1: (verify pin)
+# This command returns the number of remaining authentication attempts in the
+# form of a status that has the form 63cX, where X is the number of remaining
+# attempts. Such a status word can be easily confused with the response to a
+# case #4 APDU. This test checks if the transport layer correctly distinguishes
+# the between APDU case #1 and APDU case #4.
+apdu 0020000A --expect-sw 63c? --expect-response-regex '^$'
+
 # Case #2: (status)
 # No command data field, Response data field present
 apdu 80F2000000 --expect-sw 9000 --expect-response-regex '^[a-fA-F0-9]+$'
 
+# Case #2: (verify pin)
+# (see also above). This test checks if the transport layer is also able to
+# distinguish correctly between APDU case #2 (with zero length response) and
+# APDU case #4.
+apdu 0020000A00 --expect-sw 63c? --expect-response-regex '^$'
+
 # Case #3: (terminal capability)
 # Command data field present, No response data field
 apdu 80AA000005a903830180 --expect-sw 9000 --expect-response-regex '^$'

tests/pySim-smpp2sim_test/pySim-smpp2sim_test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Utility to verify the functionality of pySim-trace.py
+# Utility to verify the functionality of pySim-smpp2sim.py
 #
 # (C) 2026 by sysmocom - s.f.m.c. GmbH
 # All Rights Reserved

tests/unittests/smdpp_data

@@ -0,0 +1 @@
+../../smdpp-data

tests/unittests/test_bip_relay.py

@@ -0,0 +1,417 @@
+#!/usr/bin/env python3
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: Eric Wild
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import socket
+import threading
+import time
+import unittest
+
+from osmocom.utils import b2h, h2b
+
+from pySim.sms import SMS_SUBMIT, AddressField
+from pySim.cat import (ProactiveCommand, CommandDetails, DeviceIdentities,
+                       BearerDescription, BufferSize, UiccTransportLevel,
+                       OtherAddress, ChannelData, ChannelDataLength, ChannelStatus,
+                       Result, LocationInformation)
+
+from pySim.bip import Proact, ProactChannels, terminal_profile
+
+
+class _EchoServer:
+    """behold, my tiny threaded TCP echo server listening on 127.0.0.1:<port>"""
+    def __init__(self):
+        self._srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self._srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self._srv.bind(('127.0.0.1', 0))
+        self._srv.listen(1)
+        self.port = self._srv.getsockname()[1]
+        self.accepted = threading.Event()
+        self._conns = []
+        self._stop = False
+        threading.Thread(target=self._run, daemon=True).start()
+
+    def _run(self):
+        try:
+            conn, _ = self._srv.accept()
+        except OSError:
+            return
+        self._conns.append(conn)
+        self.accepted.set()
+        while not self._stop:
+            try:
+                data = conn.recv(4096)
+            except OSError:
+                break
+            if not data:
+                break
+            conn.sendall(data)
+
+    def close(self):
+        self._stop = True
+        for s in [self._srv] + self._conns:
+            try:
+                s.close()
+            except OSError:
+                pass
+
+
+def _pcmd(children_tlvs):
+    """Assemble D0 proactive-command TLV from child IE bytes,
+    decode it like transport does after a FETCH"""
+    body = b''.join(children_tlvs)
+    pdu = h2b('D0') + bytes([len(body)]) + body
+    return ProactiveCommand().from_tlv(pdu)
+
+
+def _open_channel(port, ip='127.0.0.1', cmd_nr=1):
+    a, b, c, d = (int(x) for x in ip.split('.'))
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'open_channel',
+                                'command_qualifier': 3}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': 'terminal'}).to_tlv(),
+        BearerDescription(decoded={'bearer_type': 'default', 'bearer_parameters': b''}).to_tlv(),
+        BufferSize(decoded=1024).to_tlv(),
+        UiccTransportLevel(decoded={'protocol_type': 'tcp_uicc_client_remote',
+                                    'port_number': port}).to_tlv(),
+        OtherAddress(decoded={'type_of_address': 'ipv4',
+                              'address': bytes([a, b, c, d])}).to_tlv(),
+    ])
+
+
+def _open_channel_raw(extra_ies, cmd_nr=1):
+    """OPEN CHANNEL with only the head data"""
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'open_channel',
+                                'command_qualifier': 3}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': 'terminal'}).to_tlv(),
+        BearerDescription(decoded={'bearer_type': 'default', 'bearer_parameters': b''}).to_tlv(),
+        BufferSize(decoded=1024).to_tlv(),
+    ] + extra_ies)
+
+
+def _send_data(payload, chan='channel_1', cmd_nr=1):
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'send_data',
+                                'command_qualifier': 1}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': chan}).to_tlv(),
+        ChannelData(decoded=b2h(payload)).to_tlv(),
+    ])
+
+
+def _receive_data(length, chan='channel_1', cmd_nr=1):
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'receive_data',
+                                'command_qualifier': 0}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': chan}).to_tlv(),
+        ChannelDataLength(decoded=length).to_tlv(),
+    ])
+
+
+def _close_channel(chan='channel_1', cmd_nr=1):
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'close_channel',
+                                'command_qualifier': 0}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': chan}).to_tlv(),
+    ])
+
+
+def _first(til, cls):
+    return next((x for x in til if isinstance(x, cls)), None)
+
+
+class BipRelayRoundTripTest(unittest.TestCase):
+    """Drive the fixed Proact handlers (blocking sockets) with synthetic
+    proactive commands against a local echo server and assert a byte round-trip
+    plus the channel bookkeeping / error handling."""
+
+    def setUp(self):
+        self.echo = _EchoServer()
+        self.addCleanup(self.echo.close)
+        self.events = []
+        self.proact = Proact(data_available_sink=self.events.append)
+        self.addCleanup(self._close_all_channels)
+
+    def _close_all_channels(self):
+        for chan in list(self.proact.channels.channels.values()):
+            try:
+                chan.close()
+            except Exception:
+                pass
+
+    def _open(self, cmd_nr=1):
+        til = self.proact.handle_OpenChannel(_open_channel(self.echo.port, cmd_nr=cmd_nr))
+        # every TLV in the response must serialise (the transport does exactly
+        # this to post the TERMINAL RESPONSE)
+        b''.join(x.to_tlv() for x in til)
+        return til
+
+    def test_open_send_receive_roundtrip(self):
+        # OPEN CHANNEL -> socket connected, channel 1 opened, link established
+        til = self._open()
+        self.assertTrue(self.echo.accepted.wait(timeout=2.0))
+        self.assertIn(1, self.proact.channels.channels)
+        cd = _first(til, CommandDetails)
+        self.assertEqual(cd.decoded['type_of_command'], 'open_channel')
+        # TS 102 223 6.8.2 TERMINAL RESPONSE device id: terminal -> UICC
+        self.assertEqual(b2h(_first(til, DeviceIdentities).to_tlv()), '82028281')
+        # channel status: channel 1, link established
+        self.assertEqual(_first(til, ChannelStatus).decoded, '8100')
+        self.assertEqual(_first(til, Result).decoded['general_result'], 'performed_successfully')
+
+        # SEND DATA -> bytes written to the socket, echo server sends them back
+        payload = b'Hello SCP81 relay - opaque TLS record bytes'
+        til = self.proact.handle_SendData(_send_data(payload))
+        b''.join(x.to_tlv() for x in til)
+        # channel data length in the response = free Tx space, FF = ">255"
+        self.assertEqual(_first(til, ChannelDataLength).decoded, 255)
+        self.assertEqual(_first(til, Result).decoded['general_result'], 'performed_successfully')
+
+        # RECEIVE DATA -> drain the bytes back to the "card". real card
+        # uses data-available event, we poll the buffer
+        # and may need several RECEIVE DATA commands, as the spec allows.
+        got = bytearray()
+        deadline = time.monotonic() + 3.0
+        while len(got) < len(payload) and time.monotonic() < deadline:
+            chan = self.proact.channels.channels[1]
+            chan.wait_rx(1.0)
+            til = self.proact.handle_ReceiveData(_receive_data(len(payload) - len(got)))
+            b''.join(x.to_tlv() for x in til)
+            self.assertEqual(b2h(_first(til, DeviceIdentities).to_tlv()), '82028281')
+            got += h2b(_first(til, ChannelData).decoded)
+        self.assertEqual(bytes(got), payload, "byte round-trip through the BIP relay")
+
+        # CLOSE CHANNEL -> socket closed, bookkeeping cleared
+        til = self.proact.handle_CloseChannel(_close_channel())
+        b''.join(x.to_tlv() for x in til)
+        self.assertEqual(_first(til, Result).decoded['general_result'], 'performed_successfully')
+        self.assertNotIn(1, self.proact.channels.channels)
+
+    def test_data_available_event_envelope(self):
+        # The empty->non-empty Rx transition raises ENVELOPE EVENT DOWNLOAD
+        self._open()
+        self.assertTrue(self.echo.accepted.wait(timeout=2.0))
+        payload = b'PONG'
+        self.proact.handle_SendData(_send_data(payload))
+        chan = self.proact.channels.channels[1]
+        self.assertGreater(chan.wait_rx(2.0), 0)
+        # give the reader thread a beat to invoke the sink
+        deadline = time.monotonic() + 2.0
+        while not self.events and time.monotonic() < deadline:
+            time.sleep(0.01)
+        self.assertEqual(len(self.events), 1, "one data-available event on the empty->non-empty edge")
+        env = h2b(self.events[0])
+        # d6 0e | 99 01 09 (event: data available) | 82 02 82 81 terminal->UICC
+        #       | b8 02 81 00 (channel 1 established) | b7 01 XX bytes available
+        self.assertEqual(b2h(env[:15]), 'd60e99010982028281b8028100b701')
+        self.assertGreaterEqual(env[15], 1)
+        self.assertLessEqual(env[15], len(payload))
+
+    def test_channel_number_from_device_identities(self):
+        # Two channels, not the old hardcoded 1
+        e2 = _EchoServer()
+        self.addCleanup(e2.close)
+        self.proact.handle_OpenChannel(_open_channel(self.echo.port))
+        # open a second channel with a second echo server
+        til2 = self.proact.handle_OpenChannel(_open_channel(e2.port))
+        self.assertEqual(sorted(self.proact.channels.channels), [1, 2])
+        self.assertEqual(_first(til2, ChannelStatus).decoded, '8200')  # channel 2, established
+
+        # SEND DATA addressed to channel_2 must reach the second socket
+        self.assertTrue(e2.accepted.wait(timeout=2.0))
+        self.proact.handle_SendData(_send_data(b'two', chan='channel_2'))
+        chan2 = self.proact.channels.channels[2]
+        self.assertGreater(chan2.wait_rx(2.0), 0)
+        til = self.proact.handle_ReceiveData(_receive_data(3, chan='channel_2'))
+        self.assertEqual(h2b(_first(til, ChannelData).decoded), b'two')
+        # ..and nothing on chan 1
+        self.assertEqual(self.proact.channels.channels[1].available_rx(), 0)
+
+    def test_commands_on_closed_channel_report_bip_error(self):
+        # SEND/RECEIVE/CLOSE on a channel that was never opened must be rejected
+        # with a BIP error
+        for til in (self.proact.handle_SendData(_send_data(b'x', chan='channel_4')),
+                    self.proact.handle_ReceiveData(_receive_data(1, chan='channel_4')),
+                    self.proact.handle_CloseChannel(_close_channel(chan='channel_4'))):
+            b''.join(x.to_tlv() for x in til)
+            res = _first(til, Result).decoded
+            self.assertEqual(res['general_result'], 'bearer_independent_protocol_error')
+            self.assertEqual(res['additional_information'], 'channel_id_not_valid')
+
+    def test_receive_more_than_available_is_missing_info(self):
+        # terminal must NOT wait if fewer than the requested bytes are buffered,
+        # eturns what it has with "performed with missing information".
+        self._open()
+        self.assertTrue(self.echo.accepted.wait(timeout=2.0))
+        til = self.proact.handle_ReceiveData(_receive_data(10))
+        b''.join(x.to_tlv() for x in til)
+        self.assertEqual(_first(til, Result).decoded['general_result'],
+                         'performed_with_missing_information')
+        self.assertEqual(h2b(_first(til, ChannelData).decoded), b'')
+        self.assertEqual(_first(til, ChannelDataLength).decoded, 0)
+
+
+class OpenChannelRefusalTest(unittest.TestCase):
+    """Refusal is a TERMINAL RESPONSE, not an exception, raising takes the whole
+    proactive session down and leaves the card wondering why"""
+
+    ADDR = OtherAddress(decoded={'type_of_address': 'ipv4', 'address': bytes([127, 0, 0, 1])})
+    TCP = UiccTransportLevel(decoded={'protocol_type': 'tcp_uicc_client_remote', 'port_number': 1234})
+
+    def setUp(self):
+        self.proact = Proact()
+        self.addCleanup(self._close_all_channels)
+
+    def _close_all_channels(self):
+        for chan in list(self.proact.channels.channels.values()):
+            try:
+                chan.close()
+            except Exception:
+                pass
+
+    def _assert_refused(self, til, additional_information, chan_nr=0):
+        b''.join(x.to_tlv() for x in til)               # must serialise, the transport posts it
+        res = _first(til, Result).decoded
+        self.assertEqual(res['general_result'], 'bearer_independent_protocol_error')
+        self.assertEqual(res['additional_information'], additional_information)
+        self.assertEqual(_first(til, ChannelStatus).decoded, '%02x00' % chan_nr)   # 8.56
+        self.assertIsNotNone(_first(til, BearerDescription))                       # 6.8.20
+        self.assertIsNotNone(_first(til, BufferSize))                              # 6.8.21
+        self.assertEqual(b2h(_first(til, DeviceIdentities).to_tlv()), '82028281')  # 6.8.2
+
+    def test_transport_level(self):
+        cases = [[self.ADDR.to_tlv()]]                             # absent, 6.6.27.x Optional
+        for proto in ('udp_uicc_client_remote', 'tcp_uicc_server', 'udp_uicc_client_local',
+                      'tcp_uicc_client_local', 'direct_channel'):   # not TCP client remote
+            tl = UiccTransportLevel(decoded={'protocol_type': proto, 'port_number': 1234})
+            cases.append([tl.to_tlv(), self.ADDR.to_tlv()])
+        for extra in cases:
+            with self.subTest(extra=b2h(extra[0])):
+                self._assert_refused(self.proact.handle_OpenChannel(_open_channel_raw(extra)),
+                                     'requested_uicc_if_transp_level_not_available')
+
+    def test_destination_address(self):
+        v6 = OtherAddress(decoded={'type_of_address': 'ipv6', 'address': bytes(16)})
+        for extra in ([self.TCP.to_tlv()],                         # absent
+                      [self.TCP.to_tlv(), v6.to_tlv()]):           # not IPv4
+            with self.subTest(extra=len(extra)):
+                self._assert_refused(self.proact.handle_OpenChannel(_open_channel_raw(extra)),
+                                     'no_specific_cause')
+
+    def test_no_channel_left(self):
+        for _ in range(7):                                         # 6.4.27.2, 6.4.27.3
+            self.proact.channels.channel_create()
+        cmd = _open_channel_raw([self.TCP.to_tlv(), self.ADDR.to_tlv()])
+        self._assert_refused(self.proact.handle_OpenChannel(cmd), 'no_channel_availabile')
+
+    def test_connect_failure(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)      # port nothing listens on
+        s.bind(('127.0.0.1', 0))
+        dead_port = s.getsockname()[1]
+        s.close()
+        til = self.proact.handle_OpenChannel(_open_channel(dead_port))
+        self._assert_refused(til, 'channel_closed', chan_nr=1)      # 6.4.30
+        self.assertEqual(self.proact.channels.channels, {})        # channel given back
+
+
+class ProvideLocalInformationTest(unittest.TestCase):
+    """TS 102 223 6.8.7: only 00 gets a data object; the rest keeps the empty result."""
+
+    def _cmd(self, qualifier):
+        return _pcmd([
+            CommandDetails(decoded={'command_number': 1, 'type_of_command': 'provide_local_info',
+                                    'command_qualifier': qualifier}).to_tlv(),
+            DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': 'terminal'}).to_tlv()])
+
+    def test_location(self):
+        til = Proact().handle_ProvideLocalInformation(self._cmd(0x00))
+        b''.join(x.to_tlv() for x in til)
+        self.assertEqual(_first(til, Result).decoded['general_result'], 'performed_successfully')
+        self.assertEqual(b2h(_first(til, LocationInformation).to_tlv()), '930762f21000010001')
+        self.assertEqual(b2h(_first(til, DeviceIdentities).to_tlv()), '82028281')   # 6.8.2
+
+    def test_other_qualifiers_get_no_data_object(self):
+        for qualifier in (0x01, 0x03, 0x04, 0x1a):
+            with self.subTest(command_qualifier=qualifier):
+                til = Proact().handle_ProvideLocalInformation(self._cmd(qualifier))
+                b''.join(x.to_tlv() for x in til)
+                self.assertIsNone(_first(til, LocationInformation))
+
+    def test_location_is_configurable(self):
+        til = Proact(location=h2b('26f8100539')).handle_ProvideLocalInformation(self._cmd(0x00))
+        self.assertEqual(b2h(_first(til, LocationInformation).to_tlv()), '930526f8100539')
+
+
+class TerminalProfileTest(unittest.TestCase):
+    """TS 102 223 5.2, one bit per CAT facility"""
+
+    def setUp(self):
+        self.profile = terminal_profile()
+
+    def byte(self, n):
+        return self.profile[n - 1]                      # 1-based, as 5.2 numbers them
+
+    def test_announced(self):
+        self.assertEqual(len(self.profile), 32)
+        self.assertEqual(self.byte(1), 0x13)            # profile download, SMS-PP download b2+b5
+        self.assertEqual(self.byte(4), 0x02)            # SEND SHORT MESSAGE
+        self.assertEqual(self.byte(5) & 0x01, 0x01)     # SET UP EVENT LIST
+        self.assertEqual(self.byte(6), 0x0c)            # events: data available, channel status
+        self.assertEqual(self.byte(12), 0x1f)           # OPEN/CLOSE CHANNEL, RECEIVE/SEND DATA, STATUS
+        self.assertEqual(self.byte(13) >> 5, ProactChannels.MAX_CHANNELS)
+        self.assertEqual(self.byte(14), 0x60)           # class ND, class NK
+        self.assertEqual(self.byte(17), 0x01)           # TCP, UICC client mode, remote
+
+    def test_not_announced(self):
+        self.assertEqual(self.byte(3) & 0x60, 0)        # POLL INTERVAL, POLLING OFF
+        self.assertEqual(self.byte(4) & 0xc0, 0)        # PROVIDE LOCAL INFORMATION, NMR
+        self.assertEqual(self.byte(12) & 0xe0, 0)       # SERVICE SEARCH/INFORMATION, DECLARE SERVICE
+        self.assertEqual(self.byte(14) & 0x1f, 0)       # no characters down the display
+        for n in (7, 9, 10, 11, 15, 16, 18):            # class "a", class "d", display, ESN/IMEISV
+            self.assertEqual(self.byte(n), 0)
+
+    def test_channel_count(self):
+        self.assertEqual(terminal_profile(3)[12] >> 5, 3)
+        with self.assertRaises(ValueError):             # 8.56: 1 to 7
+            terminal_profile(8)
+
+
+if __name__ == "__main__":
+    unittest.main()
+
+
+class BipSinkTest(unittest.TestCase):
+    """Both sinks are optional, a driver with no SMS path at all must not crash and burn
+    with a card that sends one, and one that has one must get the PDU."""
+
+    def _submit(self):
+        return SMS_SUBMIT(tp_da=AddressField('12345', 'unknown', 'unknown'),
+                          tp_ud=b'\x01\x02', tp_udl=2, tp_dcs=0xf6)
+
+    def test_sinks_default_to_none(self):
+        p = Proact()
+        self.assertIsNone(p.sms_sink)
+
+    def test_mo_sms_goes_to_the_sink(self):
+        seen = []
+        Proact(sms_sink=seen.append).send_sms_via_smpp(self._submit())
+        self.assertEqual(len(seen), 1)
+
+    def test_no_sms_sink_drops_instead_of_raising(self):
+        with self.assertLogs('pySim.bip', level='INFO'):
+            Proact().send_sms_via_smpp(self._submit())

tests/unittests/test_card_key_provider.py

@@ -20,7 +20,8 @@ class TestCardKeyProviderCsv(unittest.TestCase):
                        "KIK3" : "00010204040506070809488B0C0D0E0F"}
 
         csv_file_path = os.path.dirname(os.path.abspath(__file__)) + "/test_card_key_provider.csv"
-        card_key_provider_register(CardKeyProviderCsv(csv_file_path, column_keys))
+        card_key_field_cryptor = CardKeyFieldCryptor(column_keys)
+        card_key_provider_register(CardKeyProviderCsv(csv_file_path, card_key_field_cryptor))
         super().__init__(*args, **kwargs)
 
     def test_card_key_provider_get(self):

tests/unittests/test_cat.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Tests for the CAT (Card Application Toolkit) COMPREHENSION-TLV data objects"""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# IEs not properly coverd by test_tlvs.py
+
+
+import unittest
+
+from osmocom.utils import b2h, h2b
+
+from pySim.cat import IMEI, IMEISV, AccessTechnology, SupportedRadioAccessTechnologies
+
+
+class IMEI_Test(unittest.TestCase):
+    """TS 102 223 8.20: the IMEI IE is 8 bytes, coded as valie part of Mobile Identity IE from 124 008"""
+
+    IMEI_15 = '123456789012345'
+    ENCODED = '94081a32547698103254'
+
+    def test_encode_is_eight_bytes(self):
+        """15 digits in 8 byte: 16 nibbles, one is type/parity framing."""
+        tlv = IMEI(decoded=self.IMEI_15).to_tlv()
+        self.assertEqual(b2h(tlv), self.ENCODED)
+        self.assertEqual(tlv[1], 0x08)               # spec len 8
+        self.assertEqual(len(tlv) - 2, 8)
+
+    def test_first_octet_framing(self):
+        """TS 24.008 table 10.5.4"""
+        octet1 = IMEI(decoded=self.IMEI_15).to_tlv()[2]
+        self.assertEqual(octet1 & 0x07, 2)           # IMEI
+        self.assertEqual((octet1 >> 3) & 0x01, 1)    # odd
+        self.assertEqual(octet1 >> 4, 1)             # digit 1
+
+    def test_decodes_to_the_raw_imei(self):
+        """strip framing nibble"""
+        ie = IMEI()
+        ie.from_tlv(h2b(self.ENCODED))
+        self.assertEqual(ie.decoded, self.IMEI_15)
+
+    def test_even_digit_count_uses_the_end_mark(self):
+        """"end marker, IMEISV case"""
+        ie = IMEI(decoded='1234567890123456')
+        tlv = ie.to_tlv()
+        self.assertEqual(tlv[2] >> 3 & 0x01, 0)      # even
+        self.assertEqual(tlv[-1] >> 4, 0x0f)         # end mark
+        back = IMEI()
+        back.from_tlv(tlv)
+        self.assertEqual(back.decoded, '1234567890123456')
+
+
+class IMEISV_Test(unittest.TestCase):
+    """TS 102 223 8.74, no fixed len, end marker"""
+
+    IMEISV_16 = '1234567890123456'
+    ENCODED = 'e2091332547698103254f6'
+
+    def test_encode(self):
+        self.assertEqual(b2h(IMEISV(decoded=self.IMEISV_16).to_tlv()), self.ENCODED)
+
+    def test_type_of_identity_and_end_mark(self):
+        value = IMEISV(decoded=self.IMEISV_16).to_tlv()[2:]
+        self.assertEqual(value[0] & 0x07, 3)         # IMEISV
+        self.assertEqual((value[0] >> 3) & 0x01, 0)  # even
+        self.assertEqual(value[-1] >> 4, 0x0f)       # end mark
+        self.assertEqual(len(value), 9)
+
+    def test_decode(self):
+        ie = IMEISV()
+        ie.from_tlv(h2b(self.ENCODED))
+        self.assertEqual(ie.decoded, self.IMEISV_16)
+
+
+class SupportedRadioAccessTechnologies_Test(unittest.TestCase):
+    """TS 102 223 8.105"""
+
+    def test_encode_technology_enabled(self):
+        """The flag used to have a bitmask of 0 so enabled -> 00 (that is disabled..)"""
+        ie = SupportedRadioAccessTechnologies(
+            decoded=[{'technology': 'eutran', 'state': {'enabled': True}}])
+        self.assertEqual(b2h(ie.to_tlv()), 'b4020801')
+
+    def test_encode_technology_disabled(self):
+        ie = SupportedRadioAccessTechnologies(
+            decoded=[{'technology': 'eutran', 'state': {'enabled': False}}])
+        self.assertEqual(b2h(ie.to_tlv()), 'b4020800')
+
+    def test_decode_technology(self):
+        """old 0 bitmask = all enabled, no way to disable"""
+        for encoded, enabled in [('b4020800', False), ('b4020801', True)]:
+            with self.subTest(encoded=encoded):
+                ie = SupportedRadioAccessTechnologies()
+                ie.from_tlv(h2b(encoded))
+                self.assertEqual(ie.decoded[0]['technology'], 'eutran')
+                self.assertEqual(ie.decoded[0]['state']['enabled'], enabled)
+
+    def test_decode_technology_multiple(self):
+        ie = SupportedRadioAccessTechnologies()
+        ie.from_tlv(h2b('b40408010000'))
+        self.assertEqual([(e['technology'], e['state']['enabled']) for e in ie.decoded],
+                         [('eutran', True), ('gsm', False)])
+
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unittests/test_configurable_parameters.py

@@ -0,0 +1,640 @@
+#!/usr/bin/env python3
+
+# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: Neels Hofmeyr
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import enum
+import io
+import sys
+import unittest
+from importlib import resources
+from osmocom.utils import hexstr
+from pySim.esim.saip import ProfileElementSequence
+import pySim.esim.saip.personalization as p13n
+import smdpp_data.upp
+
+import xo
+update_expected_output = False
+
+def valstr(val):
+    if isinstance(val, io.BytesIO):
+        val = val.getvalue()
+    if isinstance(val, bytearray):
+        val = bytes(val)
+    return f'{val!r}'
+
+def valtypestr(val):
+    if isinstance(val, dict):
+        types = []
+        for v in val.values():
+            types.append(f'{type(v).__name__}')
+
+        val_type = '{' + ', '.join(types) + '}'
+    else:
+        val_type = f'{type(val).__name__}'
+    return f'{valstr(val)}:{val_type}'
+
+class ConfigurableParameterTest(unittest.TestCase):
+
+    def test_parameters(self):
+
+        upp_fnames = (
+                'TS48v5_SAIP2.1A_NoBERTLV.der',
+                'TS48v5_SAIP2.3_BERTLV_SUCI.der',
+               )
+
+        class Paramtest:
+            def __init__(self, param_cls, val, expect_val, expect_clean_val=None):
+                self.param_cls = param_cls
+                self.val = val
+                self.expect_clean_val = expect_clean_val
+                self.expect_val = expect_val
+
+        param_tests = [
+            Paramtest(param_cls=p13n.Imsi, val='123456',
+                      expect_clean_val=str('123456'),
+                      expect_val={'IMSI': hexstr('123456'),
+                                  'IMSI-ACC': '0040'}),
+            Paramtest(param_cls=p13n.Imsi, val=int(123456),
+                      expect_val={'IMSI': hexstr('123456'),
+                                  'IMSI-ACC': '0040'}),
+
+            Paramtest(param_cls=p13n.Imsi, val='123456789012345',
+                      expect_clean_val=str('123456789012345'),
+                      expect_val={'IMSI': hexstr('123456789012345'),
+                                  'IMSI-ACC': '0020'}),
+            Paramtest(param_cls=p13n.Imsi, val=int(123456789012345),
+                      expect_val={'IMSI': hexstr('123456789012345'),
+                                  'IMSI-ACC': '0020'}),
+
+            Paramtest(param_cls=p13n.Puk1,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+            Paramtest(param_cls=p13n.Puk1,
+                      val=int(12345678),
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+
+            Paramtest(param_cls=p13n.Puk2,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+
+            Paramtest(param_cls=p13n.Pin1,
+                      val='1234',
+                      expect_clean_val=b'1234\xff\xff\xff\xff',
+                      expect_val='1234'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val='123456',
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val=int(1234),
+                      expect_clean_val=b'1234\xff\xff\xff\xff',
+                      expect_val='1234'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val=int(123456),
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val=int(12345678),
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+
+            Paramtest(param_cls=p13n.Adm1,
+                      val='1234',
+                      expect_clean_val=b'1234\xff\xff\xff\xff',
+                      expect_val='1234'),
+            Paramtest(param_cls=p13n.Adm1,
+                      val='123456',
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+            Paramtest(param_cls=p13n.Adm1,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+            Paramtest(param_cls=p13n.Adm1,
+                      val=int(123456),
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val='Milenage',
+                      expect_clean_val=1,
+                      expect_val='Milenage'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val='TUAK',
+                      expect_clean_val=2,
+                      expect_val='TUAK'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val='usim-test',
+                      expect_clean_val=3,
+                      expect_val='usim_test'),
+
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val=1,
+                      expect_clean_val=1,
+                      expect_val='Milenage'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val=2,
+                      expect_clean_val=2,
+                      expect_val='TUAK'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val=3,
+                      expect_clean_val=3,
+                      expect_val='usim_test'),
+
+            Paramtest(param_cls=p13n.K,
+                      val='01020304050607080910111213141516',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=int(11020304050607080910111213141516),
+                      expect_clean_val=b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='11020304050607080910111213141516'),
+
+            Paramtest(param_cls=p13n.Opc,
+                      val='01020304050607080910111213141516',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.Opc,
+                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.Opc,
+                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.Opc,
+                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+
+            Paramtest(param_cls=p13n.SmspTpScAddr,
+                      val='+1234567',
+                      expect_clean_val=(True, '1234567'),
+                      expect_val='+1234567'),
+            Paramtest(param_cls=p13n.SmspTpScAddr,
+                      val=1234567,
+                      expect_clean_val=(False, '1234567'),
+                      expect_val='1234567'),
+
+            Paramtest(param_cls=p13n.TuakNumberOfKeccak,
+                      val='123',
+                      expect_clean_val=123,
+                      expect_val='123'),
+            Paramtest(param_cls=p13n.TuakNumberOfKeccak,
+                      val=123,
+                      expect_clean_val=123,
+                      expect_val='123'),
+
+            Paramtest(param_cls=p13n.MilenageRotationConstants,
+                      val='0a 0b 0c 01 02',
+                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_val='0a0b0c0102'),
+            Paramtest(param_cls=p13n.MilenageRotationConstants,
+                      val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_val='0a0b0c0102'),
+            Paramtest(param_cls=p13n.MilenageRotationConstants,
+                      val=bytearray(b'\x0a\x0b\x0c\x01\x02'),
+                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_val='0a0b0c0102'),
+
+            Paramtest(param_cls=p13n.MilenageXoringConstants,
+                      val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+                          ' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+                          ' cccccccccccccccccccccccccccccccc'
+                          ' 11111111111111111111111111111111'
+                          ' 22222222222222222222222222222222',
+                      expect_clean_val=b'\xaa' * 16
+                                       + b'\xbb' * 16
+                                       + b'\xcc' * 16
+                                       + b'\x11' * 16
+                                       + b'\x22' * 16,
+                      expect_val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+                                 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+                                 'cccccccccccccccccccccccccccccccc'
+                                 '11111111111111111111111111111111'
+                                 '22222222222222222222222222222222'),
+            Paramtest(param_cls=p13n.MilenageXoringConstants,
+                      val=b'\xaa' * 16
+                          + b'\xbb' * 16
+                          + b'\xcc' * 16
+                          + b'\x11' * 16
+                          + b'\x22' * 16,
+                      expect_clean_val=b'\xaa' * 16
+                                       + b'\xbb' * 16
+                                       + b'\xcc' * 16
+                                       + b'\x11' * 16
+                                       + b'\x22' * 16,
+                      expect_val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+                                 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+                                 'cccccccccccccccccccccccccccccccc'
+                                 '11111111111111111111111111111111'
+                                 '22222222222222222222222222222222'),
+
+            Paramtest(param_cls=p13n.MncLen,
+                      val='2',
+                      expect_clean_val=2,
+                      expect_val='2'),
+            Paramtest(param_cls=p13n.MncLen,
+                      val=3,
+                      expect_clean_val=3,
+                      expect_val='3'),
+
+            ]
+
+        for sdkey_cls in (
+            # thin out the number of tests, as a compromise between completeness and test runtime
+            p13n.SdKeyScp02Kvn20AesDek,
+           #p13n.SdKeyScp02Kvn20AesEnc,
+           #p13n.SdKeyScp02Kvn20AesMac,
+           #p13n.SdKeyScp02Kvn21AesDek,
+            p13n.SdKeyScp02Kvn21AesEnc,
+           #p13n.SdKeyScp02Kvn21AesMac,
+           #p13n.SdKeyScp02Kvn22AesDek,
+           #p13n.SdKeyScp02Kvn22AesEnc,
+            p13n.SdKeyScp02Kvn22AesMac,
+           #p13n.SdKeyScp02KvnffAesDek,
+           #p13n.SdKeyScp02KvnffAesEnc,
+           #p13n.SdKeyScp02KvnffAesMac,
+            p13n.SdKeyScp03Kvn30AesDek,
+           #p13n.SdKeyScp03Kvn30AesEnc,
+           #p13n.SdKeyScp03Kvn30AesMac,
+           #p13n.SdKeyScp03Kvn31AesDek,
+            p13n.SdKeyScp03Kvn31AesEnc,
+           #p13n.SdKeyScp03Kvn31AesMac,
+           #p13n.SdKeyScp03Kvn32AesDek,
+           #p13n.SdKeyScp03Kvn32AesEnc,
+            p13n.SdKeyScp03Kvn32AesMac,
+           #p13n.SdKeyScp80Kvn01AesDek,
+           #p13n.SdKeyScp80Kvn01AesEnc,
+           #p13n.SdKeyScp80Kvn01AesMac,
+            p13n.SdKeyScp80Kvn01DesDek,
+           #p13n.SdKeyScp80Kvn01DesEnc,
+           #p13n.SdKeyScp80Kvn01DesMac,
+           #p13n.SdKeyScp80Kvn02AesDek,
+            p13n.SdKeyScp80Kvn02AesEnc,
+           #p13n.SdKeyScp80Kvn02AesMac,
+           #p13n.SdKeyScp80Kvn02DesDek,
+           #p13n.SdKeyScp80Kvn02DesEnc,
+            p13n.SdKeyScp80Kvn02DesMac,
+           #p13n.SdKeyScp80Kvn03AesDek,
+           #p13n.SdKeyScp80Kvn03AesEnc,
+           #p13n.SdKeyScp80Kvn03AesMac,
+            p13n.SdKeyScp80Kvn03DesDek,
+           #p13n.SdKeyScp80Kvn03DesEnc,
+           #p13n.SdKeyScp80Kvn03DesMac,
+            p13n.SdKeyScp81Kvn40AesDek,
+           #p13n.SdKeyScp81Kvn40Tlspsk,
+           #p13n.SdKeyScp81Kvn41AesDek,
+            p13n.SdKeyScp81Kvn41Tlspsk,
+           #p13n.SdKeyScp81Kvn42AesDek,
+           #p13n.SdKeyScp81Kvn42Tlspsk,
+            ):
+
+            for key_len in sdkey_cls.allow_len:
+                val = '0102030405060708091011121314151617181920212223242526272829303132'
+                expect_clean_val = (b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'
+                                    b'\x17\x18\x19\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x30\x31\x32')
+                expect_val = '0102030405060708091011121314151617181920212223242526272829303132'
+
+                val = val[:key_len*2]
+                expect_clean_val = expect_clean_val[:key_len]
+                expect_val = val
+
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                # test bytes input
+                val = expect_clean_val
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                # test bytearray input
+                val = bytearray(expect_clean_val)
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                # test BytesIO input
+                val = io.BytesIO(expect_clean_val)
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                if key_len == 16:
+                    # test huge integer input.
+                    # needs to start with nonzero.. stupid
+                    val = 11020304050607080910111213141516
+                    expect_clean_val = (b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16')
+                    expect_val = '11020304050607080910111213141516'
+                    param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+        outputs = []
+
+        for upp_fname in upp_fnames:
+            test_idx = -1
+            try:
+
+                der = resources.read_binary(smdpp_data.upp, upp_fname)
+
+                for t in param_tests:
+                    test_idx += 1
+                    logloc = f'{upp_fname} {t.param_cls.__name__}(val={valtypestr(t.val)})'
+
+                    param = None
+                    try:
+                        param = t.param_cls()
+                        param.input_value = t.val
+                        param.validate()
+                    except ValueError as e:
+                        raise ValueError(f'{logloc}: {e}') from e
+
+                    clean_val = param.value
+                    logloc = f'{logloc} clean_val={valtypestr(clean_val)}'
+                    if t.expect_clean_val is not None and t.expect_clean_val != clean_val:
+                        raise ValueError(f'{logloc}: expected'
+                                         f' expect_clean_val={valtypestr(t.expect_clean_val)}')
+
+                    # on my laptop, deepcopy is about 30% slower than decoding the DER from scratch:
+                    # pes = copy.deepcopy(orig_pes)
+                    pes = ProfileElementSequence.from_der(der)
+                    try:
+                        param.apply(pes)
+                    except ValueError as e:
+                        raise ValueError(f'{logloc} apply_val(clean_val): {e}') from e
+
+                    changed_der = pes.to_der()
+
+                    pes2 = ProfileElementSequence.from_der(changed_der)
+
+                    read_back_val = t.param_cls.get_value_from_pes(pes2)
+
+                    # compose log string to show the precise type of dict values
+                    if isinstance(read_back_val, dict):
+                        types = set()
+                        for v in read_back_val.values():
+                            types.add(f'{type(v).__name__}')
+
+                        read_back_val_type = '{' + ', '.join(types) + '}'
+                    else:
+                        read_back_val_type = f'{type(read_back_val).__name__}'
+
+                    logloc = (f'{logloc} read_back_val={valtypestr(read_back_val)}')
+
+                    if isinstance(read_back_val, dict) and not t.param_cls.get_name() in read_back_val.keys():
+                        raise ValueError(f'{logloc}: expected to find name {t.param_cls.get_name()!r} in read_back_val')
+
+                    expect_val = t.expect_val
+                    if not isinstance(expect_val, dict):
+                        expect_val = { t.param_cls.get_name(): expect_val }
+                    if read_back_val != expect_val:
+                        raise ValueError(f'{logloc}: expected {expect_val=!r}:{type(t.expect_val).__name__}')
+
+                    ok = logloc.replace(' clean_val', '\n\tclean_val'
+                              ).replace(' read_back_val', '\n\tread_back_val'
+                              ).replace('=', '=\t'
+                              )
+                    output = f'\nok: {ok}'
+                    outputs.append(output)
+                    print(output)
+
+            except Exception as e:
+                raise RuntimeError(f'Error while testing UPP {upp_fname} {test_idx=}: {e}') from e
+
+        output = '\n'.join(outputs) + '\n'
+        xo_name = 'test_configurable_parameters'
+        if update_expected_output:
+            with resources.path(xo, xo_name) as xo_path:
+                with open(xo_path, 'w', encoding='utf-8') as f:
+                    f.write(output)
+        else:
+            xo_str = resources.read_text(xo, xo_name)
+            if xo_str != output:
+                at = 0
+                while at < len(output):
+                    if output[at] == xo_str[at]:
+                        at += 1
+                        continue
+                    break
+
+                raise RuntimeError(f'output differs from expected output at position {at}: "{output[at:at+20]}" != "{xo_str[at:at+20]}"')
+
+
+class TestValidateVal(unittest.TestCase):
+    """validate_val() tests for various ConfigurableParameter subclasses."""
+
+    def _ok(self, cls, val, expected=None):
+        result = cls.validate_val(val)
+        if expected is not None:
+            self.assertEqual(result, expected)
+        return result
+
+    def _err(self, cls, val):
+        with self.assertRaises(ValueError):
+            cls.validate_val(val)
+
+    # --- Iccid ---
+
+    def test_iccid_18digits_adds_luhn(self):
+        result = self._ok(p13n.Iccid, '998877665544332211')
+        self.assertIsInstance(result, str)
+        self.assertEqual(len(result), 19)
+        self.assertTrue(result.isdecimal())
+
+    def test_iccid_19digits_passthrough(self):
+        result = self._ok(p13n.Iccid, '9988776655443322110')
+        self.assertIsInstance(result, str)
+        self.assertEqual(len(result), 19)
+
+    def test_iccid_too_short(self):
+        self._err(p13n.Iccid, '12345678901234567')  # 17 digits
+
+    def test_iccid_too_long(self):
+        self._err(p13n.Iccid, '1' * 21)
+
+    def test_iccid_non_digits(self):
+        self._err(p13n.Iccid, '99887766554433221X')
+
+    # --- Imsi ---
+
+    def test_imsi_valid_short(self):
+        self._ok(p13n.Imsi, '001010', '001010')
+
+    def test_imsi_valid_long(self):
+        self._ok(p13n.Imsi, '001010123456789', '001010123456789')
+
+    def test_imsi_too_short(self):
+        self._err(p13n.Imsi, '12345')  # 5 digits, min is 6
+
+    def test_imsi_too_long(self):
+        self._err(p13n.Imsi, '1' * 16)
+
+    def test_imsi_non_digits(self):
+        self._err(p13n.Imsi, '00101A123456789')
+
+    # --- Pin1 ---
+
+    def test_pin1_4digits(self):
+        # DecimalHexParam encodes each digit as its ASCII byte, then rpad to 8 bytes with 0xff
+        self._ok(p13n.Pin1, '1234', b'1234\xff\xff\xff\xff')
+
+    def test_pin1_8digits(self):
+        self._ok(p13n.Pin1, '12345678', b'12345678')
+
+    def test_pin1_too_short(self):
+        self._err(p13n.Pin1, '123')
+
+    def test_pin1_too_long(self):
+        self._err(p13n.Pin1, '123456789')
+
+    def test_pin1_non_digits(self):
+        self._err(p13n.Pin1, '123A')
+
+    # --- Puk1 ---
+
+    def test_puk1_8digits(self):
+        self._ok(p13n.Puk1, '12345678', b'12345678')
+
+    def test_puk1_wrong_length(self):
+        self._err(p13n.Puk1, '1234567')   # 7 digits
+        self._err(p13n.Puk1, '123456789')  # 9 digits
+
+    def test_puk1_non_digits(self):
+        self._err(p13n.Puk1, '1234567X')
+
+    # --- K (BinaryParam) ---
+
+    def test_k_valid_hex_str(self):
+        self._ok(p13n.K, '000102030405060708090a0b0c0d0e0f',
+                 b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f')
+
+    def test_k_valid_bytes(self):
+        raw = bytes(range(16))
+        self._ok(p13n.K, raw, raw)
+
+    def test_k_wrong_length(self):
+        self._err(p13n.K, '00' * 15)  # 15 bytes, allow_len requires 16 or 32
+
+    def test_k_non_hex(self):
+        self._err(p13n.K, 'gg' * 16)
+
+    def test_k_odd_hex_digits(self):
+        self._err(p13n.K, '0' * 31)   # odd number of hex digits
+
+
+class TestEnumParam(unittest.TestCase):
+    """Tests for the EnumParam machinery, using AlgorithmID as the concrete subclass."""
+
+    # --- validate_val ---
+
+    def test_validate_by_name_exact(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val('Milenage'), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val('TUAK'), 2)
+        self.assertEqual(p13n.AlgorithmID.validate_val('usim_test'), 3)
+
+    def test_validate_by_int(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val(1), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val(2), 2)
+        self.assertEqual(p13n.AlgorithmID.validate_val(3), 3)
+
+    def test_validate_fuzzy_case(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val('milenage'), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val('MILENAGE'), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val('tuak'), 2)
+
+    def test_validate_fuzzy_hyphen_underscore(self):
+        # 'usim-test' has a hyphen; enum member is 'usim_test' — must fuzzy-match
+        self.assertEqual(p13n.AlgorithmID.validate_val('usim-test'), 3)
+
+    def test_validate_invalid_name(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.validate_val('unknown')
+
+    def test_validate_invalid_int(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.validate_val(99)
+
+    def test_validate_returns_int(self):
+        result = p13n.AlgorithmID.validate_val('Milenage')
+        self.assertIsInstance(result, int)
+        self.assertNotIsInstance(result, enum.Enum)
+
+    # --- map_name_to_val ---
+
+    def test_map_name_exact(self):
+        self.assertEqual(p13n.AlgorithmID.map_name_to_val('Milenage'), 1)
+
+    def test_map_name_fuzzy(self):
+        self.assertEqual(p13n.AlgorithmID.map_name_to_val('milenage'), 1)
+        self.assertEqual(p13n.AlgorithmID.map_name_to_val('usim-test'), 3)
+
+    def test_map_name_strict_raises(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.map_name_to_val('unknown', strict=True)
+
+    def test_map_name_nonstrict_returns_none(self):
+        self.assertIsNone(p13n.AlgorithmID.map_name_to_val('unknown', strict=False))
+
+    # --- map_val_to_name ---
+
+    def test_map_val_known(self):
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(1), 'Milenage')
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(2), 'TUAK')
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(3), 'usim_test')
+
+    def test_map_val_unknown_nonstrict(self):
+        self.assertIsNone(p13n.AlgorithmID.map_val_to_name(99))
+
+    def test_map_val_unknown_strict(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.map_val_to_name(99, strict=True)
+
+    # --- name_normalize ---
+
+    def test_name_normalize(self):
+        self.assertEqual(p13n.AlgorithmID.name_normalize('Milenage'), 'Milenage')
+        self.assertEqual(p13n.AlgorithmID.name_normalize('milenage'), 'Milenage')
+        self.assertEqual(p13n.AlgorithmID.name_normalize('usim-test'), 'usim_test')
+
+    # --- clean_name_str ---
+
+    def test_clean_name_str(self):
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim-test'), 'usimtest')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim_test'), 'usimtest')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('Milenage'), 'milenage')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('foo bar!'), 'foobar')
+
+
+if __name__ == "__main__":
+    if '-u' in sys.argv:
+        update_expected_output = True
+        sys.argv.remove('-u')
+    unittest.main()

tests/unittests/test_decode_select_resp.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Philipp Maier <pmaier@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+import os
+from pySim.profile import CardProfile
+from pySim.ts_51_011 import CardProfileSIM
+from pySim.ts_102_221 import CardProfileUICC
+
+class TestDecodeSelectResponse_CardProfile(unittest.TestCase):
+
+    def decode_select_response(self, card_Profile: CardProfile, testcases: list[dict]):
+        for testcase in testcases:
+            resp_hex = testcase['resp_hex']
+            decoded = card_Profile.decode_select_response(resp_hex)
+            if testcase['decoded']:
+                self.assertEqual(decoded, testcase['decoded'])
+            else:
+                print("no testvector to compare against, assuming the following output is correct:")
+                print("resp_hex:", resp_hex)
+                print("decoded:", decoded)
+
+    def test_CardProfileSIM(self):
+        testcases = [
+            # MF
+            {"resp_hex" : "000000003f000100000000000981020c0400838a838a",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'mf'}}, 'proprietary_info': {'available_memory': 0}, 'file_id': '3f00', 'file_characteristics': '81', 'num_direct_child_df': 2, 'num_direct_child_ef': 12, 'num_chv_unblock_adm_codes': 4}},
+            # DF.TELECOM
+            {"resp_hex" : "000000007f100200000000000981000d0400838a838a",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'df'}}, 'proprietary_info': {'available_memory': 0}, 'file_id': '7f10', 'file_characteristics': '81', 'num_direct_child_df': 0, 'num_direct_child_ef': 13, 'num_chv_unblock_adm_codes': 4}},
+            # EF.MSISDN
+            {"resp_hex" : "000000346f40040011ffff0102011a",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'working_ef', 'structure': 'linear_fixed'}, 'record_len': 26, 'num_of_rec': 2}, 'proprietary_info': {}, 'file_id': '6f40', 'file_size': 52, 'access_conditions': '11ffff', 'life_cycle_status_int': 'creation'}},
+            # EF.ICCID
+            {"resp_hex" : "0000000a2fe204000cffff01020000",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'working_ef', 'structure': 'transparent'}}, 'proprietary_info': {}, 'file_id': '2fe2', 'file_size': 10, 'access_conditions': '0cffff', 'life_cycle_status_int': 'creation'}},
+        ]
+        self.decode_select_response(CardProfileSIM, testcases)
+
+    def test_CardProfileUICC(self):
+        testcases = [
+            # MF
+            {"resp_hex" : "622c8202782183023f00a50c80017183040003a7388701018a01058b032f0601c60c90016083010183010a83010b",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'df', 'structure': 'no_info_given'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'?\x00', 'proprietary_information': {'uicc_characteristics': b'q', 'available_memory': 239416, 'supported_filesystem_commands': {'terminal_capability': True}}, 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'/\x06', 'ef_arr_record_nr': 1}, 'pin_status_template_do': [{'ps_do': b'`'}, {'key_reference': 1}, {'key_reference': 10}, {'key_reference': 11}]}},
+            # ADF.USIM
+            {"resp_hex" : "623d8202782183027fd0840ca0000000871002ff49ff0589a50c80017183040003a7388701018a01058b032f0601c60f90017083010183018183010a83010b",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'df', 'structure': 'no_info_given'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'\x7f\xd0', 'df_name': b'\xa0\x00\x00\x00\x87\x10\x02\xffI\xff\x05\x89', 'proprietary_information': {'uicc_characteristics': b'q', 'available_memory': 239416, 'supported_filesystem_commands': {'terminal_capability': True}}, 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'/\x06', 'ef_arr_record_nr': 1}, 'pin_status_template_do': [{'ps_do': b'p'}, {'key_reference': 1}, {'key_reference': 129}, {'key_reference': 10}, {'key_reference': 11}]}},
+            # ADF.ISIM
+            {"resp_hex" : "623d8202782183027fb0840ca0000000871004ff49ff0589a50c80017183040003a7388701018a01058b032f0601c60f90017083010183018183010a83010b",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'df', 'structure': 'no_info_given'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'\x7f\xb0', 'df_name': b'\xa0\x00\x00\x00\x87\x10\x04\xffI\xff\x05\x89', 'proprietary_information': {'uicc_characteristics': b'q', 'available_memory': 239416, 'supported_filesystem_commands': {'terminal_capability': True}}, 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'/\x06', 'ef_arr_record_nr': 1}, 'pin_status_template_do': [{'ps_do': b'p'}, {'key_reference': 1}, {'key_reference': 129}, {'key_reference': 10}, {'key_reference': 11}]}},
+            # EF.IMSI
+            {"resp_hex" : "62178202412183026f078a01058b036f060a80020009880138",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'working_ef', 'structure': 'transparent'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'o\x07', 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'o\x06', 'ef_arr_record_nr': 10}, 'file_size': 9, 'short_file_identifier': 7}},
+            # EF.ECC
+            {"resp_hex" : "621a82054221000e0283026fb78a01058b036f06088002001c880108",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'working_ef', 'structure': 'linear_fixed'}, 'record_len': 14, 'num_of_rec': 2}, 'file_identifier': b'o\xb7', 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'o\x06', 'ef_arr_record_nr': 8}, 'file_size': 28, 'short_file_identifier': 1}},
+        ]
+        self.decode_select_response(CardProfileUICC, testcases)
+
+if __name__ == "__main__":
+	unittest.main()

tests/unittests/test_esim_saip.py

@@ -21,7 +21,7 @@ import copy
 from osmocom.utils import h2b, b2h
 
 from pySim.esim.saip import *
-from pySim.esim.saip.personalization import *
+from pySim.esim.saip import personalization
 from pprint import pprint as pp
 
 
@@ -55,14 +55,56 @@ class SaipTest(unittest.TestCase):
     def test_personalization(self):
         """Test some of the personalization operations."""
         pes = copy.deepcopy(self.pes)
-        params = [Puk1('01234567'), Puk2(98765432), Pin1('1111'), Pin2(2222), Adm1('11111111'),
-                  K(h2b('000102030405060708090a0b0c0d0e0f')), Opc(h2b('101112131415161718191a1b1c1d1e1f'))]
+        params = [personalization.Puk1('01234567'),
+                  personalization.Puk2(98765432),
+                  personalization.Pin1('1111'),
+                  personalization.Pin2(2222),
+                  personalization.Adm1('11111111'),
+                  personalization.K(h2b('000102030405060708090a0b0c0d0e0f')),
+                  personalization.Opc(h2b('101112131415161718191a1b1c1d1e1f'))]
         for p in params:
             p.validate()
             p.apply(pes)
         # TODO: we don't actually test the results here, but we just verify there is no exception
         pes.to_der()
 
+    def test_personalization2(self):
+        """Test some of the personalization operations."""
+        cls = personalization.SdKeyScp80Kvn01DesEnc
+        pes = ProfileElementSequence.from_der(self.per_input)
+        prev_val = tuple(cls.get_values_from_pes(pes))
+        print(f'{prev_val=}')
+        self.assertTrue(prev_val)
+
+        set_val = '42342342342342342342342342342342'
+        param = cls(set_val)
+        param.validate()
+        param.apply(pes)
+
+        get_val1 = tuple(cls.get_values_from_pes(pes))
+        print(f'{get_val1=} {set_val=}')
+        self.assertEqual(get_val1, ({cls.name: set_val},))
+
+        get_val1b = tuple(cls.get_values_from_pes(pes))
+        print(f'{get_val1b=} {set_val=}')
+        self.assertEqual(get_val1b, ({cls.name: set_val},))
+
+        der = pes.to_der()
+
+        get_val1c = tuple(cls.get_values_from_pes(pes))
+        print(f'{get_val1c=} {set_val=}')
+        self.assertEqual(get_val1c, ({cls.name: set_val},))
+
+        # assertTrue to not dump the entire der.
+        # Expecting the modified DER to be different. If this assertion fails, then no change has happened in the output
+        # DER and the ConfigurableParameter subclass is buggy.
+        self.assertTrue(der != self.per_input)
+
+        pes2 = ProfileElementSequence.from_der(der)
+        get_val2 = tuple(cls.get_values_from_pes(pes2))
+        print(f'{get_val2=} {set_val=}')
+        self.assertEqual(get_val2, ({cls.name: set_val},))
+
     def test_constructor_encode(self):
         """Test that DER-encoding of PE created by "empty" constructor works without raising exception."""
         for cls in [ProfileElementMF, ProfileElementPuk, ProfileElementPin, ProfileElementTelecom,

tests/unittests/test_files.py

@@ -27,6 +27,7 @@ import pySim.ts_31_102
 import pySim.ts_31_103
 import pySim.ts_51_011
 import pySim.sysmocom_sja2
+import pySim.sysmocom_sjs1
 import pySim.gsm_r
 import pySim.cdma_ruim
 
@@ -176,12 +177,11 @@ class TransRecEF_Test(unittest.TestCase):
 
 
     def test_de_encode_record(self):
-        """Test the decoder and encoder for a transparent record-oriented EF.  Performs first a decoder
-        test, and then re-encodes the decoded data, comparing the re-encoded data with the
-        initial input data.
+        """Test the decoder and encoder for a transparent record-oriented EF at the whole-file
+        level.  Performs first a decode test, then re-encodes and compares with the input.
 
         Requires the given TransRecEF subclass to have a '_test_de_encode' attribute,
-        containing a list of tuples. Each tuple has to be a 2-tuple (hexstring, decoded_dict).
+        containing a list of 2-tuples (hexstring, decoded_list).
         """
         for c in self.classes:
             name = get_qualified_name(c)
@@ -192,14 +192,12 @@ class TransRecEF_Test(unittest.TestCase):
                         encoded = t[0]
                         decoded = t[1]
                         logging.debug("Testing decode of %s", name)
-                        re_dec = inst.decode_record_hex(encoded)
+                        re_dec = inst.decode_hex(encoded)
                         self.assertEqual(decoded, re_dec)
                         # re-encode the decoded data
                         logging.debug("Testing re-encode of %s", name)
-                        re_enc = inst.encode_record_hex(re_dec, len(encoded)//2)
+                        re_enc = inst.encode_hex(re_dec, len(encoded)//2)
                         self.assertEqual(encoded.upper(), re_enc.upper())
-                    # there's no point in testing padded input, as TransRecEF have a fixed record
-                    # size and we cannot ever receive more input data than that size.
 
 
 class TransparentEF_Test(unittest.TestCase):

tests/unittests/test_fs_coverage.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Verify that every CardProfile / CardApplication subclass with EF/DF content,
+and every standalone CardDF subclass (one not reachable as a child of any profile
+or application), is either listed in docs/pysim_fs_sphinx.py::SECTIONS or
+explicitly EXCLUDED."""
+
+import unittest
+import importlib
+import inspect
+import pkgutil
+import sys
+import os
+
+# Make docs/pysim_fs_sphinx.py importable without a full Sphinx build.
+_DOCS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'docs')
+sys.path.insert(0, os.path.abspath(_DOCS_DIR))
+
+import pySim                                                       # noqa: E402
+from pySim.filesystem import CardApplication, CardDF, CardMF, CardADF  # noqa: E402
+from pySim.profile import CardProfile                              # noqa: E402
+from pysim_fs_sphinx import EXCLUDED, SECTIONS                     # noqa: E402
+
+
+class TestFsCoverage(unittest.TestCase):
+    """Ensure SECTIONS + EXCLUDED together account for all classes with content."""
+
+    # Base CardDF types that are not concrete filesystem objects on their own.
+    _DF_BASE_TYPES = frozenset([CardDF, CardMF, CardADF])
+
+    @staticmethod
+    def _collect_reachable_df_types(obj) -> set:
+        """Return the set of all CardDF *types* reachable as children of *obj*."""
+        result = set()
+        if isinstance(obj, CardProfile):
+            children = obj.files_in_mf
+        elif isinstance(obj, CardApplication):
+            result.add(type(obj.adf))
+            children = list(obj.adf.children.values())
+        elif isinstance(obj, CardDF):
+            children = list(obj.children.values())
+        else:
+            return result
+        queue = list(children)
+        while queue:
+            child = queue.pop()
+            if isinstance(child, CardDF):
+                result.add(type(child))
+                queue.extend(child.children.values())
+        return result
+
+    @staticmethod
+    def _has_content(obj) -> bool:
+        """Return True if *obj* owns any EFs/DFs."""
+        if isinstance(obj, CardProfile):
+            return bool(obj.files_in_mf)
+        if isinstance(obj, CardApplication):
+            return bool(obj.adf.children)
+        return False
+
+    def test_all_profiles_and_apps_covered(self):
+        # build a set of (module, class-name) pairs that are already accounted for
+        covered = {(mod, cls) for (_, mod, cls) in SECTIONS}
+        accounted_for = covered | EXCLUDED
+
+        uncovered = []
+        reachable_df_types = set()
+        loaded_modules = {}
+
+        for modinfo in pkgutil.walk_packages(pySim.__path__, prefix='pySim.'):
+            modname = modinfo.name
+            try:
+                module = importlib.import_module(modname)
+            except Exception: # skip inport errors, if any
+                continue
+            loaded_modules[modname] = module
+
+            for name, cls in inspect.getmembers(module, inspect.isclass):
+                # skip classes that are merely imported by this module
+                if cls.__module__ != modname:
+                    continue
+                # examine only subclasses of CardProfile and CardApplication
+                if not issubclass(cls, (CardProfile, CardApplication)):
+                    continue
+                # skip the abstract base classes themselves
+                if cls in (CardProfile, CardApplication):
+                    continue
+                # classes that require constructor arguments cannot be probed
+                try:
+                    obj = cls()
+                except Exception:
+                    continue
+
+                # collect all CardDF types reachable from this profile/application
+                # (used below to identify standalone DFs)
+                reachable_df_types |= self._collect_reachable_df_types(obj)
+
+                if self._has_content(obj) and (modname, name) not in accounted_for:
+                    uncovered.append((modname, name))
+
+        # check standalone CardDFs (such as DF.EIRENE or DF.SYSTEM)
+        for modname, module in loaded_modules.items():
+            for name, cls in inspect.getmembers(module, inspect.isclass):
+                if cls.__module__ != modname:
+                    continue
+                if not issubclass(cls, CardDF):
+                    continue
+                if cls in self._DF_BASE_TYPES:
+                    continue
+                if cls in reachable_df_types:
+                    continue
+                try:
+                    obj = cls()
+                except Exception:
+                    continue
+                if obj.children and (modname, name) not in accounted_for:
+                    uncovered.append((modname, name))
+
+        if uncovered:
+            lines = [
+                'The following classes have EFs/DFs, but not listed in SECTIONS or EXCLUDED:',
+                *(f'  {modname}.{name}' for modname, name in sorted(uncovered)),
+                'Please modify docs/pysim_fs_sphinx.py accordingly',
+            ]
+            self.fail('\n'.join(lines))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/unittests/test_globalplatform.py

@@ -17,7 +17,10 @@
 
 import unittest
 import logging
+import hashlib
+from types import SimpleNamespace
 from osmocom.utils import b2h, h2b
+from osmocom.tlv import bertlv_encode_len
 
 from pySim.global_platform import *
 from pySim.global_platform.scp import *
@@ -283,6 +286,41 @@ class SCP03_Test_AES256_33(SCP03_Test, unittest.TestCase):
 # FIXME: test auth with random (0x60) vs pseudo-random (0x70) challenge
 
 
+class KeyComponentBlock_Test(unittest.TestCase):
+    """Tests for the kcb of GP CardSpec v2.3
+    - Table 11-70 kcv that required padding, preceded by its clear-text length
+    - Table 11-71 no padding required"""
+
+    def setUp(self):
+        # SCP02 (3DES DEK, 8 byte blocks), same vectors as SCP02_Test
+        self.scp02 = SCP02(card_keys=ck_3des_70)
+        self.scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
+        self.scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
+        self.scp02.gen_ext_auth_apdu()
+        # SCP03 (AES DEK, 16 byte blocks), same vectors as SCP03_Test_AES128_11
+        self.scp03 = SCP03(card_keys=KEYSET_AES128)
+        self.scp03.gen_init_update_apdu(h2b('b13e5f938fc108c4'))
+        self.scp03.parse_init_update_resp(h2b('000000000000000000003003703eb51047495b249f66c484c1d2ef1948000002'))
+        self.scp03.gen_ext_auth_apdu(0x11)
+
+    def test_encrypt_decrypt_key(self):
+        for scp in (self.scp02, self.scp03):
+            bs = scp.sk.blocksize
+            for keylen in range(1, 3 * bs + 1):
+                with self.subTest(scp=type(scp).__name__, keylen=keylen):
+                    key = bytes(range(keylen))
+                    kcb = scp.encrypt_key(key)
+                    if keylen % bs:
+                        # Table 11-70: <length of clear key component> || <encrypted padded value>
+                        self.assertEqual(kcb[0], keylen)
+                        self.assertEqual((len(kcb) - 1) % bs, 0)
+                        self.assertEqual(len(kcb) - 1, keylen + (bs - keylen % bs))
+                    else:
+                        # Table 11-71: only the encrypted key component value
+                        self.assertEqual(len(kcb), keylen)
+                    self.assertEqual(scp.decrypt_key(kcb), key)
+
+
 class SCP03_KCV_Test(unittest.TestCase):
     def test_kcv(self):
         self.assertEqual(compute_kcv('aes', KEYSET_AES128.enc), h2b('C35280'))
@@ -290,13 +328,549 @@ class SCP03_KCV_Test(unittest.TestCase):
         self.assertEqual(compute_kcv('aes', KEYSET_AES128.dek), h2b('840DE5'))
 
 
+class PutKey_PSK_Test(unittest.TestCase):
+    """Tests for the PUT KEY command data field encoding, in particular the PSK TLS ('85') key data
+    field defined by GlobalPlatform Amendment B (Remote Application Management over HTTP) Table 3-13."""
+
+    # the PUT KEY encoder we exercise
+    C = ADF_SD.AddlShellCommands
+
+    # SCP80 TLS-PSK example key from the do_put_key docstring (16 bytes)
+    PSK_CLEAR = h2b('303132333435363738393a3b3c3d3e3f')
+    # its DEK ciphertext + Table 3-13 KCV with SCP02 session set up below
+    PSK_CIPHERED = h2b('15abf1fe16ccc5aa13743394442942cd')
+    PSK_KCV = h2b('06125d')  # = SHA-1(PSK_CLEAR)[:3]
+
+    def setUp(self):
+        # SCP02 with the same vectors as SCP02_Test, so that the whole PUT KEY data field is reproducible.
+        self.scp02 = SCP02(card_keys=ck_3des_70)
+        self.scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
+        self.scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
+        self.scp02.gen_ext_auth_apdu()
+
+    def test_psk_kcv_is_sha1(self):
+        # GP Amendment B Table 3-13: KCV = 3 most significant bytes of SHA-1(clear key)
+        self.assertEqual(compute_kcv('tls_psk', self.PSK_CLEAR), hashlib.sha1(self.PSK_CLEAR).digest()[:3])
+        self.assertEqual(compute_kcv('tls_psk', self.PSK_CLEAR), self.PSK_KCV)
+
+    def test_encode_psk_framing_golden(self):
+        # assert the exact Table 3-13 layout
+        #   85 | L1 | L2 | <ciphered> | 03 | <SHA-1(clear)[:3]>
+        clear = self.PSK_CLEAR
+        ciphered = h2b('aabbccddeeff00112233445566778899')  # arbitrary 16-byte ciphertext
+        kcv = hashlib.sha1(clear).digest()[:3]
+        field = self.C.encode_key_data_psk(clear, ciphered, kcv)
+        #                            85   L1   L2   <---------- ciphered ----------->  03  <-kcv->
+        self.assertEqual(b2h(field),'85' '11' '10' 'aabbccddeeff00112233445566778899' '03' + b2h(kcv))
+        self.assertEqual(b2h(field),'851110aabbccddeeff0011223344556677889903' + '06125d')
+
+    def test_psk_golden_over_scp02(self):
+        # Full PUT KEY data field (KVN 0x40 + single PSK key) enciphered with the SCP02 DEK.
+        keys = [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR,
+                 'kcv': compute_kcv('tls_psk', self.PSK_CLEAR)}]
+        data = self.C.build_put_key_data(0x40, keys, self.scp02)
+        self.assertEqual(b2h(data),
+                         '40' '85' '11' '10' + b2h(self.PSK_CIPHERED) + '03' + b2h(self.PSK_KCV))
+
+    def test_wrong_basic_format_differs(self):
+        # regression test, the generic "Basic format" does NOT match Table 3-13 for a PSK key
+        # rejected by card with with 6a88
+        wrong_basic = self.C.encode_key_data_basic('tls_psk', self.PSK_CIPHERED, b'')
+        right_psk = self.C.encode_key_data_psk(self.PSK_CLEAR, self.PSK_CIPHERED, self.PSK_KCV)
+        self.assertEqual(b2h(wrong_basic), '8510' + b2h(self.PSK_CIPHERED) + '00')
+        self.assertEqual(b2h(right_psk), '8511' '10' + b2h(self.PSK_CIPHERED) + '03' + b2h(self.PSK_KCV))
+        self.assertNotEqual(wrong_basic, right_psk)
+
+    def test_key_component_block_length_is_bertlv(self):
+        # GP CardSpec v2.3.1 Section 11.8.2.3.1: all lengths ofPUT KEY are always BER TLV coded
+        for kcb_len, exp_len_field in [(127, '7f'), (128, '8180'), (129, '8181'), (256, '820100')]:
+            with self.subTest(kcb_len=kcb_len):
+                kcb = bytes(kcb_len)
+                field = self.C.encode_key_data_basic('rsa_modulus_n', kcb, b'')
+                self.assertEqual(b2h(field), 'a2' + exp_len_field + b2h(kcb) + '00')
+                # 85 field of Amendment B Table 3-13 uses the same coding
+                # single byte inner length (clear key < 128) == block kcb_len bytes long
+                psk = self.C.encode_key_data_psk(bytes(120), bytes(kcb_len - 1), b'')
+                self.assertEqual(b2h(psk)[:2 + len(exp_len_field)], '85' + exp_len_field)
+
+    def test_basic_format_unchanged(self):
+        # as before
+        for kt, clear in [('des', h2b('404142434445464748494a4b4c4d4e4f')),
+                          ('aes', h2b('000102030405060708090a0b0c0d0e0f'))]:
+            ciph = self.scp02.encrypt_key(clear)
+            kcv = compute_kcv(kt, clear)
+            via_construct = build_construct(self.C.KeyDataBasic, {'key_type': kt, 'kcb': b2h(ciph), 'kcv': b2h(kcv)})
+            via_helper = self.C.encode_key_data_basic(kt, ciph, kcv)
+            self.assertEqual(via_helper, via_construct)
+
+    def test_psk_padding_no_double_length(self):
+        # A PSK key whose length is not a multiple of the DEK block size (DES: 8) is right-padded before
+        # ciphering. Table 3-13 states the clear key length (L2) in the '85' DO itself, so the ciphered
+        # key field is the bare cryptogram:
+        # - ciphered field == padded ciphertext (no duplicated length prefix),
+        # - clear key == first L2 bytes.
+        for keylen in (18, 20):
+            with self.subTest(keylen=keylen):
+                clear = bytes(range(keylen))
+                padded_len = keylen + (-keylen % 8)
+                field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': clear,
+                                                          'kcv': compute_kcv('tls_psk', clear)}], self.scp02)[1:]
+                self.assertEqual(field[0], 0x85)
+                l1 = field[1]
+                l2 = field[2]
+                self.assertEqual(l2, keylen)                # single-byte BER length of clear key
+                ciphered = field[3:3 + (l1 - 1)]            # value = L2 (1 byte) || ciphered key
+                self.assertEqual(len(ciphered), padded_len) # padded to the 8-byte DES block size
+                self.assertEqual(l1, 1 + padded_len)        # no duplicated length prefix
+                self.assertEqual(self.scp02.dek_decrypt(ciphered)[:keylen], clear)
+
+    def test_psk_clear_key_is_not_padded_in_place(self):
+        # padding the bytearray in place would make L2 the padded length,
+        # then stored as key material and rejected thanks to the KCV
+        clear = h2b('000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d')  # 30, not %8
+        kcv = compute_kcv('tls_psk', clear)
+        field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': clear,
+                                                  'kcv': kcv}], self.scp02)[1:]
+        self.assertEqual(len(clear), 30)
+        self.assertEqual(field[2], 30)                       # L2 == clear key length, not 32
+        self.assertEqual(self.scp02.dek_decrypt(field[3:3 + field[1] - 1])[:30], clear)
+
+    def test_kcv_suppressed(self):
+        # --suppress-key-check -> KCV length 00 and no KCV bytes
+        field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR,
+                                                  'kcv': b''}], self.scp02)[1:]
+        self.assertEqual(b2h(field), '8511' '10' + b2h(self.PSK_CIPHERED) + '00')
+
+    def test_multikey_psk_plus_des_dek(self):
+        # load a PSK TLS key (KID 1, Amendment B format) together with its DES DEK
+        # (KID 2, Basic format) in one PUT KEY.
+        # Verify the concatenated data field parses back into the two components with proper type formats.
+        dek = h2b('404142434445464748494a4b4c4d4e4f')
+        keys = [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR, 'kcv': compute_kcv('tls_psk', self.PSK_CLEAR)},
+                {'key_type': 'des', 'clear_key': dek, 'kcv': compute_kcv('des', dek)}]
+        data = self.C.build_put_key_data(0x40, keys, self.scp02)
+
+        b = data
+        self.assertEqual(b[0], 0x40)            # KVN
+        b = b[1:]
+        # component 1: PSK TLS (Table 3-13)
+        self.assertEqual(b[0], 0x85)
+        self.assertEqual(b[1], 0x11)            # L1 = 17
+        self.assertEqual(b[2], 0x10)            # L2 = 16 (clear key length)
+        self.assertEqual(b[3:3 + 16], self.PSK_CIPHERED)
+        self.assertEqual(b[3 + 16], 0x03)       # KCV length
+        self.assertEqual(b[3 + 16 + 1:3 + 16 + 1 + 3], self.PSK_KCV)
+        b = b[3 + 16 + 1 + 3:]
+        # component 2: DES DEK (Basic format)
+        self.assertEqual(b[0], 0x80)            # key type des
+        kcb_len = b[1]
+        self.assertEqual(kcb_len, 16)
+        self.assertEqual(b[2:2 + kcb_len], self.scp02.encrypt_key(dek))
+        b = b[2 + kcb_len:]
+        self.assertEqual(b[0], 0x03)            # KCV length
+        self.assertEqual(b[1:1 + 3], compute_kcv('des', dek))
+        self.assertEqual(b[1 + 3:], b'')        # no trailing bytes
+
+    def test_no_scp_leaves_key_clear(self):
+        # During personalization (no SCP) the key is not enciphered, framing still follows Table 3-13.
+        field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR,
+                                                  'kcv': self.PSK_KCV}], None)[1:]
+        self.assertEqual(b2h(field), '8511' '10' + b2h(self.PSK_CLEAR) + '03' + b2h(self.PSK_KCV))
+
+
+class PutKey_Length_Test(unittest.TestCase):
+    """Tests for the length of the PUT KEY command APDU.  Lc of GP CardSpec v2.3 Table 11-64 is a
+    single byte, so an oversized key data field cannot be sent."""
+
+    class PutKeyOnly(ADF_SD.AddlShellCommands):
+        """ADF_SD.AddlShellCommands with a canned scc to drive put_key()"""
+        def __init__(self, scp=None, max_cmd_len=255):
+            super().__init__()
+            self.sent = []
+            self.scc = SimpleNamespace(scp=scp, max_cmd_len=max_cmd_len,
+                                       send_apdu_checksw=lambda pdu: (self.sent.append(pdu), ('', '9000'))[1])
+
+        @property
+        def _cmd(self):
+            return SimpleNamespace(lchan=SimpleNamespace(scc=self.scc))
+
+    # KVN, key type, two byte BER length of the key component block, KCV length; KCV suppressed
+    FRAMING = 1 + 1 + 2 + 1
+
+    @staticmethod
+    def key(nbytes: int):
+        return [{'key_type': 'rsa_modulus_n', 'clear_key': bytes(nbytes), 'kcv': b''}]
+
+    def test_lc_matches_data_field(self):
+        # largest key component block that still fits without a secure channel
+        sd = self.PutKeyOnly()
+        sd.put_key(0, 0x40, 1, self.key(255 - self.FRAMING))
+        apdu = sd.sent[0]
+        self.assertEqual(apdu[:8], '80D80001')
+        lc = int(apdu[8:10], 16)
+        self.assertEqual(lc, 255)                       # Lc ...
+        self.assertEqual(len(apdu[10:-2]) // 2, lc)     # ... and it matches the actual data field
+
+    def test_oversized_key_data_raises(self):
+        # real world fat example: RSA-2048 modulus does not fit, led to 3 nibble Lc 106,
+        # which silently shifted and broke the whole APDU by half a byte.
+        sd = self.PutKeyOnly()
+        with self.assertRaises(ValueError) as ctx:
+            sd.put_key(0, 0x40, 1, self.key(256))
+        self.assertIn('262', str(ctx.exception))
+        self.assertIn('255', str(ctx.exception))
+        self.assertEqual(sd.sent, [])                   # nothing was sent to the card
+
+    def test_secure_channel_overhead_lowers_the_limit(self):
+        # scc.max_cmd_len shrinks by the C-MAC + encryption padding of active SCP
+        sd = self.PutKeyOnly(max_cmd_len=239)
+        sd.put_key(0, 0x40, 1, self.key(239 - self.FRAMING))
+        self.assertEqual(int(sd.sent[0][8:10], 16), 239)
+        with self.assertRaises(ValueError):
+            sd.put_key(0, 0x40, 1, self.key(239 - self.FRAMING + 1))
+
+
 class Install_param_Test(unittest.TestCase):
     def test_gen_install_parameters(self):
         load_parameters = gen_install_parameters(256, 256, '010001001505000000000000000000000000')
         self.assertEqual(load_parameters, 'c900ef1cc8020100c7020100ca12010001001505000000000000000000000000')
 
-        load_parameters = gen_install_parameters(None, None, '')
+        load_parameters = gen_install_parameters()
         self.assertEqual(load_parameters, 'c900')
 
+class SCP_Overhead_Test(unittest.TestCase):
+    """SCP.overhead varies according to the current security level:
+    C-MAC + at level >= 3 the worst-case padding!
+    """
+
+    def _scp02(self, security_level):
+        scp = SCP02(card_keys=ck_3des_70)
+        scp.sk = Scp02SessionKeys(0x0001, ck_3des_70)
+        scp.security_level = security_level
+        return scp
+
+    def _scp03(self, security_level, s_mode=8):
+        scp = SCP03(card_keys=KEYSET_AES128, s_mode=s_mode)
+        scp.sk = Scp03SessionKeys(KEYSET_AES128, b'\x00' * s_mode, b'\x11' * s_mode)
+        scp.security_level = security_level
+        return scp
+
+    def test_scp02(self):
+        self.assertEqual(self._scp02(0x00).overhead, 0)   # no wrapping at all
+        self.assertEqual(self._scp02(0x01).overhead, 8)   # C-MAC
+        self.assertEqual(self._scp02(0x03).overhead, 16)  # C-MAC + C-DEC: pad80 to 8, largest fit 239
+
+    def test_scp03_s8(self):
+        self.assertEqual(self._scp03(0x00).overhead, 0)
+        self.assertEqual(self._scp03(0x01).overhead, 8)
+        self.assertEqual(self._scp03(0x03).overhead, 16)  # pad80 to 16 within 247 -> 240, minus pad byte
+        self.assertEqual(self._scp03(0x33).overhead, 16)  # R-MAC/R-ENC add no *command* overhead
+
+    def test_scp03_s16(self):
+        self.assertEqual(self._scp03(0x01, s_mode=16).overhead, 16)
+        self.assertEqual(self._scp03(0x03, s_mode=16).overhead, 32)  # pad80 to 16 within 239 -> 224, minus pad byte
+
+
+class SCP_Lc_Limit_Test_Base(unittest.TestCase):
+    """Test wrap_cmd_apdu() boundary handling: data of (255 - overhead) must produce Lc <= 255 else ValueError"""
+
+    def _load_apdu(self, data_len):
+        return h2b('80E80000') + bytes([data_len]) + b'\xa5' * data_len
+
+    def _check_boundary(self, scp):
+        fits = 255 - scp.overhead
+        wrapped = scp.wrap_cmd_apdu(self._load_apdu(fits))
+        self.assertLessEqual(wrapped[4], 255)
+        self.assertEqual(len(wrapped), 5 + wrapped[4])  # case #3: header + Lc bytes, no Le
+        with self.assertRaises(ValueError) as ctx:
+            scp.wrap_cmd_apdu(self._load_apdu(fits + 1))
+        self.assertIn('Lc', str(ctx.exception))
+
+
+class SCP02_Lc_Limit_Test(SCP_Lc_Limit_Test_Base):
+    """Same session vectors as SCP02_Auth_Test"""
+
+    def setUp(self):
+        self.scp02 = SCP02(card_keys=ck_3des_70)
+        self.scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
+        self.scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
+        self.scp02.gen_ext_auth_apdu()
+
+    def test_cmac_only(self):
+        self.scp02.security_level = 0x01
+        self._check_boundary(self.scp02)  # 247 fits, 248 raises
+
+    def test_cmac_cdec(self):
+        self.scp02.security_level = 0x03
+        self._check_boundary(self.scp02)  # 239 fits (-> Lc 248), 240 raises (would be 256)
+
+    def test_cmac_cdec_wrapped_lc(self):
+        # my actual failing case: 240 bytes at level 3
+        self.scp02.security_level = 0x03
+        wrapped = self.scp02.wrap_cmd_apdu(self._load_apdu(239))
+        self.assertEqual(wrapped[4], 248)  # 239 -> pad80 -> 240 ciphertext + 8 mac
+
+
+class SCP03_Lc_Limit_Test(SCP_Lc_Limit_Test_Base):
+    """Session keys derived directly"""
+
+    def _scp03(self, security_level, s_mode):
+        scp = SCP03(card_keys=KEYSET_AES128, s_mode=s_mode)
+        scp.sk = Scp03SessionKeys(KEYSET_AES128, b'\x00' * s_mode, b'\x11' * s_mode)
+        scp.security_level = security_level
+        return scp
+
+    def test_s8_cmac_only(self):
+        self._check_boundary(self._scp03(0x01, 8))    # 247 fits, 248 raises
+
+    def test_s8_cmac_cdec(self):
+        self._check_boundary(self._scp03(0x03, 8))    # 239 fits, 240 raises
+
+    def test_s16_cmac_only(self):
+        self._check_boundary(self._scp03(0x01, 16))   # 239 fits, 240 raises
+
+    def test_s16_cmac_cdec(self):
+        self._check_boundary(self._scp03(0x03, 16))   # 223 fits, 224 raises
+
+
+class _FakeSccForLoad:
+    """mock lchan.scc: records LOAD APDUs, optionally wrapping them through a real SCP
+    instance first where the Lc overflow used to blow up"""
+
+    def __init__(self, max_cmd_len=255, scp=None):
+        self.max_cmd_len = max_cmd_len
+        self.scp = scp
+        self.sent = []
+        self.wrapped = []
+
+    def send_apdu_checksw(self, apdu, sw='9000'):
+        self.sent.append(apdu.lower())
+        if self.scp:
+            self.wrapped.append(self.scp.wrap_cmd_apdu(h2b(apdu)))
+        return ('', '9000')
+
+
+class Load_ChunkLen_Test(unittest.TestCase):
+    """ADF_SD.load() chunking: block size must use scc.max_cmd_len"""
+
+    payload = b'\xaa' * 500  # actual real world case LOAD TLV: C4 + 8201f4 + 500 = 504 total
+
+    def _sd(self, scc):
+        cmd = type('_Cmd', (), {'lchan': type('_Lchan', (), {'scc': scc})(),
+                                'poutput': lambda self, *args: None})()
+        # cmd2 CommandSet has a r/o _cmd property -> shadow it
+        _SD = type('_SD', (ADF_SD.AddlShellCommands,), {'_cmd': cmd})
+        return _SD.__new__(_SD)
+
+    def _blocks(self, scc):
+        """Get (p1, p2, lc) from LOAD APDU"""
+        for apdu in scc.sent:
+            self.assertEqual(apdu[0:4], '80e8')
+            yield int(apdu[4:6], 16), int(apdu[6:8], 16), int(apdu[8:10], 16)
+
+    def test_default_no_scp(self):
+        """Without SCP the old 240 byte block size is kept, no idea what else might rely on this number"""
+        scc = _FakeSccForLoad(max_cmd_len=255)
+        self._sd(scc).load(self.payload)
+        blocks = list(self._blocks(scc))
+        self.assertEqual([b[2] for b in blocks], [240, 240, 24])
+        self.assertEqual([b[0] for b in blocks], [0x00, 0x00, 0x80])  # P1: last block flagged
+        self.assertEqual([b[1] for b in blocks], [0, 1, 2])           # P2: block num
+
+    def test_default_scp02_level3(self):
+        """max_cmd_len 239 (SCP02 lvl 3) squeezes the blocks"""
+        scc = _FakeSccForLoad(max_cmd_len=239)
+        self._sd(scc).load(self.payload)
+        self.assertEqual([b[2] for b in list(self._blocks(scc))], [239, 239, 26])
+
+    def test_explicit_chunk_len(self):
+        scc = _FakeSccForLoad(max_cmd_len=255)
+        self._sd(scc).load(self.payload, chunk_len=100)
+        self.assertEqual([b[2] for b in list(self._blocks(scc))], [100] * 5 + [4])
+
+    def test_explicit_chunk_len_too_large(self):
+        scc = _FakeSccForLoad(max_cmd_len=239)
+        with self.assertRaises(ValueError):
+            self._sd(scc).load(self.payload, chunk_len=240)
+        self.assertEqual(scc.sent, [])  # nothing sent!
+
+    def test_explicit_chunk_len_zero(self):
+        scc = _FakeSccForLoad(max_cmd_len=255)
+        with self.assertRaises(ValueError):
+            self._sd(scc).load(self.payload, chunk_len=0)
+
+    def test_end_to_end_scp02_level3(self):
+        """original failure: 286 byte CAP + SCP02 lvl 3"""
+        scp02 = SCP02(card_keys=ck_3des_70)
+        scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
+        scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
+        scp02.gen_ext_auth_apdu()
+        scp02.security_level = 0x03
+        scc = _FakeSccForLoad(max_cmd_len=255 - scp02.overhead, scp=scp02)
+        self._sd(scc).load(b'\x5a' * 286)
+        self.assertEqual(len(scc.sent), 2)  # 289 byte TLV in blocks of 239
+        for wrapped in scc.wrapped:
+            self.assertLessEqual(wrapped[4], 255)
+
+# Real Card Data (GET DATA '66'), as returned by sja5 + euicc
+CARD_DATA_V211 = ('6631732f06072a864886fc6b01600c060a2a864886fc6b0202010163090607'
+                  '2a864886fc6b03640b06092a864886fc6b040215')
+CARD_DATA_V22 = ('663b733906072a864886fc6b01600b06092a864886fc6b020202630906072a86'
+                 '4886fc6b03640b06092a864886fc6b040370640b06092a864886fc6b04810400')
+
+
+class _FakeScc:
+    """mock lchan.scc: replays scripted (data, sw) pairs + records the APDUs sent."""
+
+    def __init__(self, responses, card_data=CARD_DATA_V211):
+        self._responses = list(responses)
+        self._card_data = card_data
+        self.sent = []
+
+    def get_data(self, cla, tag):
+        if self._card_data is None:
+            raise SwMatchError('6a88', '9000')
+        return self._card_data, '9000'
+
+    def send_apdu(self, apdu):
+        self.sent.append(apdu.lower())
+        if not self._responses:
+            raise AssertionError('get_status sent unexpected APDU: %s' % apdu)
+        return self._responses.pop(0)
+
+
+class GpVersion_Test(unittest.TestCase):
+    """GP version from Card Recognition Data, which v2.1.1/v2.3.1 section 7.4.1.3
+    require to be present. The OID under tag 60 is {globalPlatform 2 v...}."""
+
+    def test_decode_real_cards(self):
+        self.assertEqual(decode_gp_version(h2b(CARD_DATA_V211)), (2, 1, 1))
+        self.assertEqual(decode_gp_version(h2b(CARD_DATA_V22)), (2, 2))
+
+    def test_unknown_oid_is_none(self):
+        self.assertIsNone(decode_gp_version(h2b('66097307060512345678')))
+
+    def test_tag_lists_follow_the_spec_tables(self):
+        """table 11-36 applications, table 11-37 for load files"""
+        self.assertEqual(b2h(get_status_tag_list('isd')), '5c074f9f70c5cfc4cc')
+        self.assertEqual(b2h(get_status_tag_list('applications')), '5c074f9f70c5cfc4cc')
+        self.assertEqual(b2h(get_status_tag_list('files')), '5c054f9f70cecc')
+        self.assertEqual(b2h(get_status_tag_list('files_and_modules')), '5c064f9f70ce84cc')
+        # C5 never load files, 84 never applications
+        self.assertNotIn('c5', b2h(get_status_tag_list('files')))
+        self.assertNotIn('84', b2h(get_status_tag_list('applications'))[4:])
+
+
+class GetStatus_Pagination_Test(unittest.TestCase):
+    """GPC v2.3.1 section 11.4.3.2 table 11-38 GET STATUS pagination test
+
+    Card answers 6310 when further matches are pending; command reissued with
+    P2 bit 1 "next occurrence" set. Tied to T=0 handling pySim/transport, which
+    used to swallow that 6310 and replied with GET RESPONSE, so page 2 was never fetched."""
+
+    ENTRY_1 = 'e3074f05a000000151'
+    ENTRY_2 = 'e3074f05a000000152'
+
+    def _sd(self, responses, card_data=CARD_DATA_V211):
+        scc = _FakeScc(responses, card_data)
+        cmd = type('_Cmd', (), {'lchan': type('_Lchan', (), {'scc': scc})()})()
+        # cmd2 strikes again, CommandSet exposes _cmd as a read only property, needs shadowing
+        _SD = type('_SD', (ADF_SD.AddlShellCommands,), {'_cmd': cmd})
+        return _SD.__new__(_SD), scc
+
+    def _aids(self, grd_list):
+        return [b2h(grd.to_dict()['gp_registry_related_data'][0]['application_aid']) for grd in grd_list]
+
+    def test_single_page(self):
+        sd, scc = self._sd([(self.ENTRY_1, '9000')])
+        grd_list = sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f24002024f0000'])
+        self.assertEqual(self._aids(grd_list), ['a000000151'])
+
+    def test_two_pages(self):
+        """6310 -> reissue with P2 bit 1 set -> 9000, both pages in result"""
+        sd, scc = self._sd([(self.ENTRY_1, '6310'), (self.ENTRY_2, '9000')])
+        grd_list = sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f24002024f0000',
+                                    '80f24003024f0000'])
+        self.assertEqual(self._aids(grd_list), ['a000000151', 'a000000152'])
+
+    def test_three_pages_keep_p2_next_occurrence(self):
+        sd, scc = self._sd([(self.ENTRY_1, '6310'), (self.ENTRY_2, '6310'), (self.ENTRY_1, '9000')])
+        grd_list = sd.get_status('applications')
+        self.assertEqual([a[6:8] for a in scc.sent], ['02', '03', '03'])
+        self.assertEqual(len(grd_list), 3)
+
+    def test_no_match_returns_empty(self):
+        """6A88 "referenced data not found" is empty result not failure."""
+        sd, _scc = self._sd([('', '6a88')])
+        self.assertEqual(sd.get_status('applications'), [])
+
+    def test_v211_card_gets_no_tag_list(self):
+        """v2.1.1 section 9.4.2.3 has no tag list,not send a tag list"""
+        sd, scc = self._sd([(self.ENTRY_1, '9000')], card_data=CARD_DATA_V211)
+        sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f24002024f0000'])
+        self.assertNotIn('5c', scc.sent[0][8:])
+
+    def test_v22_card_gets_a_tag_list(self):
+        sd, scc = self._sd([(self.ENTRY_1, '9000')], card_data=CARD_DATA_V22)
+        sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f240020b4f005c074f9f70c5cfc4cc00'])
+
+    def test_unknown_version_gets_no_tag_list(self):
+        """If the card will not say, assume the conservative form that works everywhere."""
+        sd, scc = self._sd([(self.ENTRY_1, '9000')], card_data=None)
+        sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f24002024f0000'])
+
+    def test_v22_card_rejecting_tag_list_falls_back(self):
+        """card announcing v2.2+ that still answers 6A80 to the tag list."""
+        sd, scc = self._sd([('', '6a80'), (self.ENTRY_1, '9000')], card_data=CARD_DATA_V22)
+        grd_list = sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f240020b4f005c074f9f70c5cfc4cc00',
+                                    '80f24002024f0000'])
+        self.assertEqual(self._aids(grd_list), ['a000000151'])
+
+    def test_aid_search_qualifier(self):
+        sd, scc = self._sd([(self.ENTRY_1, '9000')])
+        sd.get_status('applications', 'a000000087')
+        self.assertEqual(scc.sent, ['80f24002074f05a00000008700'])
+
+    def test_6a80_is_reported_on_a_v211_card(self):
+        """no tag list -> 6A80 is error"""
+        sd, _scc = self._sd([('', '6a80')], card_data=CARD_DATA_V211)
+        with self.assertRaises(SwMatchError) as ctx:
+            sd.get_status('applications')
+        self.assertEqual(ctx.exception.sw_actual, '6a80')
+
+    def test_unexpected_sw_is_not_silently_truncated(self):
+        """partial is not complete result"""
+        sd, _scc = self._sd([(self.ENTRY_1, '6310'), ('', '6982')])
+        with self.assertRaises(SwMatchError) as ctx:
+            sd.get_status('applications')
+        self.assertEqual(ctx.exception.sw_actual, '6982')
+
+    def test_v22_card_answering_6a88_to_the_tag_list_falls_back(self):
+        """6A88 is the other GET STATUS error condition of table 11-39, section 11.4.2.3
+        says we may get get an error status. 6A88 to the tag-list attempt should be retried
+        without it or we get nothing"""
+        sd, scc = self._sd([('', '6a88'), (self.ENTRY_1, '9000')], card_data=CARD_DATA_V22)
+        grd_list = sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f240020b4f005c074f9f70c5cfc4cc00',
+                                    '80f24002024f0000'])
+        self.assertEqual(self._aids(grd_list), ['a000000151'])
+
+    def test_v22_card_with_a_genuinely_empty_subset(self):
+        """...and when the retry answers 6A88, the list really is empty."""
+        sd, scc = self._sd([('', '6a88'), ('', '6a88')], card_data=CARD_DATA_V22)
+        self.assertEqual(sd.get_status('applications'), [])
+        self.assertEqual(len(scc.sent), 2)
+
+    def test_6a88_after_a_page_keeps_that_page(self):
+        """6A88 is "no more matches" after we have data, we're done"""
+        sd, _scc = self._sd([(self.ENTRY_1, '6310'), ('', '6a88')], card_data=CARD_DATA_V22)
+        self.assertEqual(self._aids(sd.get_status('applications')), ['a000000151'])
+
 if __name__ == "__main__":
 	unittest.main()

tests/unittests/test_log.py

@@ -20,11 +20,20 @@
 
 import unittest
 import logging
+import cmd2
+from packaging import version
 from pySim.log import PySimLogger
 import io
 import sys
 from inspect import currentframe, getframeinfo
 
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
+    from cmd2 import Color # pylint: disable=no-name-in-module
+    YELLOW = Color.YELLOW
+else: # cmd2>=2.6.2
+    from cmd2 import Fg # pylint: disable=no-name-in-module
+    YELLOW = Fg.YELLOW
+
 log = PySimLogger.get(__name__)
 
 TEST_MSG_DEBUG = "this is a debug message"
@@ -37,6 +46,17 @@ expected_message = None
 
 class PySimLogger_Test(unittest.TestCase):
 
+    def setUp(self):
+        # PySimLogger.setup() is global, so a print callback left installed here fires for
+        # every PySimLogger message emitted by any test module that runs later in the same process
+        # ... where it asserts against a stale 'expected_message' and fails a test that has nothing
+        # to do with logging. Great fun!
+        # Restore before each test.
+        saved = (PySimLogger.print_callback, PySimLogger.verbose)
+        def _restore():
+            PySimLogger.print_callback, PySimLogger.verbose = saved
+        self.addCleanup(_restore)
+
     def __test_01_safe_defaults_one(self, callback, message:str):
         # When log messages are sent to an unconfigured PySimLogger class, we expect the unmodified message being
         # logged to stdout, just as if it were printed via a normal print() statement.
@@ -117,5 +137,18 @@ class PySimLogger_Test(unittest.TestCase):
         expected_message = "CRITICAL: " + TEST_MSG_CRITICAL
         log.critical(TEST_MSG_CRITICAL)
 
+    def test_05_color(self):
+        # A color is either
+        # - raw escape sequence
+        # - cmd2 color object
+        global expected_message
+        expected_message = "\033[33mWARNING: " + TEST_MSG_WARNING + "\033[0m"
+
+        PySimLogger.setup(self._test_print_callback, {logging.WARN: "\033[33m"})
+        log.warning(TEST_MSG_WARNING)
+
+        PySimLogger.setup(self._test_print_callback, {logging.WARN: YELLOW})
+        log.warning(TEST_MSG_WARNING)                   # don't leak cmd2 Color StrEnum
+
 if __name__ == '__main__':
     unittest.main()

tests/unittests/test_ota.py

@@ -300,5 +300,292 @@ class SmsOtaTestCase(OtaTestCase):
                 self.assertEqual(d.last_status_word, t['response']['last_status_word'])
                 self.assertEqual(d.last_response_data, t['response']['last_response_data'])
 
+
+######################################################################
+# Expanded Remote Application data format (ETSI TS 102 226 Section 5.2)
+######################################################################
+
+class BerTlvLengthTestCase(unittest.TestCase):
+    """The definite-length BER-TLV length field (ISO/IEC 8825-1) used by the
+    expanded format, incl. the multi-byte (>127) forms (0x81xx / 0x82xxxx)."""
+    def test_roundtrip(self):
+        # (length value, expected encoded bytes)
+        vectors = [
+            (0,     '00'),
+            (1,     '01'),
+            (127,   '7f'),
+            (128,   '8180'),
+            (198,   '81c6'),   # big ~198 byte GET STATUS registry from a sja5
+            (255,   '81ff'),
+            (256,   '820100'),
+            (65535, '82ffff'),
+        ]
+        for length, encoded in vectors:
+            with self.subTest(length=length):
+                built = BerTlvLen.build(length)
+                self.assertEqual(b2h(built), encoded)
+                self.assertEqual(BerTlvLen.parse(built), length)
+
+
+class ExpandedCmdTestCase(unittest.TestCase):
+    """Command Scripting template TS 102 226 5.2.1"""
+
+    def test_single_capdu_golden(self):
+        # GP GET STATUS, Le=00, TS 102 226 5.2.1.1 R-APDU
+        out = encode_expanded_cmd(h2b('80f24002024f0000'))
+        # aa = TS 101 220 table 7.18 Command Scripting template tag
+        # 0a = length 10
+        # 22 = TS 101 220 table 7.19 C-APDU tag
+        # 08 = length
+        # + C-APDU
+        self.assertEqual(b2h(out), 'aa0a220880f24002024f0000')
+
+    def test_multi_capdu_golden(self):
+        out = encode_expanded_cmd([h2b('80f24002024f0000'), h2b('00a40004023f0000')])
+        self.assertEqual(b2h(out), 'aa14220880f24002024f0000220800a40004023f0000')
+
+    def test_multibyte_length_golden(self):
+        # C-APDU: 4 header + 1 Lc + 195 data = 200 bytes.
+        # 200 byte C-APDU forces long form BER lengths:
+        # C-APDU TLV, 200 -> 81c8 + template 203 -> 81cb
+        capdu = h2b('80f24000') + bytes([195]) + bytes(range(195))
+        self.assertEqual(len(capdu), 200)
+        out = encode_expanded_cmd(capdu)
+        # aa 81 cb | 22 81 c8 | <200 byte capdu>
+        self.assertEqual(b2h(out[:6]), 'aa81cb2281c8')
+        self.assertEqual(out[6:], capdu)
+
+    def test_roundtrip(self):
+        for apdus in [[h2b('80f24002024f0000')],
+                      [h2b('00a40004023f00'), h2b('80f24002024f0000')],
+                      [h2b('00'*250)]]:
+            with self.subTest(n=len(apdus)):
+                out = encode_expanded_cmd(apdus)
+                parsed = ExpandedCmd.parse(out)
+                self.assertEqual([h2b(c.c_apdu) for c in parsed.commands], apdus)
+
+
+class ExpandedRespTestCase(unittest.TestCase):
+    """Decoding of the Response Scripting template (TS 102 226 5.2.2)."""
+
+    def test_registry_golden(self):
+        # real card case: GET STATUS returns a ~198 byte registry TLV + SW 9000
+        # R-APDU = 198 data + 2 SW = 200/81c8
+        # 'number of executed' TLV 80 01 01.
+        registry = bytes(range(198))
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data=b2h(registry), status_word='9000'))])))
+        # ab | 81 ce | 80 01 01 | 23 81 c8 | <198 data> 90 00
+        self.assertEqual(b2h(data[:9]), 'ab81ce8001012381c8')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(dec.number_of_commands, 1)
+        self.assertEqual(len(dec.commands), 1)
+        self.assertEqual(dec.last_status_word, '9000')
+        self.assertEqual(dec.last_response_data, b2h(registry))
+
+    def test_status_only_golden(self):
+        # last command, no response data, SW 6132
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data='', status_word='6132'))])))
+        self.assertEqual(b2h(data), 'ab0780010123026132')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(dec.last_status_word, '6132')
+        self.assertEqual(dec.last_response_data, '')
+
+    def test_multi_command(self):
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=2),
+                    responses=[dict(r_apdu=dict(response_data='6f21', status_word='9000')),
+                               dict(r_apdu=dict(response_data='', status_word='6a82'))])))
+        dec = decode_expanded_resp(data)
+        self.assertEqual(dec.number_of_commands, 2)
+        self.assertEqual([(c.status_word, c.response_data) for c in dec.commands],
+                         [('9000', '6f21'), ('6a82', '')])
+        # last == final R-APDU, error status included
+        self.assertEqual(dec.last_status_word, '6a82')
+        self.assertEqual(dec.last_response_data, '')
+
+    def test_bad_format(self):
+        # ab | 06 | 80 01 01 | 90 01 01
+        data = h2b('ab06800101900101')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(str(dec.bad_format), 'unknown_tag')
+        self.assertIsNone(dec.last_status_word)
+
+    def test_immediate_action_error(self):
+        # ab | 06 | 80 01 01 | 81 01 01
+        data = h2b('ab06800101810101')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(str(dec.immediate_action_response), 'suspension_error')
+
+    def test_script_chaining_error(self):
+        # ab | 06 | 80 01 01 | 83 01 02
+        data = h2b('ab06800101830102')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(str(dec.script_chaining_response), 'not_supported')
+
+    def test_truncation_is_flagged(self):
+        """TS 102 226 5.2.1.1: SW 62F1 means the C-APDU response data was truncated, and
+        "this shall terminate the processing of the command list"
+         halves are invisible in the R-APDU list, truncated + aborted script must not pass as complete"""
+        # second command truncated -> processing stopped at that point
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=2),
+                    responses=[dict(r_apdu=dict(response_data='6f21', status_word='9000')),
+                               dict(r_apdu=dict(response_data='aabb', status_word='62f1'))])))
+        dec = decode_expanded_resp(data)
+        self.assertTrue(dec.truncated)
+        self.assertEqual(dec.last_status_word, '62f1')
+
+    def test_untruncated_response_is_not_flagged(self):
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data='6f21', status_word='9000'))])))
+        self.assertFalse(decode_expanded_resp(data).truncated)
+        # 62xx that is not 62F1 is warning, not truncation
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data='', status_word='6282'))])))
+        self.assertFalse(decode_expanded_resp(data).truncated)
+
+
+class ExpandedIndefiniteTestCase(unittest.TestCase):
+    """Indef len coding of expanded format TS 102 226 tables
+    5.2a/5.10a; cmd tag AE, resp tag AF.
+    Golden vectors captured from live eUICC over SCP81/HTTPS."""
+
+    def test_cmd_single_golden(self):
+        # RAM GET DATA 80CA00E000 -> AE 80 | 22 05 80ca00e000 | 00 00
+        out = encode_expanded_cmd(h2b('80ca00e000'), length_coding='indefinite')
+        self.assertEqual(b2h(out), 'ae80220580ca00e0000000')
+
+    def test_cmd_multi_golden(self):
+        # RFM: SELECT MF / SELECT EF.ICCID / READ BINARY, each in one C-APDU
+        # TLV, wrapped in indef Command Scripting template
+        out = encode_expanded_cmd([h2b('00a4000c023f00'), h2b('00a4000c022fe2'),
+                                   h2b('00b000000a')], length_coding='indefinite')
+        self.assertEqual(b2h(out),
+                         'ae80220700a4000c023f00220700a4000c022fe2220500b000000a0000')
+
+    def test_cmd_definite_is_default(self):
+        # The default/explicit definite keeps the tag AA
+        self.assertEqual(encode_expanded_cmd(h2b('80ca00e000')),
+                         encode_expanded_cmd(h2b('80ca00e000'), length_coding='definite'))
+        self.assertEqual(b2h(encode_expanded_cmd(h2b('80ca00e000'))), 'aa07220580ca00e000')
+
+    def test_cmd_invalid_length_coding(self):
+        with self.assertRaises(ValueError):
+            encode_expanded_cmd(h2b('80ca00e000'), length_coding='bogus')
+
+    def test_resp_rfm_golden(self):
+        # AF 80 | 23 02 9000 | 23 02 9000 | 23 0c <ICCID> 9000 | 00 00
+        # indef res has no "number of executed" TLV.
+        dec = decode_expanded_resp(h2b(
+            'af80' '23029000' '23029000' '230c988812010000408608149000' '0000'))
+        self.assertEqual(len(dec.commands), 3)
+        self.assertEqual([(c.status_word, c.response_data) for c in dec.commands],
+                         [('9000', ''), ('9000', ''), ('9000', '98881201000040860814')])
+        self.assertEqual(dec.last_status_word, '9000')
+        self.assertEqual(dec.last_response_data, '98881201000040860814')
+        # report the R-APDU count instead
+        self.assertEqual(dec.number_of_commands, 3)
+
+    def test_resp_ram_golden(self):
+        # RAM GET DATA: R-APDU carrying the SD key info TLV + SW.
+        resp = ('af80' '2334e030c00403308810c00402308810c00401308810c00402408810'
+                'c00401408510c00403018810c00402018810c004010188109000' '0000')
+        dec = decode_expanded_resp(h2b(resp))
+        self.assertEqual(len(dec.commands), 1)
+        self.assertEqual(dec.last_status_word, '9000')
+        self.assertEqual(dec.last_response_data,
+                         'e030c00403308810c00402308810c00401308810c00402408810'
+                         'c00401408510c00403018810c00402018810c00401018810')
+
+    def test_resp_truncated_is_rejected(self):
+        # last byte chopped off: the end-of-contents marker is incomplete
+        good = h2b('af80' '23029000' '230c988812010000408608149000' '0000')
+        for cut in (1, 2, 3):
+            with self.subTest(cut=cut):
+                with self.assertRaises(ValueError):
+                    decode_expanded_resp(good[:-cut])
+
+    def test_resp_definite_still_parses(self):
+        # same decoder still handles the definite AB template.
+        dec = decode_expanded_resp(h2b('ab0780010123029000'))
+        self.assertEqual(dec.number_of_commands, 1)
+        self.assertEqual(dec.last_status_word, '9000')
+
+    def test_resp_indefinite_bad_format(self):
+        # AF 80 | 90 01 01 | 00 00 unknown_tag no R-APDU
+        dec = decode_expanded_resp(h2b('af8090010100 00'.replace(' ', '')))
+        self.assertEqual(str(dec.bad_format), 'unknown_tag')
+        self.assertIsNone(dec.last_status_word)
+
+    def test_resp_missing_eoc_raises(self):
+        # AF 80 | 23 02 9000 without end-of-contents.
+        with self.assertRaises(ValueError):
+            decode_expanded_resp(h2b('af8023029000'))
+
+
+class ExpandedSmsPipelineTestCase(unittest.TestCase):
+    """expanded format + TS 102 225 SMS security witj 3DES keyset,
+    to ensure remote_format does not affect the compact path"""
+    def __init__(self, methodName='runTest', **kwargs):
+        super().__init__(methodName, **kwargs)
+        self.od = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3,
+                            kic=h2b('C21DD66ACAC13CB3BC8B331B24AFB57B'),
+                            algo_auth='triple_des_cbc2', kid_idx=3,
+                            kid=h2b('12110C78E678C25408233076AA033615'))
+        self.dialect = OtaDialectSms()
+        self.tar = h2b('000000')
+
+    def test_cmd_expanded_secured_roundtrip(self):
+        spi = SPI_CC_POR_CIPHERED_CC
+        enc = self.dialect.encode_cmd(self.od, self.tar, spi, h2b('80f24002024f0000'),
+                                      remote_format='expanded')
+        # decode_cmd returns opaque 'Command Scripting template'
+        dec_tar, dec_spi, dec_secured = self.dialect.decode_cmd(self.od, enc)
+        self.assertEqual(b2h(dec_tar), b2h(self.tar))
+        self.assertEqual(dec_spi, spi)
+        self.assertEqual(b2h(dec_secured), 'aa0a220880f24002024f0000')
+
+    def test_cmd_expanded_list(self):
+        spi = SPI_CC_POR_CIPHERED_CC
+        enc = self.dialect.encode_cmd(self.od, self.tar, spi,
+                                      [h2b('80f24002024f0000'), h2b('00a40004023f0000')],
+                                      remote_format='expanded')
+        _, _, dec_secured = self.dialect.decode_cmd(self.od, enc)
+        parsed = ExpandedCmd.parse(dec_secured)
+        self.assertEqual([c.c_apdu for c in parsed.commands],
+                         ['80f24002024f0000', '00a40004023f0000'])
+
+    def test_resp_expanded_plaintext(self):
+        # plaintext (u:nciphered + no CC) expanded response SMS
+        # containing a 198 byte GP registry + SW 9000 as above, decode it through decode_resp().
+        spi = SPI_CC_POR_UNCIPHERED_NOCC
+        registry = bytes(range(198))
+        secured = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data=b2h(registry), status_word='9000'))])))
+        rpl = 1 + 3 + 5 + 1 + 1 + len(secured)   # RHL-STS + secured data
+        resp_body = rpl.to_bytes(2, 'big') + b'\x0a' + self.tar + b'\x00'*5 + b'\x00' + b'\x00' + secured
+        sms = b'\x02\x71\x00' + resp_body
+        r, dec = self.dialect.decode_resp(self.od, spi, sms, remote_format='expanded')
+        self.assertEqual(r.response_status, 'por_ok')
+        self.assertEqual(dec.number_of_commands, 1)
+        self.assertEqual(dec.last_status_word, '9000')
+        self.assertEqual(dec.last_response_data, b2h(registry))
+
+    def test_compact_still_default(self):
+        # no remote_format -> compact default
+        spi = SPI_CC_POR_UNCIPHERED_NOCC
+        r, d = self.dialect.decode_resp(self.od, spi, '027100000e0ab000110000000000000001612f')
+        self.assertEqual(d.number_of_commands, 1)
+        self.assertEqual(d.last_status_word, '612f')
+        self.assertEqual(d.last_response_data, '')
+
+
 if __name__ == "__main__":
 	unittest.main()

tests/unittests/test_param_src.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+
+# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: Neels Hofmeyr
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import math
+from importlib import resources
+import unittest
+from pySim.esim.saip import param_source
+
+import xo
+update_expected_output = False
+
+class D:
+    mandatory = set()
+    optional = set()
+
+    def __init__(self, **kwargs):
+        if (set(kwargs.keys()) - set(self.optional)) != set(self.mandatory):
+            raise RuntimeError(f'{self.__class__.__name__}.__init__():'
+                               f' {set(kwargs.keys())=!r} - {self.optional=!r} != {self.mandatory=!r}')
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+        for k in self.optional:
+            if not hasattr(self, k):
+                setattr(self, k, None)
+
+decimals = '0123456789'
+hexadecimals = '0123456789abcdefABCDEF'
+
+class FakeRandom:
+    vals = b'\xab\xcfm\xf0\x98J_\xcf\x96\x87fp5l\xe7f\xd1\xd6\x97\xc1\xf9]\x8c\x86+\xdb\t^ke\xc1r'
+    i = 0
+
+    @classmethod
+    def next(cls):
+        cls.i = (cls.i + 1) % len(cls.vals)
+        return cls.vals[cls.i]
+
+    @staticmethod
+    def randint(a, b):
+        d = b - a
+        n_bytes = math.ceil(math.log(d, 2))
+        r = int.from_bytes( bytes(FakeRandom.next() for i in range(n_bytes)) )
+        return a + (r % (b - a))
+
+    @staticmethod
+    def randbytes(n):
+        return bytes(FakeRandom.next() for i in range(n))
+
+
+class ParamSourceTest(unittest.TestCase):
+
+    def test_param_source(self):
+
+        class Paramtest(D):
+            mandatory = (
+                    'param_source',
+                    'n',
+                    'expect',
+                    )
+            optional = (
+                    'expect_arg',
+                    'csv_rows',
+                    )
+            param_source: param_source.ParamSource
+            n: int
+            expect: object
+            expect_arg: object
+            csv_rows: object
+
+        def expect_const(t, vals):
+            return tuple(t.expect_arg) == tuple(vals)
+
+        def expect_random(t, vals):
+            chars = t.expect_arg.get('digits')
+            repetitions = (t.n - len(set(vals)))
+            if repetitions:
+                raise RuntimeError(f'expect_random: there are {repetitions} repetitions in the returned values: {vals}')
+            for val_i in range(len(vals)):
+                v = vals[val_i]
+                val_minlen = t.expect_arg.get('val_minlen')
+                val_maxlen = t.expect_arg.get('val_maxlen')
+                if len(v) < val_minlen or len(v) > val_maxlen:
+                    raise RuntimeError(f'expect_random: invalid length {len(v)} for value [{val_i}]: {v!r}, expecting'
+                                       f' {val_minlen}..{val_maxlen}')
+
+                if chars is not None and not all(c in chars for c in v):
+                    raise RuntimeError(f'expect_random: invalid char in value [{val_i}]: {v!r}')
+            return True
+
+        param_source_tests = [
+            Paramtest(param_source=param_source.ConstantSource.from_str('123'),
+                      n=3,
+                      expect=expect_const,
+                      expect_arg=('123', '123', '123')),
+            Paramtest(param_source=param_source.RandomDigitSource.from_str('12345'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': decimals,
+                                  'val_minlen': 5,
+                                  'val_maxlen': 5}),
+            Paramtest(param_source=param_source.RandomDigitSource.from_str('1..999'),
+                      n=10,
+                      expect=expect_random,
+                      expect_arg={'digits': decimals,
+                                  'val_minlen': 1,
+                                  'val_maxlen': 3}),
+            Paramtest(param_source=param_source.RandomDigitSource.from_str('001..999'),
+                      n=10,
+                      expect=expect_random,
+                      expect_arg={'digits': decimals,
+                                  'val_minlen': 3,
+                                  'val_maxlen': 3}),
+            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('12345678'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': hexadecimals,
+                                  'val_minlen': 8,
+                                  'val_maxlen': 8}),
+            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('0*8'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': hexadecimals,
+                                  'val_minlen': 8,
+                                  'val_maxlen': 8}),
+            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('00*4'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': hexadecimals,
+                                  'val_minlen': 8,
+                                  'val_maxlen': 8}),
+            Paramtest(param_source=param_source.IncDigitSource.from_str('10001'),
+                      n=3,
+                      expect=expect_const,
+                      expect_arg=('10001', '10002', '10003')),
+            Paramtest(param_source=param_source.CsvSource('column_name'),
+                      n=3,
+                      expect=expect_const,
+                      expect_arg=('first val', 'second val', 'third val'),
+                      csv_rows=(
+                          {'column_name': 'first val'},
+                          {'column_name': 'second val'},
+                          {'column_name': 'third val'},
+                      )),
+            ]
+
+        outputs = []
+
+        for t in param_source_tests:
+            try:
+                if hasattr(t.param_source, 'random_impl'):
+                    t.param_source.random_impl = FakeRandom
+
+                vals = []
+                for i in range(t.n):
+                    csv_row = None
+                    if t.csv_rows is not None:
+                        csv_row = t.csv_rows[i]
+                    vals.append( t.param_source.get_next(csv_row=csv_row) )
+                if not t.expect(t, vals):
+                    raise RuntimeError(f'invalid values returned: returned {vals}')
+                output = f'ok: {t.param_source.__class__.__name__} {vals=!r}'
+                outputs.append(output)
+                print(output)
+            except RuntimeError as e:
+                raise RuntimeError(f'{t.param_source.__class__.__name__} {t.n=} {t.expect.__name__}({t.expect_arg!r}): {e}') from e
+
+        output = '\n'.join(outputs) + '\n'
+        xo_name = 'test_param_src'
+        if update_expected_output:
+            with resources.path(xo, xo_name) as xo_path:
+                with open(xo_path, 'w', encoding='utf-8') as f:
+                    f.write(output)
+        else:
+            xo_str = resources.read_text(xo, xo_name)
+            if xo_str != output:
+                at = 0
+                while at < len(output):
+                    if output[at] == xo_str[at]:
+                        at += 1
+                        continue
+                    break
+
+                raise RuntimeError(f'output differs from expected output at position {at}: {xo_str[at:at+128]!r}')
+
+if __name__ == "__main__":
+    if '-u' in sys.argv:
+        update_expected_output = True
+        sys.argv.remove('-u')
+    unittest.main()

tests/unittests/test_smpp_ota_tool.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+""" test for smpp-ota-tool SMS handling, specifically the multi part sms OTA response"""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os.path
+import importlib.util
+import unittest
+
+from osmocom.utils import h2b, b2h
+
+from pySim.ota import OtaKeyset, OtaDialectSms, ExpandedRemoteResp
+from pySim.sms import ConcatenatedSmsReassembler, UserDataHeader
+
+# import the hyphenated contrib script as a module to get at SmppHandler
+# why do people name python files like that? why does everything have to be so hard?
+_TOOL_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'contrib', 'smpp-ota-tool.py')
+_spec = importlib.util.spec_from_file_location('smpp_ota_tool', _TOOL_PATH)
+smpp_ota_tool = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(smpp_ota_tool)
+SmppHandler = smpp_ota_tool.SmppHandler
+
+
+class _FakePdu:
+    """Minimal mock for smpplib deliver_sm pdu."""
+    def __init__(self, short_message):
+        self.short_message = short_message
+
+
+class MultipartRelayTestCase(unittest.TestCase):
+    """message_received_handler must return the reassembled application
+    response and survive POR messages."""
+
+    # 3DES test keyset from tests/unittests/test_ota.py) used to make the
+    # handler happy. responses are plaintext, tests do not depend on keys.
+    def _handler(self, remote_format='expanded'):
+        h = object.__new__(SmppHandler)
+        h.client = None
+        h.ota_dialect = OtaDialectSms()
+        h.ota_keyset = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3,
+                                 kic=h2b('C21DD66ACAC13CB3BC8B331B24AFB57B'),
+                                 algo_auth='triple_des_cbc2', kid_idx=3,
+                                 kid=h2b('12110C78E678C25408233076AA033615'))
+        h.tar = h2b('000000')
+        # unciphered, no CC, PoR required
+        h.spi = {'counter': 'no_counter', 'ciphering': False, 'rc_cc_ds': 'no_rc_cc_ds',
+                 'por_in_submit': False, 'por': 'por_required',
+                 'por_shall_be_ciphered': False, 'por_rc_cc_ds': 'no_rc_cc_ds'}
+        h.remote_format = remote_format
+        h.reassembler = ConcatenatedSmsReassembler()
+        h.response = None
+        return h
+
+    @staticmethod
+    def _plaintext_resp_sms(secured: bytes, sts: int = 0x00) -> bytes:
+        """Build a plaintext (unciphered, no-CC) OTA SMS response packet in the
+        canonical single-part form (UDH 02 71 00 + response packet)."""
+        rpl = 1 + 3 + 5 + 1 + 1 + len(secured)   # RHL-STS + secured data
+        body = (rpl.to_bytes(2, 'big') + b'\x0a' + h2b('000000') + b'\x00' * 5
+                + b'\x00' + bytes([sts]) + secured)
+        return b'\x02\x71\x00' + body
+
+    @staticmethod
+    def _expanded_secured(response_data_hex: str, sw: str = '9000') -> bytes:
+        return ExpandedRemoteResp.build(dict(body=dict(
+            num_executed=dict(number_of_commands=1),
+            responses=[dict(r_apdu=dict(response_data=response_data_hex, status_word=sw))])))
+
+    @staticmethod
+    def _fragment_2(tpud: bytes, ref: int, first_len: int):
+        """Split 02 71 00 + body TP-UD into two SMS parts:
+        - part1 carries the OTA (0x71) IE
+        - part2 only concatenat IE
+        matches sja5 interaction"""
+        assert tpud[:3] == b'\x02\x71\x00'
+        body = tpud[3:]
+        ota_ie = {'iei': 0x71, 'length': 0, 'value': b''}
+
+        def concat(seq):
+            return {'iei': 0x00, 'length': 3, 'value': bytes([ref, 2, seq])}
+        p1 = UserDataHeader([concat(1), ota_ie]).to_bytes() + body[:first_len]
+        p2 = UserDataHeader([concat(2)]).to_bytes() + body[first_len:]
+        return p1, p2
+
+    # ground truth: TP-User-Data captured from a sja5
+    REAL_PART1 = h2b('070003010201710000e412000000df63afe4b06db21e2113be1be09e9b66f1c113ae841cca2d030064ec16b5b80ee5ce824604a4568109d25a82fb74a325df6f911bd0a4f858ece2c770039002c480269fc65953f5fd93ebbe528d97838bac4389a7303db2b073a37a9a1a51890457f41b49fc7905ce337e83449b65560501b8b845fe63339d557a928f2643')
+    REAL_PART2 = h2b('050003010202fd9c4e50ec40fb4427af518e9c08697405d91fbb6e9fa0b0935f48a560e15f2f3f27a2e44ef3a47280acce77f030fb70eb3df863c159177e2c0e3e53052fc7bb7ed171a491ded3ab7921861176a04305bc09fcf526c07bf6bb48a19e67cf18be5bc1')
+    REAL_REASSEMBLED = '02710000e412000000df63afe4b06db21e2113be1be09e9b66f1c113ae841cca2d030064ec16b5b80ee5ce824604a4568109d25a82fb74a325df6f911bd0a4f858ece2c770039002c480269fc65953f5fd93ebbe528d97838bac4389a7303db2b073a37a9a1a51890457f41b49fc7905ce337e83449b65560501b8b845fe63339d557a928f2643fd9c4e50ec40fb4427af518e9c08697405d91fbb6e9fa0b0935f48a560e15f2f3f27a2e44ef3a47280acce77f030fb70eb3df863c159177e2c0e3e53052fc7bb7ed171a491ded3ab7921861176a04305bc09fcf526c07bf6bb48a19e67cf18be5bc1'
+
+    def test_real_card_parts_reassemble(self):
+        """two real card TP-UDs recombine into 233-byte single part packet:
+        UDH 02 71 00 + response packet"""
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self.REAL_PART1))
+        out = r.add(self.REAL_PART2)
+        self.assertEqual(len(out), 233)
+        self.assertEqual(b2h(out), self.REAL_REASSEMBLED)
+
+    def test_multipart_response_not_overwritten_by_por(self):
+        """reassembled application response must survive the ENVELOPE
+        trailing POR which contains no R-APDU"""
+        registry = bytes(range(198))
+        app = self._plaintext_resp_sms(self._expanded_secured(b2h(registry)))
+        part1, part2 = self._fragment_2(app, ref=0x42, first_len=132)
+        # single part form must be too fat -> both parts must be concatenated
+        self.assertGreater(len(app), 140)
+        # ENVELOPE PoR: por_ok, but no app R-APDU
+        inline_por = self._plaintext_resp_sms(b'', sts=0x00)
+
+        h = self._handler()
+        # arrival order
+        self.assertIsNone(h.message_received_handler(_FakePdu(part1)))
+        h.message_received_handler(_FakePdu(part2))
+        h.message_received_handler(_FakePdu(inline_por))
+
+        # self.response must be app response, not the PoR!
+        self.assertIsNotNone(h.response)
+        res, decoded = h.response
+        self.assertEqual(res.response_status, 'por_ok')
+        self.assertIsNotNone(decoded)
+        self.assertEqual(decoded.last_response_data, b2h(registry))
+        self.assertEqual(decoded.last_status_word, '9000')
+
+    def test_undecodable_response_does_not_crash(self):
+        """response the handler can't decode must not escape out of the poll()
+        loop which would kill the tool, it must be ignored"""
+        # por_ok with a not expanded 'secured data' -> expanded parse raises
+        bad = self._plaintext_resp_sms(h2b('01612f'), sts=0x00)
+        h = self._handler(remote_format='expanded')
+        # must NOT raise
+        self.assertIsNone(h.message_received_handler(_FakePdu(bad)))
+        self.assertIsNone(h.response)
+
+    def test_undecodable_por_after_good_response(self):
+        """real app response followed by undecodable PoR:
+        - good response is saved
+        - tool does not crash."""
+        registry = bytes(range(120))
+        app = self._plaintext_resp_sms(self._expanded_secured(b2h(registry)))
+        part1, part2 = self._fragment_2(app, ref=0x07, first_len=110)
+        bad_por = self._plaintext_resp_sms(h2b('deadbeef'), sts=0x00)
+
+        h = self._handler()
+        h.message_received_handler(_FakePdu(part1))
+        h.message_received_handler(_FakePdu(part2))
+        self.assertIsNone(h.message_received_handler(_FakePdu(bad_por)))  # no crash
+        res, decoded = h.response
+        self.assertIsNotNone(decoded)
+        self.assertEqual(decoded.last_response_data, b2h(registry))
+
+    def test_single_part_response_still_works(self):
+        """small response that fits one SMS turns into self.response, handled as before"""
+        h = self._handler()
+        sms = self._plaintext_resp_sms(self._expanded_secured('abcd', sw='9000'))
+        self.assertLessEqual(len(sms), 140)
+        h.message_received_handler(_FakePdu(sms))
+        res, decoded = h.response
+        self.assertIsNotNone(decoded)
+        self.assertEqual(decoded.last_response_data, 'abcd')
+        self.assertEqual(decoded.last_status_word, '9000')
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/unittests/test_sms.py

@@ -103,3 +103,126 @@ class Test_DELIVER(unittest.TestCase):
         self.assertEqual(d.tp_pid, 0x7f)
         self.assertEqual(d.tp_dcs, 0xf6)
         self.assertEqual(d.tp_udl, 8)
+
+
+class Test_ConcatenatedSmsReassembler(unittest.TestCase):
+    """3GPP TS 23.040 9.2.3.24 reassembly of multi-part SMS.
+
+    An OTA response that exceeds a single SHORT MESSAGE is delivered in several parts using
+    the SEND SHORT MESSAGE proactive command. The receiver must recombine the individual
+    parts into a single part before decoding."""
+
+    OTA_IE = {'iei': 0x71, 'length': 0, 'value': b''}
+
+    @staticmethod
+    def _concat8(ref, tot, seq):
+        return {'iei': 0x00, 'length': 3, 'value': bytes([ref, tot, seq])}
+
+    @staticmethod
+    def _concat16(ref, tot, seq):
+        return {'iei': 0x08, 'length': 4, 'value': ref.to_bytes(2, 'big') + bytes([tot, seq])}
+
+    @staticmethod
+    def _part(ies, frag):
+        return UserDataHeader(ies).to_bytes() + frag
+
+    def test_ground_truth_udh(self):
+        # part 1 UDH observed from sja5: 07 00 03 01 02 01 71 00
+        built = self._part([self._concat8(1, 2, 1), self.OTA_IE], b'')
+        self.assertEqual(b2h(built), '0700030102017100')
+
+    def test_ground_truth_udh_16bit(self):
+        # 9.2.3.24.8: 08 | 08 04 <ref16> <total> <seq> | 71 00
+        built = self._part([self._concat16(0x1234, 2, 1), self.OTA_IE], b'')
+        self.assertEqual(b2h(built), '080804123402017100')
+
+    def test_single_part_passthrough(self):
+        r = ConcatenatedSmsReassembler()
+        single = h2b('027100') + bytes(range(20))
+        self.assertEqual(r.add(single), single)
+
+    def test_two_part(self):
+        # second segment contains only the concat IE, no OTA IE
+        pkt = bytes(range(60))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1), self.OTA_IE], pkt[:35])))
+        out = r.add(self._part([self._concat8(1, 2, 2)], pkt[35:]))
+        self.assertEqual(out, h2b('027100') + pkt)
+
+    def test_out_of_order(self):
+        pkt = bytes(range(60))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(5, 2, 2), self.OTA_IE], pkt[35:])))
+        out = r.add(self._part([self._concat8(5, 2, 1), self.OTA_IE], pkt[:35]))
+        self.assertEqual(out, h2b('027100') + pkt)
+
+    def test_three_part_out_of_order(self):
+        pkt = bytes(range(90))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(7, 3, 3)], pkt[60:])))
+        self.assertIsNone(r.add(self._part([self._concat8(7, 3, 1), self.OTA_IE], pkt[:30])))
+        out = r.add(self._part([self._concat8(7, 3, 2)], pkt[30:60]))
+        self.assertEqual(out, h2b('027100') + pkt)
+
+    def test_16bit_reference(self):
+        pkt = bytes(range(40))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat16(0x1234, 2, 1), self.OTA_IE], pkt[:20])))
+        out = r.add(self._part([self._concat16(0x1234, 2, 2)], pkt[20:]))
+        self.assertEqual(out, h2b('027100') + pkt)
+
+    def test_interleaved_references(self):
+        # two concurrent concatenation sets at the same time
+        pkt = bytes(range(60))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1), self.OTA_IE], pkt[:35])))
+        self.assertIsNone(r.add(self._part([self._concat8(9, 2, 1), self.OTA_IE], b'\xaa')))
+        self.assertEqual(r.add(self._part([self._concat8(1, 2, 2)], pkt[35:])), h2b('027100') + pkt)
+        self.assertEqual(r.add(self._part([self._concat8(9, 2, 2)], b'\xbb')), h2b('027100') + b'\xaa\xbb')
+
+    def test_reserved_concat_ie_is_ignored(self):
+        # TS 23.040 9.2.3.24.1:
+        # - a total of 0
+        # - or a sequence number that is 0 or > total
+        # means "the receiving entity shall ignore the whole Information Element"
+        # the message is handed back unchanged as a single part msg and not rejected
+        # so the caller can handle the problem
+        r = ConcatenatedSmsReassembler()
+        for tot, seq in [(2, 3),    # seq > total
+                         (2, 0),    # seq == 0
+                         (0, 1)]:   # total == 0
+            with self.subTest(total=tot, seq=seq):
+                part = self._part([self._concat8(1, tot, seq)], b'\x00')
+                self.assertEqual(r.add(part), part)
+        # nothing buffered so later valid set still reassembles properly
+        self.assertEqual(r.sets, {})
+        pkt = bytes(range(40))
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1), self.OTA_IE], pkt[:20])))
+        self.assertEqual(r.add(self._part([self._concat8(1, 2, 2)], pkt[20:])), h2b('027100') + pkt)
+
+    def test_inconsistent_totals_do_not_crash(self):
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(1, 3, 3)], b'\x33')))
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1)], b'\x11')))
+        self.assertEqual(r.add(self._part([self._concat8(1, 2, 2)], b'\x22')),
+                         h2b('00') + b'\x11\x22')             # complete total=2 set
+        self.assertIn((0x00, 1, 3), r.sets)                   # total=3 set still waits
+
+    def test_incomplete_sets_are_capped(self):
+        r = ConcatenatedSmsReassembler(max_sets=2)
+        for ref in (1, 2, 3):
+            self.assertIsNone(r.add(self._part([self._concat8(ref, 2, 1)], bytes([ref]))))
+        self.assertEqual(sorted(k[1] for k in r.sets), [2, 3])      # oldest evicted
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 2)], b'\x11')))
+        self.assertEqual(sorted(k[1] for k in r.sets), [1, 3])
+        self.assertEqual(r.add(self._part([self._concat8(3, 2, 2)], b'\x33')), h2b('00') + b'\x03\x33')
+
+    def test_same_reference_in_both_ie_forms(self):
+        # the refno only unique per IE form (9.2.3.24.1 vs .8) -> two sets
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1)], b'\x0a')))
+        self.assertIsNone(r.add(self._part([self._concat16(1, 2, 2)], b'\x1b')))
+        self.assertEqual(r.add(self._part([self._concat16(1, 2, 1)], b'\x0b')),
+                         h2b('00') + b'\x0b\x1b')
+        self.assertEqual(r.add(self._part([self._concat8(1, 2, 2)], b'\x1a')),
+                         h2b('00') + b'\x0a\x1a')

tests/unittests/test_tlvs.py

@@ -21,6 +21,7 @@ import logging
 from osmocom.utils import b2h, h2b, all_subclasses
 from osmocom.tlv import *
 
+import pySim.cat
 import pySim.iso7816_4
 import pySim.ts_102_221
 import pySim.ts_102_222

tests/unittests/test_transport.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+
+"""Transport (as in t0/t1) tests"""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from osmocom.utils import h2b, b2h
+from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
+from pySim.transport import ProactiveHandler, LinkBaseTpdu
+
+
+def _send_short_message_pcmd():
+    """proactive SEND SHORT MESSAGE:
+    D0 | CommandDetails(cmd 1, t 0x13, q 0) | DeviceIdentities(uicc->network)
+       | dummy SMS_TPDU"""
+    body = h2b('8103011300' + '82028183' + '8B04DEADBEEF')
+    pdu = h2b('D0') + bytes([len(body)]) + body
+    pcmd = ProactiveCommand()
+    decoded = pcmd.from_tlv(pdu)
+    return pcmd, decoded
+
+
+class Test_prepare_response(unittest.TestCase):
+    """TERMINAL RESPONSE.
+    multi-part OTA response crash regression test."""
+
+    def setUp(self):
+        self.h = ProactiveHandler.__new__(ProactiveHandler)
+
+    def test_on_decoded_command(self):
+        _pcmd, decoded = _send_short_message_pcmd()
+        til = self.h.prepare_response(decoded)
+        self.assertEqual([type(c).__name__ for c in til],
+                         ['CommandDetails', 'DeviceIdentities', 'Result'])
+        # command details echoed, device id inverted, result OK
+        self.assertEqual(b2h(til[0].to_tlv()), '8103011300')
+        self.assertEqual(b2h(til[1].to_tlv()), '82028381')
+        self.assertEqual(b2h(til[2].to_tlv()), '830100')
+
+    def test_on_collection_resolves_via_decoded(self):
+        # Check that ProactiveCommand collection (empty .children) still works
+        pcmd, _decoded = _send_short_message_pcmd()
+        self.assertEqual(list(getattr(pcmd, 'children', []) or []), [])
+        til = self.h.prepare_response(pcmd)
+        self.assertEqual([type(c).__name__ for c in til],
+                         ['CommandDetails', 'DeviceIdentities', 'Result'])
+        self.assertEqual(b2h(til[0].to_tlv()), '8103011300')
+        self.assertEqual(b2h(til[1].to_tlv()), '82028381')
+        self.assertEqual(b2h(til[2].to_tlv()), '830100')
+
+    def test_missing_command_details_raises_clear_error(self):
+        class _NoChildren:
+            children = []
+        with self.assertRaises(ValueError) as ctx:
+            self.h.prepare_response(_NoChildren())
+        self.assertIn('CommandDetails', str(ctx.exception))
+
+
+class FakeTpduLink(LinkBaseTpdu):
+    """mock LinkBaseTpdu that replays a list of (data, sw) responses + records every TPDU that
+    the T=0 state machine sends. Secretly sending more TPDUs than intended is the error,
+    designed to test "unsolicited GET RESPONSE" mishaps"""
+
+    def __init__(self, responses):
+        super().__init__()
+        self._responses = list(responses)
+        self.sent = []
+
+    def send_tpdu(self, tpdu):
+        self.sent.append(tpdu.lower())
+        if not self._responses:
+            raise AssertionError('T=0 layer sent an unpexpected TPDU: %s (total so far: %s)'
+                                 % (tpdu, self.sent))
+        return self._responses.pop(0)
+
+    def __str__(self):
+        return 'FakeTpduLink'
+
+    def wait_for_card(self, timeout=None, newcardonly=False):
+        pass
+
+    def connect(self):
+        pass
+
+    def get_atr(self):
+        return '3b00'
+
+    def disconnect(self):
+        pass
+
+    def _reset_card(self):
+        pass
+
+
+# GP GET STATUS, wrapped in SCP02 CLA 84, Case #4.
+GET_STATUS = '84f22002094f005c054f9f70c5cc' + '00'
+GET_STATUS_TPDU = '84f22002094f005c054f9f70c5cc'
+
+# generic #4 SELECT by DF name command
+CASE4 = '00a4040c07a0000000871002' + '00'
+CASE4_TPDU = '00a4040c07a0000000871002'
+
+
+class Test_send_apdu_T0(unittest.TestCase):
+    """regression tests for the T=0 state machine in LinkBaseTpdu.__send_apdu_T0()"""
+
+    def _exchange(self, apdu, responses, strict=True, protocol=0):
+        link = FakeTpduLink(responses)
+        link.apdu_strict = strict
+        link.set_tpdu_format(protocol)
+        data, sw = link._send_apdu(apdu)
+        return link, data, sw
+
+    #### TS 102 221 section 7.3.1.1 TPDU construction
+
+    def test_case1_gets_le_appended(self):
+        link, data, sw = self._exchange('00200001', [('', '9000')])
+        self.assertEqual(link.sent, ['0020000100'])
+        self.assertEqual((data, sw), ('', '9000'))
+
+    def test_case3_passed_through_unmodified(self):
+        apdu = '00200001081122334455667788'
+        link, _data, sw = self._exchange(apdu, [('', '9000')])
+        self.assertEqual(link.sent, [apdu])
+        self.assertEqual(sw, '9000')
+
+    def test_case4_le_stripped(self):
+        link, data, sw = self._exchange(CASE4, [('', '9000')])
+        self.assertEqual(link.sent, [CASE4_TPDU])
+        self.assertEqual((data, sw), ('', '9000'))
+
+    #### TS 102 221 7.3.1.1.4 4a GP GET RESPONSE for 61xx / 9fxx
+
+    def test_61xx_fetches_response(self):
+        link, data, sw = self._exchange(CASE4, [('', '6103'), ('a1b2c3', '9000')])
+        self.assertEqual(link.sent, [CASE4_TPDU, '00c0000003'])
+        self.assertEqual((data, sw), ('a1b2c3', '9000'))
+
+    def test_61xx_chained(self):
+        link, data, sw = self._exchange(CASE4,
+                                        [('', '6102'), ('aabb', '6102'), ('ccdd', '9000')])
+        self.assertEqual(link.sent, [CASE4_TPDU, '00c0000002', '00c0000002'])
+        self.assertEqual((data, sw), ('aabbccdd', '9000'))
+
+    def test_9fxx_fetches_response(self):
+        link, data, sw = self._exchange(CASE4, [('', '9f04'), ('deadbeef', '9000')])
+        self.assertEqual(link.sent, [CASE4_TPDU, '00c0000004'])
+        self.assertEqual((data, sw), ('deadbeef', '9000'))
+
+    def test_get_response_inherits_cla(self):
+        """GET RESPONSE must reuse  CLA of command"""
+        link, _data, _sw = self._exchange(GET_STATUS, [('', '6102'), ('aabb', '9000')])
+        self.assertEqual(link.sent, [GET_STATUS_TPDU, '84c0000002'])
+
+    def test_9100_terminates(self):
+        """9100 is final status word, not fetch trigger"""
+        link, data, sw = self._exchange(CASE4, [('', '9100')])
+        self.assertEqual(link.sent, [CASE4_TPDU])
+        self.assertEqual((data, sw), ('', '9100'))
+
+    def test_error_sw_terminates(self):
+        link, data, sw = self._exchange(CASE4, [('', '6982')])
+        self.assertEqual(link.sent, [CASE4_TPDU])
+        self.assertEqual((data, sw), ('', '6982'))
+
+    def test_no_status_word_raises(self):
+        with self.assertRaises(ValueError):
+            self._exchange(CASE4, [('', None)])
+
+    #### TS 102 221 7.3.1.1.4 4b dummy GET RESPONSE
+
+    def test_clause_4b_warning_before_data_bootstraps(self):
+        """warning SW returned for the _command_ TPDU triggers dummy GET RESPONSE (Le=00)"""
+        for warn in ('6200', '6281', '62f1', '6300', '63f1'):
+            with self.subTest(sw=warn):
+                link, data, sw = self._exchange(CASE4,
+                                                [('', warn), ('', '6103'), ('a1b2c3', '9000')])
+                self.assertEqual(link.sent, [CASE4_TPDU, '00c0000000', '00c0000003'])
+                self.assertEqual((data, sw), ('a1b2c3', '9000'))
+
+    def test_warning_after_data_terminates(self):
+        """Once the response has been fetched a warning status word is the final result of the command"""
+        for warn in ('6281', '6283', '63c2', '6300', '62f1', '63f1', '6310'):
+            with self.subTest(sw=warn):
+                link, data, sw = self._exchange(CASE4, [('', '6102'), ('aabb', warn)])
+                self.assertEqual(link.sent, [CASE4_TPDU, '00c0000002'])
+                self.assertEqual((data, sw), ('aabb', warn))
+
+    def test_no_dummy_get_response_when_command_already_returned_data(self):
+        """warning that arrives together with response data (for example 6282 on a case #2 read) is final, too"""
+        link, data, sw = self._exchange('00b0000004', [('01020304', '6282')], strict=False)
+        self.assertEqual(link.sent, ['00b0000004'])
+        self.assertEqual((data, sw), ('01020304', '6282'))
+
+    def test_repeated_warning_does_not_loop(self):
+        """warning -> dummy GET RESPONSE -> warning again must terminate"""
+        link, data, sw = self._exchange(CASE4, [('', '6281'), ('', '6281')])
+        self.assertEqual(link.sent, [CASE4_TPDU, '00c0000000'])
+        self.assertEqual((data, sw), ('', '6281'))
+
+    #### fixed GlobalPlatform GET STATUS pagination
+
+    def test_gp_6310_reaches_the_caller(self):
+        """GET STATUS answers 6310"""
+        link, data, sw = self._exchange(GET_STATUS, [('', '6104'), ('e3024f00', '6310')])
+        self.assertEqual(link.sent, [GET_STATUS_TPDU, '84c0000004'])
+        self.assertEqual((data, sw), ('e3024f00', '6310'))
+
+    def test_gp_get_status_two_pages(self):
+        """Both GET STATUS pages, page 1 6310, reissued with P2 bit 1 set, page 2 9000."""
+        page1 = self._exchange(GET_STATUS, [('', '6104'), ('e3024f00', '6310')])
+        self.assertEqual(page1[1:], ('e3024f00', '6310'))
+        page2 = self._exchange('84f22003094f005c054f9f70c5cc00',
+                               [('', '6104'), ('e3024f01', '9000')])
+        self.assertEqual(page2[0].sent, ['84f22003094f005c054f9f70c5cc', '84c0000004'])
+        self.assertEqual(page2[1:], ('e3024f01', '9000'))
+
+    #### 6cxx and apdu_strict
+
+    def test_6cxx_reissues_command_with_correct_length(self):
+        link, data, sw = self._exchange('00b0000000', [('', '6c04'), ('01020304', '9000')])
+        self.assertEqual(link.sent, ['00b0000000', '00b0000004'])
+        self.assertEqual((data, sw), ('01020304', '9000'))
+
+    def test_strict_mode_does_not_auto_fetch_for_case3(self):
+        apdu = '00200001081122334455667788'
+        link, data, sw = self._exchange(apdu, [('', '6104')], strict=True)
+        self.assertEqual(link.sent, [apdu])
+        self.assertEqual((data, sw), ('', '6104'))
+
+    def test_non_strict_mode_auto_fetches_for_case3(self):
+        apdu = '00200001081122334455667788'
+        link, data, sw = self._exchange(apdu, [('', '6104'), ('aabbccdd', '9000')], strict=False)
+        self.assertEqual(link.sent, [apdu, '00c0000004'])
+        self.assertEqual((data, sw), ('aabbccdd', '9000'))
+
+    #### T=1 briefly
+
+    def test_t1_is_passed_through(self):
+        """T=1 has no GET RESPONSE"""
+        link, data, sw = self._exchange(GET_STATUS, [('e3024f00', '6310')], protocol=1)
+        self.assertEqual(link.sent, [GET_STATUS.lower()])
+        self.assertEqual((data, sw), ('e3024f00', '6310'))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unittests/xo/test_param_src

@@ -0,0 +1,9 @@
+ok: ConstantSource vals=['123', '123', '123']
+ok: RandomDigitSource vals=['13987', '49298', '55670']
+ok: RandomDigitSource vals=['650', '580', '49', '885', '497', '195', '320', '137', '245', '663']
+ok: RandomDigitSource vals=['638', '025', '232', '779', '826', '972', '650', '580', '049', '885']
+ok: RandomHexDigitSource vals=['6b65c172', 'abcf6df0', '984a5fcf']
+ok: RandomHexDigitSource vals=['96876670', '356ce766', 'd1d697c1']
+ok: RandomHexDigitSource vals=['f95d8c86', '2bdb095e', '6b65c172']
+ok: IncDigitSource vals=['10001', '10002', '10003']
+ok: CsvSource vals=['first val', 'second val', 'third val']