sysmocom_sjs1: CardModel for the sysmoUSIM-SJS1

SJS1 model lifted off the the legacy
SysmoUSIMSJS1.program(), ADM is ADM1 with CHV 0x0a,
already used by the generic UiccCardBase default.

Change-Id: I32b011c68de5f990e709718b313b6f28a31ca78b

contrib/scp81_admin_server.py

@@ -0,0 +1,524 @@
+#!/usr/bin/env python3
+"""TLS-PSK HTTP Remote Administration Server for SCP81 / RAM over HTTP"""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# The card (specifically a SD supporting SCP81) is the TLS client:
+# - it opens a TCP connection to this server,
+# - performs a TLS handshake authenticated with a PSK,
+# - and then drives the HTTP admin loop of to fetch remote APDU command strings
+# - and posts back their responses.
+# This program is the server side of that exchange, it:
+# - accepts the PSK-TLS connection,
+# - hands the card a queue of commands
+# - and logs the decoded responses.
+#
+# The two TS 102 226 annex B figure B.1 administration modes are supported over the
+# same session, selected with --mode:
+#   ram  GP Amendment B RAM:
+#           command is handled by (--targeted-application) a SD
+#   rfm  ETSI TS 102 226 RFM/RAM:
+#           command is routed to the Receiving/RFM Application specified by
+#           --targeted-application, for example UICC-filesystem/USIM-ADF RFM app.
+#
+# Remote APDU command/response bodies use the Expanded Remote Application
+# data format.
+#
+
+import ssl
+import socket
+import logging
+import argparse
+import threading
+from pathlib import Path
+from typing import List, Optional, Callable, Dict, Tuple
+
+from osmocom.utils import h2b, b2h
+
+from pySim.ota import encode_expanded_cmd, decode_expanded_resp
+
+logger = logging.getLogger(Path(__file__).stem)
+
+# Amendment B section 3.4
+ADMIN_PROTOCOL = 'globalplatform-remote-admin/1.0'
+CT_COMMAND = 'application/vnd.globalplatform.card-content-mgt;version=1.0'
+CT_RESPONSE = 'application/vnd.globalplatform.card-content-mgt-response;version=1.0'
+
+#  TS 102 226 annex B, figure B.1 RFM/RAM over HTTPS content types
+CT_RFM_COMMAND = 'application/vnd.etsi.scp.command-data;version=1.0'
+CT_RFM_RESPONSE = 'application/vnd.etsi.scp.response-data;version=1.0'
+
+MODE_CONTENT_TYPE = {
+    'ram': CT_COMMAND,          # GP Amendment B RAM: target = a Security Domain
+    'rfm': CT_RFM_COMMAND,      # ETSI TS 102 226 RFM/RAM: target = an application
+}
+
+# Amendment B Table 3-2. The 3DES and NULL suites are left out and can be enabled with
+# --ciphers / --seclevel.
+DEFAULT_CIPHERS = ':'.join([
+    'PSK-AES128-CBC-SHA256',    # TLS_PSK_WITH_AES_128_CBC_SHA256, TLS 1.2
+    'PSK-AES128-CBC-SHA',       # TLS_PSK_WITH_AES_128_CBC_SHA, TLS 1.0/1.1
+])
+
+TLS_VERSION_MAP = {
+    '1.0': ssl.TLSVersion.TLSv1,
+    '1.1': ssl.TLSVersion.TLSv1_1,
+    '1.2': ssl.TLSVersion.TLSv1_2,
+    '1.3': ssl.TLSVersion.TLSv1_3,
+}
+
+
+def format_aid(aid: str) -> str:
+    """AID -> //aid/<RID>/<PIX> for X-Admin-Targeted-Application from Amendment B section 3.4.2
+      First 5 bytes RID, the PIX the remainder, string in //aid/ notation is passed through."""
+    if aid.startswith('//aid/'):
+        return aid
+    aid = aid.replace(' ', '').lower()
+    if len(aid) < 10:
+        raise ValueError('AID %r is shorter than the 5 byte RID' % aid)
+    rid, pix = aid[:10], aid[10:]
+    return '//aid/%s/%s' % (rid, pix)
+
+
+def make_ssl_context(psk: bytes, identity: str, *,
+                     ciphers: str = DEFAULT_CIPHERS,
+                     min_tls: str = '1.2', max_tls: str = '1.3',
+                     seclevel: Optional[int] = None,
+                     identity_hint: Optional[str] = None,
+                     allow_any_identity: bool = False,
+                     extra_psks: Optional[Dict[str, bytes]] = None) -> ssl.SSLContext:
+    """PSK SSLContext, resolvesg the key from the client psk_identity
+
+    extra_psks can carry additional identity->key mappings.
+    allow_any_identity can be used for debugging
+    """
+    # the PSK callback must return immutable bytes, h2b() gives a bytearray
+    psk = bytes(psk)
+    keymap: Dict[str, bytes] = {identity: psk}
+    if extra_psks:
+        keymap.update({k: bytes(v) for k, v in extra_psks.items()})
+
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    ctx.minimum_version = TLS_VERSION_MAP[min_tls]
+    ctx.maximum_version = TLS_VERSION_MAP[max_tls]
+    cipher_str = ciphers
+    if seclevel is not None:
+        # @SECLEVEL=0 is to enable NULL/3DES/legacy PSK suites
+        cipher_str = '%s:@SECLEVEL=%d' % (ciphers, seclevel)
+    if cipher_str:
+        ctx.set_ciphers(cipher_str)
+
+    def psk_server_callback(client_identity: Optional[str]) -> bytes:
+        if allow_any_identity:
+            logger.info('PSK handshake: identity=%r (ACEPTING ANY!)', client_identity)
+            return psk
+        key = keymap.get(client_identity)
+        if key is None:
+            logger.warning('PSK handshake: unknown identity %r (known: %r) -> rejecting',
+                           client_identity, list(keymap.keys()))
+            return b''    # empty PSK aborts handshake
+        logger.info('PSK handshake: identity=%r resolved', client_identity)
+        return key
+
+    ctx.set_psk_server_callback(psk_server_callback, identity_hint=identity_hint)
+    return ctx
+
+
+class HttpRequest:
+    """A parsed HTTP request (request line + headers + body)."""
+    __slots__ = ('method', 'uri', 'version', 'headers', 'body')
+
+    def __init__(self, method: str, uri: str, version: str,
+                 headers: Dict[str, str], body: bytes):
+        self.method = method
+        self.uri = uri
+        self.version = version
+        self.headers = headers      # lower cased field names
+        self.body = body
+
+    def get(self, name: str, default=None) -> Optional[str]:
+        return self.headers.get(name.lower(), default)
+
+
+# http.client bound on a single HTTP line.
+MAX_LINE = 65536
+
+
+def _read_line(rfile) -> bytes:
+    """readline() with a bound. reaching the bound without a
+    terminator means the card is out of sync somehow, not that the line is long."""
+    line = rfile.readline(MAX_LINE)
+    if line and not line.endswith(b'\n'):
+        raise ValueError('HTTP line longer than %u bytes' % MAX_LINE)
+    return line
+
+
+def _read_chunked_body(rfile) -> bytes:
+    """Read a Transfer-Encoding: chunked body. Amendment B section 3.4.1 lets
+    the card send its response string with either a Content-Length or chunked"""
+    out = bytearray()
+    while True:
+        size_line = _read_line(rfile)
+        if not size_line:
+            break
+        size = int(size_line.split(b';', 1)[0].strip() or b'0', 16)
+        if size == 0:
+            # consume trailer headers up to the terminating blank line
+            while _read_line(rfile) not in (b'\r\n', b'\n', b''):
+                pass
+            break
+        chunk = rfile.read(size)
+        if len(chunk) != size:
+            raise ValueError('chunked body ended after %u of %u bytes' % (len(chunk), size))
+        out += chunk
+        _read_line(rfile)   # trailing CRLF after the chunk data
+    return bytes(out)
+
+
+def read_http_request(rfile, send: Optional[Callable[[bytes], None]] = None) -> Optional[HttpRequest]:
+    """Read one HTTP request from a buffered binary reader or None when closed"""
+    request_line = _read_line(rfile)
+    if not request_line:
+        return None
+    parts = request_line.rstrip(b'\r\n').decode('iso-8859-1').split(' ')
+    if len(parts) < 3:
+        raise ValueError('Malformed HTTP request line: %r' % request_line)
+    method, uri, version = parts[0], parts[1], parts[2]
+
+    headers: Dict[str, str] = {}
+    while True:
+        line = _read_line(rfile)
+        if line in (b'\r\n', b'\n', b''):
+            break
+        name, _, value = line.rstrip(b'\r\n').decode('iso-8859-1').partition(':')
+        headers[name.strip().lower()] = value.strip()
+
+    # Expect: 100-continue waits for the response before it sends the body,
+    # and RFC 2616 8.2.3 (Amendment B references RFC 2616 as [HTTP])
+    # requires the server to send it. Amendment B 3.4.1 does not mention this
+    # header, tho, might be useless.
+    if send and '100-continue' in headers.get('expect', '').lower():
+        logger.info('-> 100 Continue ')
+        send(b'HTTP/1.1 100 Continue\r\n\r\n')
+
+    body = b''
+    te = headers.get('transfer-encoding', '').lower()
+    if 'chunked' in te:
+        body = _read_chunked_body(rfile)
+    elif 'content-length' in headers:
+        n = int(headers['content-length'])
+        if n:
+            body = rfile.read(n)
+    return HttpRequest(method, uri, version, headers, body)
+
+
+def build_http_response(status_line: str, headers: List[Tuple[str, str]],
+                        body: bytes = b'') -> bytes:
+    """Serialise  HTTP response.  status_line 'HTTP/1.1 200 OK'."""
+    lines = [status_line]
+    lines += ['%s: %s' % (name, value) for name, value in headers]
+    head = ('\r\n'.join(lines) + '\r\n\r\n').encode('iso-8859-1')
+    return head + body
+
+
+def format_decoded_response(dec) -> str:
+    """hand over the data"""
+    bits = ['%u command(s) executed' % dec.number_of_commands]
+    for i, c in enumerate(dec.commands):
+        data = c.response_data or '-'
+        bits.append('  R-APDU[%u]: SW=%s data=%s' % (i, c.status_word, data))
+    if dec.get('truncated'):
+        bits.append('  TRUNCATED: an R-APDU returned SW 62F1, so the card cut the response data '
+                    'short and stopped executing the rest of the script ') # TS 102 226 5.2.1.1
+    if dec.bad_format is not None:
+        bits.append('  bad-format: %s' % dec.bad_format)
+    if dec.immediate_action_response is not None:
+        bits.append('  immediate-action-response: %s' % dec.immediate_action_response)
+    if dec.script_chaining_response is not None:
+        bits.append('  script-chaining-response: %s' % dec.script_chaining_response)
+    return '\n'.join(bits)
+
+
+class AdminSession:
+
+    def __init__(self, command_bodies: List[bytes],
+                 next_uri: Optional[str] = None,
+                 targeted_application: Optional[str] = None,
+                 on_response: Optional[Callable[[object], None]] = None,
+                 content_type: str = CT_COMMAND):
+        self.pending: List[bytes] = list(command_bodies)
+        self.next_uri = next_uri            # None -> echo the request URI
+        self.targeted_application = targeted_application
+        self.on_response = on_response
+        self.content_type = content_type    # Content-Type for the command body
+        self.responses: List[object] = []   # decoded Containers, in order
+
+    def record_response(self, dec) -> None:
+        self.responses.append(dec)
+        if self.on_response:
+            self.on_response(dec)
+
+
+def run_admin_loop(rfile, send: Callable[[bytes], None], session: AdminSession) -> AdminSession:
+    """Drive the admin loop for one connection.
+    Just keep answering the card POST requests with the next queued command
+    (200 OK + Expanded command body) until the queue is empty, end session with 204 No Content."""
+    while True:
+        req = read_http_request(rfile, send)
+        if req is None:
+            logger.info('connection closed by card')
+            return session
+
+        if req.method != 'POST':
+            logger.warning('unexpected method %s %s -> 405', req.method, req.uri)
+            send(build_http_response('HTTP/1.1 405 Method Not Allowed',
+                                     [('X-Admin-Protocol', ADMIN_PROTOCOL),
+                                      ('Connection', 'close')]))
+            return session
+
+        proto = req.get('x-admin-protocol')
+        if proto and proto != ADMIN_PROTOCOL:
+            logger.warning('card X-Admin-Protocol=%r (expected %r)', proto, ADMIN_PROTOCOL)
+        status = req.get('x-admin-script-status')
+        resume = req.get('x-admin-resume')
+        logger.info('POST %s from=%r status=%r resume=%r body=%uB',
+                    req.uri, req.get('x-admin-from'), status, resume, len(req.body))
+
+        # section 3.4.1:
+        # - body with "X-Admin-Script-Status: ok" carries the previous command
+        # response string (Expanded Remote response format);
+        # - other status values carry no body ().
+        if req.body:
+            # Expanded Remote response:
+            # - GP Amd B 'card-content-mgt-response'
+            # - ETSI 'scp.response-data'
+            logger.debug('  response Content-Type=%r raw body (%uB): %s',
+                         req.get('content-type'), len(req.body), b2h(req.body))
+            if status in (None, 'ok'):
+                try:
+                    dec = decode_expanded_resp(req.body)
+                    session.record_response(dec)
+                    logger.info('card response:\n%s', format_decoded_response(dec))
+                except Exception as e:
+                    logger.error('failed to decode response body %s: %s', b2h(req.body), e)
+            else:
+                logger.warning('body present with status=%r; ignoring', status) #  section 3.4.1
+        elif status and status != 'ok':
+            logger.info('card reported script-status=%r (no response body)', status)
+
+        if session.pending:
+            body = session.pending.pop(0)
+            next_uri = session.next_uri or req.uri
+            headers = [('X-Admin-Protocol', ADMIN_PROTOCOL),
+                       ('X-Admin-Next-URI', next_uri),
+                       ('Content-Type', session.content_type)]
+            if session.targeted_application:
+                headers.append(('X-Admin-Targeted-Application', session.targeted_application))
+            headers.append(('Content-Length', str(len(body))))
+            logger.info('-> 200 OK, next command (%uB): %s', len(body), b2h(body))
+            send(build_http_response('HTTP/1.1 200 OK', headers, body))
+        else:
+            # section 3.4.2: No more commands, end session
+            # No Content-Type or body for 204
+            logger.info('-> 204 No Content, ending administration session')
+            send(build_http_response('HTTP/1.1 204 No Content',
+                                     [('X-Admin-Protocol', ADMIN_PROTOCOL),
+                                      ('Connection', 'close')]))
+            return session
+
+
+class Scp81AdminServer:
+    """Threaded TLS-PSK server that runs the Amendment B admin loop against each
+    connecting card."""
+
+    def __init__(self, host: str, port: int, ssl_ctx: ssl.SSLContext,
+                 command_bodies: List[bytes],
+                 next_uri: Optional[str] = None,
+                 targeted_application: Optional[str] = None,
+                 on_response: Optional[Callable[[object], None]] = None,
+                 on_session_end: Optional[Callable[[AdminSession], None]] = None,
+                 content_type: str = CT_COMMAND):
+        self.host = host
+        self.port = port
+        self.ssl_ctx = ssl_ctx
+        self.command_bodies = command_bodies
+        self.next_uri = next_uri
+        self.targeted_application = targeted_application
+        self.content_type = content_type
+        self.on_response = on_response
+        self.on_session_end = on_session_end
+        self._sock: Optional[socket.socket] = None
+        self._stop = threading.Event()
+
+    def bind(self) -> int:
+        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self._sock.bind((self.host, self.port))
+        self._sock.listen(5)
+        self._sock.settimeout(0.5)
+        self.port = self._sock.getsockname()[1]
+        return self.port
+
+    def serve_forever(self) -> None:
+        if self._sock is None:
+            self.bind()
+        logger.info('SCP81 admin server listening on %s:%u (%u command(s) queued)',
+                    self.host, self.port, len(self.command_bodies))
+        while not self._stop.is_set():
+            try:
+                conn, addr = self._sock.accept()
+            except socket.timeout:
+                continue
+            except OSError:
+                break
+            threading.Thread(target=self._handle, args=(conn, addr), daemon=True).start()
+
+    def shutdown(self) -> None:
+        self._stop.set()
+        if self._sock is not None:
+            self._sock.close()
+
+    def _handle(self, conn: socket.socket, addr) -> None:
+        try:
+            tls = self.ssl_ctx.wrap_socket(conn, server_side=True)
+        except (ssl.SSLError, OSError) as e:
+            logger.warning('TLS-PSK handshake with %s failed: %s', addr, e)
+            try:
+                conn.close()
+            except OSError:
+                pass
+            return
+        logger.info('TLS-PSK established with %s: %s / %s', addr, tls.version(), tls.cipher())
+        session = AdminSession(self.command_bodies, next_uri=self.next_uri,
+                               targeted_application=self.targeted_application,
+                               on_response=self.on_response,
+                               content_type=self.content_type)
+        try:
+            rfile = tls.makefile('rb')
+            run_admin_loop(rfile, tls.sendall, session)
+        except (ssl.SSLError, OSError, ValueError) as e:
+            logger.warning('session with %s aborted: %s', addr, e)
+        finally:
+            try:
+                tls.close()
+            except OSError:
+                pass
+        logger.info('session with %s ended: %u response(s) collected', addr, len(session.responses))
+        if self.on_session_end:
+            self.on_session_end(session)
+
+
+def build_command_bodies(apdus: List[bytes], batch: bool = False,
+                         length_coding: str = 'definite') -> List[bytes]:
+    """wrpa apdus
+    each C-APDU -> one cmd message + one HTTP response per APDU
+    batch=True -> all C-APDUs in one Command Scripting template.
+    length_coding selects the definite or indefinite Command Scripting template."""
+    if not apdus:
+        return []
+    if batch:
+        return [encode_expanded_cmd(apdus, length_coding=length_coding)]
+    return [encode_expanded_cmd(a, length_coding=length_coding) for a in apdus]
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='TLS-PSK HTTP Remote Administration Server for SCP81 / RAM over HTTP')
+    parser.add_argument('--host', default='0.0.0.0', help='Host/IP to bind to (default: 0.0.0.0)')
+    parser.add_argument('--port', type=int, default=8443, help='TCP port to bind to (default: 8443)')
+    parser.add_argument('--psk', required=True,
+                        help='PSK TLS key, Amendment B key type 85 as hex')
+    parser.add_argument('--psk-identity', required=True,
+                        help='Expected PSK identity string presented by the card')
+    parser.add_argument('--psk-identity-hint', default=None,
+                        help='Optional PSK identity hint to send to the card (default: none)')
+    parser.add_argument('--allow-any-identity', action='store_true',
+                        help='DEBUG: Accept any psk_identity')
+    parser.add_argument('--ciphers', default=DEFAULT_CIPHERS,
+                        help='OpenSSL cipher string for TLS<=1.2')
+    parser.add_argument('--min-tls', default='1.2', choices=sorted(TLS_VERSION_MAP),
+                        help='Minimum TLS version (default: 1.2)')
+    parser.add_argument('--max-tls', default='1.3', choices=sorted(TLS_VERSION_MAP),
+                        help='Maximum TLS version (default: 1.3)')
+    parser.add_argument('--seclevel', type=int, default=None,
+                        help='OpenSSL @SECLEVEL to force (0 to enable NULL/3DES/legacy PSK)')
+    parser.add_argument('--uri', default=None,
+                        help='X-Admin-Next-URI to hand the card (default: request URI)')
+    parser.add_argument('--mode', choices=sorted(MODE_CONTENT_TYPE), default='ram',
+                        help='"ram" = GP Amendment B RAM to a SD (default), '
+                             '"rfm" = TS 102 226 RFM/RAM to the --targeted-application.')
+    parser.add_argument('--targeted-application', default=None,
+                        help='X-Admin-Targeted-Application AID (hex). '
+                             'Required by --mode rfm, optional for --mode ram')
+    parser.add_argument('--length-coding', choices=('definite', 'indefinite'), default='definite',
+                        help='Expanded format length coding "definite" "indefinite"')
+    parser.add_argument('--apdu', action='append', default=[], metavar='HEX',
+                        help='one of many C-APDU (hex) to send, executed in order')
+    parser.add_argument('--apdu-file', default=None,
+                        help='File with one C-APDU (hex) per line to push (# comments allowed)')
+    parser.add_argument('--batch', action='store_true',
+                        help='All C-APDUs in one large command message')
+    parser.add_argument('--raw-cmd', action='append', default=[], metavar='HEX',
+                        help='Debug, raw command')
+    parser.add_argument('-v', '--verbose', action='store_true', help='enable debug output')
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
+                        format='%(asctime)s %(levelname)s %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S')
+
+    if args.mode == 'rfm' and not args.targeted_application:
+        parser.error('--mode rfm requires --targeted-application <RFM Application AID>')
+    content_type = MODE_CONTENT_TYPE[args.mode]
+
+    apdus: List[bytes] = [h2b(a) for a in args.apdu]
+    if args.apdu_file:
+        for line in Path(args.apdu_file).read_text().splitlines():
+            line = line.split('#', 1)[0].strip()
+            if line:
+                apdus.append(h2b(line))
+    command_bodies = build_command_bodies(apdus, batch=args.batch,
+                                          length_coding=args.length_coding)
+    command_bodies += [h2b(r) for r in args.raw_cmd]
+    if not command_bodies:
+        logger.warning('no C-APDUs: the server will answer the first POST with 204...')
+
+    targeted = format_aid(args.targeted_application) if args.targeted_application else None
+    logger.info('mode=%s content-type=%s length-coding=%s targeted-application=%s',
+                args.mode, content_type, args.length_coding, targeted or '(none)')
+
+    ssl_ctx = make_ssl_context(h2b(args.psk), args.psk_identity,
+                               ciphers=args.ciphers,
+                               min_tls=args.min_tls, max_tls=args.max_tls,
+                               seclevel=args.seclevel,
+                               identity_hint=args.psk_identity_hint,
+                               allow_any_identity=args.allow_any_identity)
+
+    server = Scp81AdminServer(args.host, args.port, ssl_ctx, command_bodies,
+                              next_uri=args.uri, targeted_application=targeted,
+                              content_type=content_type)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        logger.info('shutting down')
+        server.shutdown()
+
+
+if __name__ == '__main__':
+    main()

contrib/scp81_trigger.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""scp81_trigger.py -- build the OTA packet that asks the card to open an SCP81 admin session."""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# Prints the apdu line for AdmSessTriggerParams TLV as the sms secured data, Expanded RFM mode,
+# to be fed into pysim_shell.py
+#
+# security params supplied either
+# - in the trigger
+# - from the cards data object,
+# trigger wins when both are present.
+# --no-sec omits them from the trigger so the stored ones are used.
+#
+# example params:
+#    --psk-id 'PSK Identity 123' --kvn 0x41 --kid-ref 5
+#    --ip 127.0.0.1 --port 8080 --buffer 512
+#    --host 172.96.0.1 --uri '/server/adminagent?cmd=1'
+
+import argparse
+import sys
+
+from osmocom.utils import b2h                                          # noqa: E402
+from pySim.cat import (sms_pp_download_envelope, BearerDescription,  # noqa: E402
+                       BufferSize, UiccTransportLevel, OtherAddress)
+from pySim.global_platform.http import (AdmSessTriggerParams, AdmSessionParams,  # noqa: E402
+                                        SecurityParams, HttpPostParams, RasConnectionParams,
+                                        AdminHostParam, AdminUriParam)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--psk-id", help="PSK identity for ClientHello (required, unless --no-sec)")
+    ap.add_argument("--kvn", type=lambda s: int(s, 0), help="key version of the PSK (required, unless --no-sec)")
+    ap.add_argument("--kid-ref", type=lambda s: int(s, 0), help="PSK key id (required, unless --no-sec)")
+    ap.add_argument("--host", help="HTTP Host header (required, unless --no-http)")
+    ap.add_argument("--uri", help="HTTP request URI (required, unless --no-http)")
+    ap.add_argument("--ip", help="administration server address, BIP (required, unless --no-conn)")
+    ap.add_argument("--port", type=int, help="administration server port (required, unless --no-conn)")
+    ap.add_argument("--buffer", type=int, help="BIP buffer size (required, unless --no-conn)")
+    ap.add_argument("--no-conn", action="store_true", help="omit the connection params (tag 0x84)")
+    ap.add_argument("--no-sec", action="store_true", help="omit the security params (tag 0x85)")
+    ap.add_argument("--no-http", action="store_true", help="omit the HTTP POST params (tag 0x89)")
+
+    args = ap.parse_args()
+
+    missing = []
+    if not args.no_conn:
+        missing += [n for n in ('ip', 'port', 'buffer') if getattr(args, n) is None]
+    if not args.no_sec:
+        missing += [n for n in ('psk_id', 'kvn', 'kid_ref') if getattr(args, n) is None]
+    if not args.no_http:
+        missing += [n for n in ('host', 'uri') if getattr(args, n) is None]
+    if missing:
+        ap.error("pass every value required: %s." % " ".join("--" + n.replace('_', '-') for n in missing))
+
+    session = []
+    if not args.no_conn:
+        session.append(RasConnectionParams(children=[
+            BearerDescription(decoded={'bearer_type': 'default', 'bearer_parameters': ''}),
+            BufferSize(decoded=args.buffer),
+            UiccTransportLevel(decoded={'protocol_type': 'tcp_uicc_client_remote',
+                                        'port_number': args.port}),
+            OtherAddress(decoded={'type_of_address': 'ipv4',
+                                  'address': bytes(int(b) for b in args.ip.split("."))})]))
+    if not args.no_sec:
+        session.append(SecurityParams(decoded={'psk_id': args.psk_id.encode(), 'kvn': args.kvn,
+                                               'kid': args.kid_ref, 'sha_type': None}))
+    if not args.no_http:
+        session.append(HttpPostParams(children=[AdminHostParam(decoded=args.host),
+                                                AdminUriParam(decoded=args.uri)]))
+    trig = AdmSessTriggerParams(children=[AdmSessionParams(children=session)]).to_tlv()
+
+    # stderr for logs, stdout for data
+    print("# trigger TLV   %d B  %s" % (len(trig), trig.hex()), file=sys.stderr)
+    print("# %-13s %d B  %s" % ("secured data", len(trig), trig.hex()), file=sys.stderr)
+    print(trig.hex())
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

contrib/smpp-ota-tool.py

@@ -24,7 +24,8 @@ import smpplib.gsm
 import smpplib.client
 import smpplib.consts
 import time
-from pySim.ota import OtaKeyset, OtaDialectSms, OtaAlgoCrypt, OtaAlgoAuth, CNTR_REQ, RC_CC_DS, POR_REQ
+from pySim.ota import OtaKeyset, OtaDialectSms, OtaAlgoCrypt, OtaAlgoAuth, OtaCheckError, CNTR_REQ, RC_CC_DS, POR_REQ
+from pySim.sms import ConcatenatedSmsReassembler
 from pySim.utils import b2h, h2b, is_hexstr
 from pathlib import Path
 
@@ -70,6 +71,8 @@ option_parser.add_argument("--por-req", choices=POR_REQ.decmapping.values(), def
 option_parser.add_argument('--src-addr', default='12', type=str, help='SMS source address (MSISDN)')
 option_parser.add_argument('--dest-addr', default='23', type=str, help='SMS destination address (MSISDN)')
 option_parser.add_argument('--timeout', default=10, type=int, help='Maximum response waiting time')
+option_parser.add_argument('--format', choices=['compact', 'expanded'], default='compact',
+                           help="Remote Application data format: 'compact' or 'expanded'")
 option_parser.add_argument('-a', '--apdu', action='append', required=True, type=is_hexstr, help='C-APDU to send')
 
 class SmppHandler:
@@ -77,7 +80,8 @@ class SmppHandler:
 
     def __init__(self, host: str, port: int,
                  system_id: str, password: str,
-                 ota_keyset: OtaKeyset, spi: dict, tar: bytes):
+                 ota_keyset: OtaKeyset, spi: dict, tar: bytes,
+                 remote_format: str = 'compact'):
         """
         Initialize connection to SMPP server and set static OTA SMS-TPDU ciphering parameters
         Args:
@@ -88,6 +92,7 @@ class SmppHandler:
                 ota_keyset: OTA keyset to be used for SMS-TPDU ciphering
                 spi: Security Parameter Indicator (SPI) to be used for SMS-TPDU ciphering
                 tar: Toolkit Application Reference (TAR) of the targeted card application
+                remote_format: Remote Application data format ('compact' or 'expanded', TS 102 226)
         """
 
         # Create and connect SMPP client
@@ -103,26 +108,58 @@ class SmppHandler:
         self.ota_keyset = ota_keyset
         self.tar = tar
         self.spi = spi
+        self.remote_format = remote_format
+        self.reassembler = ConcatenatedSmsReassembler()
 
     def __del__(self):
         if self.client:
             self.client.unbind()
             self.client.disconnect()
 
+    def _decode_resp(self, tpud: bytes) -> tuple:
+        """Decode a response SMS-TPDU into (response_packet, decoded).
+
+        Retry to decoding with ciphering disabled (in case the card has problems to decode the SMS-TDPU
+        we have sent, the response will contain an unencrypted error message)
+        """
+        try:
+            return self.ota_dialect.decode_resp(self.ota_keyset, self.spi, tpud,
+                                                remote_format=self.remote_format)
+        except (ValueError, OtaCheckError):
+            spi = self.spi.copy()
+            spi['por_shall_be_ciphered'] = False
+            spi['por_rc_cc_ds'] = 'no_rc_cc_ds'
+            return self.ota_dialect.decode_resp(self.ota_keyset, spi, tpud,
+                                                remote_format=self.remote_format)
+
     def message_received_handler(self, pdu):
-        if pdu.short_message:
-            logger.info("SMS-TPDU received: %s", b2h(pdu.short_message))
-            try:
-                dec = self.ota_dialect.decode_resp(self.ota_keyset, self.spi, pdu.short_message)
-            except ValueError:
-                # Retry to decoding with ciphering disabled (in case the card has problems to decode the SMS-TDPU
-                # we have sent, the response will contain an unencrypted error message)
-                spi = self.spi.copy()
-                spi['por_shall_be_ciphered'] = False
-                spi['por_rc_cc_ds'] = 'no_rc_cc_ds'
-                dec = self.ota_dialect.decode_resp(self.ota_keyset, spi, pdu.short_message)
-            logger.info("SMS-TPDU decoded: %s", dec)
-            self.response = dec
+        if not pdu.short_message:
+            return None
+        logger.info("SMS-TPDU received: %s", b2h(pdu.short_message))
+        tpud = self.reassembler.add(pdu.short_message)
+        if tpud is None:
+            logger.info("SMS-TPDU is part of concat message, waiting for more parts...")
+            return None
+        if tpud != pdu.short_message:
+            logger.info("SMS-TPDU reassembled: %s", b2h(tpud))
+        try:
+            res, decoded = self._decode_resp(tpud)
+        except Exception as e:
+            # for example ENVELOPE POR
+            logger.warning("Ignoring undecodable resp SMS-TPDU (%s: %s)", type(e).__name__, e)
+            return None
+        logger.info("SMS-TPDU decoded: %s", (res, decoded))
+        # large app response as reassembled SEND SHORT MESSAGE, but
+        # the ENVELOPE itself returns a POR without R-APDU.
+        # smpplib poll() drains all pending SMS in one call, so that PoR is processed
+        # right after the real response and would overwrite it,
+        # which leaves transceive_apdu with no last_response_data to return.
+        # Only allow a response that has no application data (decoded == None)
+        # if we do not already have a real one.
+        if decoded is None and self.response is not None and self.response[1] is not None:
+            logger.info("ignoring status response to keep earlier app response")
+            return None
+        self.response = (res, decoded)
         return None
 
     def message_sent_handler(self, pdu):
@@ -183,10 +220,14 @@ class SmppHandler:
                 tuple containing the last response data and the last status word as byte strings
         """
 
-        logger.info("C-APDU sending: %s...", b2h(apdu))
+        if isinstance(apdu, (list, tuple)):
+            logger.info("C-APDU(s) sending: %s...", [b2h(a) for a in apdu])
+        else:
+            logger.info("C-APDU sending: %s...", b2h(apdu))
 
         # translate to Secured OTA RFM
-        secured = self.ota_dialect.encode_cmd(self.ota_keyset, self.tar, self.spi, apdu=apdu)
+        secured = self.ota_dialect.encode_cmd(self.ota_keyset, self.tar, self.spi, apdu=apdu,
+                                              remote_format=self.remote_format)
         # add user data header
         tpdu = b'\x02\x70\x00' + secured
         # send via SMPP
@@ -200,6 +241,17 @@ class SmppHandler:
                 container_dict = dict(container)
                 resp = container_dict.get('last_response_data')
                 sw = container_dict.get('last_status_word')
+                # expanded format: decoded response carries
+                # per command R-APDU list; log each one.
+                for i, cmd in enumerate(container_dict.get('commands') or []):
+                    logger.info("R-APDU[%u] received: %s %s", i,
+                                cmd['response_data'], cmd['status_word'])
+                if container_dict.get('truncated'):
+                    logger.warning("Response was TRUNCATED (SW 62F1): the card cut the response "
+                                   "data short and did not execute the rest of the script")
+                if container_dict.get('bad_format') is not None:
+                    logger.warning("Response contains a Bad format TLV: %s",
+                                   container_dict['bad_format'])
         if resp is None:
             raise ValueError("Response does not contain any last_response_data, no R-APDU received!")
         if sw is None:
@@ -233,8 +285,14 @@ if __name__ == '__main__':
            'por_shall_be_ciphered': not opts.por_no_ciphering,
            'por_rc_cc_ds': opts.por_rc_cc_ds,
            'por': opts.por_req}
-    apdu = h2b("".join(opts.apdu))
+    if opts.format == 'expanded':
+        # TS 102 226 5.2.1.1: wrap each apdu in its own C-APDU TLV
+        apdu = [h2b(a) for a in opts.apdu]
+    else:
+        # compact: C-APDUs are concatenated as single command string
+        apdu = h2b("".join(opts.apdu))
 
-    smpp_handler = SmppHandler(opts.host, opts.port, opts.system_id, opts.password, ota_keyset, spi, h2b(opts.tar))
+    smpp_handler = SmppHandler(opts.host, opts.port, opts.system_id, opts.password, ota_keyset, spi,
+                               h2b(opts.tar), remote_format=opts.format)
     resp, sw = smpp_handler.transceive_apdu(apdu, opts.src_addr, opts.dest_addr, opts.timeout)
     print("%s %s" % (b2h(resp), b2h(sw)))

docs/smpp-ota-tool.rst

@@ -170,6 +170,35 @@ ensures that a message can only be sent once.
 .. note:: The replay-protection-counter is implemented as a 5 byte integer value (see also ETSI TS 102 225, Table 3).
 	  When the counter has reached its maximum, it will not overflow nor can it be reset.
 
+Expanded remote application data format
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`smpp-ota-tool` uses the TS 102 226 section 5.1 compact remote application data format by default. This
+format concatenates C-APDUs into one command string and only the result of the LAST executed command is reported back.
+Retrieving the response data therefore requires a GET RESPONSE C-APDU, and only a single GET RESPONSE command may occur per script.
+
+The TS 102 226 section 5.2 expanded remote application data format removes these limitations: Each C-APDU is
+wrapped in its own C-APDU TLV inside a Command Scripting template, and the response is a Response Scripting template that contains one R-APDU TLV with the full response data and status word per executed command. To use it, pass
+``--format expanded``; every ``--apdu`` argument then becomes its own C-APDU TLV.
+
+.. note:: The expanded format does not use GET RESPONSE. To retrieve response data from a case 2 or case 4
+	  command, include an ``Le`` field in the C-APDU. i.e. ``Le='00'`` instructs the card to return all available
+	  response data in the R-APDU, with no 256-byte limit (TS 102 226, section 5.2.1.1). Without the ``Le``
+	  field no response data is returned, except a status word for the last command!.
+
+For example, a GP GET STATUS of all applications (``80F24002024F00``) returns a registry that can be much
+larger than 256 bytes. In the compact format the card would only answer with ``61xx`` procedure bytes. In the expanded
+format, appending ``Le='00'`` (i.e. ``80F24002024F0000``) makes the card return the whole registry in one exchange:
+
+::
+
+   $ PYTHONPATH=./ ./contrib/smpp-ota-tool.py --kic <KIC> --kid <KID> --kid-idx 1 --kic-idx 1 \
+       --algo-crypt triple_des_cbc2 --algo-auth triple_des_cbc2 --tar 000000 --cntr-req no_counter \
+       --format expanded --apdu 80F24002024F0000
+
+The response data (a concatenation of GlobalPlatform registry TLVs) can then be decoded with
+``pySim.global_platform.GpRegistryRelatedData.from_tlv()``.
+
 smpp-ota-tool syntax
 ~~~~~~~~~~~~~~~~~~~~
 

osmo-smdpp.py

@@ -136,6 +136,52 @@ from pySim.esim.x509_cert import CertAndPrivkey, CertificateSet, cert_get_subjec
 import logging # noqa: E402
 logger = logging.getLogger(__name__)
 
+
+def _disable_twisted_alpn_if_incompatible():
+    """Twisted <-> pyOpenSSL TLS compatibility guard applied at import.
+
+    Twisted TLSMemoryBIOFactory applies ALPN by setting the 'select' callback
+    on the SSL Context after it has already created a Connection from that
+    Context (_createConnection -> _applyProtocolNegotiation).
+    pyOpenSSL >= 25.0.0 makes a Context immutable once it has been used and
+    raises, which aborts every inbound TLS handshake, client sees unexpected-EOF
+    / decode_error that looks like a cert/cipher problem but is not.
+    pyOpenSSL < 25 does not import against recent cryptography, so downgrading
+    it is not a fix.
+
+    This server only speaks HTTP/1.1 anyway, so ALPN negotiation is not
+    needed.
+    """
+    def _major(v):
+        import re
+        m = re.match(r'\d+', (v or '').strip())
+        return int(m.group()) if m else 0
+
+    try:
+        import OpenSSL
+    except Exception:
+        return  # no pyOpenSSL ???
+    pyossl_ver = getattr(OpenSSL, '__version__', '0')
+    if _major(pyossl_ver) < 25:
+        return  # pre-25 pyOpenSSL allows mutating a used Context
+
+    try:
+        import twisted
+        from twisted.protocols import tls
+    except Exception:
+        return
+    factory = getattr(tls, 'TLSMemoryBIOFactory', None)
+    if factory is None or not hasattr(factory, '_applyProtocolNegotiation'):
+        return  # Twisted already fixed
+
+    factory._applyProtocolNegotiation = lambda self, connection: None
+    logger.warning("Disabled Twisted ALPN negotiation: Twisted %s + "
+                   "pyOpenSSL %s are incompatible for it",
+                   getattr(twisted, '__version__', '?'), pyossl_ver)
+
+
+_disable_twisted_alpn_if_incompatible()
+
 # HACK: make this configurable
 DATA_DIR = './smdpp-data'
 HOSTNAME = 'testsmdpplus1.example.com' # must match certificates!
@@ -479,7 +525,7 @@ class SmDppHttpServer:
         """See ES9+ InitiateAuthentication SGP.22 Section 5.6.1"""
         # Verify that the received address matches its own SM-DP+ address, where the comparison SHALL be
         # case-insensitive. Otherwise, the SM-DP+ SHALL return a status code "SM-DP+ Address - Refused".
-        if content['smdpAddress'] != self.server_hostname:
+        if content['smdpAddress'].lower() != self.server_hostname.lower():
            raise ApiError('8.8.1', '3.8', 'Invalid SM-DP+ Address')
 
         euiccChallenge = b64decode(content['euiccChallenge'])
@@ -640,7 +686,7 @@ class SmDppHttpServer:
                 # look up profile based on matchingID.  We simply check if a given file exists for now..
                 path = os.path.join(self.upp_dir, matchingId) + '.der'
                 # prevent directory traversal attack
-                if os.path.commonprefix((os.path.realpath(path),self.upp_dir)) != self.upp_dir:
+                if os.path.commonpath((os.path.realpath(path),self.upp_dir)) != self.upp_dir:
                     raise ApiError('8.2.6', '3.8', 'Refused')
                 if not os.path.isfile(path) or not os.access(path, os.R_OK):
                     raise ApiError('8.2.6', '3.8', 'Refused')
@@ -870,12 +916,17 @@ def main(argv):
                         action='store_true', default=False)
     parser.add_argument("-m", "--in-memory", help="Use ephermal in-memory session storage (for concurrent runs)",
                         action='store_true', default=False)
+    parser.add_argument("--smdp-address", default=HOSTNAME,
+                        help="ES9+ SM-DP+ address advertised, defaults to \"%(default)s\". "
+                             "Include the TLS port (e.g. %(default)s:8443) when binding a port other "
+                             "than 443, so it matches the address the LPA connects to. "
+                             "The TLS certificate identity is unaffected.")
     args = parser.parse_args()
 
     logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARNING)
 
     common_cert_path = os.path.join(DATA_DIR, args.certdir)
-    hs = SmDppHttpServer(server_hostname=HOSTNAME, ci_certs_path=os.path.join(common_cert_path, 'CertificateIssuer'), common_cert_path=common_cert_path, use_brainpool=args.brainpool)
+    hs = SmDppHttpServer(server_hostname=args.smdp_address, ci_certs_path=os.path.join(common_cert_path, 'CertificateIssuer'), common_cert_path=common_cert_path, use_brainpool=args.brainpool)
     if(args.nossl):
         hs.app.run(args.host, args.port)
     else:
@@ -904,7 +955,7 @@ def main(argv):
             with open(cert_pempath, 'wb') as pem_file:
                 pem_file.write(pem_cert)
 
-        SERVER_STRING = f'ssl:{args.port}:privateKey={cert_skpath}:certKey={cert_pempath}:dhParameters={dhparam_path}'
+        SERVER_STRING = f'ssl:{args.port}:privateKey={cert_skpath}:certKey={cert_pempath}:dhParameters={dhparam_path}:interface={args.host}'
         print(SERVER_STRING)
 
         hs.app.run(host=HOSTNAME, port=args.port, endpoint_description=SERVER_STRING)

pySim-prog.py

@@ -816,7 +816,7 @@ if __name__ == '__main__':
             print("")
             print("Card programming failed with an exception:")
             print("---------------------8<---------------------")
-            traceback.print_exc()
+            print(traceback.format_exc().rstrip())
             print("---------------------8<---------------------")
             print("")
             rc = -1

pySim-shell.py

@@ -24,21 +24,21 @@ import traceback
 import re
 import cmd2
 from packaging import version
-from cmd2 import style
-
 import logging
 from pySim.log import PySimLogger
 from osmocom.utils import auto_uint8
 
-# cmd2 >= 2.3.0 has deprecated the bg/fg in favor of Bg/Fg :(
-if version.parse(cmd2.__version__) < version.parse("2.3.0"):
-    from cmd2 import fg, bg # pylint: disable=no-name-in-module
-    RED = fg.red
-    YELLOW = fg.yellow
-    LIGHT_RED = fg.bright_red
-    LIGHT_GREEN = fg.bright_green
+# cmd2 >= 3.0 replaced Fg + style() with Color + stylize()
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
+    from cmd2 import Color, stylize # pylint: disable=no-name-in-module
+    RED = Color.RED
+    YELLOW = Color.YELLOW
+    LIGHT_RED = Color.BRIGHT_RED
+    LIGHT_GREEN = Color.BRIGHT_GREEN
+    def style(text, fg=None, bg=None, bold=False): # pylint: disable=function-redefined
+        return stylize(text, fg) if fg else text
 else:
-    from cmd2 import Fg, Bg # pylint: disable=no-name-in-module
+    from cmd2 import style, Fg # pylint: disable=no-name-in-module
     RED = Fg.RED
     YELLOW = Fg.YELLOW
     LIGHT_RED = Fg.LIGHT_RED
@@ -69,50 +69,26 @@ from pySim.ts_102_222 import Ts102222Commands
 from pySim.gsm_r import DF_EIRENE
 from pySim.cat import ProactiveCommand
 
-from pySim.card_key_provider import CardKeyProviderCsv, CardKeyProviderPgsql
-from pySim.card_key_provider import card_key_provider_register, card_key_provider_get_field, card_key_provider_get
+from pySim.card_key_provider import card_key_provider_argparse_add_args, card_key_provider_init
+from pySim.card_key_provider import card_key_provider_get_field, card_key_provider_get
 
 from pySim.app import init_card
 
 log = PySimLogger.get(Path(__file__).stem)
 
-class Cmd2Compat(cmd2.Cmd):
-    """Backwards-compatibility wrapper around cmd2.Cmd to support older and newer
-    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
-    def run_editor(self, file_path: Optional[str] = None) -> None:
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            return self._run_editor(file_path) # pylint: disable=no-member
-        else:
-            return super().run_editor(file_path) # pylint: disable=no-member
-
-class Settable2Compat(cmd2.Settable):
-    """Backwards-compatibility wrapper around cmd2.Settable to support older and newer
-    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
-    def __init__(self, name, val_type, description, settable_object, **kwargs):
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            super().__init__(name, val_type, description, **kwargs) # pylint: disable=no-value-for-parameter
-        else:
-            super().__init__(name, val_type, description, settable_object, **kwargs) # pylint: disable=too-many-function-args
-
-class PysimApp(Cmd2Compat):
+class PysimApp(cmd2.Cmd):
     CUSTOM_CATEGORY = 'pySim Commands'
     BANNER = """Welcome to pySim-shell!
 (C) 2021-2023 by Harald Welte, sysmocom - s.f.m.c. GmbH and contributors
 Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/shell.html """
 
     def __init__(self, verbose, card, rs, sl, ch, script=None):
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            kwargs = {'use_ipython': True}
-        else:
-            kwargs = {'include_ipy': True}
-
         self.verbose = verbose
         PySimLogger.setup(self.poutput, {logging.WARN: YELLOW})
         self._onchange_verbose('verbose', False, self.verbose)
 
-        # pylint: disable=unexpected-keyword-arg
         super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
-                         auto_load_commands=False, startup_script=script, **kwargs)
+                         auto_load_commands=False, startup_script=script, include_ipy=True)
         self.intro = style(self.BANNER, fg=RED)
         self.default_category = 'pySim-shell built-in commands'
         self.card = None
@@ -125,21 +101,27 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         self.numeric_path = False
         self.conserve_write = True
         self.json_pretty_print = True
-        self.apdu_trace = False
+        self.apdu_trace = getattr(sl, 'apdu_tracer', None) is not None
         self.apdu_strict = False
 
-        self.add_settable(Settable2Compat('numeric_path', bool, 'Print File IDs instead of names', self,
-                                          onchange_cb=self._onchange_numeric_path))
-        self.add_settable(Settable2Compat('conserve_write', bool, 'Read and compare before write', self,
-                                          onchange_cb=self._onchange_conserve_write))
-        self.add_settable(Settable2Compat('json_pretty_print', bool, 'Pretty-Print JSON output', self))
-        self.add_settable(Settable2Compat('apdu_trace', bool, 'Trace and display APDUs exchanged with card', self,
-                                          onchange_cb=self._onchange_apdu_trace))
-        self.add_settable(Settable2Compat('apdu_strict', bool,
-                                          'Strictly apply APDU format according to ISO/IEC 7816-3, table 12', self))
-        self.add_settable(Settable2Compat('verbose', bool,
-                                          'Enable/disable verbose logging', self,
-                                          onchange_cb=self._onchange_verbose))
+        self.add_settable(cmd2.Settable('numeric_path', bool,
+                                        'Print File IDs instead of names',
+                                        self, onchange_cb=self._onchange_numeric_path))
+        self.add_settable(cmd2.Settable('conserve_write', bool,
+                                        'Read and compare before write',
+                                        self, onchange_cb=self._onchange_conserve_write))
+        self.add_settable(cmd2.Settable('json_pretty_print', bool,
+                                        'Pretty-Print JSON output',
+                                        self))
+        self.add_settable(cmd2.Settable('apdu_trace', bool,
+                                        'Trace and display APDUs exchanged with card',
+                                        self, onchange_cb=self._onchange_apdu_trace))
+        self.add_settable(cmd2.Settable('apdu_strict', bool,
+                                        'Strictly apply APDU format according to ISO/IEC 7816-3, table 12',
+                                        self))
+        self.add_settable(cmd2.Settable('verbose', bool,
+                                        'Enable/disable verbose logging',
+                                        self, onchange_cb=self._onchange_verbose))
         self.equip(card, rs)
 
     def equip(self, card, rs):
@@ -228,8 +210,10 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         def __init__(self, cmd2_app):
             self.cmd2 = cmd2_app
 
-        def trace_response(self, cmd, sw, resp):
+        def trace_command(self, cmd):
             self.cmd2.poutput("-> %s %s" % (cmd[:10], cmd[10:]))
+
+        def trace_response(self, cmd, sw, resp):
             self.cmd2.poutput("<- %s: %s" % (sw, resp))
 
     def update_prompt(self):
@@ -367,7 +351,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
             self.poutput("")
             self.poutput("Card initialization (%s) failed with an exception:" % str(self.sl))
             self.poutput("---------------------8<---------------------")
-            traceback.print_exc()
+            self.poutput(traceback.format_exc().rstrip())
             self.poutput("---------------------8<---------------------")
             self.poutput("")
             return -1
@@ -481,7 +465,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
                 self.poutput("")
                 self.poutput("Card handling (%s) failed with an exception:" % str(self.sl))
                 self.poutput("---------------------8<---------------------")
-                traceback.print_exc()
+                self.poutput(traceback.format_exc().rstrip())
                 self.poutput("---------------------8<---------------------")
                 self.poutput("")
                 fail_count = fail_count + 1
@@ -1146,23 +1130,14 @@ global_group.add_argument("--skip-card-init", help="Skip all card/profile initia
 global_group.add_argument("--verbose", help="Enable verbose logging",
                           action='store_true', default=False)
 
-card_key_group = option_parser.add_argument_group('Card Key Provider Options')
-card_key_group.add_argument('--csv', metavar='FILE',
-                            default="~/.osmocom/pysim/card_data.csv",
-                            help='Read card data from CSV file')
-card_key_group.add_argument('--pgsql', metavar='FILE',
-                            default="~/.osmocom/pysim/card_data_pgsql.cfg",
-                            help='Read card data from PostgreSQL database (config file)')
-card_key_group.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
-                            help=argparse.SUPPRESS, dest='column_key')
-card_key_group.add_argument('--column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
-                            help='per-column AES transport key', dest='column_key')
-
 adm_group = global_group.add_mutually_exclusive_group()
-adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM1', dest='pin_adm', default=None,
+adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM', dest='pin_adm', default=None,
                        help='ADM PIN used for provisioning (overwrites default)')
-adm_group.add_argument('-A', '--pin-adm-hex', metavar='PIN_ADM1_HEX', dest='pin_adm_hex', default=None,
+adm_group.add_argument('-A', '--pin-adm-hex', metavar='PIN_ADM_HEX', dest='pin_adm_hex', default=None,
                        help='ADM PIN used for provisioning, as hex string (16 characters long)')
+global_group.add_argument('--pin-adm-type',
+                          choices=[x for x in pin_names.values() if x.startswith('ADM')],
+                          help='Override ADM number. Default is card-model-specific, usually 1')
 
 option_parser.add_argument('-e', '--execute-command', action='append', default=[],
                            help='A pySim-shell command that will be executed at startup')
@@ -1170,6 +1145,7 @@ option_parser.add_argument("command", nargs='?',
                            help="A pySim-shell command that would optionally be executed at startup")
 option_parser.add_argument('command_args', nargs=argparse.REMAINDER,
                            help="Optional Arguments for command")
+card_key_provider_argparse_add_args(option_parser)
 
 if __name__ == '__main__':
     startup_errors = False
@@ -1178,16 +1154,8 @@ if __name__ == '__main__':
     # Ensure that we are able to print formatted warnings from the beginning.
     PySimLogger.setup(print, {logging.WARN: YELLOW}, opts.verbose)
 
-    # Register csv-file as card data provider, either from specified CSV
-    # or from CSV file in home directory
-    column_keys = {}
-    for par in opts.column_key:
-        name, key = par.split(':')
-        column_keys[name] = key
-    if os.path.isfile(os.path.expanduser(opts.csv)):
-        card_key_provider_register(CardKeyProviderCsv(os.path.expanduser(opts.csv), column_keys))
-    if os.path.isfile(os.path.expanduser(opts.pgsql)):
-        card_key_provider_register(CardKeyProviderPgsql(os.path.expanduser(opts.pgsql), column_keys))
+    # Init card key provider for automatic card key retrieval
+    card_key_provider_init(opts)
 
     # Init card reader driver
     sl = init_reader(opts, proactive_handler = Proact())
@@ -1208,7 +1176,7 @@ if __name__ == '__main__':
         startup_errors = True
         print("Card initialization (%s) failed with an exception:" % str(sl))
         print("---------------------8<---------------------")
-        traceback.print_exc()
+        print(traceback.format_exc().rstrip())
         print("---------------------8<---------------------")
         if not opts.noprompt:
             print("(you may still try to recover from this manually by using the 'equip' command.)")
@@ -1219,18 +1187,15 @@ if __name__ == '__main__':
 
     # If the user supplies an ADM PIN at via commandline args authenticate
     # immediately so that the user does not have to use the shell commands
-    pin_adm = sanitize_pin_adm(opts.pin_adm, opts.pin_adm_hex)
-    if pin_adm:
-        if not card:
-            print("Card error, cannot do ADM verification with supplied ADM pin now.")
-        try:
-            card._scc.verify_chv(card._adm_chv_num, h2b(pin_adm))
-        except Exception as e:
-            startup_errors = True
-            print("ADM verification (%s) failed with an exception:" % str(pin_adm))
-            print("---------------------8<---------------------")
-            print(e)
-            print("---------------------8<---------------------")
+    pin_adm_type = ""
+    if opts.pin_adm_type:
+        pin_adm_type = "--adm-type %s" % opts.pin_adm_type
+    if opts.pin_adm:
+        app.onecmd_plus_hooks("verify_adm %s %s" %
+                              (opts.pin_adm, pin_adm_type), add_to_history = False)
+    elif opts.pin_adm_hex:
+        app.onecmd_plus_hooks("verify_adm %s --pin-is-hex %s" %
+                              (opts.pin_adm_hex, pin_adm_type), add_to_history = False)
 
     # Run optional commands
     for c in opts.execute_command:

pySim-smpp2sim.py

@@ -30,10 +30,13 @@
 
 import argparse
 import logging
+import socket
+import threading
+import time
 import colorlog
 
 from twisted.protocols import basic
-from twisted.internet import defer, endpoints, protocol, reactor, task
+from twisted.internet import defer, endpoints, reactor, task
 from twisted.cred.portal import IRealm
 from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
 from twisted.cred.portal import Portal
@@ -47,13 +50,16 @@ from smpp.pdu import pdu_types, operations, pdu_encoding
 
 from pySim.sms import SMS_DELIVER, SMS_SUBMIT, AddressField
 
+from pySim.bip import Proact, terminal_profile
 from pySim.transport import LinkBase, ProactiveHandler, argparse_add_reader_args, init_reader, ApduTracer
 from pySim.commands import SimCardCommands
 from pySim.cards import UiccCardBase
 from pySim.exceptions import *
+from pySim.cat import sms_pp_download_envelope
 from pySim.cat import ProactiveCommand, SendShortMessage, SMS_TPDU, SMSPPDownload, BearerDescription
 from pySim.cat import DeviceIdentities, Address, OtherAddress, UiccTransportLevel, BufferSize
 from pySim.cat import ChannelStatus, ChannelData, ChannelDataLength
+from pySim.cat import EventList, EventDownload, Result
 from pySim.utils import b2h, h2b
 
 logger = logging.getLogger(__name__)
@@ -71,224 +77,6 @@ class MyApduTracer(ApduTracer):
         print("-> %s %s" % (cmd[:10], cmd[10:]))
         print("<- %s: %s" % (sw, resp))
 
-class TcpProtocol(protocol.Protocol):
-    def dataReceived(self, data):
-        pass
-
-    def connectionLost(self, reason):
-        pass
-
-
-def tcp_connected_callback(p: protocol.Protocol):
-    """called by twisted TCP client."""
-    logger.error("%s: connected!" % p)
-
-class ProactChannel:
-    """Representation of a single protective channel."""
-    def __init__(self, channels: 'ProactChannels', chan_nr: int):
-        self.channels = channels
-        self.chan_nr = chan_nr
-        self.ep = None
-
-    def close(self):
-        """Close the channel."""
-        if self.ep:
-            self.ep.disconnect()
-        self.channels.channel_delete(self.chan_nr)
-
-class ProactChannels:
-    """Wrapper class for maintaining state of proactive channels."""
-    def __init__(self):
-        self.channels = {}
-
-    def channel_create(self) -> ProactChannel:
-        """Create a new proactive channel, allocating its integer number."""
-        for i in range(1, 9):
-            if not i in self.channels:
-                self.channels[i] = ProactChannel(self, i)
-                return self.channels[i]
-        raise ValueError('Cannot allocate another channel: All channels active')
-
-    def channel_delete(self, chan_nr: int):
-        del self.channels[chan_nr]
-
-class Proact(ProactiveHandler):
-    #def __init__(self, smpp_factory):
-    #    self.smpp_factory = smpp_factory
-    def __init__(self):
-        self.channels = ProactChannels()
-
-    @staticmethod
-    def _find_first_element_of_type(instlist, cls):
-        for i in instlist:
-            if isinstance(i, cls):
-                return i
-        return None
-
-    """Call-back which the pySim transport core calls whenever it receives a
-    proactive command from the SIM."""
-    def handle_SendShortMessage(self, pcmd: ProactiveCommand):
-        # {'smspp_download': [{'device_identities': {'source_dev_id': 'network',
-        #                                            'dest_dev_id': 'uicc'}},
-        #                     {'address': {'ton_npi': {'ext': True,
-        #                                              'type_of_number': 'international',
-        #                                              'numbering_plan_id': 'isdn_e164'},
-        #                                  'call_number': '79'}},
-        #                     {'sms_tpdu': {'tpdu': '40048111227ff6407070611535004d02700000481516011212000001fe4c0943aea42e45021c078ae06c66afc09303608874b72f58bacadb0dcf665c29349c799fbb522e61709c9baf1890015e8e8e196e36153106c8b92f95153774'}}
-        #                    ]}
-        """Card requests sending a SMS. We need to pass it on to the ESME via SMPP."""
-        logger.info("SendShortMessage")
-        logger.info(pcmd)
-        # Relevant parts in pcmd: Address, SMS_TPDU
-        addr_ie = Proact._find_first_element_of_type(pcmd.children, Address)
-        sms_tpdu_ie = Proact._find_first_element_of_type(pcmd.children, SMS_TPDU)
-        raw_tpdu = sms_tpdu_ie.decoded['tpdu']
-        submit = SMS_SUBMIT.from_bytes(raw_tpdu)
-        submit.tp_da = AddressField(addr_ie.decoded['call_number'], addr_ie.decoded['ton_npi']['type_of_number'],
-                                    addr_ie.decoded['ton_npi']['numbering_plan_id'])
-        logger.info(submit)
-        self.send_sms_via_smpp(submit)
-
-    def handle_OpenChannel(self, pcmd: ProactiveCommand):
-        """Card requests opening a new channel via a UDP/TCP socket."""
-        # {'open_channel': [{'command_details': {'command_number': 1,
-        #                                        'type_of_command': 'open_channel',
-        #                                        'command_qualifier': 3}},
-        #                   {'device_identities': {'source_dev_id': 'uicc',
-        #                                          'dest_dev_id': 'terminal'}},
-        #                   {'bearer_description': {'bearer_type': 'default',
-        #                                           'bearer_parameters': ''}},
-        #                   {'buffer_size': 1024},
-        #                   {'uicc_transport_level': {'protocol_type': 'tcp_uicc_client_remote',
-        #                                             'port_number': 32768}},
-        #                   {'other_address': {'type_of_address': 'ipv4',
-        #                                      'address': '01020304'}}
-        #                  ]}
-        logger.info("OpenChannel")
-        logger.info(pcmd)
-        transp_lvl_ie = Proact._find_first_element_of_type(pcmd.children, UiccTransportLevel)
-        other_addr_ie = Proact._find_first_element_of_type(pcmd.children, OtherAddress)
-        bearer_desc_ie = Proact._find_first_element_of_type(pcmd.children, BearerDescription)
-        buffer_size_ie = Proact._find_first_element_of_type(pcmd.children, BufferSize)
-        if transp_lvl_ie.decoded['protocol_type'] != 'tcp_uicc_client_remote':
-            raise ValueError('Unsupported protocol_type')
-        if other_addr_ie.decoded.get('type_of_address', None) != 'ipv4':
-            raise ValueError('Unsupported type_of_address')
-        ipv4_bytes = h2b(other_addr_ie.decoded['address'])
-        ipv4_str = '%u.%u.%u.%u' % (ipv4_bytes[0], ipv4_bytes[1], ipv4_bytes[2], ipv4_bytes[3])
-        port_nr = transp_lvl_ie.decoded['port_number']
-        print("%s:%u" % (ipv4_str, port_nr))
-        channel = self.channels.channel_create()
-        channel.ep = endpoints.TCP4ClientEndpoint(reactor, ipv4_str, port_nr)
-        channel.prot = TcpProtocol()
-        d = endpoints.connectProtocol(channel.ep, channel.prot)
-        # FIXME: why is this never called despite the client showing the inbound connection?
-        d.addCallback(tcp_connected_callback)
-
-        # Terminal Response example: [
-        #  {'command_details': {'command_number': 1,
-        #                       'type_of_command': 'open_channel',
-        #                       'command_qualifier': 3}},
-        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
-        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
-        #  {'channel_status': '8100'},
-        #  {'bearer_description': {'bearer_type': 'default', 'bearer_parameters': ''}},
-        #  {'buffer_size': 1024}
-        # ]
-        return self.prepare_response(pcmd) + [ChannelStatus(decoded='8100'), bearer_desc_ie, buffer_size_ie]
-
-    def handle_CloseChannel(self, pcmd: ProactiveCommand):
-        """Close a channel."""
-        logger.info("CloseChannel")
-        logger.info(pcmd)
-
-    def handle_ReceiveData(self, pcmd: ProactiveCommand):
-        """Receive/read data from the socket."""
-        # {'receive_data': [{'command_details': {'command_number': 1,
-        #                                        'type_of_command': 'receive_data',
-        #                                        'command_qualifier': 0}},
-        #                   {'device_identities': {'source_dev_id': 'uicc',
-        #                                          'dest_dev_id': 'channel_1'}},
-        #                   {'channel_data_length': 9}
-        #                  ]}
-        logger.info("ReceiveData")
-        logger.info(pcmd)
-        # Terminal Response example: [
-        #  {'command_details': {'command_number': 1,
-        #                       'type_of_command': 'receive_data',
-        #                       'command_qualifier': 0}},
-        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
-        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
-        #  {'channel_data': '16030100040e000000'},
-        #  {'channel_data_length': 0}
-        # ]
-        return self.prepare_response(pcmd) + []
-
-    def handle_SendData(self, pcmd: ProactiveCommand):
-        """Send/write data received from the SIM to the socket."""
-        # {'send_data': [{'command_details': {'command_number': 1,
-        #                                     'type_of_command': 'send_data',
-        #                                     'command_qualifier': 1}},
-        #                {'device_identities': {'source_dev_id': 'uicc',
-        #                                       'dest_dev_id': 'channel_1'}},
-        #                {'channel_data': '160301003c010000380303d0f45e12b52ce5bb522750dd037738195334c87a46a847fe2b6886cada9ea6bf00000a00ae008c008b00b0002c010000050001000101'}
-        #               ]}
-        logger.info("SendData")
-        logger.info(pcmd)
-        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
-        chan_data_ie = Proact._find_first_element_of_type(pcmd.children, ChannelData)
-        chan_str = dev_id_ie.decoded['dest_dev_id']
-        chan_nr = 1 # FIXME
-        chan = self.channels.channels.get(chan_nr, None)
-        # FIXME chan.prot.transport.write(h2b(chan_data_ie.decoded))
-        # Terminal Response example: [
-        #  {'command_details': {'command_number': 1,
-        #                       'type_of_command': 'send_data',
-        #                       'command_qualifier': 1}},
-        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
-        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
-        #  {'channel_data_length': 255}
-        # ]
-        return self.prepare_response(pcmd) + [ChannelDataLength(decoded=255)]
-
-    def handle_SetUpEventList(self, pcmd: ProactiveCommand):
-        # {'set_up_event_list': [{'command_details': {'command_number': 1,
-        #                                             'type_of_command': 'set_up_event_list',
-        #                                             'command_qualifier': 0}},
-        #                        {'device_identities': {'source_dev_id': 'uicc',
-        #                                               'dest_dev_id': 'terminal'}},
-        #                        {'event_list': ['data_available', 'channel_status']}
-        #                       ]}
-        logger.info("SetUpEventList")
-        logger.info(pcmd)
-        # Terminal Response example: [
-        #  {'command_details': {'command_number': 1,
-        #                       'type_of_command': 'set_up_event_list',
-        #                       'command_qualifier': 0}},
-        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
-        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}}
-        # ]
-        return self.prepare_response(pcmd)
-
-    def getChannelStatus(self, pcmd: ProactiveCommand):
-        logger.info("GetChannelStatus")
-        logger.info(pcmd)
-        return self.prepare_response(pcmd) + []
-
-    def send_sms_via_smpp(self, submit: SMS_SUBMIT):
-        # while in a normal network the phone/ME would *submit* a message to the SMSC,
-        # we are actually emulating the SMSC itself, so we must *deliver* the message
-        # to the ESME
-        deliver = SMS_DELIVER.from_submit(submit)
-        deliver_smpp = deliver.to_smpp()
-
-        hackish_global_smpp.sendDataRequest(deliver_smpp)
-#       # obtain the connection/binding of system_id to be used for delivering MO-SMS to the ESME
-#       connection = smpp_server.getBoundConnections[system_id].getNextBindingForDelivery()
-#       connection.sendDataRequest(deliver_smpp)
-
-
-
 def dcs_is_8bit(dcs):
     if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
                                    pdu_types.DataCodingDefault.OCTET_UNSPECIFIED):
@@ -323,6 +111,11 @@ class MyServer:
         smppEndpoint = endpoints.TCP6ServerEndpoint(reactor, tcp_port, interface=bind_ip)
         smppEndpoint.listen(self.factory)
         self.tp = self.scc = self.card = None
+        # Serialise card/APDU access.
+        # - SMPP handler drives the card from reactor thread
+        # - BIP relay data-available path drives it from socket reader thread.
+        # The transport is not re-entrant, both must take this lock.
+        self._card_lock = threading.Lock()
 
     def connect_to_card(self, tp: LinkBase):
         self.tp = tp
@@ -333,8 +126,22 @@ class MyServer:
         self.scc.sel_ctrl = "0004"
         self.card.read_aids()
         self.card.select_adf_by_aid(adf='usim')
-        # FIXME: create a more realistic profile than ffffff
-        self.scc.terminal_profile('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff')
+        self.scc.terminal_profile(b2h(terminal_profile()))
+        # Connect the BIP relay inbound path to the card.
+        # relay socket receives data -> ME initiated ENVELOPE EVENT DOWNLOA
+        # -> triggers RECEIVE DATA proactive session.
+        # FIXME this cross-thread push to the card is exercised only with real hardware
+        # the card free tests cover socket relay + envelope construction, not delivery.
+        handler = getattr(tp, 'proactive_handler', None)
+        if isinstance(handler, Proact):
+            handler.data_available_sink = self._deliver_data_available
+
+    def _deliver_data_available(self, envelope_hex: str):
+        """push ME initiated ENVELOPE EVENT DOWNLOAD to the card"""
+        with self._card_lock:
+            logger.info("ENVELOPE(Data available): %s" % envelope_hex)
+            (data, sw) = self.scc.envelope(envelope_hex)
+            logger.info("SW %s: %s" % (sw, data))
 
     def _msgHandler(self, system_id, smpp, pdu):
         """Handler for incoming messages received via SMPP from ESME."""
@@ -362,14 +169,12 @@ class MyServer:
         tpdu = SMS_DELIVER.from_smpp_submit(pdu)
         logger.info(tpdu)
         # 2) wrap into the CAT ENVELOPE for SMS-PP-Download
-        tpdu_ie = SMS_TPDU(decoded={'tpdu': b2h(tpdu.to_bytes())})
-        addr_ie = Address(decoded={'ton_npi': {'ext':False, 'type_of_number':'unknown', 'numbering_plan_id':'unknown'}, 'call_number': '0123456'})
-        dev_ids = DeviceIdentities(decoded={'source_dev_id': 'network', 'dest_dev_id': 'uicc'})
-        sms_dl = SMSPPDownload(children=[dev_ids, addr_ie, tpdu_ie])
+        sms_dl = sms_pp_download_envelope(tpdu)
         # 3) send to the card
         envelope_hex = b2h(sms_dl.to_tlv())
         logger.info("ENVELOPE: %s" % envelope_hex)
-        (data, sw) = self.scc.envelope(envelope_hex)
+        with self._card_lock:
+            (data, sw) = self.scc.envelope(envelope_hex)
         logger.info("SW %s: %s" % (sw, data))
         if sw in ['9200', '9300']:
             # TODO send back RP-ERROR message with TP-FCS == 'SIM Application Toolkit Busy'
@@ -416,7 +221,8 @@ if __name__ == '__main__':
 
     opts = option_parser.parse_args()
 
-    tp = init_reader(opts, proactive_handler = Proact())
+    tp = init_reader(opts, proactive_handler = Proact(
+        sms_sink=lambda pdu: hackish_global_smpp.sendDataRequest(pdu)))
     if tp is None:
         exit(1)
     tp.connect()

pySim-trace.py

@@ -117,7 +117,7 @@ class Tracer:
             try:
                 apdu = self.source.read()
                 apdu_counter = apdu_counter + 1
-            except StopIteration:
+            except (StopIteration, KeyboardInterrupt):
                 print("%i APDUs parsed, stop iteration." % apdu_counter)
                 return 0
 

pySim/app.py

@@ -26,11 +26,15 @@ from pySim.cdma_ruim import CardProfileRUIM
 from pySim.ts_102_221 import CardProfileUICC
 from pySim.utils import all_subclasses
 from pySim.exceptions import SwMatchError
+from pySim.log import PySimLogger
 
-# we need to import this module so that the SysmocomSJA2 sub-class of
-# CardModel is created, which will add the ATR-based matching and
-# calling of SysmocomSJA2.add_files.  See  CardModel.apply_matching_models
+log = PySimLogger.get(__name__)
+
+# we need to import these modules so that the SysmocomSJA2 / SysmocomSJS1
+# sub-classes of CardModel are created, which will add the ATR-based matching
+# and calling of their add_files.  See CardModel.apply_matching_models
 import pySim.sysmocom_sja2
+import pySim.sysmocom_sjs1
 
 # we need to import these modules so that the various sub-classes of
 # CardProfile are created, which will be used in init_card() to iterate
@@ -54,7 +58,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
 
     # Wait up to three seconds for a card in reader and try to detect
     # the card type.
-    print("Waiting for card...")
+    log.info("Waiting for card...")
     sl.wait_for_card(3)
 
     # The user may opt to skip all card initialization. In this case only the
@@ -66,7 +70,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
     generic_card = False
     card = card_detect(scc)
     if card is None:
-        print("Warning: Could not detect card type - assuming a generic card type...")
+        log.warning("Could not detect card type - assuming a generic card type...")
         card = SimCardBase(scc)
         generic_card = True
 
@@ -76,7 +80,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
         # just means that pySim was unable to recognize the card profile. This
         # may happen in particular with unprovisioned cards that do not have
         # any files on them yet.
-        print("Unsupported card type!")
+        log.warning("Unsupported card type!")
         return None, card
 
     # ETSI TS 102 221, Table 9.3 specifies a default for the PIN key
@@ -87,7 +91,7 @@ def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState,
     if generic_card and isinstance(profile, CardProfileUICC):
         card._adm_chv_num = 0x0A
 
-    print("Info: Card is of type: %s" % str(profile))
+    log.info("Card is of type: %s", str(profile))
 
     # FIXME: this shouldn't really be here but somewhere else/more generic.
     # We cannot do it within pySim/profile.py as that would create circular

pySim/ara_m.py

@@ -300,6 +300,51 @@ class ADF_ARAM(CardADF):
                                'major': v_major, 'minor': v_minor, 'patch': v_patch}}])
         return ADF_ARAM.xceive_apdu_tlv(scc, '80cadf21', cmd_do, ResponseAramConfigDO)
 
+    @staticmethod
+    def store_ref_ar_do(scc, aid:Hexstr, aid_empty:bool, device_app_id:Hexstr, pkg_ref:str,
+                        apdu_filter:Hexstr, apdu_never:bool, apdu_always:bool,
+                        nfc_always:bool, nfc_never:bool, android_permissions:Hexstr):
+        # REF
+        ref_do_content = []
+        if aid is not None:
+            ref_do_content += [{'aid_ref_do': aid}]
+        elif aid_empty:
+            ref_do_content += [{'aid_ref_empty_do': None}]
+        ref_do_content += [{'dev_app_id_ref_do': device_app_id}]
+        if pkg_ref:
+            ref_do_content += [{'pkg_ref_do': {'package_name_string': pkg_ref}}]
+        # AR
+        ar_do_content = []
+        if apdu_never:
+            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
+        elif apdu_always:
+            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
+        elif apdu_filter:
+            if len(apdu_filter) % 16:
+                raise ValueError(f'Invalid non-modulo-16 length of APDU filter: {len(apdu_filter)}')
+            offset = 0
+            apdu_filter_list = []
+            while offset < len(apdu_filter):
+                apdu_filter_list += [{'header': apdu_filter[offset:offset+8],
+                                      'mask': apdu_filter[offset+8:offset+16]}]
+                offset += 16 # Move offset to the beginning of the next apdu_filter object
+            ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter_list}}]
+        if nfc_never:
+            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
+        elif nfc_always:
+            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
+        if android_permissions:
+            ar_do_content += [{'perm_ar_do': {'permissions': android_permissions}}]
+        d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
+        csrado = CommandStoreRefArDO()
+        csrado.from_val_dict(d)
+        return ADF_ARAM.store_data(scc, csrado)
+
+    @staticmethod
+    def aram_delete_all(scc):
+        deldo = CommandDelete()
+        return ADF_ARAM.store_data(scc, deldo)
+
     @with_default_category('Application-Specific Commands')
     class AddlShellCommands(CommandSet):
         def do_aram_get_all(self, _opts):
@@ -334,58 +379,25 @@ class ADF_ARAM(CardADF):
         apdu_grp.add_argument(
             '--apdu-filter', help='APDU filter: multiple groups of 8 hex bytes (4 byte CLA/INS/P1/P2 followed by 4 byte mask)')
         nfc_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
-        nfc_grp.add_argument('--nfc-always', action='store_true',
-                             help='NFC event access is allowed')
         nfc_grp.add_argument('--nfc-never', action='store_true',
                              help='NFC event access is not allowed')
+        nfc_grp.add_argument('--nfc-always', action='store_true',
+                             help='NFC event access is allowed')
         store_ref_ar_do_parse.add_argument(
             '--android-permissions', help='Android UICC Carrier Privilege Permissions (8 hex bytes)')
 
         @cmd2.with_argparser(store_ref_ar_do_parse)
         def do_aram_store_ref_ar_do(self, opts):
             """Perform STORE DATA [Command-Store-REF-AR-DO] to store a (new) access rule."""
-            # REF
-            ref_do_content = []
-            if opts.aid is not None:
-                ref_do_content += [{'aid_ref_do': opts.aid}]
-            elif opts.aid_empty:
-                ref_do_content += [{'aid_ref_empty_do': None}]
-            ref_do_content += [{'dev_app_id_ref_do': opts.device_app_id}]
-            if opts.pkg_ref:
-                ref_do_content += [{'pkg_ref_do': {'package_name_string': opts.pkg_ref}}]
-            # AR
-            ar_do_content = []
-            if opts.apdu_never:
-                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
-            elif opts.apdu_always:
-                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
-            elif opts.apdu_filter:
-                if len(opts.apdu_filter) % 16:
-                    raise ValueError(f'Invalid non-modulo-16 length of APDU filter: {len(opts.apdu_filter)}')
-                offset = 0
-                apdu_filter = []
-                while offset < len(opts.apdu_filter):
-                    apdu_filter += [{'header': opts.apdu_filter[offset:offset+8],
-                                     'mask': opts.apdu_filter[offset+8:offset+16]}]
-                    offset += 16 # Move offset to the beginning of the next apdu_filter object
-                ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter}}]
-            if opts.nfc_always:
-                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
-            elif opts.nfc_never:
-                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
-            if opts.android_permissions:
-                ar_do_content += [{'perm_ar_do': {'permissions': opts.android_permissions}}]
-            d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
-            csrado = CommandStoreRefArDO()
-            csrado.from_val_dict(d)
-            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, csrado)
+            res_do = ADF_ARAM.store_ref_ar_do(self._cmd.lchan.scc, opts.aid, opts.aid_empty, opts.device_app_id,
+                                              opts.pkg_ref, opts.apdu_filter, opts.apdu_never, opts.apdu_always,
+                                              opts.nfc_always, opts.nfc_never, opts.android_permissions)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
         def do_aram_delete_all(self, _opts):
             """Perform STORE DATA [Command-Delete[all]] to delete all access rules."""
-            deldo = CommandDelete()
-            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, deldo)
+            res_do = ADF_ARAM.aram_delete_all(self._cmd.lchan.scc)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
@@ -394,7 +406,6 @@ class ADF_ARAM(CardADF):
             (Proprietary feature that is specific to sysmocom's fork of Bertrand Martel’s ARA-M implementation.)"""
             self._cmd.lchan.scc.send_apdu_checksw('80e2900001A1', '9000')
 
-
 # SEAC v1.1 Section 4.1.2.2 + 5.1.2.2
 sw_aram = {
     'ARA-M': {

pySim/bip.py

@@ -0,0 +1,627 @@
+# -*- coding: utf-8 -*-
+"""Bearer Independent Protocol relay"""
+
+#
+# (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# A ProactiveHandler with TCP sockets that backs the BIP channels,
+# so a card can run its own IP session (SCP81/HTTPS, CAT_TP, ...)
+#
+# Currently used by pySim-smpp2sim.py which connects the SMS path to its SMPP server.
+# Other drivers can pass their own sinks:
+#
+#     handler = Proact(data_available_sink=..., sms_sink=...)
+#     tp = init_reader(opts, proactive_handler=handler)
+
+
+import logging
+import socket
+import threading
+import time
+
+from osmocom.utils import b2h, h2b
+
+from pySim.transport import ProactiveHandler
+from pySim.sms import SMS_DELIVER, SMS_SUBMIT, AddressField
+from pySim.cat import (ProactiveCommand, SendShortMessage, SMS_TPDU, SMSPPDownload,
+                       BearerDescription, DeviceIdentities, Address, OtherAddress,
+                       UiccTransportLevel, BufferSize, ChannelStatus, ChannelData,
+                       ChannelDataLength, EventList, EventDownload, Result,
+                       CommandDetails, LocationInformation)
+
+logger = logging.getLogger(__name__)
+
+# PROVIDE LOCAL INFORMATION location, GERAN TS 31.111 8.19.1
+# - 3 byte PLMN of TS 24.008 10.5.1.3 -> 262-01
+# - 2 byte LAC and a 2 byte cid.
+DEFAULT_LOCATION = h2b('62f21000010001')
+
+
+def terminal_profile(num_channels: int = 7) -> bytes:
+    """TERMINAL PROFILE for what we implement, TS 102 223 5.2 and annex T.
+
+    Annex T table T.1 lists what a Connected Entity, a CAT client that is not the modem
+    which is pretty much what we are, may announce, and its inverse is what only a modem may announce.
+    """
+    if not 0 <= num_channels <= ProactChannels.MAX_CHANNELS:
+        raise ValueError('num_channels must be 0..%u' % ProactChannels.MAX_CHANNELS)
+    profile = bytearray(32)
+    # 1 (Download): b1 profile download, b2+b5 SMS-PP data download. Both of the latter, per the
+    # note in TS 31.111 5.2: "several bits may need to be set to 1 for the support of the same
+    # facility ... because of backward compatibility with SAT". The relay is OTA over SMS-PP.
+    profile[0] = 0x01 | 0x02 | 0x10
+    profile[1] = 0x01                    # 2 (Other): b1 command result
+    profile[2] = 0x80                    # 3: b8 REFRESH (empty result is a valid answer, 6.4.7)
+    profile[3] = 0x02                    # 4: b2 SEND SHORT MESSAGE (the OTA response path)
+    profile[4] = 0x01                    # 5: b1 SET UP EVENT LIST
+    profile[5] = 0x04 | 0x08             # 6: b3 Event Data available, b4 Event Channel status
+    # 12 (class "e"): b1..b5 OPEN CHANNEL, CLOSE CHANNEL, RECEIVE DATA, SEND DATA, GET CHANNEL
+    # STATUS.
+    profile[11] = 0x1f
+    # 13 (class "e" supported bearers): b2 GPRS, and b6..b8 the number of channels.
+    profile[12] = 0x02 | (num_channels << 5)
+    profile[13] = 0x40 | 0x20            # 14: b6 no display capability, b7 no keypad available
+    profile[16] = 0x01                   # 15: b1 TCP, UICC in client mode, remote connection
+    return bytes(profile)
+
+
+class ProactChannel:
+    """One BIP channel, TS 102 223 class "e", backed by a blocking TCP socket.
+
+    Created by ProactChannels.channel_create(). A reader thread fills the Rx buffer from the
+    socket, the Proact handlers drain it (RECEIVE DATA) and write to it (SEND DATA). Payload
+    is opaque, TLS or CAT_TP run on the card.
+
+    Args:
+        channels: the owning ProactChannels, notified of data arrival and of close()
+        chan_nr: channel number 1..7 as used in the Device identities
+    """
+    # Why blocking sockets and not Twisted endpoints, considering we have twisted?
+    # The proactive-command loop lives in a blocking while-loop,
+    # "pySim.transport.LinkBase.send_apdu_checksw" that runs on the Twisted reactor thread.
+    # A Twisted async TCP client only makes any progress when the reactor uhh... reacts, but
+    # the reactor is stuck in that loop for the whole proactive session -> the
+    # connectProtocol() Deferred never fires while we are handling OPEN/SEND/RECEIVE CHANNEL.
+    # Plain blocking sockets just work: connect() in handle_OpenChannel, send() in
+    # handle_SendData, recv() feeding a buffer for handle_ReceiveData. No need to make it
+    # harder than it has to be to handle the "massive" T0 bandwidth..
+    # how much we try to read off the socket per recv()
+    RECV_CHUNK = 4096
+
+    def __init__(self, channels: 'ProactChannels', chan_nr: int):
+        self.channels = channels
+        self.chan_nr = chan_nr
+        self.sock = None
+        # TS 102 223 says the terminal keeps an Rx buffer per channel; RECEIVE
+        # DATA drains it, and it is filled asynchronously as the peer sends.
+        self.rx_buf = bytearray()
+        self._rx_lock = threading.Lock()
+        self._reader = None
+        self._closing = False
+        self.peer_closed = False
+
+    def connect(self, host: str, port: int, timeout: float = 10.0):
+        """Open the blocking TCP socket and start the background Rx reader."""
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            s.settimeout(timeout)
+            s.connect((host, port))
+            # Back to blocking mode for the reader thread.
+            # CLOSE CHANNEL unblocks the pending recv() via shutdown().
+            s.settimeout(None)
+        except OSError:
+            s.close()
+            raise
+        self.sock = s
+        self._reader = threading.Thread(target=self._rx_loop,
+                                        name='bip-rx-%d' % self.chan_nr, daemon=True)
+        self._reader.start()
+
+    def _rx_loop(self):
+        """Continuously read from the socket into rx_buf, like a real ME.
+
+        TS 102 223 7.5.10.1 says the event is raised 'only if the targeted channel buffer is
+        empty when new data arrives in it', so the data available hook fires on the
+        empty->non-empty transition only. That is enough: every RECEIVE DATA response tells
+        the card how many bytes remain, so it keeps fetching until the buffer is empty, and
+        the next event restarts it when more data arrives."""
+        while not self._closing:
+            try:
+                data = self.sock.recv(self.RECV_CHUNK)
+            except (OSError, ValueError):
+                break
+            if not data:
+                self.peer_closed = True
+                break
+            with self._rx_lock:
+                was_empty = len(self.rx_buf) == 0
+                self.rx_buf.extend(data)
+            if was_empty and not self._closing:
+                self.channels.notify_data_available(self)
+
+    def send(self, data: bytes):
+        """Tx, write bytes to the socket == SEND DATA"""
+        self.sock.sendall(data)
+
+    def available_rx(self) -> int:
+        """Number of bytes waiting in the Rx buffer, what RECEIVE DATA can return right now."""
+        with self._rx_lock:
+            return len(self.rx_buf)
+
+    def take_rx(self, n: int):
+        """Take up to n bytes out of the Rx buffer. Returns (bytes, bytes still remaining)."""
+        with self._rx_lock:
+            chunk = bytes(self.rx_buf[:n])
+            del self.rx_buf[:n]
+            remaining = len(self.rx_buf)
+        return chunk, remaining
+
+    def wait_rx(self, timeout: float) -> int:
+        """wait up to timeout seconds until the rxbuf has data
+        returns the number of bytes available
+        Cards have a "data available" event, card free callers use
+        this to wait for the echoed bytes."""
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            avail = self.available_rx()
+            if avail or self.peer_closed:
+                return avail
+            time.sleep(0.005)
+        return self.available_rx()
+
+    def close(self):
+        """Close channel: stop reader, close socket, drop bookkeeping."""
+        self._closing = True
+        if self.sock is not None:
+            try:
+                self.sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+            try:
+                self.sock.close()
+            except OSError:
+                pass
+        # CLOSE CHANNEL synchronously handled inside the rx reader thread
+        # (data-available -> ENVELOPE -> FETCH -> handle_CloseChannel -> close),
+        # so close() can be called on the reader thread.
+        # Joining self raises "cannot join current thread" so better skip i..
+        # setting _closing + shutting down the socket already makes _rx_loop
+        # return on the next iteration anyway.
+        if self._reader is not None and self._reader is not threading.current_thread():
+            self._reader.join(timeout=1.0)
+        self.channels.channel_delete(self.chan_nr)
+
+class ProactChannels:
+    """The open BIP channels of one terminal, keyed by channel number.
+
+    Args:
+        on_data_available: callback(chan: ProactChannel), invoked from the channel's reader
+            thread when data arrives in an empty Rx buffer. Proact turns it into an
+            ENVELOPE EVENT DOWNLOAD (data available).
+    """
+
+    # TS 102 223 8.56 channel identifier in 3 bits as "1 to 7", 0 == no channel available
+    # TERMINAL PROFILE has to agree with byte 13 , "number of channels supported by terminal"
+    MAX_CHANNELS = 7
+
+    def __init__(self, on_data_available=None):
+        self.channels = {}
+        self._on_data_available = on_data_available
+
+    def channel_create(self) -> ProactChannel:
+        """Create a new proactive channel, allocating its integer number."""
+        for i in range(1, self.MAX_CHANNELS + 1):
+            if not i in self.channels:
+                self.channels[i] = ProactChannel(self, i)
+                return self.channels[i]
+        raise ValueError('Cannot allocate another channel: All channels active')
+
+    def channel_delete(self, chan_nr: int):
+        """Forget a channel, called by ProactChannel.close()."""
+        self.channels.pop(chan_nr, None)
+
+    def notify_data_available(self, chan: ProactChannel):
+        """Run the on_data_available callback for chan, if one was given."""
+        if self._on_data_available:
+            self._on_data_available(chan)
+
+class Proact(ProactiveHandler):
+    """ProactiveHandler that answers the BIP proactive commands with TCP sockets.
+
+    The transport calls the handle_* methods with the decoded proactive command and posts
+    the returned IE list as TERMINAL RESPONSE.
+
+    Args:
+        data_available_sink: callback(envelope_hex: str), called from a channel reader thread
+            with an encoded ENVELOPE EVENT DOWNLOAD (data available). The caller forwards it
+            to the card with the ENVELOPE command, the card then FETCHes RECEIVE DATA.
+            None: the event is only logged (card free / test mode).
+        sms_sink: callback(pdu), called with the SMPP deliver_sm of a SEND SHORT MESSAGE
+            the card issued; pySim-smpp2sim.py hands it to its SMPP server.
+            None: the SMS is logged and dropped.
+        location: Location information returned in PROVIDE LOCAL INFORMATION (location).
+    """
+    def __init__(self, data_available_sink=None, sms_sink=None, location: bytes = DEFAULT_LOCATION):
+        self.data_available_sink = data_available_sink
+        self.sms_sink = sms_sink
+        self.location = location
+        self.channels = ProactChannels(on_data_available=self._on_channel_data_available)
+
+    def handle_ProvideLocalInformation(self, pcmd: ProactiveCommand):
+        """only location
+
+        TS 102 223 6.8.7 says TERMINAL RESPONSE to PROVIDE LOCAL INFORMATION "shall"
+        contain the data object the command qualifier (6.6.15) asked for. At least answer '00',
+        location information, usually requested.
+
+        answering "terminal currently unable to process - no service", which is a handset
+        out of coverage makes SJA5 believe it and postpones the entire session!
+        it registers a location status event, starts a ten minute timer and waits for coverage."""
+        cmd_det_ie = Proact._find_first_element_of_type(pcmd.children, CommandDetails)
+        if cmd_det_ie is not None and cmd_det_ie.decoded['command_qualifier'] == 0x00:
+            return self.prepare_response(pcmd) + [LocationInformation(decoded=self.location)]
+        return self.prepare_response(pcmd)
+
+    def receive_fetch(self, pcmd: ProactiveCommand):
+        """Answer anything this handler has no specific handler for.
+
+        A card coming up will usually issue PROVIDE LOCAL INFORMATION,
+        POLL INTERVAL or TIMER MANAGEMENT before it gets anywhere near a BIP channel,
+        whatever the TERMINAL PROFILE announces.
+
+        Note that this is not the spec-correct answer. TS 102 223 6.8.7
+        says a successful TERMINAL RESPONSE to PROVIDE LOCAL INFORMATION "shall" carry the
+        requested Local information data object, and 6.8.13/6.8.14 says the same for TIMER
+        MANAGEMENT, this returns empty results for all of them, which works with real cards.
+
+        Always "performed_successfully", never "command_beyond_terminal_capability" because
+        answering that to PROVIDE LOCAL INFORMATION makes a card refuse to open the session.
+        """
+        logger.info("no handler for %s, answering performed_successfully",
+                    type(pcmd.decoded).__name__)
+        return self.prepare_response(pcmd, 'performed_successfully')
+
+    @staticmethod
+    def _find_first_element_of_type(instlist, cls):
+        for i in instlist:
+            if isinstance(i, cls):
+                return i
+        return None
+
+    @staticmethod
+    def _channel_nr_from_dev_ids(dev_id_ie: DeviceIdentities) -> int:
+        """Maps id like channel_1 -> channel number.
+        TS 102 223 Section 8.7 says low nibble is channel number,
+        channel-N = 0x21..0x27"""
+        dest = dev_id_ie.decoded['dest_dev_id']
+        return DeviceIdentities.DEV_IDS.inverse[dest] & 0x0f
+
+    def _channel_for(self, dev_id_ie: DeviceIdentities):
+        """Resolve the ProactChannel addressed by a command dev id, or None"""
+        return self.channels.channels.get(self._channel_nr_from_dev_ids(dev_id_ie), None)
+
+    @staticmethod
+    def _channel_status(chan_nr: int, established: bool = True) -> str:
+        """TS 102 223 Section 8.56 channel status value for the
+        default/network bearer:
+        - byte 3 low 3 bits = channel id
+        - bit 8 = link established
+        - byte 4 = 00 no further info"""
+        b3 = (0x80 if established else 0x00) | (chan_nr & 0x07)
+        return '%02x00' % b3
+
+    def _bip_response_head(self, pcmd: ProactiveCommand,
+                           general_result: str = 'performed_successfully',
+                           additional_information: str = ''):
+        """CommandDetails / DeviceIdentities / Result head part of a BIP TERMINAL
+        RESPONSE. Built on prepare_response() but with two changes:
+
+        - Device identities forced source=terminal, dest=UICC.
+          TS 102 223 6.8.2 mandates for every TERMINAL RESPONSE
+          prepare_response() inverts the commands device id, which is
+          right for a uicc->terminal command but would yield a wrong
+          channel_N->UICC for the channel addressed BIP commands.
+
+        - Result is recreated for non success cases. prepare_response()
+          hard codes empty "additional information", but for enum results
+          like BIP error -> AddlInfoBip the empty value cannot be encoded at
+          all, so we always ask prepare_response() for a success Result
+          and swap for a properly encoded one here."""
+        head = self.prepare_response(pcmd, 'performed_successfully')
+        for i, ie in enumerate(head):
+            if isinstance(ie, DeviceIdentities):
+                head[i] = DeviceIdentities(decoded={'source_dev_id': 'terminal',
+                                                    'dest_dev_id': 'uicc'})
+            elif isinstance(ie, Result) and general_result != 'performed_successfully':
+                res = Result()
+                res.from_dict({'result': {'general_result': general_result,
+                                          'additional_information': additional_information}})
+                head[i] = res
+        return head
+
+    def _build_data_available_envelope(self, chan: ProactChannel) -> bytes:
+        """TS 102 223 7.5.10.2 ENVELOPE EVENT DOWNLOAD
+        Event list, Device id terminal->UICC, Channel status,
+        Channel data length (bytes available or FF for > 255)."""
+        avail = min(chan.available_rx(), 0xff)
+        ed = EventDownload(children=[
+            EventList(decoded=['data_available']),
+            DeviceIdentities(decoded={'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}),
+            ChannelStatus(decoded=self._channel_status(chan.chan_nr)),
+            ChannelDataLength(decoded=avail),
+        ])
+        return ed.to_tlv()
+
+    def _on_channel_data_available(self, chan: ProactChannel):
+        """rx reader thread hook: socket data arrived while the channel buffer
+        was empty. card uses ENVELOPE EVENT DOWNLOAD + responds by FETCHing RECEIVE DATA
+        proactive command. Card free only builds and logs"""
+        envelope_hex = b2h(self._build_data_available_envelope(chan))
+        logger.info("channel %u: %u byte(s) available -> ENVELOPE(Data available) %s",
+                    chan.chan_nr, chan.available_rx(), envelope_hex)
+        if self.data_available_sink:
+            self.data_available_sink(envelope_hex)
+
+    # handle_*: called by the transport with the decoded proactive command, the returned IE
+    # list becomes the TERMINAL RESPONSE.
+    def handle_SendShortMessage(self, pcmd: ProactiveCommand):
+        # {'smspp_download': [{'device_identities': {'source_dev_id': 'network',
+        #                                            'dest_dev_id': 'uicc'}},
+        #                     {'address': {'ton_npi': {'ext': True,
+        #                                              'type_of_number': 'international',
+        #                                              'numbering_plan_id': 'isdn_e164'},
+        #                                  'call_number': '79'}},
+        #                     {'sms_tpdu': {'tpdu': '40048111227ff6407070611535004d02700000481516011212000001fe4c0943aea42e45021c078ae06c66afc09303608874b72f58bacadb0dcf665c29349c799fbb522e61709c9baf1890015e8e8e196e36153106c8b92f95153774'}}
+        #                    ]}
+        """SEND SHORT MESSAGE: hand the MO-SMS to sms_sink, answer with success so the card
+        continues with the next part of a multi part response."""
+        logger.info("SendShortMessage")
+        logger.info(pcmd)
+        # Relevant parts in pcmd: Address, SMS_TPDU
+        addr_ie = Proact._find_first_element_of_type(pcmd.children, Address)
+        sms_tpdu_ie = Proact._find_first_element_of_type(pcmd.children, SMS_TPDU)
+        raw_tpdu = sms_tpdu_ie.decoded['tpdu']
+        submit = SMS_SUBMIT.from_bytes(raw_tpdu)
+        submit.tp_da = AddressField(addr_ie.decoded['call_number'], addr_ie.decoded['ton_npi']['type_of_number'],
+                                    addr_ie.decoded['ton_npi']['numbering_plan_id'])
+        logger.info(submit)
+        self.send_sms_via_smpp(submit)
+        # Return a successful TERMINAL RESPONSE.
+        # This is important:
+        # - without it the transport cannot complete the proactive command
+        # - for a multi part OTA response, the card would never be asked to give us
+        # the remaining SMS chunks.
+        # 'pcmd' is a decoded SendShortMessage IE, which contains CommandDetails and
+        # DeviceIdentities that prepare_response() echoes/inverts.
+        return self.prepare_response(pcmd)
+
+    def handle_OpenChannel(self, pcmd: ProactiveCommand):
+        """OPEN CHANNEL: connect a TCP socket to the given address and port, allocate a
+        channel number and report it in the Channel status of the response."""
+        # {'open_channel': [{'command_details': {'command_number': 1,
+        #                                        'type_of_command': 'open_channel',
+        #                                        'command_qualifier': 3}},
+        #                   {'device_identities': {'source_dev_id': 'uicc',
+        #                                          'dest_dev_id': 'terminal'}},
+        #                   {'bearer_description': {'bearer_type': 'default',
+        #                                           'bearer_parameters': ''}},
+        #                   {'buffer_size': 1024},
+        #                   {'uicc_transport_level': {'protocol_type': 'tcp_uicc_client_remote',
+        #                                             'port_number': 32768}},
+        #                   {'other_address': {'type_of_address': 'ipv4',
+        #                                      'address': '01020304'}}
+        #                  ]}
+        logger.info("OpenChannel")
+        logger.info(pcmd)
+        transp_lvl_ie = Proact._find_first_element_of_type(pcmd.children, UiccTransportLevel)
+        other_addr_ie = Proact._find_first_element_of_type(pcmd.children, OtherAddress)
+        bearer_desc_ie = Proact._find_first_element_of_type(pcmd.children, BearerDescription)
+        buffer_size_ie = Proact._find_first_element_of_type(pcmd.children, BufferSize)
+
+        def refuse(additional_information: str, chan_nr: int = 0):
+            """TERMINAL RESPONSE refusing the OPEN CHANNEL
+
+            - always a BIP error, only the cause byte of TS 102 223 8.12.11 differs
+            - chan_nr 0 -> "no channel available" in the Channel status, 8.56
+            - 6.8.18, 6.8.20, 6.8.21 want chan status, Bearer desc and buf size
+              in a successful or unsuccessful response
+            """
+            ies = [ChannelStatus(decoded=self._channel_status(chan_nr, established=False))]
+            ies += [ie for ie in (bearer_desc_ie, buffer_size_ie) if ie is not None]
+            return self._bip_response_head(pcmd, 'bearer_independent_protocol_error',
+                                           additional_information) + ies
+
+        # UICC/terminal interface transport level is Optional, TS 102 223 6.6.27.x. Absent means
+        # the CAT application runs its own network and transport layer, which we do not do.
+        if transp_lvl_ie is None or transp_lvl_ie.decoded['protocol_type'] != 'tcp_uicc_client_remote':
+            logger.warning("OpenChannel: unsupported UICC/terminal interface transport level (%s) "
+                           "-> refusing", transp_lvl_ie.decoded if transp_lvl_ie else '(absent)')
+            return refuse('requested_uicc_if_transp_level_not_available')
+        if other_addr_ie is None or other_addr_ie.decoded.get('type_of_address', None) != 'ipv4':
+            # No cause byte fits a wrong address family. '06' is about the transport level data
+            # object, and 8.12.11 leaves '14' ("IPv4 only allowed") reserved by 3GPP, so '00'.
+            logger.warning("OpenChannel: unsupported data destination address (%s) -> refusing",
+                           other_addr_ie.decoded if other_addr_ie else '(absent)')
+            return refuse('no_specific_cause')
+        addr_bytes = h2b(other_addr_ie.decoded['address']) if isinstance(
+                other_addr_ie.decoded['address'], str) else other_addr_ie.decoded['address']
+        ipv4_str = '%u.%u.%u.%u' % (addr_bytes[0], addr_bytes[1], addr_bytes[2], addr_bytes[3])
+        port_nr = transp_lvl_ie.decoded['port_number']
+        logger.info("OpenChannel: connecting to %s:%u", ipv4_str, port_nr)
+        try:
+            channel = self.channels.channel_create()
+        except ValueError:
+            # TS 102 223 6.4.27.2 and 6.4.27.3: no channel left -> BIP error
+            logger.warning("OpenChannel: all %u channels are in use -> refusing",
+                           len(self.channels.channels))
+            return refuse('no_channel_availabile')
+        # yes, blocking connect()
+        try:
+            channel.connect(ipv4_str, port_nr)
+        except OSError as e:
+            logger.warning("OpenChannel: connect to %s:%u failed: %s", ipv4_str, port_nr, e)
+            self.channels.channel_delete(channel.chan_nr)
+            # TS 102 223 6.4.30 is the only clause naming a cause for a link that could not be
+            # established: BIP error, channel closed. 6.4.27.4 lists no error cases at all.
+            return refuse('channel_closed', channel.chan_nr)
+
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'open_channel',
+        #                       'command_qualifier': 3}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_status': '8100'},
+        #  {'bearer_description': {'bearer_type': 'default', 'bearer_parameters': ''}},
+        #  {'buffer_size': 1024}
+        # ]
+        return self._bip_response_head(pcmd) + [
+                ChannelStatus(decoded=self._channel_status(channel.chan_nr)),
+                bearer_desc_ie, buffer_size_ie]
+
+    def handle_CloseChannel(self, pcmd: ProactiveCommand):
+        """CLOSE CHANNEL: close the socket of the addressed channel and free its number."""
+        logger.info("CloseChannel")
+        logger.info(pcmd)
+        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
+        chan = self._channel_for(dev_id_ie)
+        if chan is None:
+            # channel closed / invalid
+            return self._bip_response_head(pcmd, 'bearer_independent_protocol_error',
+                                           'channel_id_not_valid')
+        chan.close()
+        return self._bip_response_head(pcmd)
+
+    def handle_ReceiveData(self, pcmd: ProactiveCommand):
+        """RECEIVE DATA: the card fetches up to Channel data length bytes from the Rx buffer
+        of the addressed channel, the response also carries how many bytes remain."""
+        # {'receive_data': [{'command_details': {'command_number': 1,
+        #                                        'type_of_command': 'receive_data',
+        #                                        'command_qualifier': 0}},
+        #                   {'device_identities': {'source_dev_id': 'uicc',
+        #                                          'dest_dev_id': 'channel_1'}},
+        #                   {'channel_data_length': 9}
+        #                  ]}
+        logger.info("ReceiveData")
+        logger.info(pcmd)
+        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
+        req_len_ie = Proact._find_first_element_of_type(pcmd.children, ChannelDataLength)
+        chan = self._channel_for(dev_id_ie)
+        if chan is None:
+            return self._bip_response_head(pcmd, 'bearer_independent_protocol_error',
+                                           'channel_id_not_valid')
+        # TS 102 223 8.54: RECEIVE DATA contains the requested count the card wants
+        requested = req_len_ie.decoded if req_len_ie is not None else chan.available_rx()
+        data, remaining = chan.take_rx(requested)
+        # TS 102 223 6.4.29:
+        # - return data available in the Rx buffer + num bytes still remaining (FF if > 255)
+        # - if fewer than requested available terminal must NOT wait, report and returns what we have
+        general_result = 'performed_successfully'
+        if len(data) < requested:
+            general_result = 'performed_with_missing_information'
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'receive_data',
+        #                       'command_qualifier': 0}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_data': '16030100040e000000'},
+        #  {'channel_data_length': 0}
+        # ]
+        return self._bip_response_head(pcmd, general_result) + [
+                ChannelData(decoded=b2h(data)),
+                ChannelDataLength(decoded=min(remaining, 0xff))]
+
+    def handle_SendData(self, pcmd: ProactiveCommand):
+        """SEND DATA: write the Channel data of the command to the socket of the addressed
+        channel."""
+        # {'send_data': [{'command_details': {'command_number': 1,
+        #                                     'type_of_command': 'send_data',
+        #                                     'command_qualifier': 1}},
+        #                {'device_identities': {'source_dev_id': 'uicc',
+        #                                       'dest_dev_id': 'channel_1'}},
+        #                {'channel_data': '160301003c010000380303d0f45e12b52ce5bb522750dd037738195334c87a46a847fe2b6886cada9ea6bf00000a00ae008c008b00b0002c010000050001000101'}
+        #               ]}
+        logger.info("SendData")
+        logger.info(pcmd)
+        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
+        chan_data_ie = Proact._find_first_element_of_type(pcmd.children, ChannelData)
+        chan = self._channel_for(dev_id_ie)
+        if chan is None:
+            return self._bip_response_head(pcmd, 'bearer_independent_protocol_error',
+                                           'channel_id_not_valid')
+        # lets accept hexstrings as well
+        payload = chan_data_ie.decoded
+        if isinstance(payload, str):
+            payload = h2b(payload)
+        # command_qualifier bit 1 selects 'send immediately' / Tx-buffer store and forward
+        # For TCP stream all we have is a socket and TCP takes care of segmentation,
+        # so just send.
+        chan.send(payload)
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'send_data',
+        #                       'command_qualifier': 1}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_data_length': 255}
+        # ]
+        # TS 102 223 6.4.30 / 8.54 Channel data length = free space tx buf; FF == > 255 available
+        return self._bip_response_head(pcmd) + [ChannelDataLength(decoded=255)]
+
+    def handle_SetUpEventList(self, pcmd: ProactiveCommand):
+        """SET UP EVENT LIST: acknowledged, data available and channel status are always on."""
+        # {'set_up_event_list': [{'command_details': {'command_number': 1,
+        #                                             'type_of_command': 'set_up_event_list',
+        #                                             'command_qualifier': 0}},
+        #                        {'device_identities': {'source_dev_id': 'uicc',
+        #                                               'dest_dev_id': 'terminal'}},
+        #                        {'event_list': ['data_available', 'channel_status']}
+        #                       ]}
+        logger.info("SetUpEventList")
+        logger.info(pcmd)
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'set_up_event_list',
+        #                       'command_qualifier': 0}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}}
+        # ]
+        return self.prepare_response(pcmd)
+
+    def getChannelStatus(self, pcmd: ProactiveCommand):
+        logger.info("GetChannelStatus")
+        logger.info(pcmd)
+        return self.prepare_response(pcmd) + []
+
+    def send_sms_via_smpp(self, submit: SMS_SUBMIT):
+        # while in a normal network the phone/ME would *submit* a message to the SMSC,
+        # we are actually emulating the SMSC itself, so we must *deliver* the message
+        # to the ESME
+        deliver = SMS_DELIVER.from_submit(submit)
+        deliver_smpp = deliver.to_smpp()
+
+        if self.sms_sink is None:
+            logger.info('no sms_sink: dropping MO-SMS %s', deliver_smpp)
+            return
+        self.sms_sink(deliver_smpp)
+#       # obtain the connection/binding of system_id to be used for delivering MO-SMS to the ESME
+#       connection = smpp_server.getBoundConnections[system_id].getNextBindingForDelivery()
+#       connection.sendDataRequest(deliver_smpp)
+
+
+

pySim/card_key_provider.py

@@ -33,10 +33,12 @@ from Cryptodome.Cipher import AES
 from osmocom.utils import h2b, b2h
 from pySim.log import PySimLogger
 
+import os
 import abc
 import csv
 import logging
 import yaml
+import argparse
 
 log = PySimLogger.get(__name__)
 
@@ -130,6 +132,31 @@ class CardKeyFieldCryptor:
         cipher = AES.new(h2b(self.transport_keys[field_name.upper()]), AES.MODE_CBC, self.__IV)
         return b2h(cipher.encrypt(h2b(plaintext_val)))
 
+    @staticmethod
+    def argparse_add_args(arg_parser: argparse.ArgumentParser):
+        arg_parser.add_argument('--column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                                help='per-column AES transport key', dest='column_key')
+        # Depprecated argument, replaced by --column-key (see above)
+        arg_parser.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                                help=argparse.SUPPRESS, dest='column_key')
+
+    @staticmethod
+    def transport_keys_from_opts(opts: argparse.Namespace) -> dict:
+        """
+        Transport keys are passed via the commandline using the '--column-key' option. Each column requires a
+        dedicated transport key. This method can be used to extract the column keys parameters from the commandline
+        options into a dict that can be directly passed to the construtor with the transport_keys argument.
+
+        Args:
+                opts: parsed commandline options (Namespace)
+        """
+
+        transport_keys = {}
+        for par in opts.column_key:
+            name, key = par.split(':')
+            transport_keys[name] = key
+        return transport_keys
+
 class CardKeyProvider(abc.ABC):
     """Base class, not containing any concrete implementation."""
 
@@ -148,24 +175,33 @@ class CardKeyProvider(abc.ABC):
                 fond None shall be returned.
         """
 
+    @staticmethod
+    def argparse_add_args(arg_parser: argparse.ArgumentParser):
+        """
+        Add the commandline arguments relevant for this card key provider.
+
+        Args:
+                arg_parser : argument parser group
+        """
+
     def __str__(self):
         return type(self).__name__
 
 class CardKeyProviderCsv(CardKeyProvider):
     """Card key provider implementation that allows to query against a specified CSV file."""
 
-    def __init__(self, csv_filename: str, transport_keys: dict):
+    def __init__(self, csv_filename: str, field_cryptor: CardKeyFieldCryptor):
         """
         Args:
                 csv_filename : file name (path) of CSV file containing card-individual key/data
-                transport_keys : (see class CardKeyFieldCryptor)
+                field_cryptor : (see class CardKeyFieldCryptor)
         """
         log.info("Using CSV file as card key data source: %s" % csv_filename)
         self.csv_file = open(csv_filename, 'r')
         if not self.csv_file:
             raise RuntimeError("Could not open CSV file '%s'" % csv_filename)
         self.csv_filename = csv_filename
-        self.crypt = CardKeyFieldCryptor(transport_keys)
+        self.crypt = field_cryptor
 
     def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         self.csv_file.seek(0)
@@ -188,14 +224,20 @@ class CardKeyProviderCsv(CardKeyProvider):
             return None
         return return_dict
 
+    @staticmethod
+    def argparse_add_args(arg_parser: argparse.ArgumentParser):
+        arg_parser.add_argument('--csv', metavar='FILE',
+                                default="~/.osmocom/pysim/card_data.csv",
+                                help='Read card data from CSV file')
+
 class CardKeyProviderPgsql(CardKeyProvider):
     """Card key provider implementation that allows to query against a specified PostgreSQL database table."""
 
-    def __init__(self, config_filename: str, transport_keys: dict):
+    def __init__(self, config_filename: str, field_cryptor: CardKeyFieldCryptor):
         """
         Args:
                 config_filename : file name (path) of CSV file containing card-individual key/data
-                transport_keys : (see class CardKeyFieldCryptor)
+                field_cryptor : (see class CardKeyFieldCryptor)
         """
         import psycopg2
         log.info("Using SQL database as card key data source: %s" % config_filename)
@@ -212,7 +254,7 @@ class CardKeyProviderPgsql(CardKeyProvider):
                                          host=config.get('host'))
             self.tables = config.get('table_names')
             log.info("Card key database tables: %s" % str(self.tables))
-            self.crypt = CardKeyFieldCryptor(transport_keys)
+            self.crypt = field_cryptor
 
     def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         import psycopg2
@@ -252,6 +294,11 @@ class CardKeyProviderPgsql(CardKeyProvider):
             result[k] = self.crypt.decrypt_field(k, result.get(k))
         return result
 
+    @staticmethod
+    def argparse_add_args(arg_parser: argparse.ArgumentParser):
+        arg_parser.add_argument('--pgsql', metavar='FILE',
+                                default="~/.osmocom/pysim/card_data_pgsql.cfg",
+                                help='Read card data from PostgreSQL database (config file)')
 
 def card_key_provider_register(provider: CardKeyProvider, provider_list=card_key_providers):
     """Register a new card key provider.
@@ -305,3 +352,19 @@ def card_key_provider_get_field(field: str, key: str, value: str, provider_list=
     fields = [field]
     result = card_key_provider_get(fields, key, value, card_key_providers)
     return result.get(field.upper())
+
+def card_key_provider_argparse_add_args(arg_parser: argparse.ArgumentParser):
+    """Add card key provider commandline options to the given argument parser"""
+    card_key_group = arg_parser.add_argument_group('Card Key Provider Options')
+    CardKeyProviderCsv.argparse_add_args(card_key_group)
+    CardKeyProviderPgsql.argparse_add_args(card_key_group)
+    CardKeyFieldCryptor.argparse_add_args(card_key_group)
+
+def card_key_provider_init(opts: argparse.Namespace):
+    """Initialize card key provider depending on the user provided commandline options"""
+    transport_keys = CardKeyFieldCryptor.transport_keys_from_opts(opts)
+    card_key_field_cryptor = CardKeyFieldCryptor(transport_keys)
+    if os.path.isfile(os.path.expanduser(opts.csv)):
+        card_key_provider_register(CardKeyProviderCsv(os.path.expanduser(opts.csv), card_key_field_cryptor))
+    if os.path.isfile(os.path.expanduser(opts.pgsql)):
+        card_key_provider_register(CardKeyProviderPgsql(os.path.expanduser(opts.pgsql), card_key_field_cryptor))

pySim/cat.py

@@ -22,7 +22,7 @@ from typing import List
 from bidict import bidict
 from construct import Int8ub, Int16ub, Byte, BitsInteger
 from construct import Struct, Enum, BitStruct, this
-from construct import Switch, GreedyRange, FlagsEnum
+from construct import Switch, GreedyRange, FlagsEnum, Adapter
 from osmocom.tlv import TLV_IE, COMPR_TLV_IE, BER_TLV_IE, TLV_IE_Collection
 from osmocom.construct import PlmnAdapter, BcdAdapter, GsmStringAdapter, TonNpi, GsmString, Bytes, GreedyBytes
 from osmocom.utils import b2h, h2b
@@ -318,11 +318,58 @@ class FileList(COMPR_TLV_IE, tag=0x92):
 
 # TS 102 223 Section 8.19
 class LocationInformation(COMPR_TLV_IE, tag=0x93):
-    pass
+    # 8.19: coding is per access technology, and the lengths differ (TS 131.111 8.19.1-.4: GERAN 7,
+    # UTRAN/E-UTRAN 9, NG-RAN 11) with nothing in the IE to say which -> keep the value opaque.
+    _construct = GreedyBytes
 
-# TS 102 223 Section 8.20
+class MobileIdentityAdapter(Adapter):
+    """TS 124.008 section 10.5.1.4 figure 10.5.4 + table 10.5.4
+
+    NOT a plain BCD string:
+    - bits 1-3 type of identity + odd/even bit 4
+    - digit 1 in bits 5-8, following octets contain 2 digits, low nibble first
+    - if even length: high nibble of last octet 1111
+    So IMEI IE of 8 bytes is 15 digits + framing nibble."""
+
+    # Table 10.5.4 bits 321
+    TYPE_IMSI = 1
+    TYPE_IMEI = 2
+    TYPE_IMEISV = 3
+
+    def __init__(self, subcon, type_of_identity: int):
+        super().__init__(subcon)
+        self.type_of_identity = type_of_identity
+
+    def _decode(self, obj, context, path):
+        data = bytes(obj)
+        if not data:
+            return ''
+        # TS 24.008 figure 10.5.4: octet 3 holds type of identity (b1-3), odd/even (b4) and
+        # digit 1 in its high nibble, the remaining digits follow BCD swapped from octet 4
+        odd = bool(data[0] & 0x08)          # bit 4: 1 = odd number of digits
+        digits = '%x' % (data[0] >> 4)      # bits 5-8: digit 1
+        for octet in data[1:]:
+            digits += '%x%x' % (octet & 0x0f, octet >> 4)
+        if not odd:
+            digits = digits[:-1]            # drop the 1111 end mark
+        return digits
+
+    def _encode(self, obj, context, path):
+        digits = str(obj)
+        odd = len(digits) % 2
+        first = (int(digits[0], 16) << 4) | (0x08 if odd else 0x00) | self.type_of_identity
+        rest = digits[1:] if odd else digits[1:] + 'f'
+        return bytes([first]) + bytes((int(rest[i+1], 16) << 4) | int(rest[i], 16)
+                                      for i in range(0, len(rest), 2))
+
+# TS 102 223 Section 8.20, len is fixed at 8: "The IMEI is coded [..] as the
+# value part of the Mobile Identity IE as specified in TS 124 008", and the
+# IMEI itself is the 15 digits of TS 123 003.
 class IMEI(COMPR_TLV_IE, tag=0x94):
-    _construct = BcdAdapter(GreedyBytes)
+    _test_de_encode = [
+        ( '94081a32547698103254', '123456789012345' ),
+    ]
+    _construct = MobileIdentityAdapter(GreedyBytes, MobileIdentityAdapter.TYPE_IMEI)
 
 # TS 102 223 Section 8.21
 class HelpRequest(COMPR_TLV_IE, tag=0x95):
@@ -536,9 +583,9 @@ class Aid(COMPR_TLV_IE, tag=0xAF):
 
 # TS 102 223 Section 8.61
 class AccessTechnology(COMPR_TLV_IE, tag=0xBF):
-    SingleAccessTech = Enum(Int8ub, gsm=0, tia_eia_533=1, tia_eia_136_270=2, utran=3, tetra=4,
-                                    tia_eia_95_b=5, cdma1000_1x=6, cdma2000_hrpd=7, eutran=8,
-                                    ehrpd=9, nr=0x0a)
+    SingleAccessTech = Enum(Int8ub, gsm=0, tia_eia_553=1, tia_eia_136_270=2, utran=3, tetra=4,
+                                    tia_eia_95_b=5, cdma2000_1x=6, cdma2000_hrpd=7, eutran=8,
+                                    ehrpd=9, nr=0x0a, satellite_nr=0x0b, satellite_eutran=0x0c)
     _construct = GreedyRange(SingleAccessTech)
 
 # TS 102 223 Section 8.63
@@ -596,6 +643,14 @@ class UtranEutranMeasurementQualifier(COMPR_TLV_IE, tag=0xE9):
                                 eutran_inter_rat_utran=0x08,
                                 eutran_inter_rat_nr=0x09)
 
+# TS 102 223 Section 8.74, length is not fixed, because IMEISV is 16 digits per TS 123.003
+# -> even count needs the '1111' end mark and is 9 bytes long
+class IMEISV(COMPR_TLV_IE, tag=0xE2):
+    _test_de_encode = [
+        ( 'e2091332547698103254f6', '1234567890123456' ),
+    ]
+    _construct = MobileIdentityAdapter(GreedyBytes, MobileIdentityAdapter.TYPE_IMEISV)
+
 # TS 102 223 Section 8.75
 class NetworkSearchMode(COMPR_TLV_IE, tag=0xE5):
     _construct = Enum(Int8ub, manual=0, automatic=1)
@@ -729,8 +784,12 @@ class DnsServerAddress(COMPR_TLV_IE, tag=0xC0):
 
 # TS 102 223 Section 8.105
 class SupportedRadioAccessTechnologies(COMPR_TLV_IE, tag=0xB4):
+    # 2 bytes/entry:
+    # - technology of 8.61
+    # - state byte b1 is 0 disabled/1 enabled
+    # - b2-b8 RFU.
     AccessTechTuple = Struct('technology'/AccessTechnology.SingleAccessTech,
-                             'state'/FlagsEnum(Int8ub, enabled=0))
+                             'state'/FlagsEnum(Int8ub, enabled=1))
     _construct = GreedyRange(AccessTechTuple)
 
 # TS 102 223 Section 8.107
@@ -763,6 +822,22 @@ class SMSPPDownload(BER_TLV_IE, tag=0xD1,
                     nested=[DeviceIdentities, Address, SMS_TPDU]):
     pass
 
+
+def sms_pp_download_envelope(tpdu, call_number: str = '0123456') -> SMSPPDownload:
+    """TS 31.111 Section 7.1.1.2 wrap of a SMS-DELIVER TPDU in the ENVELOPE (SMS-PP Download)
+     call_number :
+     SMSC address to report, defined in TS 31.111 7.1.1.2 as
+    "the RP_Originating_Address of the Service Centre (TS-Service-Centre-Address, 3GPP TS 24.011)"
+    its presence is Conditional, and the note there says the UICC should be fine
+    if its missing, so for remote management its presence should suffice (?).
+    """
+    return SMSPPDownload(children=[
+        DeviceIdentities(decoded={'source_dev_id': 'network', 'dest_dev_id': 'uicc'}),
+        Address(decoded={'ton_npi': {'ext': False, 'type_of_number': 'unknown',
+                                     'numbering_plan_id': 'unknown'},
+                         'call_number': call_number}),
+        SMS_TPDU(decoded={'tpdu': b2h(tpdu.to_bytes())})])
+
 # TS 101 220 Table 7.17 + 31.111 7.1.1.3
 class SMSCBDownload(BER_TLV_IE, tag=0xD2,
                     nested=[DeviceIdentities, CBSPage]):

pySim/esim/saip/__init__.py

@@ -34,7 +34,7 @@ from pySim import ts_102_222
 from pySim.utils import dec_imsi
 from pySim.ts_102_221 import FileDescriptor
 from pySim.filesystem import CardADF, Path
-from pySim.ts_31_102 import ADF_USIM
+from pySim.ts_31_102 import ADF_USIM, EF_UST, EF_SUCI_Calc_Info
 from pySim.ts_31_103 import ADF_ISIM
 from pySim.esim import compile_asn1_subdir
 from pySim.esim.saip import templates
@@ -1079,6 +1079,13 @@ class SecurityDomainKey:
                 'keyVersionNumber': bytes([self.key_version_number]),
                 'keyComponents': [k.to_saip_dict() for k in self.key_components]}
 
+    def get_key_component(self, key_type):
+        for kc in self.key_components:
+            if kc.key_type == key_type:
+                return kc.key_data
+        return None
+
+
 class ProfileElementSD(ProfileElement):
     """Class representing a securityDomain ProfileElement."""
     type = 'securityDomain'
@@ -1719,7 +1726,52 @@ class ProfileElementSequence:
         if 'BT' in ftype_list:
             svc_set.add('ber-tlv')
         # FIXME:dfLinked files (scan all files, check for non-empty Fcp.linkPath presence of DFs)
-        # TODO: 5G related bits (derive from EF.UST or file presence?)
+
+        # 5G:
+        # - When SUCI is:
+        #   - enabled (EF.UST 124 = true)
+        #   AND
+        #   - calculated in the USIM (EF.UST 125 = true),
+        #   then eUICC-Mandatory-services needs 'get-identity'.
+        # - 'get-identity' implies that the eUICC must support ONE OF profile-A OR profile-B.
+        #   (One might assume from this that, when SUCI-CalcInfo for USIM in DF.SAIP contains both key types, then no
+        #   profile-A or B services need to be requested explicitly. However, the correct logic is:)
+        # - Iff the SUCI-CalcInfo for USIM (DF.SAIP) contains a key of profile-A ("identifier": 1),
+        #   then eUICC-Mandatory-services needs 'profile-a-x25519'.
+        # - Same: profile-B ("identifier": 2) needs 'profile-b-p256'.
+        # - (When SUCI is calculated in the UE, then the eUICC does not need to provide any of these services.)
+        suci_in_usim_enabled = False
+        try:
+            f_ust = self.get_pe_for_type("usim").files["ef-ust"]
+            ust = EF_UST().decode_bin(f_ust.body)
+            suci_in_usim_enabled = ust[124]['activated'] and ust[125]['activated']
+        except (KeyError, AttributeError):
+            pass
+        if suci_in_usim_enabled:
+            svc_set.add('get-identity')
+            # now check for profile-a and profile-b presence
+            suci_calcinfo_has_profile_a = False
+            suci_calcinfo_has_profile_b = False
+            try:
+                f_sucici = self.get_pe_for_type("df-saip").files["ef-suci-calc-info-usim"]
+                sucici = EF_SUCI_Calc_Info().decode_bin(f_sucici.body) or {}
+                for prot_scheme in sucici['prot_scheme_id_list']:
+                    if not isinstance(prot_scheme, dict):
+                        continue
+                    ps_id = prot_scheme["identifier"]
+                    if ps_id == 1:
+                        suci_calcinfo_has_profile_a = True
+                    elif ps_id == 2:
+                        suci_calcinfo_has_profile_b = True
+            except (KeyError, AttributeError):
+                pass
+            if suci_calcinfo_has_profile_a:
+                # The profile has a profile-A key, so require that
+                svc_set.add('profile-a-x25519')
+            if suci_calcinfo_has_profile_b:
+                # The profile has a profile-B key, so require that
+                svc_set.add('profile-b-p256')
+
         hdr_pe = self.get_pe_for_type('header')
         # patch in the 'manual' services from the existing list:
         for old_svc in hdr_pe.decoded['eUICC-Mandatory-services'].keys():

pySim/esim/saip/batch.py

@@ -0,0 +1,362 @@
+"""Implementation of Personalization of eSIM profiles in SimAlliance/TCA Interoperable Profile:
+   Run a batch of N personalizations"""
+
+# (C) 2025-2026 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: nhofmeyr@sysmocom.de
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import copy
+import pprint
+from typing import Generator, Union
+from pySim.esim.saip.personalization import ConfigurableParameter
+from pySim.esim.saip import param_source
+from pySim.esim.saip import ProfileElementSequence, ProfileElementSD
+from pySim.global_platform import KeyUsageQualifier
+from osmocom.utils import b2h
+
+# a list of ConfigurableParameter classes and/or ConfigurableParameter class instances
+ParamList = list[Union[type[ConfigurableParameter], ConfigurableParameter]]
+
+class BatchPersonalization:
+    """Produce a series of eSIM profiles from predefined parameters.
+    Personalization parameters are derived from pysim.esim.saip.param_source.ParamSource.
+
+    Usage example:
+
+    der_input = open('some_file', 'rb').read()
+    pes = ProfileElementSequence.from_der(der_input)
+    p = BatchPersonalization(
+        n=10,
+        src_pes=pes,
+        csv_rows=get_csv_reader())
+
+    p.add_param_and_src(
+        personalization.Iccid(),
+        param_source.IncDigitSource(
+            num_digits=18,
+            first_value=123456789012340001,
+            last_value=123456789012340010))
+
+    # add more parameters here, using ConfigurableParameter and ParamSource subclass instances to define the profile
+    # ...
+
+    # generate all 10 profiles (from n=10 above)
+    for result_pes in p.generate_profiles():
+        upp = result_pes.to_der()
+        store_upp(upp)
+    """
+
+    class ParamAndSrc:
+        """tie a ConfigurableParameter to a source of actual values"""
+        def __init__(self, param: ConfigurableParameter, src: param_source.ParamSource):
+            if isinstance(param, type):
+                self.param_cls = param
+            else:
+                self.param_cls = param.__class__
+            self.src = src
+
+    def __init__(self,
+                 n: int,
+                 src_pes: ProfileElementSequence,
+                 params: list[ParamAndSrc]=None,
+                 csv_rows: Generator=None,
+                ):
+        """
+        n: number of eSIM profiles to generate.
+        src_pes: a decoded eSIM profile as ProfileElementSequence, to serve as template. This is not modified, only
+                 copied.
+        params: list of ParamAndSrc instances, defining a ConfigurableParameter and corresponding ParamSource to fill in
+                profile values.
+        csv_rows: A generator (e.g. iter(list_of_rows)) producing all CSV rows one at a time, starting with a row
+                  containing the column headers. This is compatible with the python csv.reader. Each row gets passed to
+                  ParamSource.get_next(), such that ParamSource implementations can access the row items. See
+                  param_source.CsvSource.
+        """
+        self.n = n
+        self.params = params or []
+        self.src_pes = src_pes
+        self.csv_rows = csv_rows
+
+    def add_param_and_src(self, param:ConfigurableParameter, src:param_source.ParamSource):
+        self.params.append(BatchPersonalization.ParamAndSrc(param, src))
+
+    def generate_profiles(self):
+        # get first row of CSV: column names
+        csv_columns = None
+        if self.csv_rows:
+            try:
+                csv_columns = next(self.csv_rows)
+            except StopIteration as e:
+                raise ValueError('the input CSV file appears to be empty') from e
+
+        for i in range(self.n):
+            csv_row = None
+            if self.csv_rows and csv_columns:
+                try:
+                    csv_row_list = next(self.csv_rows)
+                except StopIteration as e:
+                    raise ValueError(f'not enough rows in the input CSV for eSIM nr {i+1} of {self.n}') from e
+
+                csv_row = dict(zip(csv_columns, csv_row_list))
+
+            pes = copy.deepcopy(self.src_pes)
+
+            for p in self.params:
+                try:
+                    input_value = p.src.get_next(csv_row=csv_row)
+                    assert input_value is not None
+                    value = p.param_cls.validate_val(input_value)
+                    p.param_cls.apply_val(pes, value)
+                except Exception as e:
+                    raise ValueError(f'{p.param_cls.get_name()} fed by {p.src.name}: {e}') from e
+
+            pes.rebuild_mandatory_services()
+
+            yield pes
+
+
+class UppAudit(dict):
+    """
+    Key-value pairs collected from a single UPP DER or PES.
+
+    UppAudit itself is a dict, callers may use the standard python dict API to access key-value pairs read from the UPP.
+    """
+
+    @classmethod
+    def from_der(cls, der: bytes, params: ParamList, der_size=False, additional_sd_keys=False):
+        """return a dict of parameter name and set of selected parameter values found in a DER encoded profile. Note:
+        some ConfigurableParameter implementations return more than one key-value pair, for example, Imsi returns
+        both 'IMSI' and 'IMSI-ACC' parameters.
+
+        e.g.
+            UppAudit.from_der(my_der, [Imsi, ])
+            --> {'IMSI': {'001010000000023'}, 'IMSI-ACC': {'5'}}
+
+        (where 'IMSI' == Imsi.name)
+
+        Read all parameters listed in params. params is a list of either ConfigurableParameter classes or
+        ConfigurableParameter class instances. This calls only classmethods, so each entry in params can either be the
+        class itself, or a class-instance of, a (non-abstract) ConfigurableParameter subclass.
+        For example, params = [Imsi, ] is equivalent to params = [Imsi(), ].
+
+        For der_size=True, also include a {'der_size':12345} entry.
+
+        For additional_sd_keys=True, output also all Security Domain KVN that there are *no* ConfigurableParameter
+        subclasses for. For example, SCP80 has reserved kvn 0x01..0x0f, but we offer only Scp80Kvn01, Scp80Kvn02,
+        Scp80Kvn03. So we would not show kvn 0x04..0x0f in an audit. additional_sd_keys=True includes audits of all SD
+        key KVN there may be in the UPP. This helps to spot SD keys that may already be present in a UPP template, with
+        unexpected / unusual kvn.
+        """
+
+        # make an instance of this class
+        upp_audit = cls()
+
+        if der_size:
+            upp_audit['der_size'] = set((len(der), ))
+
+        pes = ProfileElementSequence.from_der(der)
+        for param in params:
+            try:
+                for valdict in param.get_values_from_pes(pes):
+                    upp_audit.add_values(valdict)
+            except Exception as e:
+                raise ValueError(f'Error during audit for parameter {param}: {e}') from e
+
+        if not additional_sd_keys:
+            return upp_audit
+
+        # additional_sd_keys
+        for pe in pes.pe_list:
+            if pe.type != 'securityDomain':
+                continue
+            assert isinstance(pe, ProfileElementSD)
+
+            for key in pe.keys:
+                audit_key = f'SdKey_KVN{key.key_version_number:02x}_ID{key.key_identifier:02x}'
+                kuq_bin = KeyUsageQualifier.build(key.key_usage_qualifier).hex()
+                audit_val = f'{key.key_components=!r} key_usage_qualifier=0x{kuq_bin}={key.key_usage_qualifier!r}'
+                upp_audit.add_values({audit_key: audit_val})
+
+        return upp_audit
+
+    def get_single_val(self, key, allow_absent=False, absent_val=None):
+        """
+        Return the audit's value for the given audit key (like 'IMSI' or 'IMSI-ACC').
+        Any kind of value may occur multiple times in a profile. When all of these agree to the same unambiguous value,
+        return that value. When they do not agree, raise a ValueError.
+        """
+        # key should be a string, but if someone passes a ConfigurableParameter, just use its default name
+        if ConfigurableParameter.is_super_of(key):
+            key = key.get_name()
+
+        assert isinstance(key, str)
+        v = self.get(key)
+        if v is None and allow_absent:
+            return absent_val
+        if not isinstance(v, set):
+            raise ValueError(f'audit value should be a set(), got {v!r}')
+        if len(v) != 1:
+            raise ValueError(f'expected a single value for {key}, got {v!r}')
+        v = tuple(v)[0]
+        return v
+
+    @staticmethod
+    def audit_val_to_str(v):
+        """
+        Usually, we want to see a single value in an audit. Still, to be able to collect multiple ambiguous values,
+        audit values are always python sets. Turn it into a nice string representation: only the value when it is
+        unambiguous, otherwise a list of the ambiguous values.
+        A value may also be completely absent, then return 'not present'.
+        """
+        def try_single_val(w):
+            'change single-entry sets to just the single value'
+            if isinstance(w, set):
+                if len(w) == 1:
+                    return tuple(w)[0]
+                if len(w) == 0:
+                    return None
+            return w
+
+        v = try_single_val(v)
+        if isinstance(v, bytes):
+            v = b2h(v)
+        if v is None:
+            return 'not present'
+        return str(v)
+
+    def get_val_str(self, key):
+        """Return a string of the value stored for the given key"""
+        return UppAudit.audit_val_to_str(self.get(key))
+
+    def add_values(self, src:dict):
+        """Merge a plain dict of values into self, which is a dict of sets.
+        For example from
+            self == { 'a': {123} }
+        and
+            src == { 'a': 456, 'b': 789 }
+        then after this function call:
+            self == { 'a': {123, 456}, 'b': {789} }
+        """
+        assert isinstance(src, dict)
+        for key, srcval in src.items():
+            dstvalset = self.get(key)
+            if dstvalset is None:
+                dstvalset = set()
+                self[key] = dstvalset
+            dstvalset.add(srcval)
+
+    def __str__(self):
+        return '\n'.join(f'{key}: {self.get_val_str(key)}' for key in sorted(self.keys()))
+
+class BatchAudit(list):
+    """
+    Collect UppAudit instances for a batch of UPP, for example from a personalization.BatchPersonalization.
+    Produce an output CSV.
+
+    Usage example:
+
+        ba = BatchAudit(params=(personalization.Iccid, ))
+        for upp_der in upps:
+            ba.add_audit(upp_der)
+        print(ba.summarize())
+
+        with open('output.csv', 'wb') as csv_data:
+            csv_str = io.TextIOWrapper(csv_data, 'utf-8', newline='')
+            csv.writer(csv_str).writerows( ba.to_csv_rows() )
+            csv_str.flush()
+
+    BatchAudit itself is a list, callers may use the standard python list API to access the UppAudit instances.
+    """
+
+    def __init__(self, params: ParamList):
+        assert params
+        self.params = params
+
+    def add_audit(self, upp_der:bytes):
+        audit = UppAudit.from_der(upp_der, self.params)
+        self.append(audit)
+        return audit
+
+    def summarize(self):
+        batch_audit = UppAudit()
+
+        audits = self
+
+        if len(audits) > 2:
+            val_sep = ', ..., '
+        else:
+            val_sep = ', '
+
+        first_audit = None
+        last_audit = None
+        if len(audits) >= 1:
+            first_audit = audits[0]
+        if len(audits) >= 2:
+            last_audit = audits[-1]
+
+        if first_audit:
+            if last_audit:
+                for key in first_audit.keys():
+                    first_val = first_audit.get_val_str(key)
+                    last_val = last_audit.get_val_str(key)
+
+                    if first_val == last_val:
+                        val = first_val
+                    else:
+                        val_sep_with_newline = f"{val_sep.rstrip()}\n{' ' * (len(key) + 2)}"
+                        val = val_sep_with_newline.join((first_val, last_val))
+                    batch_audit[key] = val
+            else:
+                batch_audit.update(first_audit)
+
+        return batch_audit
+
+    def to_csv_rows(self, headers=True, sort_key=None):
+        """generator that yields all audits' values as rows, useful feed to a csv.writer."""
+        columns = set()
+        for audit in self:
+            columns.update(audit.keys())
+
+        columns = tuple(sorted(columns, key=sort_key))
+
+        if headers:
+            yield columns
+
+        for audit in self:
+            yield (audit.get_single_val(col, allow_absent=True, absent_val="") for col in columns)
+
+def esim_profile_introspect(upp):
+    pes = ProfileElementSequence.from_der(upp.read())
+    d = {}
+    d['upp'] = repr(pes)
+
+    def show_bytes_as_hexdump(item):
+        if isinstance(item, bytes):
+            return b2h(item)
+        if isinstance(item, list):
+            return list(show_bytes_as_hexdump(i) for i in item)
+        if isinstance(item, tuple):
+            return tuple(show_bytes_as_hexdump(i) for i in item)
+        if isinstance(item, dict):
+            d = {}
+            for k, v in item.items():
+                d[k] = show_bytes_as_hexdump(v)
+            return d
+        return item
+
+    l = list((pe.type, show_bytes_as_hexdump(pe.decoded)) for pe in pes)
+    d['pp'] = pprint.pformat(l, width=120)
+    return d

pySim/esim/saip/param_source.py

@@ -0,0 +1,221 @@
+# Implementation of SimAlliance/TCA Interoperable Profile handling: parameter sources for batch personalization.
+#
+# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: nhofmeyr@sysmocom.de
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import secrets
+import re
+from osmocom.utils import b2h
+
+class ParamSourceExn(Exception):
+    pass
+
+class ParamSourceExhaustedExn(ParamSourceExn):
+    pass
+
+class ParamSourceUndefinedExn(ParamSourceExn):
+    pass
+
+class ParamSource:
+    """abstract parameter source. For usage, see personalization.BatchPersonalization."""
+
+    # This name should be short but descriptive, useful for a user interface, like 'random decimal digits'.
+    name = "none"
+    numeric_base = None # or 10 or 16
+
+    def __init__(self, input_str:str):
+        """Subclasses should call super().__init__(input_str) before evaluating self.input_str. Each subclass __init__()
+        may in turn manipulate self.input_str to apply expansions or decodings."""
+        self.input_str = input_str
+
+    def get_next(self, csv_row:dict=None):
+        """Subclasses implement this: return the next value from the parameter source.
+           When there are no more values from the source, raise a ParamSourceExhaustedExn.
+           This default implementation is an empty source."""
+        raise ParamSourceExhaustedExn()
+
+    @classmethod
+    def from_str(cls, input_str:str):
+        """compatibility with earlier version of ParamSource. Just use the constructor."""
+        return cls(input_str)
+
+class ConstantSource(ParamSource):
+    """one value for all"""
+    name = "constant"
+
+    def get_next(self, csv_row:dict=None):
+        return self.input_str
+
+class InputExpandingParamSource(ParamSource):
+
+    def __init__(self, input_str:str):
+        super().__init__(input_str)
+        self.input_str = self.expand_input_str(self.input_str)
+
+    @classmethod
+    def expand_input_str(cls, input_str:str):
+        # user convenience syntax '0*32' becomes '00000000000000000000000000000000'
+        if "*" not in input_str:
+            return input_str
+        # re:                "XX            *        123" with optional spaces
+        tokens = re.split(r"([^ \t]+)[ \t]*\*[ \t]*([0-9]+)", input_str)
+        if len(tokens) < 3:
+            return input_str
+        parts = []
+        for unchanged, snippet, repeat_str in zip(tokens[0::3], tokens[1::3], tokens[2::3]):
+            parts.append(unchanged)
+            repeat = int(repeat_str)
+            parts.append(snippet * repeat)
+
+        return "".join(parts)
+
+class DecimalRangeSource(InputExpandingParamSource):
+    """abstract: decimal numbers with a value range"""
+
+    numeric_base = 10
+
+    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
+        """Constructor to set up values from a (user entered) string: DecimalRangeSource(input_str).
+        Constructor to set up values directly: DecimalRangeSource(num_digits=3, first_value=123, last_value=456)
+
+        num_digits produces leading zeros when first_value..last_value are shorter.
+        """
+        assert ((input_str is not None and (num_digits, first_value, last_value) == (None, None, None))
+                or (input_str is None and None not in (num_digits, first_value, last_value)))
+
+        if input_str is not None:
+            super().__init__(input_str)
+
+            input_str = self.input_str
+
+            if ".." in input_str:
+                first_str, last_str = input_str.split('..')
+                first_str = first_str.strip()
+                last_str = last_str.strip()
+            else:
+                first_str = input_str.strip()
+                last_str = None
+
+            num_digits = len(first_str)
+            first_value = int(first_str)
+            last_value = int(last_str if last_str is not None else "9" * num_digits)
+
+        assert num_digits > 0
+        assert first_value <= last_value
+        self.num_digits = num_digits
+        self.first_value = first_value
+        self.last_value = last_value
+
+    def val_to_digit(self, val:int):
+        return "%0*d" % (self.num_digits, val)  # pylint: disable=consider-using-f-string
+
+class RandomSourceMixin:
+    random_impl = secrets.SystemRandom()
+
+class RandomDigitSource(DecimalRangeSource, RandomSourceMixin):
+    """return a different sequence of random decimal digits each"""
+    name = "random decimal digits"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.used_keys = set()
+
+    def get_next(self, csv_row:dict=None):
+        # try to generate random digits that are always different from previously produced random digits
+        for _ in range(10):
+            val = self.random_impl.randint(self.first_value, self.last_value)
+            if val not in self.used_keys:
+                break
+        self.used_keys.add(val)
+        return self.val_to_digit(val)
+
+class RandomHexDigitSource(InputExpandingParamSource, RandomSourceMixin):
+    """return a different sequence of random hexadecimal digits each"""
+    name = "random hexadecimal digits"
+    numeric_base = 16
+    def __init__(self, input_str:str):
+        super().__init__(input_str)
+        input_str = self.input_str
+
+        num_digits = len(input_str.strip())
+        if num_digits < 1:
+            raise ValueError("zero number of digits")
+        # hex digits always come in two
+        if (num_digits & 1) != 0:
+            raise ValueError(f"hexadecimal value should have even number of digits, not {num_digits}")
+        self.num_digits = num_digits
+        self.used_keys = set()
+
+    def get_next(self, csv_row:dict=None):
+        # try to generate random bytes that are always different from previously produced random bytes
+        for _ in range(10):
+            val = self.random_impl.randbytes(self.num_digits // 2)
+            if val not in self.used_keys:
+                break
+        self.used_keys.add(val)
+
+        return b2h(val)
+
+class IncDigitSource(DecimalRangeSource):
+    """incrementing sequence of digits"""
+    name = "incrementing decimal digits"
+
+    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
+        """input_str: the range of values to iterate. Format: 'FIRST..LAST' (e.g. '0001..9999') or
+        just 'FIRST' (iterates to the maximum value for the given digit width). Leading zeros in
+        FIRST determine the digit width and are preserved in returned values."""
+        super().__init__(input_str, num_digits, first_value, last_value)
+        self.next_val = None
+        self.reset()
+
+    def reset(self):
+        """Restart from the first value of the defined range passed to __init__()."""
+        self.next_val = self.first_value
+
+    def get_next(self, csv_row:dict=None):
+        val = self.next_val
+        if val is None:
+            raise ParamSourceExhaustedExn()
+
+        returnval = self.val_to_digit(val)
+
+        val += 1
+        if val > self.last_value:
+            self.next_val = None
+        else:
+            self.next_val = val
+
+        return returnval
+
+class CsvSource(ParamSource):
+    """apply a column from a CSV row, as passed in to ParamSource.get_next(csv_row)"""
+    name = "from CSV"
+
+    def __init__(self, input_str:str):
+        """input_str: the CSV column name to read values from.
+        The caller passes the current CSV row to get_next(), from which CsvSource picks the column matching
+        this name."""
+        super().__init__(input_str)
+        self.csv_column = self.input_str
+
+    def get_next(self, csv_row:dict=None):
+        val = None
+        if csv_row:
+            val = csv_row.get(self.csv_column)
+        if val is None:
+            raise ParamSourceUndefinedExn(f"no value for CSV column {self.csv_column!r}")
+        return val

pySim/esim/saip/personalization.py

@@ -16,13 +16,24 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import abc
+import enum
 import io
-from typing import List, Tuple
+import re
+from typing import List, Tuple, Generator, Optional
 
+from construct.core import StreamError
 from osmocom.tlv import camel_to_snake
-from pySim.utils import enc_iccid, enc_imsi, h2b, rpad, sanitize_iccid
-from pySim.esim.saip import ProfileElement, ProfileElementSequence
+from osmocom.utils import hexstr
+from pySim.utils import enc_iccid, dec_iccid, enc_imsi, dec_imsi, h2b, b2h, rpad, sanitize_iccid
+from pySim.ts_31_102 import EF_AD
 from pySim.ts_51_011 import EF_SMSP
+from pySim.esim.saip import param_source
+from pySim.esim.saip import ProfileElement, ProfileElementSD, ProfileElementSequence
+from pySim.esim.saip import SecurityDomainKey, SecurityDomainKeyComponent
+from pySim.global_platform import KeyUsageQualifier, KeyType
+
+def unrpad(s: hexstr, c='f') -> hexstr:
+    return hexstr(s.rstrip(c))
 
 def remove_unwanted_tuples_from_list(l: List[Tuple], unwanted_keys: List[str]) -> List[Tuple]:
     """In a list of tuples, remove all tuples whose first part equals 'unwanted_key'."""
@@ -43,7 +54,6 @@ class ClassVarMeta(abc.ABCMeta):
         x = super().__new__(metacls, name, bases, namespace)
         for k, v in kwargs.items():
             setattr(x, k, v)
-        setattr(x, 'name', camel_to_snake(name))
         return x
 
 class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
@@ -63,6 +73,7 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
     min_len: minimum length of an input str; min_len = 4
     max_len: maximum length of an input str; max_len = 8
     allow_len: permit only specific lengths; allow_len = (8, 16, 32)
+    numeric_base: indicate hex / decimal, if any; numeric_base = None; numeric_base = 10; numeric_base = 16
 
     Subclasses may change the meaning of these by overriding validate_val(), for example that the length counts
     resulting bytes instead of a hexstring length. Most subclasses will be covered by the default validate_val().
@@ -117,6 +128,8 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
     max_len = None
     allow_len = None # a list of specific lengths
     example_input = None
+    default_source = None # a param_source.ParamSource subclass
+    numeric_base = None # or 10 or 16
 
     def __init__(self, input_value=None):
         self.input_value = input_value # the raw input value as given by caller
@@ -178,19 +191,28 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
             if cls.allow_chars is not None:
                 if any(c not in cls.allow_chars for c in val):
                     raise ValueError(f"invalid characters in input value {val!r}, valid chars are {cls.allow_chars}")
+        elif isinstance(val, io.BytesIO):
+            val = val.getvalue()
+
+        if hasattr(val, '__len__'):
+            val_len = len(val)
+        else:
+            # e.g. int length
+            val_len = len(str(val))
+
         if cls.allow_len is not None:
             l = cls.allow_len
             # cls.allow_len could be one int, or a tuple of ints. Wrap a single int also in a tuple.
             if not isinstance(l, (tuple, list)):
                 l = (l,)
-            if len(val) not in l:
-                raise ValueError(f'length must be one of {cls.allow_len}, not {len(val)}: {val!r}')
+            if val_len not in l:
+                raise ValueError(f'length must be one of {cls.allow_len}, not {val_len}: {val!r}')
         if cls.min_len is not None:
-            if len(val) < cls.min_len:
-                raise ValueError(f'length must be at least {cls.min_len}, not {len(val)}: {val!r}')
+            if val_len < cls.min_len:
+                raise ValueError(f'length must be at least {cls.min_len}, not {val_len}: {val!r}')
         if cls.max_len is not None:
-            if len(val) > cls.max_len:
-                raise ValueError(f'length must be at most {cls.max_len}, not {len(val)}: {val!r}')
+            if val_len > cls.max_len:
+                raise ValueError(f'length must be at most {cls.max_len}, not {val_len}: {val!r}')
         return val
 
     @classmethod
@@ -199,6 +221,49 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
         Write the given val in the right format in all the right places in pes."""
         pass
 
+    @classmethod
+    def get_value_from_pes(cls, pes: ProfileElementSequence):
+        """Same as get_values_from_pes() but expecting a single value.
+           get_values_from_pes() may return values like this:
+             [{ 'AlgorithmID': 'Milenage' }, { 'AlgorithmID': 'Milenage' }]
+           This ensures that all these entries are identical and would return only
+              { 'AlgorithmID': 'Milenage' }.
+
+           This is relevant for any profile element that may appear multiple times in the same PES (only a few),
+           where each occurrence should reflect the same value (all currently known parameters).
+        """
+
+        val = None
+        for v in cls.get_values_from_pes(pes):
+            if val is None:
+                val = v
+            elif val != v:
+                raise ValueError(f'get_value_from_pes(): got distinct values: {val!r} != {v!r}')
+        return val
+
+    @classmethod
+    @abc.abstractmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence) -> Generator:
+        """This is what subclasses implement: yield all values from a decoded profile package.
+           Find all values in the pes, and yield them decoded to a valid cls.input_value format.
+           Should be a generator function, i.e. use 'yield' instead of 'return'.
+
+           Yielded value must be a dict(). Usually, an implementation will return only one key, like
+
+              { "ICCID": "1234567890123456789" }
+
+           Some implementations have more than one value to return, like
+
+              { "IMSI": "00101012345678", "IMSI-ACC" : "5" }
+
+           Implementation example:
+
+             for pe in pes:
+                if my_condition(pe):
+                    yield { cls.name: b2h(my_bin_value_from(pe)) }
+           """
+        pass
+
     @classmethod
     def get_len_range(cls):
         """considering all of min_len, max_len and allow_len, get a tuple of the resulting (min, max) of permitted
@@ -219,6 +284,20 @@ class ConfigurableParameter(abc.ABC, metaclass=ClassVarMeta):
             return (None, None)
         return (min(vals), max(vals))
 
+    @classmethod
+    def get_typical_input_len(cls):
+        '''return a good length to use as the visible width of a user interface input field.
+           May be overridden by subclasses.
+           This default implementation returns the maximum allowed value length -- a good fit for most subclasses.
+           '''
+        return cls.get_len_range()[1] or 16
+
+    @classmethod
+    def is_super_of(cls, other_class):
+        try:
+            return issubclass(other_class, cls)
+        except TypeError:
+            return False
 
 class DecimalParam(ConfigurableParameter):
     """Decimal digits. The input value may be a string of decimal digits like '012345', or an int. The output of
@@ -226,6 +305,7 @@ class DecimalParam(ConfigurableParameter):
     """
     allow_types = (str, int)
     allow_chars = '0123456789'
+    numeric_base = 10
 
     @classmethod
     def validate_val(cls, val):
@@ -249,6 +329,7 @@ class DecimalHexParam(DecimalParam):
     @classmethod
     def validate_val(cls, val):
         val = super().validate_val(val)
+        assert isinstance(val, str)
         val = ''.join('%02x' % ord(x) for x in val)
         if cls.rpad is not None:
             c = cls.rpad_char
@@ -256,9 +337,21 @@ class DecimalHexParam(DecimalParam):
         # a DecimalHexParam subclass expects the apply_val() input to be a bytes instance ready for the pes
         return h2b(val)
 
+    @classmethod
+    def decimal_hex_to_str(cls, val):
+        """useful for get_values_from_pes() implementations of subclasses"""
+        if isinstance(val, bytes):
+            val = b2h(val)
+        assert isinstance(val, hexstr)
+        if cls.rpad is not None:
+            c = cls.rpad_char or 'f'
+            val = unrpad(val, c)
+        return val.to_bytes().decode('ascii')
+
 class IntegerParam(ConfigurableParameter):
     allow_types = (str, int)
     allow_chars = '0123456789'
+    numeric_base = 10
 
     # two integers, if the resulting int should be range limited
     min_val = None
@@ -279,14 +372,28 @@ class IntegerParam(ConfigurableParameter):
             raise ValueError(f'Value {val} is out of range, must be [{cls.min_val}..{cls.max_val}]')
         return val
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for valdict in super().get_values_from_pes(pes):
+            for key, val in valdict.items():
+                if isinstance(val, int):
+                    valdict[key] = str(val)
+            yield valdict
+
 class BinaryParam(ConfigurableParameter):
-    allow_types = (str, io.BytesIO, bytes, bytearray)
+    allow_types = (str, io.BytesIO, bytes, bytearray, int)
     allow_chars = '0123456789abcdefABCDEF'
     strip_chars = ' \t\r\n'
+    numeric_base = 16
+    default_source = param_source.RandomHexDigitSource
 
     @classmethod
     def validate_val(cls, val):
         # take care that min_len and max_len are applied to the binary length by converting to bytes first
+        if isinstance(val, int):
+            min_len, _max_len = cls.get_len_range()
+            val = '%0*d' % (min_len, val)
+
         if isinstance(val, str):
             if cls.strip_chars is not None:
                 val = ''.join(c for c in val if c not in cls.strip_chars)
@@ -301,6 +408,82 @@ class BinaryParam(ConfigurableParameter):
         val = super().validate_val(val)
         return bytes(val)
 
+    @classmethod
+    def get_typical_input_len(cls):
+        # override to return twice the length, because of hex digits.
+        min_len, max_len = cls.get_len_range()
+        if max_len is None:
+            return None
+        # two hex characters per value octet.
+        # (maybe *3 to also allow for spaces?)
+        return max_len * 2
+
+
+class EnumParam(ConfigurableParameter):
+    """ConfigurableParameter for named integer enumeration values.
+
+    Subclasses must define a nested enum.IntEnum named 'Values' listing all valid names and their
+    integer codes.  apply_val() and get_values_from_pes() are not implemented here and this must
+    be inherited from another mixin."""
+
+    class Values(enum.IntEnum):
+        pass  # subclasses override this
+
+    @classmethod
+    def validate_val(cls, val) -> int:
+        if isinstance(val, int):
+            try:
+                return int(cls.Values(val))
+            except ValueError:
+                pass
+        elif isinstance(val, str):
+            member = cls.map_name_to_val(val, strict=False)
+            if member is not None:
+                return member
+
+        valid = ', '.join(m.name for m in cls.Values)
+        raise ValueError(f"{cls.get_name()}: invalid argument: {val!r}. Valid arguments are: {valid}")
+
+    @classmethod
+    def map_name_to_val(cls, name: str, strict=True) -> int:
+        """Return the integer value for a given enum member name. Performs an exact match first,
+        then falls back to fuzzy matching (case-insensitive, punctuation-insensitive)."""
+        try:
+            return int(cls.Values[name])
+        except KeyError:
+            pass
+
+        clean = cls.clean_name_str(name)
+        for member in cls.Values:
+            if cls.clean_name_str(member.name) == clean:
+                return int(member)
+
+        if strict:
+            valid = ', '.join(m.name for m in cls.Values)
+            raise ValueError(f"{cls.get_name()}: {name!r} is not a known value. Known values are: {valid}")
+        return None
+
+    @classmethod
+    def map_val_to_name(cls, val, strict=False) -> str:
+        """Return the enum member name for a given integer value."""
+        try:
+            return cls.Values(val).name
+        except ValueError:
+            if strict:
+                raise ValueError(f"{cls.get_name()}: {val!r} ({type(val).__name__}) is not a known value.")
+            return None
+
+    @classmethod
+    def name_normalize(cls, name: str) -> str:
+        """Map a (possibly fuzzy) name to its canonical enum member name."""
+        return cls.Values(cls.map_name_to_val(name)).name
+
+    @classmethod
+    def clean_name_str(cls, val: str) -> str:
+        """Strip punctuation and case for fuzzy name comparison.
+        Treats hyphens and underscores as equivalent (both removed)."""
+        return re.sub('[^0-9A-Za-z]', '', val).lower()
+
 
 class Iccid(DecimalParam):
     """ICCID Parameter. Input: string of decimal digits.
@@ -309,6 +492,7 @@ class Iccid(DecimalParam):
     min_len = 18
     max_len = 20
     example_input = '998877665544332211'
+    default_source = param_source.IncDigitSource
 
     @classmethod
     def validate_val(cls, val):
@@ -322,6 +506,17 @@ class Iccid(DecimalParam):
         # patch MF/EF.ICCID
         file_replace_content(pes.get_pe_for_type('mf').decoded['ef-iccid'], h2b(enc_iccid(val)))
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        padded = b2h(pes.get_pe_for_type('header').decoded['iccid'])
+        iccid = unrpad(padded)
+        yield { cls.name: iccid }
+
+        for pe in pes.get_pes_for_type('mf'):
+            iccid_f = pe.files.get('ef-iccid', None)
+            if iccid_f is not None:
+                yield { cls.name: dec_iccid(b2h(iccid_f.body)) }
+
 class Imsi(DecimalParam):
     """Configurable IMSI. Expects value to be a string of digits. Automatically sets the ACC to
     the last digit of the IMSI."""
@@ -330,6 +525,7 @@ class Imsi(DecimalParam):
     min_len = 6
     max_len = 15
     example_input = '00101' + ('0' * 10)
+    default_source = param_source.IncDigitSource
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -342,6 +538,18 @@ class Imsi(DecimalParam):
             file_replace_content(pe.decoded['ef-acc'], acc.to_bytes(2, 'big'))
         # TODO: DF.GSM_ACCESS if not linked?
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('usim'):
+            imsi_f = pe.files.get('ef-imsi', None)
+            acc_f = pe.files.get('ef-acc', None)
+            y = {}
+            if imsi_f:
+                y[cls.name] = dec_imsi(b2h(imsi_f.body))
+            if acc_f:
+                y[cls.name + '-ACC'] = b2h(acc_f.body)
+            yield y
+
 class SmspTpScAddr(ConfigurableParameter):
     """Configurable SMSC (SMS Service Centre) TP-SC-ADDR. Expects to be a phone number in national or
     international format (designated by a leading +). Automatically sets the NPI to E.164 and the TON based on
@@ -350,25 +558,45 @@ class SmspTpScAddr(ConfigurableParameter):
     name = 'SMSP-TP-SC-ADDR'
     allow_chars = '+0123456789'
     strip_chars = ' \t\r\n'
+    numeric_base = 10
     max_len = 21 # '+' and 20 digits
     min_len = 1
     example_input = '+49301234567'
+    default_source = param_source.ConstantSource
 
-    @classmethod
-    def validate_val(cls, val):
-        val = super().validate_val(val)
-        addr_str = str(val)
+    @staticmethod
+    def str_to_tuple(addr_str):
         if addr_str[0] == '+':
             digits = addr_str[1:]
             international = True
         else:
             digits = addr_str
             international = False
+        return (international, digits)
+
+    @staticmethod
+    def tuple_to_str(addr_tuple):
+        international, digits = addr_tuple
+        if international:
+            ret = '+'
+        else:
+            ret = ''
+        ret += digits
+        return ret
+
+    @classmethod
+    def validate_val(cls, val):
+        val = super().validate_val(val)
+
+        addr_tuple = cls.str_to_tuple(str(val))
+
+        international, digits = addr_tuple
         if len(digits) > 20:
             raise ValueError(f'TP-SC-ADDR must not exceed 20 digits: {digits!r}')
         if not digits.isdecimal():
             raise ValueError(f'TP-SC-ADDR must only contain decimal digits: {digits!r}')
-        return (international, digits)
+
+        return addr_tuple
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -392,98 +620,318 @@ class SmspTpScAddr(ConfigurableParameter):
             ef_smsp_dec['tp_sc_addr']['ton_npi']['type_of_number'] = 'international' if international else 'unknown'
             # ensure the parameter_indicators.tp_sc_addr is True
             ef_smsp_dec['parameter_indicators']['tp_sc_addr'] = True
-            # re-encode into the File body
-            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1)
+
+            # alpha_id padding: to make room for a human readable SMSC name that can be provisioned to the profile later
+            # on, alpha_id needs to be empty but padded 0xff to some length.
+            # - alpha_id is optional, setting alpha_id = '' ensures the IE is present.
+            # - the length of the file is 28+Y where Y is the length of the alpha_id -- here the intended length of our padding
+            #   (see 3GPP TS 31.102 4.2.27 EF.SMSP). So if we want a maximum length of alpha_id = 14, we set the total
+            #   file size to 28+14 = 42.
+            # - this file size has to go in two places: encode_record_bin() needs to know the length to encode the right
+            #   length of fillFileContent.
+            # - the f_smsp needs to show the right file size in the PES, as in
+            #      'ef-smsp': [('fileDescriptor', {'efFileSize': '2a', ...
+            #   (where 2a == 42)
+            # - To generate the right amount of fillFileContent, pass total_len=42 to encode_record_bin().
+            # - To show the right size in the PES, set f_smsp.rec_len = 42
+            ef_smsp_dec['alpha_id'] = ''
+            f_smsp.rec_len = 42
+
+            # re-encode into the File body.
+            #
             #print("SMSP  (new): %s" % f_smsp.body)
             # re-generate the pe.decoded member from the File instance
+            f_smsp.body = ef_smsp.encode_record_bin(ef_smsp_dec, 1, total_len=f_smsp.rec_len)
             pe.file2pe(f_smsp)
 
-class SdKey(BinaryParam, metaclass=ClassVarMeta):
-    """Configurable Security Domain (SD) Key.  Value is presented as bytes."""
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('usim'):
+            f_smsp = pe.files['ef-smsp']
+            ef_smsp = EF_SMSP()
+            ef_smsp_dec = ef_smsp.decode_record_bin(f_smsp.body, 1)
+
+            tp_sc_addr = ef_smsp_dec.get('tp_sc_addr', None)
+
+            digits = tp_sc_addr.get('call_number', None)
+
+            ton_npi = tp_sc_addr.get('ton_npi', None)
+            international = ton_npi.get('type_of_number', None)
+            international = (international == 'international')
+
+            yield { cls.name: cls.tuple_to_str((international, digits)) }
+
+
+class MncLen(EnumParam):
+    """MNC length.  Sets only the MNC length field in EF.AD (Administrative Data).
+    Accepted values: integer 2 or 3, digit strings '2' or '3', or enum names 'MNC2'/'MNC3'.
+    """
+    name = 'MNC-LEN'
+    example_input = '2'
+    default_source = param_source.ConstantSource
+
+    class Values(enum.IntEnum):
+        MNC2 = 2
+        MNC3 = 3
+
+    @classmethod
+    def validate_val(cls, val):
+        if isinstance(val, str) and val.isdigit():
+            val = int(val)
+        return super().validate_val(val)
+
+    @classmethod
+    def _get_f_ad(cls, pe: ProfileElement):
+        if not hasattr(pe, 'files'):
+            return None
+        f_ad = pe.files.get('ef-ad', None)
+        if f_ad and f_ad.body:
+            return f_ad
+        return None
+
+    @classmethod
+    def _decode_f_ad(cls, f_ad):
+        try:
+            ef_ad_dec = EF_AD().decode_bin(f_ad.body)
+        except StreamError:
+            return None
+        if 'mnc_len' not in ef_ad_dec:
+            return None
+        return ef_ad_dec
+
+    @classmethod
+    def apply_val(cls, pes: ProfileElementSequence, val: int):
+        for pe in pes.get_pes_for_type('usim'):
+            f_ad = cls._get_f_ad(pe)
+            if f_ad is None:
+                continue
+            # decode existing values
+            ef_ad_dec = cls._decode_f_ad(f_ad)
+            if ef_ad_dec is None:
+                continue
+            # change mnc_len
+            ef_ad_dec['mnc_len'] = val
+            # re-encode into the File body
+            f_ad.body = EF_AD().encode_bin(ef_ad_dec)
+            pe.file2pe(f_ad)
+
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('usim'):
+            f_ad = cls._get_f_ad(pe)
+            if f_ad is None:
+                continue
+            ef_ad_dec = cls._decode_f_ad(f_ad)
+            if ef_ad_dec is None:
+                continue
+            mnc_len = ef_ad_dec.get('mnc_len')
+            yield { cls.name: str(mnc_len) }
+
+
+class SdKey(BinaryParam):
+    """Configurable Security Domain (SD) Key.  Value is presented as bytes.
+       Non-abstract implementations are generated in SdKey.generate_sd_key_classes"""
     # these will be set by subclasses
     key_type = None
-    key_id = None
     kvn = None
+    key_id = None
     key_usage_qual = None
 
     @classmethod
-    def _apply_sd(cls, pe: ProfileElement, value):
-        assert pe.type == 'securityDomain'
-        for key in pe.decoded['keyList']:
-            if key['keyIdentifier'][0] == cls.key_id and key['keyVersionNumber'][0] == cls.kvn:
-                assert len(key['keyComponents']) == 1
-                key['keyComponents'][0]['keyData'] = value
-                return
-        # Could not find matching key to patch, create a new one
-        key = {
-            'keyUsageQualifier': bytes([cls.key_usage_qual]),
-            'keyIdentifier': bytes([cls.key_id]),
-            'keyVersionNumber': bytes([cls.kvn]),
-            'keyComponents': [
-                { 'keyType': bytes([cls.key_type]), 'keyData': value },
-            ]
-        }
-        pe.decoded['keyList'].append(key)
+    def apply_val(cls, pes: ProfileElementSequence, val):
+        set_components = [ SecurityDomainKeyComponent(cls.key_type, val) ]
+
+        for pe in pes.pe_list:
+            if pe.type != 'securityDomain':
+                continue
+            assert isinstance(pe, ProfileElementSD)
+
+            key = pe.find_key(key_version_number=cls.kvn, key_id=cls.key_id)
+            if not key:
+                # Could not find matching key to patch, create a new one
+                key = SecurityDomainKey(
+                        key_version_number=cls.kvn,
+                        key_id=cls.key_id,
+                        key_usage_qualifier=cls.key_usage_qual,
+                        key_components=set_components,
+                        )
+                pe.add_key(key)
+            else:
+                # A key of this KVN and ID already exists in the profile.
+
+                # Keep the key_usage_qualifier as it was in the profile, so skip this here:
+                # key.key_usage_qualifier = cls.key_usage_qual
+
+                key.key_components = set_components
 
     @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, value):
-        for pe in pes.get_pes_for_type('securityDomain'):
-            cls._apply_sd(pe, value)
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.pe_list:
+            if pe.type != 'securityDomain':
+                continue
+            assert isinstance(pe, ProfileElementSD)
 
-class SdKeyScp80_01(SdKey, kvn=0x01, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp80_01Kic(SdKeyScp80_01, key_id=0x01, key_usage_qual=0x18): # FIXME: ordering?
-    pass
-class SdKeyScp80_01Kid(SdKeyScp80_01, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp80_01Kik(SdKeyScp80_01, key_id=0x03, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp81_01(SdKey, kvn=0x81): # FIXME
-    pass
-class SdKeyScp81_01Psk(SdKeyScp81_01, key_id=0x01, key_type=0x85, key_usage_qual=0x3C):
-    pass
-class SdKeyScp81_01Dek(SdKeyScp81_01, key_id=0x02, key_type=0x88, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp02_20(SdKey, kvn=0x20, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp02_20Enc(SdKeyScp02_20, key_id=0x01, key_usage_qual=0x18):
-    pass
-class SdKeyScp02_20Mac(SdKeyScp02_20, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp02_20Dek(SdKeyScp02_20, key_id=0x03, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp03_30(SdKey, kvn=0x30, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp03_30Enc(SdKeyScp03_30, key_id=0x01, key_usage_qual=0x18):
-    pass
-class SdKeyScp03_30Mac(SdKeyScp03_30, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp03_30Dek(SdKeyScp03_30, key_id=0x03, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp03_31(SdKey, kvn=0x31, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp03_31Enc(SdKeyScp03_31, key_id=0x01, key_usage_qual=0x18):
-    pass
-class SdKeyScp03_31Mac(SdKeyScp03_31, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp03_31Dek(SdKeyScp03_31, key_id=0x03, key_usage_qual=0x48):
-    pass
-
-class SdKeyScp03_32(SdKey, kvn=0x32, key_type=0x88, permitted_len=[16,24,32]): # AES key type
-    pass
-class SdKeyScp03_32Enc(SdKeyScp03_32, key_id=0x01, key_usage_qual=0x18):
-    pass
-class SdKeyScp03_32Mac(SdKeyScp03_32, key_id=0x02, key_usage_qual=0x14):
-    pass
-class SdKeyScp03_32Dek(SdKeyScp03_32, key_id=0x03, key_usage_qual=0x48):
-    pass
+            key = pe.find_key(key_version_number=cls.kvn, key_id=cls.key_id)
+            if not key:
+                continue
+            kc = key.get_key_component(cls.key_type)
+            if kc:
+                yield { cls.name: b2h(kc) }
 
 
+    LEN_128 = (16,)
+    LEN_128_192_256 = (16, 24, 32)
+    LEN_128_256 = (16, 32)
 
+    DES =    ('DES', dict(key_type=KeyType.des, allow_len=LEN_128) )
+    AES =    ('AES', dict(key_type=KeyType.aes, allow_len=LEN_128_192_256) )
+
+    ENC =    ('ENC', dict(key_id=0x01, key_usage_qual=0x18) )
+    MAC =    ('MAC', dict(key_id=0x02, key_usage_qual=0x14) )
+    DEK =    ('DEK', dict(key_id=0x03, key_usage_qual=0x48) )
+
+    TLSPSK_PSK = ('TLSPSK', dict(key_type=KeyType.tls_psk, key_id=0x01, key_usage_qual=0x3c, allow_len=LEN_128_192_256) )
+    TLSPSK_DEK = ('DEK',    dict(key_id=0x02, key_usage_qual=0x48) )
+
+    # THIS IS THE LIST that controls which SdKeyXxx subclasses exist:
+    SD_KEY_DEFS = (
+        # name    KVN                     x variants            x variants
+        ('SCP02', (0x20, 0x21, 0x22, 0xff), (AES, ),              (ENC, MAC, DEK) ),
+        ('SCP03', (0x30, 0x31, 0x32),       (AES, ),              (ENC, MAC, DEK) ),
+        ('SCP80', (0x01, 0x02, 0x03),       (DES, AES),           (ENC, MAC, DEK) ),
+
+        # key_id=1
+        ('SCP81', (0x40, 0x41, 0x42),       (TLSPSK_PSK, ),                       ),
+        # key_id=2
+        ('SCP81', (0x40, 0x41, 0x42),       (DES, AES),           (TLSPSK_DEK, )  ),
+    )
+
+    all_implementations = None
+
+    @classmethod
+    def generate_sd_key_classes(cls, sd_key_defs=SD_KEY_DEFS):
+        '''This generates python classes to be exported in this module, as subclasses of class SdKey.
+
+        We create SdKey subclasses dynamically from a list.
+        You can list all of them via:
+          from pySim.esim.saip.personalization import SdKey
+          SdKey.all_implementations
+        or
+          print('\n'.join(sorted(f'{x.__name__}\t{x.name}' for x in SdKey.all_implementations)))
+
+        at time of writing this comment, this prints:
+
+        SdKeyScp02Kvn20AesDek	SCP02-KVN20-AES-DEK
+        SdKeyScp02Kvn20AesEnc	SCP02-KVN20-AES-ENC
+        SdKeyScp02Kvn20AesMac	SCP02-KVN20-AES-MAC
+        SdKeyScp02Kvn21AesDek	SCP02-KVN21-AES-DEK
+        SdKeyScp02Kvn21AesEnc	SCP02-KVN21-AES-ENC
+        SdKeyScp02Kvn21AesMac	SCP02-KVN21-AES-MAC
+        SdKeyScp02Kvn22AesDek	SCP02-KVN22-AES-DEK
+        SdKeyScp02Kvn22AesEnc	SCP02-KVN22-AES-ENC
+        SdKeyScp02Kvn22AesMac	SCP02-KVN22-AES-MAC
+        SdKeyScp02KvnffAesDek	SCP02-KVNff-AES-DEK
+        SdKeyScp02KvnffAesEnc	SCP02-KVNff-AES-ENC
+        SdKeyScp02KvnffAesMac	SCP02-KVNff-AES-MAC
+        SdKeyScp03Kvn30AesDek	SCP03-KVN30-AES-DEK
+        SdKeyScp03Kvn30AesEnc	SCP03-KVN30-AES-ENC
+        SdKeyScp03Kvn30AesMac	SCP03-KVN30-AES-MAC
+        SdKeyScp03Kvn31AesDek	SCP03-KVN31-AES-DEK
+        SdKeyScp03Kvn31AesEnc	SCP03-KVN31-AES-ENC
+        SdKeyScp03Kvn31AesMac	SCP03-KVN31-AES-MAC
+        SdKeyScp03Kvn32AesDek	SCP03-KVN32-AES-DEK
+        SdKeyScp03Kvn32AesEnc	SCP03-KVN32-AES-ENC
+        SdKeyScp03Kvn32AesMac	SCP03-KVN32-AES-MAC
+        SdKeyScp80Kvn01AesDek	SCP80-KVN01-AES-DEK
+        SdKeyScp80Kvn01AesEnc	SCP80-KVN01-AES-ENC
+        SdKeyScp80Kvn01AesMac	SCP80-KVN01-AES-MAC
+        SdKeyScp80Kvn01DesDek	SCP80-KVN01-DES-DEK
+        SdKeyScp80Kvn01DesEnc	SCP80-KVN01-DES-ENC
+        SdKeyScp80Kvn01DesMac	SCP80-KVN01-DES-MAC
+        SdKeyScp80Kvn02AesDek	SCP80-KVN02-AES-DEK
+        SdKeyScp80Kvn02AesEnc	SCP80-KVN02-AES-ENC
+        SdKeyScp80Kvn02AesMac	SCP80-KVN02-AES-MAC
+        SdKeyScp80Kvn02DesDek	SCP80-KVN02-DES-DEK
+        SdKeyScp80Kvn02DesEnc	SCP80-KVN02-DES-ENC
+        SdKeyScp80Kvn02DesMac	SCP80-KVN02-DES-MAC
+        SdKeyScp80Kvn03AesDek	SCP80-KVN03-AES-DEK
+        SdKeyScp80Kvn03AesEnc	SCP80-KVN03-AES-ENC
+        SdKeyScp80Kvn03AesMac	SCP80-KVN03-AES-MAC
+        SdKeyScp80Kvn03DesDek	SCP80-KVN03-DES-DEK
+        SdKeyScp80Kvn03DesEnc	SCP80-KVN03-DES-ENC
+        SdKeyScp80Kvn03DesMac	SCP80-KVN03-DES-MAC
+        SdKeyScp81Kvn40AesDek	SCP81-KVN40-AES-DEK
+        SdKeyScp81Kvn40DesDek	SCP81-KVN40-DES-DEK
+        SdKeyScp81Kvn40Tlspsk	SCP81-KVN40-TLSPSK
+        SdKeyScp81Kvn41AesDek	SCP81-KVN41-AES-DEK
+        SdKeyScp81Kvn41DesDek	SCP81-KVN41-DES-DEK
+        SdKeyScp81Kvn41Tlspsk	SCP81-KVN41-TLSPSK
+        SdKeyScp81Kvn42AesDek	SCP81-KVN42-AES-DEK
+        SdKeyScp81Kvn42DesDek	SCP81-KVN42-DES-DEK
+        SdKeyScp81Kvn42Tlspsk	SCP81-KVN42-TLSPSK
+        '''
+
+        SdKey.all_implementations = []
+
+        def camel(s):
+            return s[:1].upper() + s[1:].lower()
+
+        def do_variants(name, kvn, remaining_variants, labels=[], attrs={}):
+            'recurse to unfold as many variants as there may be'
+            if remaining_variants:
+                # not a leaf node, collect more labels and attrs
+                variants = remaining_variants[0]
+                remaining_variants = remaining_variants[1:]
+
+                for label, valdict in variants:
+                    # pass copies to recursion
+                    inner_labels = list(labels)
+                    inner_attrs = dict(attrs)
+
+                    inner_labels.append(label)
+                    inner_attrs.update(valdict)
+                    do_variants(name, kvn, remaining_variants,
+                                labels=inner_labels,
+                                attrs=inner_attrs)
+                return
+
+            # leaf node. create a new class with all the accumulated vals
+            parts = [name, f'KVN{kvn:02x}',] + labels
+            cls_label = '-'.join(p for p in parts if p)
+
+            parts = ['Sd', 'Key', name, f'Kvn{kvn:02x}'] + labels
+            clsname = ''.join(camel(p) for p in parts)
+
+            max_key_len = attrs.get('allow_len')[-1]
+
+            attrs.update({
+                'name' : cls_label,
+                'kvn': kvn,
+                'example_input': f'00*{max_key_len}',
+                })
+
+            # below line is like
+            # class SdKeyScpNNKvnXXYyyZzz(SdKey):
+            #     <set attrs>
+            cls_def = type(clsname, (cls,), attrs)
+
+            # for some unknown reason, subclassing from abc.ABC makes cls_def.__module__ == 'abc',
+            # but we don't want 'abc.SdKeyScp03Kvn32AesEnc'.
+            # Make sure it is 'pySim.esim.saip.personalization.SdKeyScp03Kvn32AesEnc'
+            cls_def.__module__ = __name__
+
+            globals()[clsname] = cls_def
+            SdKey.all_implementations.append(cls_def)
+
+
+        for items in sd_key_defs:
+            name, kvns = items[:2]
+            variants = items[2:]
+            for kvn in kvns:
+                do_variants(name, kvn, variants)
+
+# this creates all of the classes named like SdKeyScp02Kvn20AesDek to be published in this python module:
+SdKey.generate_sd_key_classes()
 
 def obtain_all_pe_from_pelist(l: List[ProfileElement], wanted_type: str) -> ProfileElement:
     return (pe for pe in l if pe.type == wanted_type)
@@ -502,7 +950,8 @@ class Puk(DecimalHexParam):
     allow_len = 8
     rpad = 16
     keyReference = None
-    example_input = '0' * allow_len
+    example_input = f'0*{allow_len}'
+    default_source = param_source.RandomDigitSource
 
     @classmethod
     def apply_val(cls, pes: ProfileElementSequence, val):
@@ -516,6 +965,14 @@ class Puk(DecimalHexParam):
         raise ValueError("input template UPP has unexpected structure:"
                          f" cannot find pukCode with keyReference={cls.keyReference}")
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        mf_pes = pes.pes_by_naa['mf'][0]
+        for pukCodes in obtain_all_pe_from_pelist(mf_pes, 'pukCodes'):
+            for pukCode in pukCodes.decoded['pukCodes']:
+                if pukCode['keyReference'] == cls.keyReference:
+                    yield { cls.name: cls.decimal_hex_to_str(pukCode['pukValue']) }
+
 class Puk1(Puk):
     name = 'PUK1'
     keyReference = 0x01
@@ -529,7 +986,8 @@ class Pin(DecimalHexParam):
     rpad = 16
     min_len = 4
     max_len = 8
-    example_input = '0' * max_len
+    example_input = f'0*{max_len}'
+    default_source = param_source.RandomDigitSource
     keyReference = None
 
     @staticmethod
@@ -551,9 +1009,24 @@ class Pin(DecimalHexParam):
             raise ValueError('input template UPP has unexpected structure:'
                              + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference}')
 
+    @classmethod
+    def _read_all_pinvalues_from_pe(cls, pe: ProfileElement):
+        "This is a separate function because subclasses may feed different pe arguments."
+        for pinCodes in obtain_all_pe_from_pelist(pe, 'pinCodes'):
+            if pinCodes.decoded['pinCodes'][0] != 'pinconfig':
+                continue
+
+            for pinCode in pinCodes.decoded['pinCodes'][1]:
+                if pinCode['keyReference'] == cls.keyReference:
+                    yield { cls.name: cls.decimal_hex_to_str(pinCode['pinValue']) }
+
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        yield from cls._read_all_pinvalues_from_pe(pes.pes_by_naa['mf'][0])
+
 class Pin1(Pin):
     name = 'PIN1'
-    example_input = '0' * 4  # PIN are usually 4 digits
+    example_input = '0*4'  # PIN are usually 4 digits
     keyReference = 0x01
 
 class Pin2(Pin1):
@@ -572,6 +1045,14 @@ class Pin2(Pin1):
                     raise ValueError('input template UPP has unexpected structure:'
                             + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference} in {naa=}')
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for naa in pes.pes_by_naa:
+            if naa not in ['usim','isim','csim','telecom']:
+                continue
+            for pe in pes.pes_by_naa[naa]:
+                yield from cls._read_all_pinvalues_from_pe(pe)
+
 class Adm1(Pin):
     name = 'ADM1'
     keyReference = 0x0A
@@ -596,26 +1077,59 @@ class AlgoConfig(ConfigurableParameter):
             raise ValueError('input template UPP has unexpected structure:'
                              f' {cls.__name__} cannot find algoParameter with key={cls.algo_config_key}')
 
-class AlgorithmID(DecimalParam, AlgoConfig):
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('akaParameter'):
+            algoConfiguration = pe.decoded['algoConfiguration']
+            if len(algoConfiguration) < 2:
+                continue
+            if algoConfiguration[0] != 'algoParameter':
+                continue
+            if not algoConfiguration[1]:
+                continue
+            val = algoConfiguration[1].get(cls.algo_config_key, None)
+            if val is None:
+                continue
+            if isinstance(val, bytes):
+                val = b2h(val)
+            # if it is an int (algorithmID), just pass thru as int
+            yield { cls.name: val }
+
+class AlgorithmID(EnumParam, AlgoConfig):
+    """use validate_val() from EnumParam, and apply_val() from AlgoConfig.
+    In get_values_from_pes(), return enum value names, not raw values."""
+    name = "Algorithm"
     algo_config_key = 'algorithmID'
-    allow_len = 1
-    example_input = 1  # Milenage
+    example_input = "Milenage"
+    default_source = param_source.ConstantSource
+
+    # as in pySim/esim/asn1/saip/PE_Definitions-3.3.1.asn
+    class Values(enum.IntEnum):
+        Milenage  = 1
+        TUAK      = 2
+        usim_test = 3  # input 'usim-test' also accepted via fuzzy matching
+
+    # EnumParam.validate_val() returns the int values from Values
 
     @classmethod
-    def validate_val(cls, val):
-        val = super().validate_val(val)
-        val = int(val)
-        valid = (1, 2, 3)
-        if val not in valid:
-            raise ValueError(f'Invalid algorithmID {val!r}, must be one of {valid}')
-        return val
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        # return enum names, not raw values.
+        # use of super(): this intends to call AlgoConfig.get_values_from_pes() so that the cls argument is this cls
+        # here (AlgorithmID); i.e. AlgoConfig.get_values_from_pes(pes) doesn't work, because AlgoConfig needs to look up
+        # cls.algo_config_key.
+        for d in super(cls, cls).get_values_from_pes(pes):
+            if cls.name in d:
+                # convert int to value string
+                val = d[cls.name]
+                d[cls.name] = cls.map_val_to_name(val, strict=True)
+            yield d
 
 class K(BinaryParam, AlgoConfig):
     """use validate_val() from BinaryParam, and apply_val() from AlgoConfig"""
     name = 'K'
     algo_config_key = 'key'
     allow_len = (128 // 8, 256 // 8) # length in bytes (from BinaryParam); TUAK also allows 256 bit
-    example_input = '00' * allow_len[0]
+    example_input = f'00*{allow_len[0]}'
 
 class Opc(K):
     name = 'OPc'
@@ -629,6 +1143,7 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):
     algo_config_key = 'rotationConstants'
     allow_len = 5 # length in bytes (from BinaryParam)
     example_input = '40 00 20 40 60'
+    default_source = param_source.ConstantSource
 
     @classmethod
     def validate_val(cls, val):
@@ -641,7 +1156,7 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):
 
 class MilenageXoringConstants(BinaryParam, AlgoConfig):
     """XOR-ing constants c1,c2,c3,c4,c5 of Milenage, 128bit each. See 3GPP TS 35.206 Sections 2.3 + 5.3.
-    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concetenation
+    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concatenation
     of::
 
      00000000000000000000000000000000
@@ -659,6 +1174,7 @@ class MilenageXoringConstants(BinaryParam, AlgoConfig):
                      ' 00000000000000000000000000000002'
                      ' 00000000000000000000000000000004'
                      ' 00000000000000000000000000000008')
+    default_source = param_source.ConstantSource
 
 class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
     """Number of iterations of Keccak-f[1600] permutation as recomended by Section 7.2 of 3GPP TS 35.231"""
@@ -667,3 +1183,4 @@ class TuakNumberOfKeccak(IntegerParam, AlgoConfig):
     min_val = 1
     max_val = 255
     example_input = '1'
+    default_source = param_source.ConstantSource

pySim/euicc.py

@@ -226,9 +226,28 @@ class Icon(BER_TLV_IE, tag=0x94):
     _construct = GreedyBytes
 class ProfileClass(BER_TLV_IE, tag=0x95):
     _construct = Enum(Int8ub, test=0, provisioning=1, operational=2)
+class ProfilePolicyRules(BER_TLV_IE, tag=0x99):
+    _construct = GreedyBytes
+class NotificationConfigurationInfo(BER_TLV_IE, tag=0xb6):
+    _construct = GreedyBytes
+
+# ProfileOwner
+class ProfileOwnerPLMN(BER_TLV_IE, tag=0x80):
+    _construct = PlmnAdapter(Bytes(3))
+class ProfileOwnerGID1(BER_TLV_IE, tag=0x81):
+    _construct = GreedyBytes
+class ProfileOwnerGID2(BER_TLV_IE, tag=0x82):
+    _construct = GreedyBytes
+class ProfileOwner(BER_TLV_IE, tag=0xb7, nested=[ProfileOwnerPLMN, ProfileOwnerGID1, ProfileOwnerGID2]):
+    _construct = GreedyBytes
+
+class SMDPPProprietaryData(BER_TLV_IE, tag=0xb8):
+    _construct = GreedyBytes
+
 class ProfileInfo(BER_TLV_IE, tag=0xe3, nested=[Iccid, IsdpAid, ProfileState, ProfileNickname,
                                                 ServiceProviderName, ProfileName, IconType, Icon,
-                                                ProfileClass]): # FIXME: more IEs
+                                                ProfileClass, ProfilePolicyRules, NotificationConfigurationInfo,
+                                                ProfileOwner, SMDPPProprietaryData]):
     pass
 class ProfileInfoSeq(BER_TLV_IE, tag=0xa0, nested=[ProfileInfo]):
     pass
@@ -444,9 +463,28 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             d = rn.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['notification_sent_resp']))
 
-        def do_get_profiles_info(self, _opts):
+        get_profiles_info_parser = argparse.ArgumentParser()
+        get_profiles_info_parser.add_argument('--all', action='store_true', help='Retrieve all known tags of a profile')
+
+        @cmd2.with_argparser(get_profiles_info_parser)
+        def do_get_profiles_info(self, opts):
             """Perform an ES10c GetProfilesInfo function."""
-            pi = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, ProfileInfoListReq(), ProfileInfoListResp)
+            if opts.all:
+                tags = [nest.tag for nest in ProfileInfo.nested_collection_cls().nested]
+                u8tags = []
+                # TODO: rework TagList to support 2 byte tags to not filter it into u8 tags
+                for tag in tags:
+                    if tag <= 255:
+                        u8tags.append(tag)
+                    elif tag <= 65535:
+                        u8tags.append(tag >> 8)
+                        u8tags.append(tag & 0xff)
+                    # Ignoring 3 byte tags
+                req = ProfileInfoListReq(children=[TagList(decoded=u8tags)])
+            else:
+                req = ProfileInfoListReq()
+
+            pi = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, req, ProfileInfoListResp)
             d = pi.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['profile_info_list_resp']))
 

pySim/exceptions.py

@@ -38,15 +38,16 @@ class SwMatchError(Exception):
     """Raised when an operation specifies an expected SW but the actual SW from
        the card doesn't match."""
 
-    def __init__(self, sw_actual: str, sw_expected: str, rs=None):
+    def __init__(self, sw_actual: str, sw_expected, rs=None):
         """
         Args:
                 sw_actual : the SW we actually received from the card (4 hex digits)
-                sw_expected : the SW we expected to receive from the card (4 hex digits)
+                sw_expected : the SW we expected to receive from the card (4 hex digits),
+                              or a list of acceptable ones
                 rs : interpreter class to convert SW to string
         """
         self.sw_actual = sw_actual
-        self.sw_expected = sw_expected
+        self.sw_expected = '/'.join(sw_expected) if isinstance(sw_expected, (list, tuple)) else sw_expected
         self.rs = rs
 
     @property

pySim/filesystem.py

@@ -44,6 +44,7 @@ from pySim.utils import sw_match, decomposeATR
 from pySim.jsonpath import js_path_modify
 from pySim.commands import SimCardCommands
 from pySim.exceptions import SwMatchError
+from pySim.log import PySimLogger
 
 # int: a single service is associated with this file
 # list: any of the listed services requires this file
@@ -52,6 +53,8 @@ CardFileService = Union[int, List[int], Tuple[int, ...]]
 
 Size = Tuple[int, Optional[int]]
 
+log = PySimLogger.get(__name__)
+
 class CardFile:
     """Base class for all objects in the smart card filesystem.
     Serve as a common ancestor to all other file types; rarely used directly.
@@ -1609,14 +1612,14 @@ class CardModel(abc.ABC):
         card_atr = scc.get_atr()
         for atr in cls._atrs:
             if atr == card_atr:
-                print("Detected CardModel:", cls.__name__)
+                log.info("Detected CardModel: %s", cls.__name__)
                 return True
         # if nothing found try to just compare the Historical Bytes of the ATR
         card_atr_hb = decomposeATR(card_atr)['hb']
         for atr in cls._atrs:
             atr_hb = decomposeATR(atr)['hb']
             if atr_hb == card_atr_hb:
-                print("Detected CardModel:", cls.__name__)
+                log.info("Detected CardModel: %s", cls.__name__)
                 return True
         return False
 

pySim/global_platform/__init__.py

@@ -18,10 +18,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
 import io
+import hashlib
 from copy import deepcopy
 from typing import Optional, List, Dict, Tuple
 from construct import Optional as COptional
 from construct import Struct, GreedyRange, FlagsEnum, Int16ub, Int24ub, Padding, Bit, Const
+from construct import Construct, stream_read, stream_write
 from Cryptodome.Random import get_random_bytes
 from Cryptodome.Cipher import DES, DES3, AES
 from osmocom.utils import *
@@ -35,6 +37,9 @@ from pySim.filesystem import *
 from pySim.profile import CardProfile
 from pySim.ota import SimFileAccessAndToolkitAppSpecParams
 from pySim.javacard import CapFile
+from pySim.log import PySimLogger
+
+log = PySimLogger.get(__name__)
 
 # GPCS Table 11-48 Load Parameter Tags
 class NonVolatileCodeMinMemoryReq(BER_TLV_IE, tag=0xC6):
@@ -148,6 +153,24 @@ sw_table = {
     },
 }
 
+class PutKeyLength(Construct):
+    """A length field of a PUT KEY data field, GP CardSpec v2.3.1 11.8.2.3.1
+    - all lengths ASN.1 BER-TLV (ITU-T X.690 Section 8.1.3)
+    - except that the length 128 may also be coded on one byte as '80' for backwards compatibility
+    80 does not introduce the indefinite form here which is unused in GP as far as i know.
+    That legacy form is accepted when parsing, but never generated, which agrees with the spec"""
+    def _parse(self, stream, context, path):
+        first = stream_read(stream, 1, path)[0]
+        if first <= 0x80:
+            return first
+        return int.from_bytes(stream_read(stream, first & 0x7f, path), 'big')
+
+    def _build(self, obj, stream, context, path):
+        data = bertlv_encode_len(obj)
+        stream_write(stream, data, len(data), path)
+        return obj
+
+
 # GlobalPlatform 2.1.1 Section 9.1.6
 KeyType = Enum(Byte,    des=0x80,
                         tls_psk=0x85,                           # v2.3.1 Section 11.1.8
@@ -512,6 +535,63 @@ class GpRegistryRelatedData(BER_TLV_IE, tag=0xe3, nested=[ApplicationAID, LifeCy
                                                           ExecutableModuleAID, AssociatedSecurityDomainAID]):
     pass
 
+# GP CS v2.3.1 Table 11-36/11-37 possible data objects requested/returned from GET STATUS for each registry entry.
+# Applications and Executable Load Files have _different_ sets, so a tag list requesting them has
+# to match the subset because 11.4.2.3 warns that asking for a data object an entry does not have
+# "may" be answered with an error status.
+GetStatusTagListIEs = {
+    # Table 11-36 GP Application Data
+    'isd':               [ApplicationAID, LifeCycleState, Privileges, ImplicitSelectionParameter,
+                          ExecutableLoadFileAID, AssociatedSecurityDomainAID],
+    'applications':      [ApplicationAID, LifeCycleState, Privileges, ImplicitSelectionParameter,
+                          ExecutableLoadFileAID, AssociatedSecurityDomainAID],
+    # Table 11-37 GP Executable Load File Data. 84 only for the subset that asks for the modules (Note 2)!
+    'files':             [ApplicationAID, LifeCycleState, ExecutableLoadFileVersionNumber,
+                          AssociatedSecurityDomainAID],
+    'files_and_modules': [ApplicationAID, LifeCycleState, ExecutableLoadFileVersionNumber,
+                          ExecutableModuleAID, AssociatedSecurityDomainAID],
+}
+
+def get_status_tag_list(subset: str) -> bytes:
+    """Encode the GET STATUS tag list for the given status subset"""
+    tags = b''.join([bertlv_encode_tag(ie.tag) for ie in GetStatusTagListIEs[subset]])
+    return b'\x5c' + bertlv_encode_len(len(tags)) + tags
+
+# GP CS v2.3.1 Appendix H.2 / Table H-1
+# oid prefix {iso(1) member-body(2) country-USA(840) globalPlatform(114283)} + card management type 2
+# afterwards GP version.
+OID_GP_CARD_MGMT_TYPE = h2b('2a864886fc6b02')
+
+def _find_tlv_value(decoded, key: str):
+    """depth first search for the nested decoded TLV_IE dict/list"""
+    if isinstance(decoded, dict):
+        for k, v in decoded.items():
+            if k == key:
+                return v
+            found = _find_tlv_value(v, key)
+            if found is not None:
+                return found
+    elif isinstance(decoded, list):
+        for item in decoded:
+            found = _find_tlv_value(item, key)
+            if found is not None:
+                return found
+    return None
+
+def decode_gp_version(card_data: bytes) -> Optional[Tuple[int, ...]]:
+    """GP version from Card Data returned by GET DATA, like (2, 1, 1) or (2, 2).
+    None if cm type OID is absent/unknown"""
+    cd = CardData()
+    cd.from_tlv(card_data)
+    ctv = _find_tlv_value(cd.to_dict(), 'card_management_type_and_version')
+    oid = _find_tlv_value(ctv, 'object_identifier') if ctv is not None else None
+    if oid is None:
+        return None
+    oid = h2b(oid) if isinstance(oid, str) else bytes(oid)
+    if not oid.startswith(OID_GP_CARD_MGMT_TYPE):
+        return None
+    return tuple(oid[len(OID_GP_CARD_MGMT_TYPE):])
+
 # Application Dedicated File of a Security Domain
 class ADF_SD(CardADF):
     StoreData = BitStruct('last_block'/Flag,
@@ -602,8 +682,8 @@ class ADF_SD(CardADF):
             See GlobalPlatform CardSpecification v2.3 Section 11.8 for details.
 
             The KCV (Key Check Values) can either be explicitly specified using `--key-check`, or will
-            otherwise be automatically generated for DES and AES keys.  You can suppress the latter using
-            `--suppress-key-check`.
+            otherwise be automatically generated for DES, AES and TLS-PSK keys.  You can suppress the
+            latter using `--suppress-key-check`.
 
             Example (SCP80 KIC/KID/KIK):
                 put_key --key-version-nr 1 --key-id 0x01    --key-type aes --key-data 000102030405060708090a0b0c0d0e0f
@@ -620,33 +700,81 @@ class ADF_SD(CardADF):
             kdb = []
             for i in range(0, len(opts.key_type)):
                 if opts.key_check and len(opts.key_check) > i:
-                    kcv = opts.key_check[i]
+                    kcv = h2b(opts.key_check[i])
                 elif opts.suppress_key_check:
-                    kcv = ''
+                    kcv = b''
                 else:
-                    kcv_bin = compute_kcv(opts.key_type[i], h2b(opts.key_data[i])) or b''
-                    kcv = b2h(kcv_bin)
-                if self._cmd.lchan.scc.scp:
-                    # encrypted key data with DEK of current SCP
-                    kcb = b2h(self._cmd.lchan.scc.scp.encrypt_key(h2b(opts.key_data[i])))
-                else:
-                    # (for example) during personalization, DEK might not be required)
-                    kcb = opts.key_data[i]
-                kdb.append({'key_type': opts.key_type[i], 'kcb': kcb, 'kcv': kcv})
+                    kcv = compute_kcv(opts.key_type[i], h2b(opts.key_data[i])) or b''
+                kdb.append({'key_type': opts.key_type[i], 'clear_key': h2b(opts.key_data[i]), 'kcv': kcv})
             p2 = opts.key_id
             if len(opts.key_type) > 1:
                 p2 |= 0x80
             self.put_key(opts.old_key_version_nr, opts.key_version_nr, p2, kdb)
 
-        # Table 11-68: Key Data Field - Format 1 (Basic Format)
-        KeyDataBasic = GreedyRange(Struct('key_type'/KeyType,
-                                          'kcb'/Prefixed(Int8ub, GreedyBytes),
-                                          'kcv'/Prefixed(Int8ub, GreedyBytes)))
+        # Table 11-68: Key Data Field - Format 1 (Basic Format).  The key component block length is
+        # BER-TLV coded (Section 11.8.2.3.1), the key check value length is always '00' - '7F'.
+        KeyDataBasic = Struct('key_type'/KeyType,
+                              'kcb'/Prefixed(PutKeyLength(), GreedyBytes),
+                              'kcv'/Prefixed(Int8ub, GreedyBytes))
 
-        def put_key(self, old_kvn:int, kvn: int, kid: int, key_dict: dict) -> bytes:
+        @classmethod
+        def encode_key_data_basic(cls, key_type: str, kcb: bytes, kcv: bytes) -> bytes:
+            """Generic Basic key data field, GP CardSpec v2.3 Table 11-68):
+                tag || L1 || <maybe L2> KCB || <1-byte length> KCV"""
+            return cls.KeyDataBasic.build({'key_type': key_type, 'kcb': kcb, 'kcv': kcv})
+
+        @classmethod
+        def encode_key_data_psk(cls, clear_key: bytes, ciphered_key: bytes, kcv: bytes) -> bytes:
+            """Single PSK TLS '85' key data field per GP Amendment B 1.2, 3.9.1 / Table 3-13:
+                85 | L1 | <L2>  <ciphered PSK key> | <KCV length> | <KCV>
+            - framing is like Basic Format, but the kcb is always GP CardSpec Table 11-70
+            so always with the length of the clear text key value, even without padding!
+            - 'ciphered_key' is DEK(block-padded clear key), no additional length prefix."""
+            kcb = bertlv_encode_len(len(clear_key)) + ciphered_key
+            return cls.encode_key_data_basic('tls_psk', kcb, kcv)
+
+        @classmethod
+        def build_put_key_data(cls, kvn: int, keys: List[dict], scp) -> bytes:
+            """Assemble the PUT KEY data field, mixed PSK + DES DEK is supported:
+            - new KVN followed by one key data field per key.
+            - tls_psk keys per GP Amendment B
+            - other key types generic Basic format
+            Param 'keys' is a dict:
+            - 'key_type' (str)
+            - 'clear_key' (bytes)
+            - 'kcv' (bytes / empty).
+            'scp' may be None (e.g. during personalization, when the DEK may not be required)."""
+            key_data = kvn.to_bytes(1, 'big')
+            for k in keys:
+                clear = k['clear_key']
+                if k['key_type'] == 'tls_psk':
+                    # len always part of the data see CardSpec Table 11-70 vs Table 11-71
+                    if scp:
+                        ciphered = scp.dek_encrypt(scp.pad_to_blocksize(clear))
+                    else:
+                        ciphered = clear
+                    key_data += cls.encode_key_data_psk(clear, ciphered, k['kcv'])
+                else:
+                    if scp:
+                        ciphered = scp.encrypt_key(clear)
+                    else:
+                        # (for example) during personalization, DEK might not be required
+                        ciphered = clear
+                    key_data += cls.encode_key_data_basic(k['key_type'], ciphered, k['kcv'])
+            return key_data
+
+        def put_key(self, old_kvn:int, kvn: int, kid: int, keys: List[dict]) -> bytes:
             """Perform the GlobalPlatform PUT KEY command in order to store a new key on the card.
             See GlobalPlatform CardSpecification v2.3 Section 11.8 for details."""
-            key_data = kvn.to_bytes(1, 'big') + build_construct(ADF_SD.AddlShellCommands.KeyDataBasic, key_dict)
+            key_data = self.build_put_key_data(kvn, keys, self._cmd.lchan.scc.scp)
+            # Lc of Table 11-64 is a single byte, while LOAD or STORE DATA splits we can't:
+            # 11.8.2.3.3 splits a key at component boundaries -> not helping here
+            max_cmd_len = self._cmd.lchan.scc.max_cmd_len
+            if len(key_data) > max_cmd_len:
+                raise ValueError('key data field of %u bytes exceeds the maximum command length of %u '
+                                 '(limited by the overhead of the current secure channel); use fewer '
+                                 'keys per command, a single key component that large needs STORE DATA' %
+                                 (len(key_data), max_cmd_len))
             hdr = "80D8%02x%02x%02x" % (old_kvn, kid, len(key_data))
             data, _sw = self._cmd.lchan.scc.send_apdu_checksw(hdr + b2h(key_data) + "00")
             return data
@@ -665,26 +793,78 @@ class ADF_SD(CardADF):
             for grd in grd_list:
                 self._cmd.poutput_json(grd.to_dict())
 
+        def gp_version(self) -> Optional[Tuple[int, ...]]:
+            """GP version the selected SD reports in its Card Recognition
+            Data, e.g. (2, 1, 1). Card Recognition Data "shall be present" v2.1.1/v2.3.1 section 7.4.1.3,
+            so this must succeed no matter the GP version. None if card did not answer GET DATA / OID unknown.
+            Cached, it cannot change during a session."""
+            if not hasattr(self, '_gp_version'):
+                self._gp_version = None
+                try:
+                    data, _sw = self._cmd.lchan.scc.get_data(cla=0x80, tag=CardData.tag)
+                    self._gp_version = decode_gp_version(h2b(data))
+                except (SwMatchError, ValueError) as e:
+                    log.warning("Could not determine GlobalPlatform version: %s", e)
+            return self._gp_version
+
         def get_status(self, subset:str, aid_search_qualifier:Hexstr = '') -> List[GpRegistryRelatedData]:
-            subset_hex = b2h(build_construct(StatusSubset, subset))
             aid = ApplicationAID(decoded=aid_search_qualifier)
-            cmd_data = aid.to_tlv() + h2b('5c054f9f70c5cc')
-            p2 = 0x02 # TLV format according to Table 11-36
+            # GPC CardSpec v2.3.1 Table 11-35 says only the AID search tag is mandatory, tag list is
+            # Optional and not present in the older v2.1.1, where section 9.4.2.3 defines the data
+            # field as the search qualifier.
+            # Cards like the sja5 implementing that old GP version reject anything else with 6A80
+            # from v2.1.1 Table 9-26 so only send a tag list to a card that announces v2.2 or later.
+            #
+            # Not sending one is not a problem on older cards, the tag list only gives us data beyond
+            # what 11.4.3.1 gives us anyway, for example the associated SD AID which matters on an eUICC
+            # where entries belong to different SD.
+            version = self.gp_version()
+            log.debug("Card Recognition Data reports GlobalPlatform %s",
+                      '.'.join(str(v) for v in version) if version else 'unknown')
+            if version is not None and version >= (2, 2):
+                try:
+                    return self._get_status(subset, aid.to_tlv() + get_status_tag_list(subset))
+                except SwMatchError as e:
+                    # Retry if v2.2 or later but rejected the tag list anyway.
+                    # 6A80 and 6A88 are the error conditions GET STATUS defines in table 11-39.
+                    # Retrying beats not ending up with a list again...
+                    if e.sw_actual not in ('6a80', '6a88'):
+                        raise
+                    log.warning("Card reports GlobalPlatform %s but answered %s to the GET STATUS tag list; "
+                                "retrying with the default search",
+                                '.'.join(str(v) for v in version), e.sw_actual)
+            return self._get_status(subset, aid.to_tlv(), empty_on_6a88=True)
+
+        def _get_status(self, subset:str, cmd_data:bytes,
+                        empty_on_6a88: bool = False) -> List[GpRegistryRelatedData]:
+            subset_hex = b2h(build_construct(StatusSubset, subset))
+            p2 = 0x02 # GPC v2.3.1 11.4.2.2 table 11-34, b2: response data structure per table 11-36
             grd_list = []
             while True:
                 hdr = "80F2%s%02x%02x" % (subset_hex, p2, len(cmd_data))
                 data, sw = self._cmd.lchan.scc.send_apdu(hdr + b2h(cmd_data) + "00")
+                if sw == '6a88':
+                    # Table 11-39 "Referenced data not found". After collecting all pages this can
+                    # only mean "nothing more matches" -> listing is complete. On the first page
+                    # it is ambiguous, empty result or bad command data field, so leave that to get_status()
+                    # which knows if a tag list was sent.
+                    if grd_list or empty_on_6a88:
+                        return grd_list
+                    raise SwMatchError(sw, ['9000', '6310'])
+                if sw not in ['9000', '6310']:
+                    # Never return a silently truncated registry
+                    raise SwMatchError(sw, ['9000', '6310'])
                 remainder = h2b(data)
                 while len(remainder):
                     # tlv sequence, each element is one GpRegistryRelatedData()
                     grd = GpRegistryRelatedData()
                     _dec, remainder = grd.from_tlv(remainder)
                     grd_list.append(grd)
-                if sw != '6310':
+                if sw == '9000':
                     return grd_list
-                else:
-                    p2 |= 0x01
-            return grd_list
+                # 6310 = more data available, table 11-38: reissue as get next occurrence(s), b1 of
+                # table 11-34. Keeps b2 unchanged.
+                p2 |= 0x01
 
         set_status_parser = argparse.ArgumentParser()
         set_status_parser.add_argument('scope', choices=list(SetStatusScope.ksymapping.values()),
@@ -826,23 +1006,32 @@ class ADF_SD(CardADF):
         load_parser_from_grp.add_argument('--from-hex', type=is_hexstr, help='load from hex string')
         load_parser_from_grp.add_argument('--from-file', type=argparse.FileType('rb', 0), help='load from binary file')
         load_parser_from_grp.add_argument('--from-cap-file', type=argparse.FileType('rb', 0), help='load from JAVA-card CAP file')
+        load_parser.add_argument('--chunk-len', type=auto_uint8, default=None,
+                                 help='Block size for the LOAD command; default: as large as the current secure channel overhead permits, at most 240')
 
         @cmd2.with_argparser(load_parser)
         def do_load(self, opts):
             """Perform a GlobalPlatform LOAD command. (We currently only support loading without DAP and
             without ciphering.)"""
             if opts.from_hex is not None:
-                self.load(h2b(opts.from_hex))
+                self.load(h2b(opts.from_hex), opts.chunk_len)
             elif opts.from_file is not None:
-                self.load(opts.from_file.read())
+                self.load(opts.from_file.read(), opts.chunk_len)
             elif opts.from_cap_file is not None:
                 cap = CapFile(opts.from_cap_file)
-                self.load(cap.get_loadfile())
+                self.load(cap.get_loadfile(), opts.chunk_len)
             else:
                 raise ValueError('load source not specified!')
 
-        def load(self, contents:bytes, chunk_len:int = 240):
-            # TODO:tune chunk_len based on the overhead of the used SCP?
+        def load(self, contents:bytes, chunk_len:Optional[int] = None):
+            # scc.max_cmd_len knows the overhead the currently active SCP
+            # 240 is the old default, keep it for now.
+            max_chunk_len = self._cmd.lchan.scc.max_cmd_len
+            if chunk_len is None:
+                chunk_len = min(240, max_chunk_len)
+            elif not 1 <= chunk_len <= max_chunk_len:
+                raise ValueError('chunk_len must be in range 1..%u (limited by the overhead of the current secure channel)' %
+                                 max_chunk_len)
             # build TLV according to GPC_SPE_034 section 11.6.2.3 / Table 11-58 for unencrypted case
             remainder = b'\xC4' + bertlv_encode_len(len(contents)) + contents
             # transfer this in various chunks to the card
@@ -881,6 +1070,8 @@ class ADF_SD(CardADF):
         install_cap_parser_inst_prm_grp.add_argument('--install-parameters-stk',
                                                      type=is_hexstr, default=None,
                                                      help='Load Parameters (ETSI TS 102 226, section 8.2.1.3.2.1)')
+        install_cap_parser.add_argument('--chunk-len', type=auto_uint8, default=None,
+                                        help='Block size for the LOAD command; default: as large as the current secure channel overhead permits, at most 240')
 
         @cmd2.with_argparser(install_cap_parser)
         def do_install_cap(self, opts):
@@ -919,7 +1110,7 @@ class ADF_SD(CardADF):
             self._cmd.poutput("step #1: install for load...")
             self.do_install_for_load("--load-file-aid %s --security-domain-aid %s" % (load_file_aid, security_domain_aid))
             self._cmd.poutput("step #2: load...")
-            self.load(load_file)
+            self.load(load_file, opts.chunk_len)
             self._cmd.poutput("step #3: install_for_install (and make selectable)...")
             self.do_install_for_install("--load-file-aid %s --module-aid %s --application-aid %s --install-parameters %s --make-selectable" %
                                         (load_file_aid, module_aid, application_aid, install_parameters))
@@ -1065,10 +1256,16 @@ def compute_kcv_aes(key:bytes) -> bytes:
     cipher = AES.new(key, AES.MODE_ECB)
     return cipher.encrypt(plaintext)
 
+def compute_kcv_psk(key:bytes) -> bytes:
+    # GP Amendment B v1.2, 3.9.1 / Table 3-13
+    # KCV of a PSK TLS key is the 3 highest-order bytes of the SHA-1 digest of the clear key value.
+    return hashlib.sha1(key).digest()
+
 # dict is keyed by the string name of the KeyType enum above in this file
 KCV_CALCULATOR = {
         'aes': compute_kcv_aes,
         'des': compute_kcv_des,
+        'tls_psk': compute_kcv_psk,
     }
 
 def compute_kcv(key_type: str, key: bytes) -> Optional[bytes]:

pySim/global_platform/scp.py

@@ -27,9 +27,9 @@ from osmocom.utils import b2h
 from osmocom.tlv import bertlv_parse_len, bertlv_encode_len
 from pySim.utils import parse_command_apdu
 from pySim.secure_channel import SecureChannel
+from pySim.log import PySimLogger
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
+log = PySimLogger.get(__name__)
 
 def scp02_key_derivation(constant: bytes, counter: int, base_key: bytes) -> bytes:
     assert len(constant) == 2
@@ -75,7 +75,7 @@ class Scp02SessionKeys:
             h = e.encrypt(strxor(h, bytes(padded_data[8*i:8*(i+1)])))
         h = d.decrypt(h)
         h = e.encrypt(h)
-        logger.debug("mac_1des(%s,icv=%s) -> %s", b2h(data), b2h(icv), b2h(h))
+        log.debug("mac_1des(%s,icv=%s) -> %s", b2h(data), b2h(icv), b2h(h))
         if self.des_icv_enc:
             self.icv = self.des_icv_enc.encrypt(h)
         else:
@@ -89,7 +89,7 @@ class Scp02SessionKeys:
         h = b'\x00' * 8
         for i in range(q):
             h = e.encrypt(strxor(h, bytes(padded_data[8*i:8*(i+1)])))
-        logger.debug("mac_3des(%s) -> %s", b2h(data), b2h(h))
+        log.debug("mac_3des(%s) -> %s", b2h(data), b2h(h))
         return h
 
     def __init__(self, counter: int, card_keys: 'GpCardKeyset', icv_encrypt=True):
@@ -182,6 +182,29 @@ class SCP(SecureChannel, abc.ABC):
         """Should we perform R-ENC?"""
         return self.security_level & 0x20
 
+    @property
+    @abc.abstractmethod
+    def mac_len(self) -> int:
+        """Length of the appended C-MAC, to be provided by derived class."""
+
+    @property
+    def overhead(self) -> int:
+        """Worst-case len that wrapping a command APDU adds to its data field at the
+        current sec level is (255 - overhead), C-MAC + C-DECRYPTION encryption padding."""
+        if not self.do_cmac:
+            return 0
+        if not self.do_cenc:
+            return self.mac_len
+        # see Secure Channel Protocol '03' Card Specification v2.3 - Amendment D v1.1.2
+        # which defers to GPCS v2.3 Section B.2 which then defers to
+        # NIST SP 800-38B for encryption and points out that
+        # the padding is, as expected, just the usual padding from NIST SP 800-38A
+        # C-DECRYPTION pads with ('80'+['00'...] at least 1 byte) up to
+        # the cipher block size + C-MAC on top -> largest usable data field
+        # is one byte less than the largest block-size multiple within 255 - mac_len.
+        bs = self.sk.blocksize
+        return 255 - ((255 - self.mac_len) // bs * bs - 1)
+
     def __str__(self) -> str:
         return "%s[%02x]" % (self.__class__.__name__, self.security_level)
 
@@ -215,11 +238,20 @@ class SCP(SecureChannel, abc.ABC):
     def gen_ext_auth_apdu(self, security_level: int = 0x01) -> bytes:
         pass
 
+    def pad_to_blocksize(self, data: bytes) -> bytes:
+        """Right pad the data with zero bytes to a multiple of the DEK cipher block size."""
+        if len(data) % self.sk.blocksize:
+            # not '+=' which would mutate the callers bytearray in place..
+            data = data + b'\x00' * (self.sk.blocksize - len(data) % self.sk.blocksize)
+        return data
+
     def encrypt_key(self, key: bytes) -> bytes:
         """Encrypt a key with the DEK."""
-        num_pad = len(key) % self.sk.blocksize
-        if num_pad:
-            return bertlv_encode_len(len(key)) + self.dek_encrypt(key + b'\x00'*num_pad)
+        if len(key) % self.sk.blocksize:
+            # The kcv is right padded before encryption and the kcb
+            # is formatted as described in Table 11-70: preceded by the actual length of the
+            # clear text kcv.
+            return bertlv_encode_len(len(key)) + self.dek_encrypt(self.pad_to_blocksize(key))
         return self.dek_encrypt(key)
 
     def decrypt_key(self, encrypted_key:bytes) -> bytes:
@@ -232,9 +264,8 @@ class SCP(SecureChannel, abc.ABC):
             # Block provides the actual length of the key component value, which allows recovering the
             # clear-text key component value after decryption of the encrypted key component value and removal
             # of padding bytes.
-            decrypted = self.dek_decrypt(encrypted_key)
-            key_len, remainder = bertlv_parse_len(decrypted)
-            return remainder[:key_len]
+            key_len, remainder = bertlv_parse_len(encrypted_key)
+            return self.dek_decrypt(remainder)[:key_len]
         else:
             # If the length of the Key Component Block is a multiple of the block size of the encryption
             # algorithm (i.e.  8 bytes for DES, 16 bytes for AES), then it shall be assumed that no padding
@@ -260,10 +291,8 @@ class SCP02(SCP):
     # Key Version Number 0x70 is a non-spec special-case of sysmoISIM-SJA2/SJA5 and possibly more sysmocom products
     # Key Version Number 0x01 is a non-spec special-case of sysmoUSIM-SJS1
     kvn_ranges = [[0x01, 0x01], [0x20, 0x2f], [0x70, 0x70]]
-
-    def __init__(self, *args, **kwargs):
-        self.overhead = 8
-        super().__init__(*args, **kwargs)
+    # C-MAC (Single DES + final 3DES, B.1.2.2) is always one full DES block
+    mac_len = 8
 
     def dek_encrypt(self, plaintext:bytes) -> bytes:
         # See also GPC section B.1.1.2, E.4.7, and E.4.1
@@ -276,10 +305,10 @@ class SCP02(SCP):
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self, card_challenge: bytes, host_challenge: bytes):
-        logger.debug("host_challenge(%s), card_challenge(%s)", b2h(host_challenge), b2h(card_challenge))
+        log.debug("host_challenge(%s), card_challenge(%s)", b2h(host_challenge), b2h(card_challenge))
         self.host_cryptogram = self.sk.calc_mac_3des(self.sk.counter.to_bytes(2, 'big') + card_challenge + host_challenge)
         self.card_cryptogram = self.sk.calc_mac_3des(self.host_challenge + self.sk.counter.to_bytes(2, 'big') + card_challenge)
-        logger.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
+        log.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
 
     def gen_init_update_apdu(self, host_challenge: bytes = b'\x00'*8) -> bytes:
         """Generate INITIALIZE UPDATE APDU."""
@@ -291,7 +320,7 @@ class SCP02(SCP):
         resp = self.constr_iur.parse(resp_bin)
         self.card_challenge = resp['card_challenge']
         self.sk = Scp02SessionKeys(resp['seq_counter'], self.card_keys)
-        logger.debug(self.sk)
+        log.debug(self.sk)
         self._compute_cryptograms(self.card_challenge, self.host_challenge)
         if self.card_cryptogram != resp['card_cryptogram']:
             raise ValueError("card cryptogram doesn't match")
@@ -311,7 +340,7 @@ class SCP02(SCP):
 
     def _wrap_cmd_apdu(self, apdu: bytes, *args, **kwargs) -> bytes:
         """Wrap Command APDU for SCP02: calculate MAC and encrypt."""
-        logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
+        log.debug("wrap_cmd_apdu(%s)", b2h(apdu))
 
         if not self.do_cmac:
             return apdu
@@ -338,10 +367,16 @@ class SCP02(SCP):
             # CMAC on modified APDU
             mlc = lc + 8
             clac = cla | CLA_SM
+        if mlc >= 256:
+            raise ValueError('Modified Lc (%u) would exceed maximum when appending 8 bytes of mac' % mlc)
         mac = self.sk.calc_mac_1des(bytes([clac]) + apdu[1:4] + bytes([mlc]) + data)
         if self.do_cenc:
+            padded_data = pad80(data, 8)
+            if len(padded_data) + 8 >= 256:
+                raise ValueError('Modified Lc (%u) would exceed maximum when appending padding and mac' %
+                                 (len(padded_data) + 8))
             k = DES3.new(self.sk.enc, DES.MODE_CBC, b'\x00'*8)
-            data = k.encrypt(pad80(data, 8))
+            data = k.encrypt(padded_data)
             lc = len(data)
 
         lc += 8
@@ -378,7 +413,7 @@ def scp03_key_derivation(constant: bytes, context: bytes, base_key: bytes, l: Op
     if l is None:
         l = len(base_key) * 8
 
-    logger.debug("scp03_kdf(constant=%s, context=%s, base_key=%s, l=%u)", b2h(constant), b2h(context), b2h(base_key), l)
+    log.debug("scp03_kdf(constant=%s, context=%s, base_key=%s, l=%u)", b2h(constant), b2h(context), b2h(base_key), l)
     output_len = l // 8
     # SCP03 Section 4.1.5 defines a different parameter order than NIST SP 800-108, so we cannot use the
     # existing Cryptodome.Protocol.KDF.SP800_108_Counter function :(
@@ -451,7 +486,7 @@ class Scp03SessionKeys:
         # This block SHALL be encrypted with S-ENC to produce the ICV for command encryption.
         cipher = AES.new(self.s_enc, AES.MODE_CBC, iv)
         icv = cipher.encrypt(data)
-        logger.debug("_get_icv(data=%s, is_resp=%s) -> icv=%s", b2h(data), is_response, b2h(icv))
+        log.debug("_get_icv(data=%s, is_resp=%s) -> icv=%s", b2h(data), is_response, b2h(icv))
         return icv
 
     # TODO: Resolve duplication with pySim.esim.bsp.BspAlgoCryptAES128 which provides pad80-wrapping
@@ -477,9 +512,13 @@ class SCP03(SCP):
 
     def __init__(self, *args, **kwargs):
         self.s_mode = kwargs.pop('s_mode', 8)
-        self.overhead = self.s_mode
         super().__init__(*args, **kwargs)
 
+    @property
+    def mac_len(self) -> int:
+        # C-MAC truncated to 8 in S8 or 16 bytes in S16 mode
+        return self.s_mode
+
     def dek_encrypt(self, plaintext:bytes) -> bytes:
         cipher = AES.new(self.card_keys.dek, AES.MODE_CBC, b'\x00'*16)
         return cipher.encrypt(plaintext)
@@ -489,12 +528,12 @@ class SCP03(SCP):
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self):
-        logger.debug("host_challenge(%s), card_challenge(%s)", b2h(self.host_challenge), b2h(self.card_challenge))
+        log.debug("host_challenge(%s), card_challenge(%s)", b2h(self.host_challenge), b2h(self.card_challenge))
         # Card + Host Authentication Cryptogram: Section 6.2.2.2 + 6.2.2.3
         context = self.host_challenge + self.card_challenge
         self.card_cryptogram = scp03_key_derivation(self.sk.DERIV_CONST_AUTH_CGRAM_CARD, context, self.sk.s_mac, l=self.s_mode*8)
         self.host_cryptogram = scp03_key_derivation(self.sk.DERIV_CONST_AUTH_CGRAM_HOST, context, self.sk.s_mac, l=self.s_mode*8)
-        logger.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
+        log.debug("host_cryptogram(%s), card_cryptogram(%s)", b2h(self.host_cryptogram), b2h(self.card_cryptogram))
 
     def gen_init_update_apdu(self, host_challenge: Optional[bytes] = None) -> bytes:
         """Generate INITIALIZE UPDATE APDU."""
@@ -514,7 +553,7 @@ class SCP03(SCP):
         self.i_param = resp['i_param']
         # derive session keys and compute cryptograms
         self.sk = Scp03SessionKeys(self.card_keys, self.host_challenge, self.card_challenge)
-        logger.debug(self.sk)
+        log.debug(self.sk)
         self._compute_cryptograms()
         # verify computed cryptogram matches received cryptogram
         if self.card_cryptogram != resp['card_cryptogram']:
@@ -529,7 +568,7 @@ class SCP03(SCP):
 
     def _wrap_cmd_apdu(self, apdu: bytes, skip_cenc: bool = False) -> bytes:
         """Wrap Command APDU for SCP03: calculate MAC and encrypt."""
-        logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
+        log.debug("wrap_cmd_apdu(%s)", b2h(apdu))
 
         if not self.do_cmac:
             return apdu
@@ -584,7 +623,7 @@ class SCP03(SCP):
         # status word: in this case only the status word shall be returned in the response. All status words
         # except '9000' and warning status words (i.e. '62xx' and '63xx') shall be interpreted as error status
         # words.
-        logger.debug("unwrap_rsp_apdu(sw=%s, rsp_apdu=%s)", sw, rsp_apdu)
+        log.debug("unwrap_rsp_apdu(sw=%s, rsp_apdu=%s)", sw, rsp_apdu)
         if not self.do_rmac:
             assert not self.do_renc
             return rsp_apdu
@@ -600,9 +639,9 @@ class SCP03(SCP):
         if self.do_renc:
             # decrypt response data
             decrypted = self.sk._decrypt(response_data)
-            logger.debug("decrypted: %s", b2h(decrypted))
+            log.debug("decrypted: %s", b2h(decrypted))
             # remove padding
             response_data = unpad80(decrypted)
-            logger.debug("response_data: %s", b2h(response_data))
+            log.debug("response_data: %s", b2h(response_data))
 
         return response_data

pySim/global_platform/uicc.py

@@ -91,6 +91,7 @@ class UiccSdInstallParams(TLV_IE_Collection, nested=[UiccScp, AcceptExtradAppsAn
 
 # Key Usage:
 # KVN 0x01 .. 0x0F reserved for SCP80
+# KVN 0x81 .. 0x8f reserved for SCP81
 # KVN 0x11 reserved for DAP specified in ETSI TS 102 226
 # KVN 0x20 .. 0x2F reserved for SCP02
 #   KID 0x01 = ENC; 0x02 = MAC; 0x03 = DEK

pySim/legacy/cards.py

@@ -152,7 +152,8 @@ class SimCard(SimCardBase):
         return sw
 
     def update_smsp(self, smsp):
-        data, sw = self._scc.update_record(EF['SMSP'], 1, rpad(smsp, 84))
+        print("using update_smsp")
+        data, sw = self._scc.update_record(EF['SMSP'], 1, smsp, leftpad=True)
         return sw
 
     def update_ad(self, mnc=None, opmode=None, ofm=None, path=EF['AD']):

pySim/log.py

@@ -24,7 +24,16 @@
 #
 
 import logging
-from cmd2 import style
+import enum
+import cmd2
+from packaging import version
+
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
+    from cmd2 import stylize as _stylize # pylint: disable=no-name-in-module
+    def _style(text, fg=None): # pylint: disable=function-redefined
+        return _stylize(text, fg) if fg else text
+else: # cmd2>=2.6.2
+    from cmd2 import style as _style # pylint: disable=no-name-in-module
 
 class _PySimLogHandler(logging.Handler):
     def __init__(self, log_callback):
@@ -44,7 +53,7 @@ class PySimLogger:
     """
 
     LOG_FMTSTR = "%(levelname)s: %(message)s"
-    LOG_FMTSTR_VERBOSE = "%(module)s.%(lineno)d -- " + LOG_FMTSTR
+    LOG_FMTSTR_VERBOSE = "%(name)s.%(lineno)d -- " + LOG_FMTSTR
     __formatter = logging.Formatter(LOG_FMTSTR)
     __formatter_verbose = logging.Formatter(LOG_FMTSTR_VERBOSE)
 
@@ -118,10 +127,10 @@ class PySimLogger:
                 formatted_message = logging.Formatter.format(PySimLogger.__formatter, record)
             color = PySimLogger.colors.get(record.levelno)
             if color:
-                if isinstance(color, str):
+                if isinstance(color, str) and not isinstance(color, enum.Enum):
                     PySimLogger.print_callback(color + formatted_message + "\033[0m")
                 else:
-                    PySimLogger.print_callback(style(formatted_message, fg = color))
+                    PySimLogger.print_callback(_style(formatted_message, fg = color))
             else:
                 PySimLogger.print_callback(formatted_message)
 

pySim/ota.py

@@ -18,10 +18,12 @@
 import zlib
 import abc
 import struct
-from typing import Optional, Tuple
-from construct import Enum, Int8ub, Int16ub, Struct, BitsInteger, BitStruct
+from typing import Optional, Tuple, List, Union
+from construct import ConstructError, Enum, Int8ub, Int16ub, Struct, BitsInteger, BitStruct
 from construct import Flag, Padding, Switch, this, PrefixedArray, GreedyRange
+from construct import Const, Prefixed, Select, Construct, SizeofError, stream_read, stream_write
 from osmocom.construct import *
+from osmocom.tlv import bertlv_encode_len
 from osmocom.utils import b2h
 
 from pySim.sms import UserDataHeader
@@ -56,6 +58,217 @@ CompactRemoteResp = Struct('number_of_commands'/Int8ub,
                            'last_status_word'/HexAdapter(Bytes(2)),
                            'last_response_data'/HexAdapter(GreedyBytes))
 
+######################################################################
+# Expanded Remote Application data format, ETSI TS 102 226 V19.0.0 (2025-11) Section 5.2
+#   5.2.1   Expanded Remote command structure
+#   5.2.1.1 C-APDU TLV
+#   5.2.1.2 Immediate Action TLV
+#   5.2.1.3 Error Action TLV
+#   5.2.1.4 Script Chaining TLV
+#   5.2.2   Expanded Remote response structure (tables 5.10 .. 5.16)
+#
+# definite length coding and indefinite length coding are supported.
+#
+# BER-TLV tag values from ETSI TS 101 220 V19.0.0 tables 7.18, 7.19, 7.20
+# C-APDU / R-APDU ETSI TS 102 223 Section 8.35 + 8.36
+# inside these the CR flag of the tag is 0 (TS 101 220 tables 7.19/7.20),
+# so tag bytes are 22 and 23 and not A2/A3.
+#
+# This layer sits above the TS 102 225 security layer.
+######################################################################
+
+class BerTlvLength(Construct):
+    """A definite-length BER-TLV length field used by the "expanded remote
+    application data format" from ISO/IEC 8825-1 referenced by TS 102 226 5.2
+
+    - short form (0..127 -> single octet)
+    - long form (128.. -> 0x8N followed by N length octets)
+    Indefinite length coding (first octet 0x80, TS 102 226 tables 5.2a/5.10a)
+    is omitted here because it is only recommended for HTTPS/CoAP transport, not SMS."""
+    def _parse(self, stream, context, path):
+        first = stream_read(stream, 1, path)[0]
+        if first < 0x80:
+            return first
+        num_octets = first & 0x7f
+        if num_octets == 0:
+            raise NotImplementedError('indefinite coding is not supported')
+        return int.from_bytes(stream_read(stream, num_octets, path), 'big')
+
+    def _build(self, obj, stream, context, path):
+        encoded = bertlv_encode_len(obj)
+        stream_write(stream, encoded, len(encoded), path)
+        return obj
+
+    def _sizeof(self, context, path):
+        raise SizeofError('BER-TLV length has a variable size?!')
+
+BerTlvLen = BerTlvLength()
+
+class _RApduValueAdapter(Adapter):
+    """Split/join value of R-APDU COMPREHENSION-TLV TS 102 223 8.36
+      [R-APDU data (x-2 bytes)] SW1 SW2."""
+    def _decode(self, obj, context, path):
+        raw = bytes(obj)
+        return Container(response_data=b2h(raw[:-2]), status_word=b2h(raw[-2:]))
+
+    def _encode(self, obj, context, path):
+        return h2b(obj['response_data']) + h2b(obj['status_word'])
+
+#### Command Scripting template TS 102 226 tables 5.2 / 5.2a, TS 101 220 tables 7.18/7.19
+#
+# The two TS 101 220 table 7.18 length codings use different template tags:
+# - definite tag AA
+# - indefinite AE
+# In both codings the inner Command TLVs use definite length coding, only the
+# surrounding template differs.
+
+# TS 102 223 8.35
+ExpandedC_APDU = Struct('_tag'/Const(b'\x22'),
+                        'c_apdu'/Prefixed(BerTlvLen, HexAdapter(GreedyBytes)))
+
+# shared by both length codings.
+ExpandedCmdItems = GreedyRange(ExpandedC_APDU)
+
+# TS 102 226 table 5.2: Command Scripting template, definite length coding only
+ExpandedCmd = Struct('_tag'/Const(b'\xaa'),
+                     'commands'/Prefixed(BerTlvLen, ExpandedCmdItems))
+
+# TS 102 226 table 5.2a: indefinite length coding, 'AE 80 <C-APDU TLVs> 00 00'. GreedyRange
+# stops at the first octet that is not a C-APDU tag, which is the end-of-contents marker.
+ExpandedCmdIndef = Struct('_tag'/Const(b'\xae'), '_indef'/Const(b'\x80'),
+                          'commands'/ExpandedCmdItems, '_eoc'/Const(b'\x00\x00'))
+
+#### Response Scripting template TS 102 226 5.2.2, tables 5.10-5.16, TS 101 220 table 7.20
+
+# TS 102 223 8.36
+ExpandedR_APDU = Struct('_tag'/Const(b'\x23'),
+                        'r_apdu'/Prefixed(BerTlvLen, _RApduValueAdapter(GreedyBytes)))
+
+# TS 102 226 table 5.11
+# Value is an integer per ISO/IEC 8825-1, likely just one octet.
+ExpandedNumExecuted = Struct('_tag'/Const(b'\x80'),
+                             'number_of_commands'/Prefixed(BerTlvLen, GreedyInteger()))
+
+# TS 102 226 table 5.12
+ExpandedBadFormat = Struct('_tag'/Const(b'\x90'),
+                           'bad_format'/Prefixed(BerTlvLen,
+                               Enum(Int8ub, unknown_tag=1, wrong_length=2, length_not_found=3)))
+
+# TS 102 226 table 5.14
+ExpandedImmediateActionResp = Struct('_tag'/Const(b'\x81'),
+                                     'immediate_action_response'/Prefixed(BerTlvLen,
+                                         Enum(Int8ub, suspension_error=1)))
+
+# TS 102 226 table 5.16
+ExpandedScriptChainingResp = Struct('_tag'/Const(b'\x83'),
+                                    'script_chaining_response'/Prefixed(BerTlvLen,
+                                        Enum(Int8ub, no_previous_script=1,
+                                             not_supported=2, unable_to_process=3)))
+
+# response TLVs shared by the def and indef Response Scripting templates
+ExpandedRespItems = GreedyRange(Select(ExpandedR_APDU,
+                                       ExpandedBadFormat,
+                                       ExpandedImmediateActionResp,
+                                       ExpandedScriptChainingResp))
+
+# - starts with the "Number of executed command TLV objects" (table 5.10/5.13/5.15)
+# - followed by a sequence of R-APDU TLVs
+# - and/or one of the error # response TLVs
+ExpandedRemoteResp = Struct('_tag'/Const(b'\xab'),
+                            'body'/Prefixed(BerTlvLen, Struct(
+                                'num_executed'/ExpandedNumExecuted,
+                                'responses'/ExpandedRespItems)))
+
+# TS 102 226 table 5.10a: indefinite length coding, no "number of executed" TLV
+ExpandedRemoteRespIndef = Struct('_tag'/Const(b'\xaf'), '_indef'/Const(b'\x80'),
+                                 'responses'/ExpandedRespItems, '_eoc'/Const(b'\x00\x00'))
+
+
+def encode_expanded_cmd(apdus: Union[bytes, List[bytes]],
+                        length_coding: str = 'definite') -> bytes:
+    """builds the Command Scripting template, TS 102 226 5.2.1
+
+    Args:
+        apdus: single C-APDU bytes or list of C-APDUs bytes. Each
+               C-APDU is wrapped into a C-APDU TLV- This function does not add
+               or modify Le.
+        length_coding: 'definite' (the default, tag 'AA', table 5.2) or
+               'indefinite' (tag 'AE', table 5.2a: 'AE 80 <cmd TLVs> 00 00').
+               Inner C-APDU TLVs use definite length coding in both cases.
+    Returns:
+        encoded Command Scripting template as bytes
+    """
+    if isinstance(apdus, (bytes, bytearray)):
+        apdus = [apdus]
+    commands = [{'c_apdu': b2h(a)} for a in apdus]
+    if length_coding == 'definite':
+        return ExpandedCmd.build({'commands': commands})
+    if length_coding == 'indefinite':
+        return ExpandedCmdIndef.build({'commands': commands})
+    raise ValueError("Invalid length_coding: %r" % length_coding)
+
+
+def decode_expanded_resp(data: bytes) -> Container:
+    """Decode a Response Scripting template, TS 102 226 5.2.2 def and indef length
+    coding
+
+    returned Container has:
+        number_of_commands   -- "number of executed command TLV objects" table 5.11
+                                 for definite coding. indefinite coding does not have
+                                 this TLV, so report the number of returned R-APDUs instead.
+        commands             -- list of Containers, one per R-APDU TLV, each
+                                with 'response_data' and 'status_word' hexstr
+        last_response_data   -- response_data of the last R-APDU or ''
+        last_status_word     -- status_word of the last R-APDU or None
+        truncated            -- True if any R-APDU has SW 62F1.
+                                5.2.1.1 states card sets that status when it had to truncate
+                                C-APDU response data, and "this shall terminate the
+                                processing of the command list".
+                                so the response is short AND the remaining commands never ran.
+        bad_format           -- error type of a trailing Bad format TLV if present
+        immediate_action_response -- Immediate Action Response TLV, if there was a suspension error
+        script_chaining_response  -- Script Chaining Response TLV, if there was a chaining error
+
+    The 'last_response_data'/'last_status_word'/'number_of_commands' keys are compatible with
+    CompactRemoteResp so existing callers keep working."""
+    if isinstance(data, str):
+        data = h2b(data)
+    try:
+        if data[:1] == b'\xaf':
+            responses = ExpandedRemoteRespIndef.parse(data)['responses']
+            num_executed = None
+        else:
+            parsed = ExpandedRemoteResp.parse(data)
+            responses = parsed['body']['responses']
+            num_executed = parsed['body']['num_executed']['number_of_commands']
+    except ConstructError as e:
+        raise ValueError('malformed Response Scripting template: %s' % e) from e
+
+    commands = []
+    bad_format = None
+    immediate_action_response = None
+    script_chaining_response = None
+    for item in responses:
+        if 'r_apdu' in item:
+            commands.append(Container(response_data=item['r_apdu']['response_data'],
+                                      status_word=item['r_apdu']['status_word']))
+        elif 'bad_format' in item:
+            bad_format = item['bad_format']
+        elif 'immediate_action_response' in item:
+            immediate_action_response = item['immediate_action_response']
+        elif 'script_chaining_response' in item:
+            script_chaining_response = item['script_chaining_response']
+    # TS 102 226 5.2.1.1: 62F1 means response of a C-APDU was truncated, processing terminated
+    truncated = any(c['status_word'].lower() == '62f1' for c in commands)
+    return Container(number_of_commands=num_executed if num_executed is not None else len(commands),
+                     commands=commands,
+                     last_response_data=commands[-1]['response_data'] if commands else '',
+                     last_status_word=commands[-1]['status_word'] if commands else None,
+                     truncated=truncated,
+                     bad_format=bad_format,
+                     immediate_action_response=immediate_action_response,
+                     script_chaining_response=script_chaining_response)
+
 RC_CC_DS = Enum(BitsInteger(2), no_rc_cc_ds=0, rc=1, cc=2, ds=3)
 CNTR_REQ = Enum(BitsInteger(2), no_counter=0, counter_no_replay_or_seq=1, counter_must_be_higher=2, counter_must_be_lower=3)
 POR_REQ = Enum(BitsInteger(2), no_por=0, por_required=1, por_only_when_error=2)
@@ -149,13 +362,23 @@ class OtaDialect(abc.ABC):
         raise ValueError("Invalid rc_cc_ds: %s" % spi['rc_cc_ds'])
 
     @abc.abstractmethod
-    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict, apdu: bytes) -> bytes:
+    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict,
+                   apdu: Union[bytes, List[bytes]], remote_format: str = 'compact') -> bytes:
+        """Encode a command for a format.
+
+        remote_format:
+        'compact' TS 102 226 5.1, DEFAULT assumes apdus are opaque already-concatenated command strings
+        'expanded' TS 102 226 5.2 wraps a single C-APDU or list of C-APDUs in a Command Scripting template."""
         pass
 
     @abc.abstractmethod
-    def decode_resp(self, otak: OtaKeyset, spi: dict, apdu: bytes) -> (object, Optional["CompactRemoteResp"]):
-        """Decode a response into a response packet and, if indicted (by a
-        response status of `"por_ok"`) a decoded response.
+    def decode_resp(self, otak: OtaKeyset, spi: dict, apdu: bytes,
+                    remote_format: str = 'compact') -> (object, Optional[object]):
+        """Decode response into response packet + a decoded response if por_ok.
+
+        remote_format:
+        'compact' -> DEFAULT TS 102 226 5.1.2 CompactRemoteResp2
+        'expanded' -> container returned by decode_expanded_resp(), TS 102 226 5.2.2
 
         The response packet's common characteristics are not fully determined,
         and (so far) completely proprietary per dialect."""
@@ -335,7 +558,16 @@ class OtaDialectSms(OtaDialect):
                                'secured_data'/GreedyBytes)
     hdr_construct = Struct('chl'/Int8ub, 'spi'/SPI, 'kic'/KIC, 'kid'/KID_CC, 'tar'/Bytes(3))
 
-    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict, apdu: bytes) -> bytes:
+    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict,
+                   apdu: Union[bytes, List[bytes]], remote_format: str = 'compact') -> bytes:
+        # as above:
+        # expanded format is a Command Scripting template wrapping the C-APDU(s)
+        # compact format passes already concatenated command string
+        if remote_format == 'expanded':
+            apdu = encode_expanded_cmd(apdu)
+        elif remote_format != 'compact':
+            raise ValueError("Invalid remote_format: %s" % remote_format)
+
         # length of signature in octets
         len_sig = self._compute_sig_len(spi)
         pad_cnt = 0
@@ -446,7 +678,10 @@ class OtaDialectSms(OtaDialect):
         return hdr_dec['tar'], spi, apdu
 
 
-    def decode_resp(self, otak: OtaKeyset, spi: dict, data: bytes) -> ("OtaDialectSms.SmsResponsePacket", Optional["CompactRemoteResp"]):
+    def decode_resp(self, otak: OtaKeyset, spi: dict, data: bytes,
+                    remote_format: str = 'compact') -> ("OtaDialectSms.SmsResponsePacket", Optional[object]):
+        if remote_format not in ('compact', 'expanded'):
+            raise ValueError("Invalid remote_format: %s ?!" % remote_format)
         if isinstance(data, str):
             data = h2b(data)
         # plain-text POR:   027100000e0ab000110000000000000001612f
@@ -492,9 +727,11 @@ class OtaDialectSms(OtaDialect):
         else:
             raise OtaCheckError('Unknown por_rc_cc_ds: %s' % spi['por_rc_cc_ds'])
 
-        # TODO: ExpandedRemoteResponse according to TS 102 226 5.2.2
         if res.response_status == 'por_ok' and len(res['secured_data']):
-            dec = CompactRemoteResp.parse(res['secured_data'])
+            if remote_format == 'expanded':
+                dec = decode_expanded_resp(res['secured_data'])
+            else:
+                dec = CompactRemoteResp.parse(res['secured_data'])
         else:
             dec = None
         return (res, dec)

pySim/sms.py

@@ -19,6 +19,7 @@
 
 import typing
 import abc
+import logging
 from bidict import bidict
 from construct import Int8ub, Byte, Bit, Flag, BitsInteger
 from construct import Struct, Enum, Tell, BitStruct, this, Padding
@@ -28,6 +29,8 @@ from osmocom.utils import Hexstr, h2b, b2h
 
 from smpp.pdu import pdu_types, operations
 
+logger = logging.getLogger(__name__)
+
 BytesOrHex = typing.Union[Hexstr, bytes]
 
 class UserDataHeader:
@@ -60,6 +63,109 @@ class UserDataHeader:
         return self._construct.build({'ies':self.ies, 'data':b''})
 
 
+class ConcatenatedSmsReassembler:
+    """3GPP TS 23.040 section 9.2.3.24 concat multi part reassembly
+
+    A large user-data payload (e.g. a big OTA response packet) is split by the
+    sending entity into several SMS,
+    each carries a
+    - "concat short messages" IE in its UDH that identifies the set (ref num),
+    - total number of parts
+    - this parts seqno.
+    supports both:
+    IEI 0x00, section 9.2.3.24.1 8-bit ref form
+    IEI 0x08, section 9.2.3.24.8 the 16-bit ref form
+
+    Feed each received TP-User-Data (UDH + payload) to add() which
+    returns the reassembled TP-User-Data once all parts of the set have arrived,
+    or None as long as parts are still missing.
+
+    A non-concatenated SMS is returned unchanged,
+    just like one where the concat IE holds a reserved value:
+    TS 23.040 9.2.3.24.1 says
+    - both a total of zero
+    - a sequence number that is zero or greater than the total
+    that "the receiving entity shall ignore the whole IE",
+    we treat the message as a single, non-concatenated one and warn, not
+    as an error, so the caller does not die.
+
+    The reassembled TP-User-Data is built with a UDH that contains
+    the non-concat IEs seen in the parts, for example the the OTA "response packet"
+    indicator IE 0x71, followed by the concatenated payloads in sequence order,
+    so exactly the single-SMS form the sender would have produced for a payload that fits
+    into one SMS.
+    This allows convenient decoding by the normal single part path."""
+
+    # IEI: Concatenated short messages, 8-bit reference number
+    # (see 3GPP TS 23.040 section 9.2.3.24 and section 9.2.3.24.1)
+    CONCAT_8BIT = 0x00
+    # IEI: Concatenated short message, 16-bit reference number
+    # (see 3GPP TS 23.040 section 9.2.3.24 and section 9.2.3.24.8)
+    CONCAT_16BIT = 0x08
+
+    def __init__(self, max_sets: int = 8):
+        # keyed by (iei, ref, total): {'parts': {seq: payload}, 'header_ies'}, insertion ordered
+        self.sets = {}
+        self.max_sets = max_sets    # incomplete sets kept, oldest is dropped beyond that
+
+    @classmethod
+    def _parse_concat_ie(cls, ies) -> typing.Optional[typing.Tuple[int, int, int, int]]:
+        """Return (iei, ref, total, seq) of the concat IE, or None"""
+        for ie in ies:
+            if ie['iei'] == cls.CONCAT_8BIT and ie['length'] == 3:
+                v = ie['value']
+                return cls.CONCAT_8BIT, v[0], v[1], v[2]
+            if ie['iei'] == cls.CONCAT_16BIT and ie['length'] == 4:
+                v = ie['value']
+                return cls.CONCAT_16BIT, int.from_bytes(v[0:2], 'big'), v[2], v[3]
+        return None
+
+    def add(self, tpud: BytesOrHex) -> typing.Optional[bytes]:
+        """Add one TP-User-Data.
+        Returns
+        - the reassembled TP-User-Data if set is complete or sms not multipart,
+        - else None"""
+        if isinstance(tpud, str):
+            tpud = h2b(tpud)
+        udh, payload = UserDataHeader.from_bytes(tpud)
+        concat = self._parse_concat_ie(udh.ies)
+        if concat is None:
+            return tpud
+        iei, ref, total, seq = concat
+        if total < 1 or seq < 1 or seq > total:
+            # TS 23.040 9.2.3.24.1 / 9.2.3.24.8, total zero or seqno zero / > total:
+            # Ignoring the IE means the message has no valid concat IE, which is a single part message.
+            # Better warn and hand it back rather than raise, so we don't kill the callers receive loop/session
+            logger.warning('Ignoring reserved concat IE (ref=%u total=%u seq=%u), treating the '
+                           'message as non-concat', ref, total, seq)
+            return tpud
+        # TS 23.040 9.2.3.24.1 Total is constant in a set, refno only unique per IE form -> both set identity
+        # - full count = seqno 1..total is present
+        # - part disagreeing on the total ends up as set that cannot complete like set with missing parts
+        key = (iei, ref, total)
+        if key not in self.sets and len(self.sets) >= self.max_sets:
+            del self.sets[next(iter(self.sets))]
+        s = self.sets.setdefault(key, {'parts': {}, 'header_ies': []})
+        s['parts'][seq] = payload
+        # - remember the non concat IEs (OTA 0x71 indicator for example)
+        # - keep first seen occurrence of each IEI,
+        # so app IE present only in the first segment is preserved independent of arrival order
+        seen = {ie['iei'] for ie in s['header_ies']}
+        for ie in udh.ies:
+            if ie['iei'] in (self.CONCAT_8BIT, self.CONCAT_16BIT):
+                continue
+            if ie['iei'] not in seen:
+                s['header_ies'].append(ie)
+                seen.add(ie['iei'])
+        if len(s['parts']) < total:
+            return None
+        # all parts present -> reassemble in seq order
+        del self.sets[(iei, ref, total)]
+        body = b''.join(s['parts'][i] for i in range(1, total + 1))
+        header = UserDataHeader(s['header_ies']).to_bytes()
+        return header + body
+
+
 def smpp_dcs_is_8bit(dcs: pdu_types.DataCoding) -> bool:
     """Determine if the given SMPP data coding scheme is 8-bit or not."""
     if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
@@ -140,8 +246,8 @@ class AddressField:
     def to_bytes(self) -> bytes:
         """Encode the AddressField into the binary representation as used in T-PDU."""
         num_digits = len(self.digits)
-        if num_digits % 2:
-            self.digits += 'f'
+        # don't store the filler nibble or get_bytes() encodes it as digit and ends up too large
+        digits = self.digits + 'f' if num_digits % 2 else self.digits
         d = {
             'addr_len': num_digits,
             'type_of_addr': {
@@ -149,7 +255,7 @@ class AddressField:
                 'type_of_number': self.ton,
                 'numbering_plan_id': self.npi,
                 },
-            'digits': self.digits,
+            'digits': digits,
             }
         return self._construct.build(d)
 

pySim/sysmocom_sjs1.py

@@ -0,0 +1,61 @@
+# coding=utf-8
+"""Utilities / Functions related to sysmocom sysmoUSIM-SJS1 cards
+
+(C) 2026 by sysmocom - s.f.m.c. GmbH
+All Rights Reserved
+
+Author: Eric Wild <ewild@sysmocom.de>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from construct import Struct, Bytes, Flag
+from osmocom.utils import *
+from osmocom.construct import *
+
+from pySim.filesystem import *
+from pySim.runtime import RuntimeState
+
+
+class EF_Ki(TransparentEF):
+    _test_de_encode = [
+        ('000102030405060708090a0b0c0d0e0f',
+         {'key': h2b('000102030405060708090a0b0c0d0e0f')}),
+    ]
+
+    def __init__(self, fid='00ff', name='EF.Ki'):
+        super().__init__(fid, name=name, desc='K/Ki authentication key', size=(16, 16))
+        self._construct = Struct('key'/Bytes(16))
+
+
+class EF_OPc(TransparentEF):
+    _test_de_encode = [
+        ('016ca53d7a0a804561646816d7b0c702fb',
+         {'use_opc_instead_of_op': True, 'op_opc': h2b('6ca53d7a0a804561646816d7b0c702fb')}),
+    ]
+
+    def __init__(self, fid='00f7', name='EF.OPc'):
+        super().__init__(fid, name=name, desc='OP/OPc for milenage', size=(17, 17))
+        self._construct = Struct('use_opc_instead_of_op'/Flag, 'op_opc'/Bytes(16))
+
+
+class SysmoUSIMSJS1(CardModel):
+    _atrs = ["3b9f96801fc78031a073be21136743200718000001a5"]
+
+    @classmethod
+    def add_files(cls, rs: RuntimeState):
+        """Add sysmoUSIM-SJS1 specific files to given RuntimeState."""
+        # the key material lives in DF.GSM shared with ADF.USIM
+        if '7f20' in rs.mf.children:
+            rs.mf.children['7f20'].add_files([EF_Ki(), EF_OPc()])

pySim/transport/__init__.py

@@ -45,8 +45,10 @@ class ApduTracer:
 
 class StdoutApduTracer(ApduTracer):
     """Minimalistic APDU tracer, printing commands to stdout."""
-    def trace_response(self, cmd, sw, resp):
+    def trace_command(self, cmd):
         log.info("-> %s %s", cmd[:10], cmd[10:])
+
+    def trace_response(self, cmd, sw, resp):
         log.info("<- %s: %s", sw, resp)
 
     def trace_reset(self):
@@ -70,10 +72,26 @@ class ProactiveHandler(abc.ABC):
         raise NotImplementedError('No handler method for %s' % pcmd.decoded)
 
     def prepare_response(self, pcmd: ProactiveCommand, general_result: str = 'performed_successfully'):
+        # TERMINAL RESPONSE per ETSI TS 102 223 section 6.8: Command details (6.8.1) echoed from the
+        # command, Device identities (6.8.2) with source and destination swapped, Result (6.8.3).
+        # pcmd can be
+        # - decoded proactive command IE (.children contains CommandDetails/DeviceIdentities)
+        # - ProactiveCommand collection wrapper (empty .children).
+        # Normalise to the children obj, so both work:
+        # - handler that passes its decoded command
+        # - fallback path that passes collection
+        children = list(getattr(pcmd, 'children', None) or [])
+        if not any(isinstance(c, CommandDetails) for c in children):
+            decoded = getattr(pcmd, 'decoded', None)
+            if decoded is not None and decoded is not pcmd:
+                children = list(getattr(decoded, 'children', None) or [])
         # The Command Details are echoed from the command that has been processed.
-        (command_details,) = [c for c in pcmd.children if isinstance(c, CommandDetails)]
+        command_details = next((c for c in children if isinstance(c, CommandDetails)), None)
         # invert the device identities
-        (command_dev_ids,) = [c for c in pcmd.children if isinstance(c, DeviceIdentities)]
+        command_dev_ids = next((c for c in children if isinstance(c, DeviceIdentities)), None)
+        if command_details is None or command_dev_ids is None:
+            raise ValueError('failed to prepare TERMINAL RESPONSE: proactive command has no '
+                             'CommandDetails/DeviceIdentities (%r)' % (pcmd,))
         rsp_dev_ids = DeviceIdentities()
         rsp_dev_ids.from_dict({'device_identities': {
                                     'dest_dev_id': command_dev_ids.decoded['source_dev_id'],
@@ -301,24 +319,67 @@ class LinkBaseTpdu(LinkBase):
 
         prev_tpdu = tpdu
         data, sw = self.send_tpdu(tpdu)
+        log.debug("T0: case #%u TPDU: %s => %s %s", case, tpdu, data or "(no data)", sw or "(no status word)")
+        if sw is None:
+            raise ValueError("no status word received")
 
-        # When we have sent the first APDU, the SW may indicate that there are response bytes
-        # available. There are two SWs commonly used for this 9fxx (sim) and 61xx (usim), where
-        # xx is the number of response bytes available.
-        # See also:
-        if sw is not None:
-            while (sw[0:2] in ['9f', '61', '62', '63']):
-                # SW1=9F: 3GPP TS 51.011 9.4.1, Responses to commands which are correctly executed
-                # SW1=61: ISO/IEC 7816-4, Table 5 — General meaning of the interindustry values of SW1-SW2
-                # SW1=62: ETSI TS 102 221 7.3.1.1.4 Clause 4b): 62xx, 63xx, 9xxx != 9000
-                tpdu_gr = tpdu[0:2] + 'c00000' + sw[2:4]
+        # After sending the APDU/TPDU the UICC/eUICC or SIM may response with a status word that indicates that further
+        # TPDUs have to be sent in order to complete the task.
+        if case == 4 or self.apdu_strict == False:
+            # In case the APDU is a case #4 APDU, the UICC/eUICC/SIM may indicate that there is response data
+            # available which has to be retrieved using a GET RESPONSE command TPDU.
+            #
+            # ETSI TS 102 221, section 7.3.1.1.4 is very cleare about the fact that the GET RESPONSE mechanism
+            # shall only apply on case #4 APDUs but unfortunately it is impossible to distinguish between case #3
+            # and case #4 when the APDU format is not strictly followed. In order to be able to detect case #4
+            # correctly the Le byte (usually 0x00) must be present, is often forgotten. To avoid problems with
+            # legacy scripts that use raw APDU strings, we will still loosely apply GET RESPONSE based on what
+            # the status word indicates. Unless the user explicitly enables the strict mode (set apdu_strict true)
+            #
+            # The dummy GET RESPONSE of clause 4b (see below) is one shot: it turns a warning SW into the 61xx
+            # that announces the response length. It is only ever a valid reaction to the SW returned for the
+            # _command_ TPDU. Once a response has been fetched there is nothing left to announce, so a warning
+            # SW is the final result of the command and has to be passed on to the caller unmodified.
+            #
+            # This matters because the 62xx/63xx range is not exclusive to ETSI TS 102 221.
+            # GPC v2.3.1 section 11.4.3.2 table 11-38 GP GET STATUS (80 F2) answers
+            # 6310 "more data available", meaning "reissue with P2 bit 1 set" as per section 11.4.2.2 table 11-34
+            # rather than "response data is waiting". Trying a random GET RESPONSE at that point
+            # makes the card answer 6982 and tears down the whole SCP session and following commands fail with 6985.
+            dummy_gr_allowed = not data
+            while True:
+                if sw in ['9000', '9100']:
+                    # A status word of 9000 (or 9100 in case there is pending data from a proactive SIM command)
+                    # indicates that either no response data was returnd or all response data has been retrieved
+                    # successfully. We may discontinue the processing at this point.
+                    break;
+                if sw[0:2] in ['61', '9f']:
+                    # A status word of 61xx or 9fxx indicates that there is (still) response data available. We
+                    # send a GET RESPONSE command with the length value indicated in the second byte of the status
+                    # word. (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4a and 3GPP TS 51.011 9.4.1 and
+                    # ISO/IEC 7816-4, Table 5)
+                    le_gr = sw[2:4]
+                elif sw[0:2] in ['62', '63'] and dummy_gr_allowed:
+                    # There are corner cases (status word is 62xx or 63xx) where the UICC/eUICC/SIM asks us
+                    # to send a dummy GET RESPONSE command. We send a GET RESPONSE command with a length of 0.
+                    # (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4b and ETSI TS 151 011, section 9.4.1)
+                    le_gr = '00'
+                else:
+                    # A status word other then the ones covered by the above logic may indicate an error. In this
+                    # case we will discontinue the processing as well.
+                    # (see also ETSI TS 102 221, section 7.3.1.1.4, clause 4c)
+                    break
+                tpdu_gr = tpdu[0:2] + 'c00000' + le_gr
                 prev_tpdu = tpdu_gr
-                d, sw = self.send_tpdu(tpdu_gr)
-                data += d
-            if sw[0:2] == '6c':
-                # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
-                tpdu_gr = prev_tpdu[0:8] + sw[2:4]
-                data, sw = self.send_tpdu(tpdu_gr)
+                data_gr, sw = self.send_tpdu(tpdu_gr)
+                log.debug("T0: GET RESPONSE TPDU: %s => %s %s", tpdu_gr, data_gr or "(no data)", sw or "(no status word)")
+                data += data_gr
+                dummy_gr_allowed = False
+        if sw[0:2] == '6c':
+            # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
+            tpdu_gr = prev_tpdu[0:8] + sw[2:4]
+            data, sw = self.send_tpdu(tpdu_gr)
+            log.debug("T0: repated case #%u TPDU: %s => %s %s", case, tpdu_gr, data or "(no data)", sw or "(no status word)")
 
         return data, sw
 

pySim/ts_102_221.py

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 from bidict import bidict
+import copy
 
 from construct import Select, Const, Bit, Struct, Int16ub, FlagsEnum, GreedyString, ValidationError
 from construct import Optional as COptional, Computed
@@ -335,6 +336,8 @@ class TerminalCapability(BER_TLV_IE, tag=0xa9, nested=[TerminalPowerSupply, Exte
 
 # ETSI TS 102 221 Section 9.2.7 + ISO7816-4 9.3.3/9.3.4
 class _AM_DO_DF(DataObject):
+    """ISO7816-4:2005 5.4.3.1 Table 16"""
+
     def __init__(self):
         super().__init__('access_mode', 'Access Mode', tag=0x80)
 
@@ -381,7 +384,7 @@ class _AM_DO_DF(DataObject):
 
 
 class _AM_DO_EF(DataObject):
-    """ISO7816-4 9.3.2 Table 18 + 9.3.3.1 Table 31"""
+    """ISO7816-4:2005 5.4.3.1 Table 17"""
 
     def __init__(self):
         super().__init__('access_mode', 'Access Mode', tag=0x80)
@@ -429,7 +432,7 @@ class _AM_DO_EF(DataObject):
 
 
 class _AM_DO_CHDR(DataObject):
-    """Command Header Access Mode DO according to ISO 7816-4 Table 32."""
+    """Command Header Access Mode DO according to ISO 7816-4:2005 5.4.3.2 Table 22."""
 
     def __init__(self, tag):
         super().__init__('command_header', 'Command Header Description', tag=tag)
@@ -543,8 +546,9 @@ class CRT_DO(DataObject):
         pin = pin_names.inverse[self.decoded]
         return b'\x83\x01' + pin.to_bytes(1, 'big') + b'\x95\x01\x08'
 
-# ISO7816-4 9.3.3 Table 33
 class SecCondByte_DO(DataObject):
+    """ISO7816-4:2005 5.4.3.1 Table 20"""
+
     def __init__(self, tag=0x9d):
         super().__init__('security_condition_byte', tag=tag)
 
@@ -732,36 +736,57 @@ class EF_ARR(LinFixedEF):
                 raise ValueError
         return by_mode
 
+    @staticmethod
+    def __get_do_sequence(decode_for_df : bool = False):
+        if decode_for_df:
+            return DataObjectSequence('arr', sequence=[AM_DO_DF, SC_DO])
+        else:
+            return DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+
     def _decode_record_bin(self, raw_bin_data, **kwargs):
-        # we can only guess if we should decode for EF or DF here :(
-        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        # we can only guess if we should decode for EF or DF here, but our caller may
+        # be able to pass us a hint:
+        arr_seq = self.__get_do_sequence(kwargs.get('decode_for_df', False))
         dec = arr_seq.decode_multi(raw_bin_data)
         # we cannot pass the result through flatten() here, as we don't have a related
         # 'un-flattening' decoder, and hence would be unable to encode :(
         return dec[0]
 
     def _encode_record_bin(self, in_json, **kwargs):
-        # we can only guess if we should decode for EF or DF here :(
-        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        # we can only guess if we should decode for EF or DF here, but our caller may
+        # be able to pass us a hint:
+        arr_seq = self.__get_do_sequence(kwargs.get('encode_for_df', False))
         return arr_seq.encode_multi(in_json)
 
     @with_default_category('File-Specific Commands')
     class AddlShellCommands(CommandSet):
-        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
+        read_arr_argparser = copy.deepcopy(LinFixedEF.ShellCommands.read_rec_dec_parser)
+        read_arr_argparser.add_argument('--decode-for-df', action='store_true',
+                                        help='Decode EF.ARR record as if used by a DF (default: EF)')
+
+        @cmd2.with_argparser(read_arr_argparser)
         def do_read_arr_record(self, opts):
             """Read one EF.ARR record in flattened, human-friendly form."""
-            (data, _sw) = self._cmd.lchan.read_record_dec(opts.RECORD_NR)
+            (hexdata, _sw) = self._cmd.lchan.read_record(opts.RECORD_NR)
+            data = self._cmd.lchan.selected_file._decode_record_bin(h2b(hexdata),
+                                                                    decode_for_df = opts.decode_for_df)
             data = self._cmd.lchan.selected_file.flatten(data)
             self._cmd.poutput_json(data, opts.oneline)
 
-        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
+        read_arrs_argparser = copy.deepcopy(LinFixedEF.ShellCommands.read_recs_dec_parser)
+        read_arrs_argparser.add_argument('--decode-for-df', action='store_true',
+                                        help='Decode EF.ARR records as if used by a DF (default: EF)')
+
+        @cmd2.with_argparser(read_arrs_argparser)
         def do_read_arr_records(self, opts):
             """Read + decode all EF.ARR records in flattened, human-friendly form."""
             num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
             # collect all results in list so they are rendered as JSON list when printing
             data_list = []
             for recnr in range(1, 1 + num_of_rec):
-                (data, _sw) = self._cmd.lchan.read_record_dec(recnr)
+                (hexdata, _sw) = self._cmd.lchan.read_record(recnr)
+                data = self._cmd.lchan.selected_file._decode_record_bin(h2b(hexdata),
+                                                                        decode_for_df = opts.decode_for_df)
                 data = self._cmd.lchan.selected_file.flatten(data)
                 data_list.append(data)
             self._cmd.poutput_json(data_list, opts.oneline)

pySim/ts_31_102.py

@@ -285,6 +285,14 @@ class EF_SUCI_Calc_Info(TransparentEF):
                                 {"hnet_pubkey_identifier": 11, "hnet_pubkey":
                                  h2b("d1bc365f4997d17ce4374e72181431cbfeba9e1b98d7618f79d48561b144672a")}]} ),
     ]
+    _test_decode = [
+        ( 'A000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
+          {"prot_scheme_id_list": [],
+           "hnet_pubkey_list": []} ),
+        ( 'A000A100FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
+          {"prot_scheme_id_list": [],
+           "hnet_pubkey_list": []} ),
+    ]
     # 3GPP TS 31.102 Section 4.4.11.8
     class ProtSchemeIdList(BER_TLV_IE, tag=0xa0):
         # FIXME: 3GPP TS 24.501 Protection Scheme Identifier
@@ -327,7 +335,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         """conversion method to generate list of {hnet_pubkey_identifier, hnet_pubkey} dicts
         from flat [{hnet_pubkey_identifier: }, {net_pubkey: }, ...] list"""
         out = []
-        while len(l):
+        while l:
             a = l.pop(0)
             b = l.pop(0)
             z = {**a, **b}
@@ -389,7 +397,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         # remaining data holds Home Network Public Key Data Object
         hpkl = EF_SUCI_Calc_Info.HnetPubkeyList()
         hpkl.from_tlv(in_bytes[pos:])
-        hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'])
+        hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'] or [])
 
         return {
             'prot_scheme_id_list': prot_scheme_id_list,

pySim/ts_51_011.py

@@ -251,6 +251,16 @@ class EF_SMSP(LinFixedEF):
                                                       "numbering_plan_id": "isdn_e164" },
                             "call_number": "4915790109999" },
             "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
+        ( 'e1ffffffffffffffffffffffff0891945197109099f9ffffff0000a9',
+          { "alpha_id": "", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
+                                                          "tp_pid": True, "tp_dcs": True, "tp_vp": True },
+            "tp_dest_addr": { "length": 255, "ton_npi": { "ext": True, "type_of_number": "reserved_for_extension",
+                                                          "numbering_plan_id": "reserved_for_extension" },
+                              "call_number": "" },
+            "tp_sc_addr": { "length": 8, "ton_npi": { "ext": True, "type_of_number": "international",
+                                                      "numbering_plan_id": "isdn_e164" },
+                            "call_number": "4915790109999" },
+            "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 4320 } ),
         ( '454e6574776f726b73fffffffffffffff1ffffffffffffffffffffffffffffffffffffffffffffffff0000a7',
           { "alpha_id": "ENetworks", "parameter_indicators": { "tp_dest_addr": False, "tp_sc_addr": True,
                                                                "tp_pid": True, "tp_dcs": True, "tp_vp": False },
@@ -331,7 +341,8 @@ class EF_SMSP(LinFixedEF):
                         'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
         DestAddr = Struct('length'/Rebuild(Int8ub, lambda ctx: EF_SMSP.dest_addr_len(ctx)),
                         'ton_npi'/TonNpi, 'call_number'/PaddedBcdAdapter(Rpad(Bytes(10))))
-        self._construct = Struct('alpha_id'/COptional(GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28)))),
+        # (see comment below)
+        self._construct = Struct('alpha_id'/GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-28))),
                                  'parameter_indicators'/InvertAdapter(BitStruct(
                                                                         Const(7, BitsInteger(3)),
                                                                         'tp_vp'/Flag,
@@ -345,6 +356,25 @@ class EF_SMSP(LinFixedEF):
                                  'tp_dcs'/Bytes(1),
                                  'tp_vp_minutes'/EF_SMSP.ValidityPeriodAdapter(Byte))
 
+    # Ensure 'alpha_id' is always present
+    def encode_record_hex(self, abstract_data: dict, record_nr: int, total_len: int = None) -> str:
+        # Problem: TS 51.011 Section 10.5.6 describes the 'alpha_id' field as optional. However, this is only true
+        # at the time when the record length of the file is set up in the file system. A card manufacturer may decide
+        # to remove the field by setting the record length to 28. Likewise, the card manaufacturer may also decide to
+        # set the field to a distinct length by setting the record length to a value greater than 28 (e.g. 14 bytes
+        # 'alpha_id' + 28 bytes). Due to the fixed nature of the record length, this eventually means that in practice
+        # 'alpha_id' is a mandatory field with a fixed length.
+        #
+        # Due to the problematic specification of 'alpha_id' as a pseudo-optional field at the beginning of a
+        # fixed-size memory, the construct definition in self._construct has been incorrectly implemented and the field
+        # has been marked as COptional. We may correct the problem by removing COptional. But to maintain compatibility,
+        # we then have to ensure that in case the field is not provided (None), it is set to an empty string ('').
+        #
+        # See also ts_31_102.py, class EF_OCI for a correct example.
+        if abstract_data['alpha_id'] is None:
+            abstract_data['alpha_id'] = ''
+        return super().encode_record_hex(abstract_data, record_nr, total_len)
+
 # TS 51.011 Section 10.5.7
 class EF_SMSS(TransparentEF):
     class MemCapAdapter(Adapter):
@@ -1233,9 +1263,11 @@ class CardProfileSIM(CardProfile):
 
     @staticmethod
     def decode_select_response(resp_hex: str) -> object:
-        # we try to build something that resembles a dict resulting from the TLV decoder
-        # of TS 102.221 (FcpTemplate), so that higher-level code only has to deal with one
-        # format of SELECT response
+        """
+        Decode the select response to a dict representation, similar to the one of TS 102.221 (see ts_102_221.py,
+        class FcpTemplate), so that higher-level code only has to deal with one respresentation. See also
+        3GPP TS 51.011, section 9.2.1
+        """
         resp_bin = h2b(resp_hex)
         struct_of_file_map = {
             0: 'transparent',
@@ -1273,13 +1305,24 @@ class CardProfileSIM(CardProfile):
                 record_len = resp_bin[14]
                 ret['file_descriptor']['record_len'] = record_len
                 ret['file_descriptor']['num_of_rec'] = ret['file_size'] // record_len
-            ret['access_conditions'] = b2h(resp_bin[8:10])
-            if resp_bin[11] & 0x01 == 0:
+            ret['access_conditions'] = b2h(resp_bin[8:11])
+
+            # Life cycle status integer, see also ETSI TS 102 221, table 11.7b
+            lcsi = resp_bin[11]
+            if lcsi == 0x00:
+                ret['life_cycle_status_int'] = 'no_information'
+            elif lcsi == 0x01:
+                ret['life_cycle_status_int'] = 'creation'
+            elif lcsi == 0x03:
+                ret['life_cycle_status_int'] = 'initialization'
+            elif lcsi & 0xFD == 0x05:
                 ret['life_cycle_status_int'] = 'operational_activated'
-            elif resp_bin[11] & 0x04:
+            elif lcsi & 0xFD == 0x04:
                 ret['life_cycle_status_int'] = 'operational_deactivated'
+            elif lcsi & 0xFC == 0x0C:
+                ret['life_cycle_status_int'] = 'termination'
             else:
-                ret['life_cycle_status_int'] = 'terminated'
+                ret['life_cycle_status_int'] = lcsi
         return ret
 
     @classmethod

pyproject.toml

@@ -4,3 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.pylint.main]
 ignored-classes = ["twisted.internet.reactor"]
+
+[tool.pylint.TYPECHECK]
+# SdKey subclasses are generated dynamically via SdKey.generate_sd_key_classes()
+generated-members = ["SdKey[A-Za-z0-9]+"]

requirements.txt

@@ -1,7 +1,7 @@
 pyscard
 pyserial
 pytlv
-cmd2>=2.6.2,<3.0
+cmd2>=2.6.2,<4.0
 jsonpath-ng
 construct>=2.10.70
 bidict

setup.py

@@ -21,7 +21,7 @@ setup(
         "pyscard",
         "pyserial",
         "pytlv",
-        "cmd2 >= 1.5.0, < 3.0",
+        "cmd2 >= 2.6.2, < 4.0",
         "jsonpath-ng",
         "construct >= 2.10.70",
         "bidict",

tests/pySim-prog_test/Fairwaves-SIM.ok

@@ -5,7 +5,7 @@ ICCID: 8988219000000117833
 IMSI: 001010000000111
 GID1: ffffffffffffffff
 GID2: ffffffffffffffff
-SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
+SMSP: ffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
 SMSC: 0015555
 SPN: Fairwaves
 Show in HPLMN: False

tests/pySim-prog_test/Wavemobile-SIM.ok

@@ -5,7 +5,7 @@ ICCID: 89445310150011013678
 IMSI: 001010000000102
 GID1: Can't read file -- SW match failed! Expected 9000 and got 6a82.
 GID2: Can't read file -- SW match failed! Expected 9000 and got 6a82.
-SMSP: e1ffffffffffffffffffffffff0581005155f5ffffffffffff000000ffffffffffffffffffffffffffff
+SMSP: ffffffffffffffffffffffffffffe1ffffffffffffffffffffffff0581005155f5ffffffffffff000000
 SMSC: 0015555
 SPN: wavemobile
 Show in HPLMN: False

tests/pySim-shell_test/apdu/test_apdu.script

@@ -7,10 +7,24 @@ set apdu_strict true
 # No command data field, No response data field present
 apdu 00700001 --expect-sw 9000 --expect-response-regex '^$'
 
+# Case #1: (verify pin)
+# This command returns the number of remaining authentication attempts in the
+# form of a status that has the form 63cX, where X is the number of remaining
+# attempts. Such a status word can be easily confused with the response to a
+# case #4 APDU. This test checks if the transport layer correctly distinguishes
+# the between APDU case #1 and APDU case #4.
+apdu 0020000A --expect-sw 63c? --expect-response-regex '^$'
+
 # Case #2: (status)
 # No command data field, Response data field present
 apdu 80F2000000 --expect-sw 9000 --expect-response-regex '^[a-fA-F0-9]+$'
 
+# Case #2: (verify pin)
+# (see also above). This test checks if the transport layer is also able to
+# distinguish correctly between APDU case #2 (with zero length response) and
+# APDU case #4.
+apdu 0020000A00 --expect-sw 63c? --expect-response-regex '^$'
+
 # Case #3: (terminal capability)
 # Command data field present, No response data field
 apdu 80AA000005a903830180 --expect-sw 9000 --expect-response-regex '^$'

tests/unittests/smdpp_data

@@ -0,0 +1 @@
+../../smdpp-data

tests/unittests/test_bip_relay.py

@@ -0,0 +1,417 @@
+#!/usr/bin/env python3
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: Eric Wild
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import socket
+import threading
+import time
+import unittest
+
+from osmocom.utils import b2h, h2b
+
+from pySim.sms import SMS_SUBMIT, AddressField
+from pySim.cat import (ProactiveCommand, CommandDetails, DeviceIdentities,
+                       BearerDescription, BufferSize, UiccTransportLevel,
+                       OtherAddress, ChannelData, ChannelDataLength, ChannelStatus,
+                       Result, LocationInformation)
+
+from pySim.bip import Proact, ProactChannels, terminal_profile
+
+
+class _EchoServer:
+    """behold, my tiny threaded TCP echo server listening on 127.0.0.1:<port>"""
+    def __init__(self):
+        self._srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self._srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self._srv.bind(('127.0.0.1', 0))
+        self._srv.listen(1)
+        self.port = self._srv.getsockname()[1]
+        self.accepted = threading.Event()
+        self._conns = []
+        self._stop = False
+        threading.Thread(target=self._run, daemon=True).start()
+
+    def _run(self):
+        try:
+            conn, _ = self._srv.accept()
+        except OSError:
+            return
+        self._conns.append(conn)
+        self.accepted.set()
+        while not self._stop:
+            try:
+                data = conn.recv(4096)
+            except OSError:
+                break
+            if not data:
+                break
+            conn.sendall(data)
+
+    def close(self):
+        self._stop = True
+        for s in [self._srv] + self._conns:
+            try:
+                s.close()
+            except OSError:
+                pass
+
+
+def _pcmd(children_tlvs):
+    """Assemble D0 proactive-command TLV from child IE bytes,
+    decode it like transport does after a FETCH"""
+    body = b''.join(children_tlvs)
+    pdu = h2b('D0') + bytes([len(body)]) + body
+    return ProactiveCommand().from_tlv(pdu)
+
+
+def _open_channel(port, ip='127.0.0.1', cmd_nr=1):
+    a, b, c, d = (int(x) for x in ip.split('.'))
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'open_channel',
+                                'command_qualifier': 3}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': 'terminal'}).to_tlv(),
+        BearerDescription(decoded={'bearer_type': 'default', 'bearer_parameters': b''}).to_tlv(),
+        BufferSize(decoded=1024).to_tlv(),
+        UiccTransportLevel(decoded={'protocol_type': 'tcp_uicc_client_remote',
+                                    'port_number': port}).to_tlv(),
+        OtherAddress(decoded={'type_of_address': 'ipv4',
+                              'address': bytes([a, b, c, d])}).to_tlv(),
+    ])
+
+
+def _open_channel_raw(extra_ies, cmd_nr=1):
+    """OPEN CHANNEL with only the head data"""
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'open_channel',
+                                'command_qualifier': 3}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': 'terminal'}).to_tlv(),
+        BearerDescription(decoded={'bearer_type': 'default', 'bearer_parameters': b''}).to_tlv(),
+        BufferSize(decoded=1024).to_tlv(),
+    ] + extra_ies)
+
+
+def _send_data(payload, chan='channel_1', cmd_nr=1):
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'send_data',
+                                'command_qualifier': 1}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': chan}).to_tlv(),
+        ChannelData(decoded=b2h(payload)).to_tlv(),
+    ])
+
+
+def _receive_data(length, chan='channel_1', cmd_nr=1):
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'receive_data',
+                                'command_qualifier': 0}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': chan}).to_tlv(),
+        ChannelDataLength(decoded=length).to_tlv(),
+    ])
+
+
+def _close_channel(chan='channel_1', cmd_nr=1):
+    return _pcmd([
+        CommandDetails(decoded={'command_number': cmd_nr, 'type_of_command': 'close_channel',
+                                'command_qualifier': 0}).to_tlv(),
+        DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': chan}).to_tlv(),
+    ])
+
+
+def _first(til, cls):
+    return next((x for x in til if isinstance(x, cls)), None)
+
+
+class BipRelayRoundTripTest(unittest.TestCase):
+    """Drive the fixed Proact handlers (blocking sockets) with synthetic
+    proactive commands against a local echo server and assert a byte round-trip
+    plus the channel bookkeeping / error handling."""
+
+    def setUp(self):
+        self.echo = _EchoServer()
+        self.addCleanup(self.echo.close)
+        self.events = []
+        self.proact = Proact(data_available_sink=self.events.append)
+        self.addCleanup(self._close_all_channels)
+
+    def _close_all_channels(self):
+        for chan in list(self.proact.channels.channels.values()):
+            try:
+                chan.close()
+            except Exception:
+                pass
+
+    def _open(self, cmd_nr=1):
+        til = self.proact.handle_OpenChannel(_open_channel(self.echo.port, cmd_nr=cmd_nr))
+        # every TLV in the response must serialise (the transport does exactly
+        # this to post the TERMINAL RESPONSE)
+        b''.join(x.to_tlv() for x in til)
+        return til
+
+    def test_open_send_receive_roundtrip(self):
+        # OPEN CHANNEL -> socket connected, channel 1 opened, link established
+        til = self._open()
+        self.assertTrue(self.echo.accepted.wait(timeout=2.0))
+        self.assertIn(1, self.proact.channels.channels)
+        cd = _first(til, CommandDetails)
+        self.assertEqual(cd.decoded['type_of_command'], 'open_channel')
+        # TS 102 223 6.8.2 TERMINAL RESPONSE device id: terminal -> UICC
+        self.assertEqual(b2h(_first(til, DeviceIdentities).to_tlv()), '82028281')
+        # channel status: channel 1, link established
+        self.assertEqual(_first(til, ChannelStatus).decoded, '8100')
+        self.assertEqual(_first(til, Result).decoded['general_result'], 'performed_successfully')
+
+        # SEND DATA -> bytes written to the socket, echo server sends them back
+        payload = b'Hello SCP81 relay - opaque TLS record bytes'
+        til = self.proact.handle_SendData(_send_data(payload))
+        b''.join(x.to_tlv() for x in til)
+        # channel data length in the response = free Tx space, FF = ">255"
+        self.assertEqual(_first(til, ChannelDataLength).decoded, 255)
+        self.assertEqual(_first(til, Result).decoded['general_result'], 'performed_successfully')
+
+        # RECEIVE DATA -> drain the bytes back to the "card". real card
+        # uses data-available event, we poll the buffer
+        # and may need several RECEIVE DATA commands, as the spec allows.
+        got = bytearray()
+        deadline = time.monotonic() + 3.0
+        while len(got) < len(payload) and time.monotonic() < deadline:
+            chan = self.proact.channels.channels[1]
+            chan.wait_rx(1.0)
+            til = self.proact.handle_ReceiveData(_receive_data(len(payload) - len(got)))
+            b''.join(x.to_tlv() for x in til)
+            self.assertEqual(b2h(_first(til, DeviceIdentities).to_tlv()), '82028281')
+            got += h2b(_first(til, ChannelData).decoded)
+        self.assertEqual(bytes(got), payload, "byte round-trip through the BIP relay")
+
+        # CLOSE CHANNEL -> socket closed, bookkeeping cleared
+        til = self.proact.handle_CloseChannel(_close_channel())
+        b''.join(x.to_tlv() for x in til)
+        self.assertEqual(_first(til, Result).decoded['general_result'], 'performed_successfully')
+        self.assertNotIn(1, self.proact.channels.channels)
+
+    def test_data_available_event_envelope(self):
+        # The empty->non-empty Rx transition raises ENVELOPE EVENT DOWNLOAD
+        self._open()
+        self.assertTrue(self.echo.accepted.wait(timeout=2.0))
+        payload = b'PONG'
+        self.proact.handle_SendData(_send_data(payload))
+        chan = self.proact.channels.channels[1]
+        self.assertGreater(chan.wait_rx(2.0), 0)
+        # give the reader thread a beat to invoke the sink
+        deadline = time.monotonic() + 2.0
+        while not self.events and time.monotonic() < deadline:
+            time.sleep(0.01)
+        self.assertEqual(len(self.events), 1, "one data-available event on the empty->non-empty edge")
+        env = h2b(self.events[0])
+        # d6 0e | 99 01 09 (event: data available) | 82 02 82 81 terminal->UICC
+        #       | b8 02 81 00 (channel 1 established) | b7 01 XX bytes available
+        self.assertEqual(b2h(env[:15]), 'd60e99010982028281b8028100b701')
+        self.assertGreaterEqual(env[15], 1)
+        self.assertLessEqual(env[15], len(payload))
+
+    def test_channel_number_from_device_identities(self):
+        # Two channels, not the old hardcoded 1
+        e2 = _EchoServer()
+        self.addCleanup(e2.close)
+        self.proact.handle_OpenChannel(_open_channel(self.echo.port))
+        # open a second channel with a second echo server
+        til2 = self.proact.handle_OpenChannel(_open_channel(e2.port))
+        self.assertEqual(sorted(self.proact.channels.channels), [1, 2])
+        self.assertEqual(_first(til2, ChannelStatus).decoded, '8200')  # channel 2, established
+
+        # SEND DATA addressed to channel_2 must reach the second socket
+        self.assertTrue(e2.accepted.wait(timeout=2.0))
+        self.proact.handle_SendData(_send_data(b'two', chan='channel_2'))
+        chan2 = self.proact.channels.channels[2]
+        self.assertGreater(chan2.wait_rx(2.0), 0)
+        til = self.proact.handle_ReceiveData(_receive_data(3, chan='channel_2'))
+        self.assertEqual(h2b(_first(til, ChannelData).decoded), b'two')
+        # ..and nothing on chan 1
+        self.assertEqual(self.proact.channels.channels[1].available_rx(), 0)
+
+    def test_commands_on_closed_channel_report_bip_error(self):
+        # SEND/RECEIVE/CLOSE on a channel that was never opened must be rejected
+        # with a BIP error
+        for til in (self.proact.handle_SendData(_send_data(b'x', chan='channel_4')),
+                    self.proact.handle_ReceiveData(_receive_data(1, chan='channel_4')),
+                    self.proact.handle_CloseChannel(_close_channel(chan='channel_4'))):
+            b''.join(x.to_tlv() for x in til)
+            res = _first(til, Result).decoded
+            self.assertEqual(res['general_result'], 'bearer_independent_protocol_error')
+            self.assertEqual(res['additional_information'], 'channel_id_not_valid')
+
+    def test_receive_more_than_available_is_missing_info(self):
+        # terminal must NOT wait if fewer than the requested bytes are buffered,
+        # eturns what it has with "performed with missing information".
+        self._open()
+        self.assertTrue(self.echo.accepted.wait(timeout=2.0))
+        til = self.proact.handle_ReceiveData(_receive_data(10))
+        b''.join(x.to_tlv() for x in til)
+        self.assertEqual(_first(til, Result).decoded['general_result'],
+                         'performed_with_missing_information')
+        self.assertEqual(h2b(_first(til, ChannelData).decoded), b'')
+        self.assertEqual(_first(til, ChannelDataLength).decoded, 0)
+
+
+class OpenChannelRefusalTest(unittest.TestCase):
+    """Refusal is a TERMINAL RESPONSE, not an exception, raising takes the whole
+    proactive session down and leaves the card wondering why"""
+
+    ADDR = OtherAddress(decoded={'type_of_address': 'ipv4', 'address': bytes([127, 0, 0, 1])})
+    TCP = UiccTransportLevel(decoded={'protocol_type': 'tcp_uicc_client_remote', 'port_number': 1234})
+
+    def setUp(self):
+        self.proact = Proact()
+        self.addCleanup(self._close_all_channels)
+
+    def _close_all_channels(self):
+        for chan in list(self.proact.channels.channels.values()):
+            try:
+                chan.close()
+            except Exception:
+                pass
+
+    def _assert_refused(self, til, additional_information, chan_nr=0):
+        b''.join(x.to_tlv() for x in til)               # must serialise, the transport posts it
+        res = _first(til, Result).decoded
+        self.assertEqual(res['general_result'], 'bearer_independent_protocol_error')
+        self.assertEqual(res['additional_information'], additional_information)
+        self.assertEqual(_first(til, ChannelStatus).decoded, '%02x00' % chan_nr)   # 8.56
+        self.assertIsNotNone(_first(til, BearerDescription))                       # 6.8.20
+        self.assertIsNotNone(_first(til, BufferSize))                              # 6.8.21
+        self.assertEqual(b2h(_first(til, DeviceIdentities).to_tlv()), '82028281')  # 6.8.2
+
+    def test_transport_level(self):
+        cases = [[self.ADDR.to_tlv()]]                             # absent, 6.6.27.x Optional
+        for proto in ('udp_uicc_client_remote', 'tcp_uicc_server', 'udp_uicc_client_local',
+                      'tcp_uicc_client_local', 'direct_channel'):   # not TCP client remote
+            tl = UiccTransportLevel(decoded={'protocol_type': proto, 'port_number': 1234})
+            cases.append([tl.to_tlv(), self.ADDR.to_tlv()])
+        for extra in cases:
+            with self.subTest(extra=b2h(extra[0])):
+                self._assert_refused(self.proact.handle_OpenChannel(_open_channel_raw(extra)),
+                                     'requested_uicc_if_transp_level_not_available')
+
+    def test_destination_address(self):
+        v6 = OtherAddress(decoded={'type_of_address': 'ipv6', 'address': bytes(16)})
+        for extra in ([self.TCP.to_tlv()],                         # absent
+                      [self.TCP.to_tlv(), v6.to_tlv()]):           # not IPv4
+            with self.subTest(extra=len(extra)):
+                self._assert_refused(self.proact.handle_OpenChannel(_open_channel_raw(extra)),
+                                     'no_specific_cause')
+
+    def test_no_channel_left(self):
+        for _ in range(7):                                         # 6.4.27.2, 6.4.27.3
+            self.proact.channels.channel_create()
+        cmd = _open_channel_raw([self.TCP.to_tlv(), self.ADDR.to_tlv()])
+        self._assert_refused(self.proact.handle_OpenChannel(cmd), 'no_channel_availabile')
+
+    def test_connect_failure(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)      # port nothing listens on
+        s.bind(('127.0.0.1', 0))
+        dead_port = s.getsockname()[1]
+        s.close()
+        til = self.proact.handle_OpenChannel(_open_channel(dead_port))
+        self._assert_refused(til, 'channel_closed', chan_nr=1)      # 6.4.30
+        self.assertEqual(self.proact.channels.channels, {})        # channel given back
+
+
+class ProvideLocalInformationTest(unittest.TestCase):
+    """TS 102 223 6.8.7: only 00 gets a data object; the rest keeps the empty result."""
+
+    def _cmd(self, qualifier):
+        return _pcmd([
+            CommandDetails(decoded={'command_number': 1, 'type_of_command': 'provide_local_info',
+                                    'command_qualifier': qualifier}).to_tlv(),
+            DeviceIdentities(decoded={'source_dev_id': 'uicc', 'dest_dev_id': 'terminal'}).to_tlv()])
+
+    def test_location(self):
+        til = Proact().handle_ProvideLocalInformation(self._cmd(0x00))
+        b''.join(x.to_tlv() for x in til)
+        self.assertEqual(_first(til, Result).decoded['general_result'], 'performed_successfully')
+        self.assertEqual(b2h(_first(til, LocationInformation).to_tlv()), '930762f21000010001')
+        self.assertEqual(b2h(_first(til, DeviceIdentities).to_tlv()), '82028281')   # 6.8.2
+
+    def test_other_qualifiers_get_no_data_object(self):
+        for qualifier in (0x01, 0x03, 0x04, 0x1a):
+            with self.subTest(command_qualifier=qualifier):
+                til = Proact().handle_ProvideLocalInformation(self._cmd(qualifier))
+                b''.join(x.to_tlv() for x in til)
+                self.assertIsNone(_first(til, LocationInformation))
+
+    def test_location_is_configurable(self):
+        til = Proact(location=h2b('26f8100539')).handle_ProvideLocalInformation(self._cmd(0x00))
+        self.assertEqual(b2h(_first(til, LocationInformation).to_tlv()), '930526f8100539')
+
+
+class TerminalProfileTest(unittest.TestCase):
+    """TS 102 223 5.2, one bit per CAT facility"""
+
+    def setUp(self):
+        self.profile = terminal_profile()
+
+    def byte(self, n):
+        return self.profile[n - 1]                      # 1-based, as 5.2 numbers them
+
+    def test_announced(self):
+        self.assertEqual(len(self.profile), 32)
+        self.assertEqual(self.byte(1), 0x13)            # profile download, SMS-PP download b2+b5
+        self.assertEqual(self.byte(4), 0x02)            # SEND SHORT MESSAGE
+        self.assertEqual(self.byte(5) & 0x01, 0x01)     # SET UP EVENT LIST
+        self.assertEqual(self.byte(6), 0x0c)            # events: data available, channel status
+        self.assertEqual(self.byte(12), 0x1f)           # OPEN/CLOSE CHANNEL, RECEIVE/SEND DATA, STATUS
+        self.assertEqual(self.byte(13) >> 5, ProactChannels.MAX_CHANNELS)
+        self.assertEqual(self.byte(14), 0x60)           # class ND, class NK
+        self.assertEqual(self.byte(17), 0x01)           # TCP, UICC client mode, remote
+
+    def test_not_announced(self):
+        self.assertEqual(self.byte(3) & 0x60, 0)        # POLL INTERVAL, POLLING OFF
+        self.assertEqual(self.byte(4) & 0xc0, 0)        # PROVIDE LOCAL INFORMATION, NMR
+        self.assertEqual(self.byte(12) & 0xe0, 0)       # SERVICE SEARCH/INFORMATION, DECLARE SERVICE
+        self.assertEqual(self.byte(14) & 0x1f, 0)       # no characters down the display
+        for n in (7, 9, 10, 11, 15, 16, 18):            # class "a", class "d", display, ESN/IMEISV
+            self.assertEqual(self.byte(n), 0)
+
+    def test_channel_count(self):
+        self.assertEqual(terminal_profile(3)[12] >> 5, 3)
+        with self.assertRaises(ValueError):             # 8.56: 1 to 7
+            terminal_profile(8)
+
+
+if __name__ == "__main__":
+    unittest.main()
+
+
+class BipSinkTest(unittest.TestCase):
+    """Both sinks are optional, a driver with no SMS path at all must not crash and burn
+    with a card that sends one, and one that has one must get the PDU."""
+
+    def _submit(self):
+        return SMS_SUBMIT(tp_da=AddressField('12345', 'unknown', 'unknown'),
+                          tp_ud=b'\x01\x02', tp_udl=2, tp_dcs=0xf6)
+
+    def test_sinks_default_to_none(self):
+        p = Proact()
+        self.assertIsNone(p.sms_sink)
+
+    def test_mo_sms_goes_to_the_sink(self):
+        seen = []
+        Proact(sms_sink=seen.append).send_sms_via_smpp(self._submit())
+        self.assertEqual(len(seen), 1)
+
+    def test_no_sms_sink_drops_instead_of_raising(self):
+        with self.assertLogs('pySim.bip', level='INFO'):
+            Proact().send_sms_via_smpp(self._submit())

tests/unittests/test_card_key_provider.py

@@ -20,7 +20,8 @@ class TestCardKeyProviderCsv(unittest.TestCase):
                        "KIK3" : "00010204040506070809488B0C0D0E0F"}
 
         csv_file_path = os.path.dirname(os.path.abspath(__file__)) + "/test_card_key_provider.csv"
-        card_key_provider_register(CardKeyProviderCsv(csv_file_path, column_keys))
+        card_key_field_cryptor = CardKeyFieldCryptor(column_keys)
+        card_key_provider_register(CardKeyProviderCsv(csv_file_path, card_key_field_cryptor))
         super().__init__(*args, **kwargs)
 
     def test_card_key_provider_get(self):

tests/unittests/test_cat.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Tests for the CAT (Card Application Toolkit) COMPREHENSION-TLV data objects"""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# IEs not properly coverd by test_tlvs.py
+
+
+import unittest
+
+from osmocom.utils import b2h, h2b
+
+from pySim.cat import IMEI, IMEISV, AccessTechnology, SupportedRadioAccessTechnologies
+
+
+class IMEI_Test(unittest.TestCase):
+    """TS 102 223 8.20: the IMEI IE is 8 bytes, coded as valie part of Mobile Identity IE from 124 008"""
+
+    IMEI_15 = '123456789012345'
+    ENCODED = '94081a32547698103254'
+
+    def test_encode_is_eight_bytes(self):
+        """15 digits in 8 byte: 16 nibbles, one is type/parity framing."""
+        tlv = IMEI(decoded=self.IMEI_15).to_tlv()
+        self.assertEqual(b2h(tlv), self.ENCODED)
+        self.assertEqual(tlv[1], 0x08)               # spec len 8
+        self.assertEqual(len(tlv) - 2, 8)
+
+    def test_first_octet_framing(self):
+        """TS 24.008 table 10.5.4"""
+        octet1 = IMEI(decoded=self.IMEI_15).to_tlv()[2]
+        self.assertEqual(octet1 & 0x07, 2)           # IMEI
+        self.assertEqual((octet1 >> 3) & 0x01, 1)    # odd
+        self.assertEqual(octet1 >> 4, 1)             # digit 1
+
+    def test_decodes_to_the_raw_imei(self):
+        """strip framing nibble"""
+        ie = IMEI()
+        ie.from_tlv(h2b(self.ENCODED))
+        self.assertEqual(ie.decoded, self.IMEI_15)
+
+    def test_even_digit_count_uses_the_end_mark(self):
+        """"end marker, IMEISV case"""
+        ie = IMEI(decoded='1234567890123456')
+        tlv = ie.to_tlv()
+        self.assertEqual(tlv[2] >> 3 & 0x01, 0)      # even
+        self.assertEqual(tlv[-1] >> 4, 0x0f)         # end mark
+        back = IMEI()
+        back.from_tlv(tlv)
+        self.assertEqual(back.decoded, '1234567890123456')
+
+
+class IMEISV_Test(unittest.TestCase):
+    """TS 102 223 8.74, no fixed len, end marker"""
+
+    IMEISV_16 = '1234567890123456'
+    ENCODED = 'e2091332547698103254f6'
+
+    def test_encode(self):
+        self.assertEqual(b2h(IMEISV(decoded=self.IMEISV_16).to_tlv()), self.ENCODED)
+
+    def test_type_of_identity_and_end_mark(self):
+        value = IMEISV(decoded=self.IMEISV_16).to_tlv()[2:]
+        self.assertEqual(value[0] & 0x07, 3)         # IMEISV
+        self.assertEqual((value[0] >> 3) & 0x01, 0)  # even
+        self.assertEqual(value[-1] >> 4, 0x0f)       # end mark
+        self.assertEqual(len(value), 9)
+
+    def test_decode(self):
+        ie = IMEISV()
+        ie.from_tlv(h2b(self.ENCODED))
+        self.assertEqual(ie.decoded, self.IMEISV_16)
+
+
+class SupportedRadioAccessTechnologies_Test(unittest.TestCase):
+    """TS 102 223 8.105"""
+
+    def test_encode_technology_enabled(self):
+        """The flag used to have a bitmask of 0 so enabled -> 00 (that is disabled..)"""
+        ie = SupportedRadioAccessTechnologies(
+            decoded=[{'technology': 'eutran', 'state': {'enabled': True}}])
+        self.assertEqual(b2h(ie.to_tlv()), 'b4020801')
+
+    def test_encode_technology_disabled(self):
+        ie = SupportedRadioAccessTechnologies(
+            decoded=[{'technology': 'eutran', 'state': {'enabled': False}}])
+        self.assertEqual(b2h(ie.to_tlv()), 'b4020800')
+
+    def test_decode_technology(self):
+        """old 0 bitmask = all enabled, no way to disable"""
+        for encoded, enabled in [('b4020800', False), ('b4020801', True)]:
+            with self.subTest(encoded=encoded):
+                ie = SupportedRadioAccessTechnologies()
+                ie.from_tlv(h2b(encoded))
+                self.assertEqual(ie.decoded[0]['technology'], 'eutran')
+                self.assertEqual(ie.decoded[0]['state']['enabled'], enabled)
+
+    def test_decode_technology_multiple(self):
+        ie = SupportedRadioAccessTechnologies()
+        ie.from_tlv(h2b('b40408010000'))
+        self.assertEqual([(e['technology'], e['state']['enabled']) for e in ie.decoded],
+                         [('eutran', True), ('gsm', False)])
+
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unittests/test_configurable_parameters.py

@@ -0,0 +1,640 @@
+#!/usr/bin/env python3
+
+# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: Neels Hofmeyr
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import enum
+import io
+import sys
+import unittest
+from importlib import resources
+from osmocom.utils import hexstr
+from pySim.esim.saip import ProfileElementSequence
+import pySim.esim.saip.personalization as p13n
+import smdpp_data.upp
+
+import xo
+update_expected_output = False
+
+def valstr(val):
+    if isinstance(val, io.BytesIO):
+        val = val.getvalue()
+    if isinstance(val, bytearray):
+        val = bytes(val)
+    return f'{val!r}'
+
+def valtypestr(val):
+    if isinstance(val, dict):
+        types = []
+        for v in val.values():
+            types.append(f'{type(v).__name__}')
+
+        val_type = '{' + ', '.join(types) + '}'
+    else:
+        val_type = f'{type(val).__name__}'
+    return f'{valstr(val)}:{val_type}'
+
+class ConfigurableParameterTest(unittest.TestCase):
+
+    def test_parameters(self):
+
+        upp_fnames = (
+                'TS48v5_SAIP2.1A_NoBERTLV.der',
+                'TS48v5_SAIP2.3_BERTLV_SUCI.der',
+               )
+
+        class Paramtest:
+            def __init__(self, param_cls, val, expect_val, expect_clean_val=None):
+                self.param_cls = param_cls
+                self.val = val
+                self.expect_clean_val = expect_clean_val
+                self.expect_val = expect_val
+
+        param_tests = [
+            Paramtest(param_cls=p13n.Imsi, val='123456',
+                      expect_clean_val=str('123456'),
+                      expect_val={'IMSI': hexstr('123456'),
+                                  'IMSI-ACC': '0040'}),
+            Paramtest(param_cls=p13n.Imsi, val=int(123456),
+                      expect_val={'IMSI': hexstr('123456'),
+                                  'IMSI-ACC': '0040'}),
+
+            Paramtest(param_cls=p13n.Imsi, val='123456789012345',
+                      expect_clean_val=str('123456789012345'),
+                      expect_val={'IMSI': hexstr('123456789012345'),
+                                  'IMSI-ACC': '0020'}),
+            Paramtest(param_cls=p13n.Imsi, val=int(123456789012345),
+                      expect_val={'IMSI': hexstr('123456789012345'),
+                                  'IMSI-ACC': '0020'}),
+
+            Paramtest(param_cls=p13n.Puk1,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+            Paramtest(param_cls=p13n.Puk1,
+                      val=int(12345678),
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+
+            Paramtest(param_cls=p13n.Puk2,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+
+            Paramtest(param_cls=p13n.Pin1,
+                      val='1234',
+                      expect_clean_val=b'1234\xff\xff\xff\xff',
+                      expect_val='1234'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val='123456',
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val=int(1234),
+                      expect_clean_val=b'1234\xff\xff\xff\xff',
+                      expect_val='1234'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val=int(123456),
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+            Paramtest(param_cls=p13n.Pin1,
+                      val=int(12345678),
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+
+            Paramtest(param_cls=p13n.Adm1,
+                      val='1234',
+                      expect_clean_val=b'1234\xff\xff\xff\xff',
+                      expect_val='1234'),
+            Paramtest(param_cls=p13n.Adm1,
+                      val='123456',
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+            Paramtest(param_cls=p13n.Adm1,
+                      val='12345678',
+                      expect_clean_val=b'12345678',
+                      expect_val='12345678'),
+            Paramtest(param_cls=p13n.Adm1,
+                      val=int(123456),
+                      expect_clean_val=b'123456\xff\xff',
+                      expect_val='123456'),
+
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val='Milenage',
+                      expect_clean_val=1,
+                      expect_val='Milenage'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val='TUAK',
+                      expect_clean_val=2,
+                      expect_val='TUAK'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val='usim-test',
+                      expect_clean_val=3,
+                      expect_val='usim_test'),
+
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val=1,
+                      expect_clean_val=1,
+                      expect_val='Milenage'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val=2,
+                      expect_clean_val=2,
+                      expect_val='TUAK'),
+            Paramtest(param_cls=p13n.AlgorithmID,
+                      val=3,
+                      expect_clean_val=3,
+                      expect_val='usim_test'),
+
+            Paramtest(param_cls=p13n.K,
+                      val='01020304050607080910111213141516',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.K,
+                      val=int(11020304050607080910111213141516),
+                      expect_clean_val=b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='11020304050607080910111213141516'),
+
+            Paramtest(param_cls=p13n.Opc,
+                      val='01020304050607080910111213141516',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.Opc,
+                      val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.Opc,
+                      val=bytearray(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+            Paramtest(param_cls=p13n.Opc,
+                      val=io.BytesIO(b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'),
+                      expect_clean_val=b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16',
+                      expect_val='01020304050607080910111213141516'),
+
+            Paramtest(param_cls=p13n.SmspTpScAddr,
+                      val='+1234567',
+                      expect_clean_val=(True, '1234567'),
+                      expect_val='+1234567'),
+            Paramtest(param_cls=p13n.SmspTpScAddr,
+                      val=1234567,
+                      expect_clean_val=(False, '1234567'),
+                      expect_val='1234567'),
+
+            Paramtest(param_cls=p13n.TuakNumberOfKeccak,
+                      val='123',
+                      expect_clean_val=123,
+                      expect_val='123'),
+            Paramtest(param_cls=p13n.TuakNumberOfKeccak,
+                      val=123,
+                      expect_clean_val=123,
+                      expect_val='123'),
+
+            Paramtest(param_cls=p13n.MilenageRotationConstants,
+                      val='0a 0b 0c 01 02',
+                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_val='0a0b0c0102'),
+            Paramtest(param_cls=p13n.MilenageRotationConstants,
+                      val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_val='0a0b0c0102'),
+            Paramtest(param_cls=p13n.MilenageRotationConstants,
+                      val=bytearray(b'\x0a\x0b\x0c\x01\x02'),
+                      expect_clean_val=b'\x0a\x0b\x0c\x01\x02',
+                      expect_val='0a0b0c0102'),
+
+            Paramtest(param_cls=p13n.MilenageXoringConstants,
+                      val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+                          ' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+                          ' cccccccccccccccccccccccccccccccc'
+                          ' 11111111111111111111111111111111'
+                          ' 22222222222222222222222222222222',
+                      expect_clean_val=b'\xaa' * 16
+                                       + b'\xbb' * 16
+                                       + b'\xcc' * 16
+                                       + b'\x11' * 16
+                                       + b'\x22' * 16,
+                      expect_val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+                                 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+                                 'cccccccccccccccccccccccccccccccc'
+                                 '11111111111111111111111111111111'
+                                 '22222222222222222222222222222222'),
+            Paramtest(param_cls=p13n.MilenageXoringConstants,
+                      val=b'\xaa' * 16
+                          + b'\xbb' * 16
+                          + b'\xcc' * 16
+                          + b'\x11' * 16
+                          + b'\x22' * 16,
+                      expect_clean_val=b'\xaa' * 16
+                                       + b'\xbb' * 16
+                                       + b'\xcc' * 16
+                                       + b'\x11' * 16
+                                       + b'\x22' * 16,
+                      expect_val='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+                                 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
+                                 'cccccccccccccccccccccccccccccccc'
+                                 '11111111111111111111111111111111'
+                                 '22222222222222222222222222222222'),
+
+            Paramtest(param_cls=p13n.MncLen,
+                      val='2',
+                      expect_clean_val=2,
+                      expect_val='2'),
+            Paramtest(param_cls=p13n.MncLen,
+                      val=3,
+                      expect_clean_val=3,
+                      expect_val='3'),
+
+            ]
+
+        for sdkey_cls in (
+            # thin out the number of tests, as a compromise between completeness and test runtime
+            p13n.SdKeyScp02Kvn20AesDek,
+           #p13n.SdKeyScp02Kvn20AesEnc,
+           #p13n.SdKeyScp02Kvn20AesMac,
+           #p13n.SdKeyScp02Kvn21AesDek,
+            p13n.SdKeyScp02Kvn21AesEnc,
+           #p13n.SdKeyScp02Kvn21AesMac,
+           #p13n.SdKeyScp02Kvn22AesDek,
+           #p13n.SdKeyScp02Kvn22AesEnc,
+            p13n.SdKeyScp02Kvn22AesMac,
+           #p13n.SdKeyScp02KvnffAesDek,
+           #p13n.SdKeyScp02KvnffAesEnc,
+           #p13n.SdKeyScp02KvnffAesMac,
+            p13n.SdKeyScp03Kvn30AesDek,
+           #p13n.SdKeyScp03Kvn30AesEnc,
+           #p13n.SdKeyScp03Kvn30AesMac,
+           #p13n.SdKeyScp03Kvn31AesDek,
+            p13n.SdKeyScp03Kvn31AesEnc,
+           #p13n.SdKeyScp03Kvn31AesMac,
+           #p13n.SdKeyScp03Kvn32AesDek,
+           #p13n.SdKeyScp03Kvn32AesEnc,
+            p13n.SdKeyScp03Kvn32AesMac,
+           #p13n.SdKeyScp80Kvn01AesDek,
+           #p13n.SdKeyScp80Kvn01AesEnc,
+           #p13n.SdKeyScp80Kvn01AesMac,
+            p13n.SdKeyScp80Kvn01DesDek,
+           #p13n.SdKeyScp80Kvn01DesEnc,
+           #p13n.SdKeyScp80Kvn01DesMac,
+           #p13n.SdKeyScp80Kvn02AesDek,
+            p13n.SdKeyScp80Kvn02AesEnc,
+           #p13n.SdKeyScp80Kvn02AesMac,
+           #p13n.SdKeyScp80Kvn02DesDek,
+           #p13n.SdKeyScp80Kvn02DesEnc,
+            p13n.SdKeyScp80Kvn02DesMac,
+           #p13n.SdKeyScp80Kvn03AesDek,
+           #p13n.SdKeyScp80Kvn03AesEnc,
+           #p13n.SdKeyScp80Kvn03AesMac,
+            p13n.SdKeyScp80Kvn03DesDek,
+           #p13n.SdKeyScp80Kvn03DesEnc,
+           #p13n.SdKeyScp80Kvn03DesMac,
+            p13n.SdKeyScp81Kvn40AesDek,
+           #p13n.SdKeyScp81Kvn40Tlspsk,
+           #p13n.SdKeyScp81Kvn41AesDek,
+            p13n.SdKeyScp81Kvn41Tlspsk,
+           #p13n.SdKeyScp81Kvn42AesDek,
+           #p13n.SdKeyScp81Kvn42Tlspsk,
+            ):
+
+            for key_len in sdkey_cls.allow_len:
+                val = '0102030405060708091011121314151617181920212223242526272829303132'
+                expect_clean_val = (b'\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16'
+                                    b'\x17\x18\x19\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x30\x31\x32')
+                expect_val = '0102030405060708091011121314151617181920212223242526272829303132'
+
+                val = val[:key_len*2]
+                expect_clean_val = expect_clean_val[:key_len]
+                expect_val = val
+
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                # test bytes input
+                val = expect_clean_val
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                # test bytearray input
+                val = bytearray(expect_clean_val)
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                # test BytesIO input
+                val = io.BytesIO(expect_clean_val)
+                param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+                if key_len == 16:
+                    # test huge integer input.
+                    # needs to start with nonzero.. stupid
+                    val = 11020304050607080910111213141516
+                    expect_clean_val = (b'\x11\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\x16')
+                    expect_val = '11020304050607080910111213141516'
+                    param_tests.append(Paramtest(param_cls=sdkey_cls, val=val, expect_clean_val=expect_clean_val, expect_val=expect_val))
+
+        outputs = []
+
+        for upp_fname in upp_fnames:
+            test_idx = -1
+            try:
+
+                der = resources.read_binary(smdpp_data.upp, upp_fname)
+
+                for t in param_tests:
+                    test_idx += 1
+                    logloc = f'{upp_fname} {t.param_cls.__name__}(val={valtypestr(t.val)})'
+
+                    param = None
+                    try:
+                        param = t.param_cls()
+                        param.input_value = t.val
+                        param.validate()
+                    except ValueError as e:
+                        raise ValueError(f'{logloc}: {e}') from e
+
+                    clean_val = param.value
+                    logloc = f'{logloc} clean_val={valtypestr(clean_val)}'
+                    if t.expect_clean_val is not None and t.expect_clean_val != clean_val:
+                        raise ValueError(f'{logloc}: expected'
+                                         f' expect_clean_val={valtypestr(t.expect_clean_val)}')
+
+                    # on my laptop, deepcopy is about 30% slower than decoding the DER from scratch:
+                    # pes = copy.deepcopy(orig_pes)
+                    pes = ProfileElementSequence.from_der(der)
+                    try:
+                        param.apply(pes)
+                    except ValueError as e:
+                        raise ValueError(f'{logloc} apply_val(clean_val): {e}') from e
+
+                    changed_der = pes.to_der()
+
+                    pes2 = ProfileElementSequence.from_der(changed_der)
+
+                    read_back_val = t.param_cls.get_value_from_pes(pes2)
+
+                    # compose log string to show the precise type of dict values
+                    if isinstance(read_back_val, dict):
+                        types = set()
+                        for v in read_back_val.values():
+                            types.add(f'{type(v).__name__}')
+
+                        read_back_val_type = '{' + ', '.join(types) + '}'
+                    else:
+                        read_back_val_type = f'{type(read_back_val).__name__}'
+
+                    logloc = (f'{logloc} read_back_val={valtypestr(read_back_val)}')
+
+                    if isinstance(read_back_val, dict) and not t.param_cls.get_name() in read_back_val.keys():
+                        raise ValueError(f'{logloc}: expected to find name {t.param_cls.get_name()!r} in read_back_val')
+
+                    expect_val = t.expect_val
+                    if not isinstance(expect_val, dict):
+                        expect_val = { t.param_cls.get_name(): expect_val }
+                    if read_back_val != expect_val:
+                        raise ValueError(f'{logloc}: expected {expect_val=!r}:{type(t.expect_val).__name__}')
+
+                    ok = logloc.replace(' clean_val', '\n\tclean_val'
+                              ).replace(' read_back_val', '\n\tread_back_val'
+                              ).replace('=', '=\t'
+                              )
+                    output = f'\nok: {ok}'
+                    outputs.append(output)
+                    print(output)
+
+            except Exception as e:
+                raise RuntimeError(f'Error while testing UPP {upp_fname} {test_idx=}: {e}') from e
+
+        output = '\n'.join(outputs) + '\n'
+        xo_name = 'test_configurable_parameters'
+        if update_expected_output:
+            with resources.path(xo, xo_name) as xo_path:
+                with open(xo_path, 'w', encoding='utf-8') as f:
+                    f.write(output)
+        else:
+            xo_str = resources.read_text(xo, xo_name)
+            if xo_str != output:
+                at = 0
+                while at < len(output):
+                    if output[at] == xo_str[at]:
+                        at += 1
+                        continue
+                    break
+
+                raise RuntimeError(f'output differs from expected output at position {at}: "{output[at:at+20]}" != "{xo_str[at:at+20]}"')
+
+
+class TestValidateVal(unittest.TestCase):
+    """validate_val() tests for various ConfigurableParameter subclasses."""
+
+    def _ok(self, cls, val, expected=None):
+        result = cls.validate_val(val)
+        if expected is not None:
+            self.assertEqual(result, expected)
+        return result
+
+    def _err(self, cls, val):
+        with self.assertRaises(ValueError):
+            cls.validate_val(val)
+
+    # --- Iccid ---
+
+    def test_iccid_18digits_adds_luhn(self):
+        result = self._ok(p13n.Iccid, '998877665544332211')
+        self.assertIsInstance(result, str)
+        self.assertEqual(len(result), 19)
+        self.assertTrue(result.isdecimal())
+
+    def test_iccid_19digits_passthrough(self):
+        result = self._ok(p13n.Iccid, '9988776655443322110')
+        self.assertIsInstance(result, str)
+        self.assertEqual(len(result), 19)
+
+    def test_iccid_too_short(self):
+        self._err(p13n.Iccid, '12345678901234567')  # 17 digits
+
+    def test_iccid_too_long(self):
+        self._err(p13n.Iccid, '1' * 21)
+
+    def test_iccid_non_digits(self):
+        self._err(p13n.Iccid, '99887766554433221X')
+
+    # --- Imsi ---
+
+    def test_imsi_valid_short(self):
+        self._ok(p13n.Imsi, '001010', '001010')
+
+    def test_imsi_valid_long(self):
+        self._ok(p13n.Imsi, '001010123456789', '001010123456789')
+
+    def test_imsi_too_short(self):
+        self._err(p13n.Imsi, '12345')  # 5 digits, min is 6
+
+    def test_imsi_too_long(self):
+        self._err(p13n.Imsi, '1' * 16)
+
+    def test_imsi_non_digits(self):
+        self._err(p13n.Imsi, '00101A123456789')
+
+    # --- Pin1 ---
+
+    def test_pin1_4digits(self):
+        # DecimalHexParam encodes each digit as its ASCII byte, then rpad to 8 bytes with 0xff
+        self._ok(p13n.Pin1, '1234', b'1234\xff\xff\xff\xff')
+
+    def test_pin1_8digits(self):
+        self._ok(p13n.Pin1, '12345678', b'12345678')
+
+    def test_pin1_too_short(self):
+        self._err(p13n.Pin1, '123')
+
+    def test_pin1_too_long(self):
+        self._err(p13n.Pin1, '123456789')
+
+    def test_pin1_non_digits(self):
+        self._err(p13n.Pin1, '123A')
+
+    # --- Puk1 ---
+
+    def test_puk1_8digits(self):
+        self._ok(p13n.Puk1, '12345678', b'12345678')
+
+    def test_puk1_wrong_length(self):
+        self._err(p13n.Puk1, '1234567')   # 7 digits
+        self._err(p13n.Puk1, '123456789')  # 9 digits
+
+    def test_puk1_non_digits(self):
+        self._err(p13n.Puk1, '1234567X')
+
+    # --- K (BinaryParam) ---
+
+    def test_k_valid_hex_str(self):
+        self._ok(p13n.K, '000102030405060708090a0b0c0d0e0f',
+                 b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f')
+
+    def test_k_valid_bytes(self):
+        raw = bytes(range(16))
+        self._ok(p13n.K, raw, raw)
+
+    def test_k_wrong_length(self):
+        self._err(p13n.K, '00' * 15)  # 15 bytes, allow_len requires 16 or 32
+
+    def test_k_non_hex(self):
+        self._err(p13n.K, 'gg' * 16)
+
+    def test_k_odd_hex_digits(self):
+        self._err(p13n.K, '0' * 31)   # odd number of hex digits
+
+
+class TestEnumParam(unittest.TestCase):
+    """Tests for the EnumParam machinery, using AlgorithmID as the concrete subclass."""
+
+    # --- validate_val ---
+
+    def test_validate_by_name_exact(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val('Milenage'), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val('TUAK'), 2)
+        self.assertEqual(p13n.AlgorithmID.validate_val('usim_test'), 3)
+
+    def test_validate_by_int(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val(1), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val(2), 2)
+        self.assertEqual(p13n.AlgorithmID.validate_val(3), 3)
+
+    def test_validate_fuzzy_case(self):
+        self.assertEqual(p13n.AlgorithmID.validate_val('milenage'), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val('MILENAGE'), 1)
+        self.assertEqual(p13n.AlgorithmID.validate_val('tuak'), 2)
+
+    def test_validate_fuzzy_hyphen_underscore(self):
+        # 'usim-test' has a hyphen; enum member is 'usim_test' — must fuzzy-match
+        self.assertEqual(p13n.AlgorithmID.validate_val('usim-test'), 3)
+
+    def test_validate_invalid_name(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.validate_val('unknown')
+
+    def test_validate_invalid_int(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.validate_val(99)
+
+    def test_validate_returns_int(self):
+        result = p13n.AlgorithmID.validate_val('Milenage')
+        self.assertIsInstance(result, int)
+        self.assertNotIsInstance(result, enum.Enum)
+
+    # --- map_name_to_val ---
+
+    def test_map_name_exact(self):
+        self.assertEqual(p13n.AlgorithmID.map_name_to_val('Milenage'), 1)
+
+    def test_map_name_fuzzy(self):
+        self.assertEqual(p13n.AlgorithmID.map_name_to_val('milenage'), 1)
+        self.assertEqual(p13n.AlgorithmID.map_name_to_val('usim-test'), 3)
+
+    def test_map_name_strict_raises(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.map_name_to_val('unknown', strict=True)
+
+    def test_map_name_nonstrict_returns_none(self):
+        self.assertIsNone(p13n.AlgorithmID.map_name_to_val('unknown', strict=False))
+
+    # --- map_val_to_name ---
+
+    def test_map_val_known(self):
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(1), 'Milenage')
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(2), 'TUAK')
+        self.assertEqual(p13n.AlgorithmID.map_val_to_name(3), 'usim_test')
+
+    def test_map_val_unknown_nonstrict(self):
+        self.assertIsNone(p13n.AlgorithmID.map_val_to_name(99))
+
+    def test_map_val_unknown_strict(self):
+        with self.assertRaises(ValueError):
+            p13n.AlgorithmID.map_val_to_name(99, strict=True)
+
+    # --- name_normalize ---
+
+    def test_name_normalize(self):
+        self.assertEqual(p13n.AlgorithmID.name_normalize('Milenage'), 'Milenage')
+        self.assertEqual(p13n.AlgorithmID.name_normalize('milenage'), 'Milenage')
+        self.assertEqual(p13n.AlgorithmID.name_normalize('usim-test'), 'usim_test')
+
+    # --- clean_name_str ---
+
+    def test_clean_name_str(self):
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim-test'), 'usimtest')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('usim_test'), 'usimtest')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('Milenage'), 'milenage')
+        self.assertEqual(p13n.AlgorithmID.clean_name_str('foo bar!'), 'foobar')
+
+
+if __name__ == "__main__":
+    if '-u' in sys.argv:
+        update_expected_output = True
+        sys.argv.remove('-u')
+    unittest.main()

tests/unittests/test_decode_select_resp.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Philipp Maier <pmaier@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+import os
+from pySim.profile import CardProfile
+from pySim.ts_51_011 import CardProfileSIM
+from pySim.ts_102_221 import CardProfileUICC
+
+class TestDecodeSelectResponse_CardProfile(unittest.TestCase):
+
+    def decode_select_response(self, card_Profile: CardProfile, testcases: list[dict]):
+        for testcase in testcases:
+            resp_hex = testcase['resp_hex']
+            decoded = card_Profile.decode_select_response(resp_hex)
+            if testcase['decoded']:
+                self.assertEqual(decoded, testcase['decoded'])
+            else:
+                print("no testvector to compare against, assuming the following output is correct:")
+                print("resp_hex:", resp_hex)
+                print("decoded:", decoded)
+
+    def test_CardProfileSIM(self):
+        testcases = [
+            # MF
+            {"resp_hex" : "000000003f000100000000000981020c0400838a838a",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'mf'}}, 'proprietary_info': {'available_memory': 0}, 'file_id': '3f00', 'file_characteristics': '81', 'num_direct_child_df': 2, 'num_direct_child_ef': 12, 'num_chv_unblock_adm_codes': 4}},
+            # DF.TELECOM
+            {"resp_hex" : "000000007f100200000000000981000d0400838a838a",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'df'}}, 'proprietary_info': {'available_memory': 0}, 'file_id': '7f10', 'file_characteristics': '81', 'num_direct_child_df': 0, 'num_direct_child_ef': 13, 'num_chv_unblock_adm_codes': 4}},
+            # EF.MSISDN
+            {"resp_hex" : "000000346f40040011ffff0102011a",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'working_ef', 'structure': 'linear_fixed'}, 'record_len': 26, 'num_of_rec': 2}, 'proprietary_info': {}, 'file_id': '6f40', 'file_size': 52, 'access_conditions': '11ffff', 'life_cycle_status_int': 'creation'}},
+            # EF.ICCID
+            {"resp_hex" : "0000000a2fe204000cffff01020000",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'file_type': 'working_ef', 'structure': 'transparent'}}, 'proprietary_info': {}, 'file_id': '2fe2', 'file_size': 10, 'access_conditions': '0cffff', 'life_cycle_status_int': 'creation'}},
+        ]
+        self.decode_select_response(CardProfileSIM, testcases)
+
+    def test_CardProfileUICC(self):
+        testcases = [
+            # MF
+            {"resp_hex" : "622c8202782183023f00a50c80017183040003a7388701018a01058b032f0601c60c90016083010183010a83010b",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'df', 'structure': 'no_info_given'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'?\x00', 'proprietary_information': {'uicc_characteristics': b'q', 'available_memory': 239416, 'supported_filesystem_commands': {'terminal_capability': True}}, 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'/\x06', 'ef_arr_record_nr': 1}, 'pin_status_template_do': [{'ps_do': b'`'}, {'key_reference': 1}, {'key_reference': 10}, {'key_reference': 11}]}},
+            # ADF.USIM
+            {"resp_hex" : "623d8202782183027fd0840ca0000000871002ff49ff0589a50c80017183040003a7388701018a01058b032f0601c60f90017083010183018183010a83010b",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'df', 'structure': 'no_info_given'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'\x7f\xd0', 'df_name': b'\xa0\x00\x00\x00\x87\x10\x02\xffI\xff\x05\x89', 'proprietary_information': {'uicc_characteristics': b'q', 'available_memory': 239416, 'supported_filesystem_commands': {'terminal_capability': True}}, 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'/\x06', 'ef_arr_record_nr': 1}, 'pin_status_template_do': [{'ps_do': b'p'}, {'key_reference': 1}, {'key_reference': 129}, {'key_reference': 10}, {'key_reference': 11}]}},
+            # ADF.ISIM
+            {"resp_hex" : "623d8202782183027fb0840ca0000000871004ff49ff0589a50c80017183040003a7388701018a01058b032f0601c60f90017083010183018183010a83010b",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'df', 'structure': 'no_info_given'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'\x7f\xb0', 'df_name': b'\xa0\x00\x00\x00\x87\x10\x04\xffI\xff\x05\x89', 'proprietary_information': {'uicc_characteristics': b'q', 'available_memory': 239416, 'supported_filesystem_commands': {'terminal_capability': True}}, 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'/\x06', 'ef_arr_record_nr': 1}, 'pin_status_template_do': [{'ps_do': b'p'}, {'key_reference': 1}, {'key_reference': 129}, {'key_reference': 10}, {'key_reference': 11}]}},
+            # EF.IMSI
+            {"resp_hex" : "62178202412183026f078a01058b036f060a80020009880138",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'working_ef', 'structure': 'transparent'}, 'record_len': None, 'num_of_rec': None}, 'file_identifier': b'o\x07', 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'o\x06', 'ef_arr_record_nr': 10}, 'file_size': 9, 'short_file_identifier': 7}},
+            # EF.ECC
+            {"resp_hex" : "621a82054221000e0283026fb78a01058b036f06088002001c880108",
+             "decoded" : {'file_descriptor': {'file_descriptor_byte': {'shareable': True, 'file_type': 'working_ef', 'structure': 'linear_fixed'}, 'record_len': 14, 'num_of_rec': 2}, 'file_identifier': b'o\xb7', 'life_cycle_status_integer': 'operational_activated', 'security_attrib_referenced': {'ef_arr_file_id': b'o\x06', 'ef_arr_record_nr': 8}, 'file_size': 28, 'short_file_identifier': 1}},
+        ]
+        self.decode_select_response(CardProfileUICC, testcases)
+
+if __name__ == "__main__":
+	unittest.main()

tests/unittests/test_esim_saip.py

@@ -21,7 +21,7 @@ import copy
 from osmocom.utils import h2b, b2h
 
 from pySim.esim.saip import *
-from pySim.esim.saip.personalization import *
+from pySim.esim.saip import personalization
 from pprint import pprint as pp
 
 
@@ -55,14 +55,56 @@ class SaipTest(unittest.TestCase):
     def test_personalization(self):
         """Test some of the personalization operations."""
         pes = copy.deepcopy(self.pes)
-        params = [Puk1('01234567'), Puk2(98765432), Pin1('1111'), Pin2(2222), Adm1('11111111'),
-                  K(h2b('000102030405060708090a0b0c0d0e0f')), Opc(h2b('101112131415161718191a1b1c1d1e1f'))]
+        params = [personalization.Puk1('01234567'),
+                  personalization.Puk2(98765432),
+                  personalization.Pin1('1111'),
+                  personalization.Pin2(2222),
+                  personalization.Adm1('11111111'),
+                  personalization.K(h2b('000102030405060708090a0b0c0d0e0f')),
+                  personalization.Opc(h2b('101112131415161718191a1b1c1d1e1f'))]
         for p in params:
             p.validate()
             p.apply(pes)
         # TODO: we don't actually test the results here, but we just verify there is no exception
         pes.to_der()
 
+    def test_personalization2(self):
+        """Test some of the personalization operations."""
+        cls = personalization.SdKeyScp80Kvn01DesEnc
+        pes = ProfileElementSequence.from_der(self.per_input)
+        prev_val = tuple(cls.get_values_from_pes(pes))
+        print(f'{prev_val=}')
+        self.assertTrue(prev_val)
+
+        set_val = '42342342342342342342342342342342'
+        param = cls(set_val)
+        param.validate()
+        param.apply(pes)
+
+        get_val1 = tuple(cls.get_values_from_pes(pes))
+        print(f'{get_val1=} {set_val=}')
+        self.assertEqual(get_val1, ({cls.name: set_val},))
+
+        get_val1b = tuple(cls.get_values_from_pes(pes))
+        print(f'{get_val1b=} {set_val=}')
+        self.assertEqual(get_val1b, ({cls.name: set_val},))
+
+        der = pes.to_der()
+
+        get_val1c = tuple(cls.get_values_from_pes(pes))
+        print(f'{get_val1c=} {set_val=}')
+        self.assertEqual(get_val1c, ({cls.name: set_val},))
+
+        # assertTrue to not dump the entire der.
+        # Expecting the modified DER to be different. If this assertion fails, then no change has happened in the output
+        # DER and the ConfigurableParameter subclass is buggy.
+        self.assertTrue(der != self.per_input)
+
+        pes2 = ProfileElementSequence.from_der(der)
+        get_val2 = tuple(cls.get_values_from_pes(pes2))
+        print(f'{get_val2=} {set_val=}')
+        self.assertEqual(get_val2, ({cls.name: set_val},))
+
     def test_constructor_encode(self):
         """Test that DER-encoding of PE created by "empty" constructor works without raising exception."""
         for cls in [ProfileElementMF, ProfileElementPuk, ProfileElementPin, ProfileElementTelecom,

tests/unittests/test_files.py

@@ -27,6 +27,7 @@ import pySim.ts_31_102
 import pySim.ts_31_103
 import pySim.ts_51_011
 import pySim.sysmocom_sja2
+import pySim.sysmocom_sjs1
 import pySim.gsm_r
 import pySim.cdma_ruim
 

tests/unittests/test_globalplatform.py

@@ -17,7 +17,10 @@
 
 import unittest
 import logging
+import hashlib
+from types import SimpleNamespace
 from osmocom.utils import b2h, h2b
+from osmocom.tlv import bertlv_encode_len
 
 from pySim.global_platform import *
 from pySim.global_platform.scp import *
@@ -283,6 +286,41 @@ class SCP03_Test_AES256_33(SCP03_Test, unittest.TestCase):
 # FIXME: test auth with random (0x60) vs pseudo-random (0x70) challenge
 
 
+class KeyComponentBlock_Test(unittest.TestCase):
+    """Tests for the kcb of GP CardSpec v2.3
+    - Table 11-70 kcv that required padding, preceded by its clear-text length
+    - Table 11-71 no padding required"""
+
+    def setUp(self):
+        # SCP02 (3DES DEK, 8 byte blocks), same vectors as SCP02_Test
+        self.scp02 = SCP02(card_keys=ck_3des_70)
+        self.scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
+        self.scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
+        self.scp02.gen_ext_auth_apdu()
+        # SCP03 (AES DEK, 16 byte blocks), same vectors as SCP03_Test_AES128_11
+        self.scp03 = SCP03(card_keys=KEYSET_AES128)
+        self.scp03.gen_init_update_apdu(h2b('b13e5f938fc108c4'))
+        self.scp03.parse_init_update_resp(h2b('000000000000000000003003703eb51047495b249f66c484c1d2ef1948000002'))
+        self.scp03.gen_ext_auth_apdu(0x11)
+
+    def test_encrypt_decrypt_key(self):
+        for scp in (self.scp02, self.scp03):
+            bs = scp.sk.blocksize
+            for keylen in range(1, 3 * bs + 1):
+                with self.subTest(scp=type(scp).__name__, keylen=keylen):
+                    key = bytes(range(keylen))
+                    kcb = scp.encrypt_key(key)
+                    if keylen % bs:
+                        # Table 11-70: <length of clear key component> || <encrypted padded value>
+                        self.assertEqual(kcb[0], keylen)
+                        self.assertEqual((len(kcb) - 1) % bs, 0)
+                        self.assertEqual(len(kcb) - 1, keylen + (bs - keylen % bs))
+                    else:
+                        # Table 11-71: only the encrypted key component value
+                        self.assertEqual(len(kcb), keylen)
+                    self.assertEqual(scp.decrypt_key(kcb), key)
+
+
 class SCP03_KCV_Test(unittest.TestCase):
     def test_kcv(self):
         self.assertEqual(compute_kcv('aes', KEYSET_AES128.enc), h2b('C35280'))
@@ -290,6 +328,208 @@ class SCP03_KCV_Test(unittest.TestCase):
         self.assertEqual(compute_kcv('aes', KEYSET_AES128.dek), h2b('840DE5'))
 
 
+class PutKey_PSK_Test(unittest.TestCase):
+    """Tests for the PUT KEY command data field encoding, in particular the PSK TLS ('85') key data
+    field defined by GlobalPlatform Amendment B (Remote Application Management over HTTP) Table 3-13."""
+
+    # the PUT KEY encoder we exercise
+    C = ADF_SD.AddlShellCommands
+
+    # SCP80 TLS-PSK example key from the do_put_key docstring (16 bytes)
+    PSK_CLEAR = h2b('303132333435363738393a3b3c3d3e3f')
+    # its DEK ciphertext + Table 3-13 KCV with SCP02 session set up below
+    PSK_CIPHERED = h2b('15abf1fe16ccc5aa13743394442942cd')
+    PSK_KCV = h2b('06125d')  # = SHA-1(PSK_CLEAR)[:3]
+
+    def setUp(self):
+        # SCP02 with the same vectors as SCP02_Test, so that the whole PUT KEY data field is reproducible.
+        self.scp02 = SCP02(card_keys=ck_3des_70)
+        self.scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
+        self.scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
+        self.scp02.gen_ext_auth_apdu()
+
+    def test_psk_kcv_is_sha1(self):
+        # GP Amendment B Table 3-13: KCV = 3 most significant bytes of SHA-1(clear key)
+        self.assertEqual(compute_kcv('tls_psk', self.PSK_CLEAR), hashlib.sha1(self.PSK_CLEAR).digest()[:3])
+        self.assertEqual(compute_kcv('tls_psk', self.PSK_CLEAR), self.PSK_KCV)
+
+    def test_encode_psk_framing_golden(self):
+        # assert the exact Table 3-13 layout
+        #   85 | L1 | L2 | <ciphered> | 03 | <SHA-1(clear)[:3]>
+        clear = self.PSK_CLEAR
+        ciphered = h2b('aabbccddeeff00112233445566778899')  # arbitrary 16-byte ciphertext
+        kcv = hashlib.sha1(clear).digest()[:3]
+        field = self.C.encode_key_data_psk(clear, ciphered, kcv)
+        #                            85   L1   L2   <---------- ciphered ----------->  03  <-kcv->
+        self.assertEqual(b2h(field),'85' '11' '10' 'aabbccddeeff00112233445566778899' '03' + b2h(kcv))
+        self.assertEqual(b2h(field),'851110aabbccddeeff0011223344556677889903' + '06125d')
+
+    def test_psk_golden_over_scp02(self):
+        # Full PUT KEY data field (KVN 0x40 + single PSK key) enciphered with the SCP02 DEK.
+        keys = [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR,
+                 'kcv': compute_kcv('tls_psk', self.PSK_CLEAR)}]
+        data = self.C.build_put_key_data(0x40, keys, self.scp02)
+        self.assertEqual(b2h(data),
+                         '40' '85' '11' '10' + b2h(self.PSK_CIPHERED) + '03' + b2h(self.PSK_KCV))
+
+    def test_wrong_basic_format_differs(self):
+        # regression test, the generic "Basic format" does NOT match Table 3-13 for a PSK key
+        # rejected by card with with 6a88
+        wrong_basic = self.C.encode_key_data_basic('tls_psk', self.PSK_CIPHERED, b'')
+        right_psk = self.C.encode_key_data_psk(self.PSK_CLEAR, self.PSK_CIPHERED, self.PSK_KCV)
+        self.assertEqual(b2h(wrong_basic), '8510' + b2h(self.PSK_CIPHERED) + '00')
+        self.assertEqual(b2h(right_psk), '8511' '10' + b2h(self.PSK_CIPHERED) + '03' + b2h(self.PSK_KCV))
+        self.assertNotEqual(wrong_basic, right_psk)
+
+    def test_key_component_block_length_is_bertlv(self):
+        # GP CardSpec v2.3.1 Section 11.8.2.3.1: all lengths ofPUT KEY are always BER TLV coded
+        for kcb_len, exp_len_field in [(127, '7f'), (128, '8180'), (129, '8181'), (256, '820100')]:
+            with self.subTest(kcb_len=kcb_len):
+                kcb = bytes(kcb_len)
+                field = self.C.encode_key_data_basic('rsa_modulus_n', kcb, b'')
+                self.assertEqual(b2h(field), 'a2' + exp_len_field + b2h(kcb) + '00')
+                # 85 field of Amendment B Table 3-13 uses the same coding
+                # single byte inner length (clear key < 128) == block kcb_len bytes long
+                psk = self.C.encode_key_data_psk(bytes(120), bytes(kcb_len - 1), b'')
+                self.assertEqual(b2h(psk)[:2 + len(exp_len_field)], '85' + exp_len_field)
+
+    def test_basic_format_unchanged(self):
+        # as before
+        for kt, clear in [('des', h2b('404142434445464748494a4b4c4d4e4f')),
+                          ('aes', h2b('000102030405060708090a0b0c0d0e0f'))]:
+            ciph = self.scp02.encrypt_key(clear)
+            kcv = compute_kcv(kt, clear)
+            via_construct = build_construct(self.C.KeyDataBasic, {'key_type': kt, 'kcb': b2h(ciph), 'kcv': b2h(kcv)})
+            via_helper = self.C.encode_key_data_basic(kt, ciph, kcv)
+            self.assertEqual(via_helper, via_construct)
+
+    def test_psk_padding_no_double_length(self):
+        # A PSK key whose length is not a multiple of the DEK block size (DES: 8) is right-padded before
+        # ciphering. Table 3-13 states the clear key length (L2) in the '85' DO itself, so the ciphered
+        # key field is the bare cryptogram:
+        # - ciphered field == padded ciphertext (no duplicated length prefix),
+        # - clear key == first L2 bytes.
+        for keylen in (18, 20):
+            with self.subTest(keylen=keylen):
+                clear = bytes(range(keylen))
+                padded_len = keylen + (-keylen % 8)
+                field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': clear,
+                                                          'kcv': compute_kcv('tls_psk', clear)}], self.scp02)[1:]
+                self.assertEqual(field[0], 0x85)
+                l1 = field[1]
+                l2 = field[2]
+                self.assertEqual(l2, keylen)                # single-byte BER length of clear key
+                ciphered = field[3:3 + (l1 - 1)]            # value = L2 (1 byte) || ciphered key
+                self.assertEqual(len(ciphered), padded_len) # padded to the 8-byte DES block size
+                self.assertEqual(l1, 1 + padded_len)        # no duplicated length prefix
+                self.assertEqual(self.scp02.dek_decrypt(ciphered)[:keylen], clear)
+
+    def test_psk_clear_key_is_not_padded_in_place(self):
+        # padding the bytearray in place would make L2 the padded length,
+        # then stored as key material and rejected thanks to the KCV
+        clear = h2b('000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d')  # 30, not %8
+        kcv = compute_kcv('tls_psk', clear)
+        field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': clear,
+                                                  'kcv': kcv}], self.scp02)[1:]
+        self.assertEqual(len(clear), 30)
+        self.assertEqual(field[2], 30)                       # L2 == clear key length, not 32
+        self.assertEqual(self.scp02.dek_decrypt(field[3:3 + field[1] - 1])[:30], clear)
+
+    def test_kcv_suppressed(self):
+        # --suppress-key-check -> KCV length 00 and no KCV bytes
+        field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR,
+                                                  'kcv': b''}], self.scp02)[1:]
+        self.assertEqual(b2h(field), '8511' '10' + b2h(self.PSK_CIPHERED) + '00')
+
+    def test_multikey_psk_plus_des_dek(self):
+        # load a PSK TLS key (KID 1, Amendment B format) together with its DES DEK
+        # (KID 2, Basic format) in one PUT KEY.
+        # Verify the concatenated data field parses back into the two components with proper type formats.
+        dek = h2b('404142434445464748494a4b4c4d4e4f')
+        keys = [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR, 'kcv': compute_kcv('tls_psk', self.PSK_CLEAR)},
+                {'key_type': 'des', 'clear_key': dek, 'kcv': compute_kcv('des', dek)}]
+        data = self.C.build_put_key_data(0x40, keys, self.scp02)
+
+        b = data
+        self.assertEqual(b[0], 0x40)            # KVN
+        b = b[1:]
+        # component 1: PSK TLS (Table 3-13)
+        self.assertEqual(b[0], 0x85)
+        self.assertEqual(b[1], 0x11)            # L1 = 17
+        self.assertEqual(b[2], 0x10)            # L2 = 16 (clear key length)
+        self.assertEqual(b[3:3 + 16], self.PSK_CIPHERED)
+        self.assertEqual(b[3 + 16], 0x03)       # KCV length
+        self.assertEqual(b[3 + 16 + 1:3 + 16 + 1 + 3], self.PSK_KCV)
+        b = b[3 + 16 + 1 + 3:]
+        # component 2: DES DEK (Basic format)
+        self.assertEqual(b[0], 0x80)            # key type des
+        kcb_len = b[1]
+        self.assertEqual(kcb_len, 16)
+        self.assertEqual(b[2:2 + kcb_len], self.scp02.encrypt_key(dek))
+        b = b[2 + kcb_len:]
+        self.assertEqual(b[0], 0x03)            # KCV length
+        self.assertEqual(b[1:1 + 3], compute_kcv('des', dek))
+        self.assertEqual(b[1 + 3:], b'')        # no trailing bytes
+
+    def test_no_scp_leaves_key_clear(self):
+        # During personalization (no SCP) the key is not enciphered, framing still follows Table 3-13.
+        field = self.C.build_put_key_data(0x40, [{'key_type': 'tls_psk', 'clear_key': self.PSK_CLEAR,
+                                                  'kcv': self.PSK_KCV}], None)[1:]
+        self.assertEqual(b2h(field), '8511' '10' + b2h(self.PSK_CLEAR) + '03' + b2h(self.PSK_KCV))
+
+
+class PutKey_Length_Test(unittest.TestCase):
+    """Tests for the length of the PUT KEY command APDU.  Lc of GP CardSpec v2.3 Table 11-64 is a
+    single byte, so an oversized key data field cannot be sent."""
+
+    class PutKeyOnly(ADF_SD.AddlShellCommands):
+        """ADF_SD.AddlShellCommands with a canned scc to drive put_key()"""
+        def __init__(self, scp=None, max_cmd_len=255):
+            super().__init__()
+            self.sent = []
+            self.scc = SimpleNamespace(scp=scp, max_cmd_len=max_cmd_len,
+                                       send_apdu_checksw=lambda pdu: (self.sent.append(pdu), ('', '9000'))[1])
+
+        @property
+        def _cmd(self):
+            return SimpleNamespace(lchan=SimpleNamespace(scc=self.scc))
+
+    # KVN, key type, two byte BER length of the key component block, KCV length; KCV suppressed
+    FRAMING = 1 + 1 + 2 + 1
+
+    @staticmethod
+    def key(nbytes: int):
+        return [{'key_type': 'rsa_modulus_n', 'clear_key': bytes(nbytes), 'kcv': b''}]
+
+    def test_lc_matches_data_field(self):
+        # largest key component block that still fits without a secure channel
+        sd = self.PutKeyOnly()
+        sd.put_key(0, 0x40, 1, self.key(255 - self.FRAMING))
+        apdu = sd.sent[0]
+        self.assertEqual(apdu[:8], '80D80001')
+        lc = int(apdu[8:10], 16)
+        self.assertEqual(lc, 255)                       # Lc ...
+        self.assertEqual(len(apdu[10:-2]) // 2, lc)     # ... and it matches the actual data field
+
+    def test_oversized_key_data_raises(self):
+        # real world fat example: RSA-2048 modulus does not fit, led to 3 nibble Lc 106,
+        # which silently shifted and broke the whole APDU by half a byte.
+        sd = self.PutKeyOnly()
+        with self.assertRaises(ValueError) as ctx:
+            sd.put_key(0, 0x40, 1, self.key(256))
+        self.assertIn('262', str(ctx.exception))
+        self.assertIn('255', str(ctx.exception))
+        self.assertEqual(sd.sent, [])                   # nothing was sent to the card
+
+    def test_secure_channel_overhead_lowers_the_limit(self):
+        # scc.max_cmd_len shrinks by the C-MAC + encryption padding of active SCP
+        sd = self.PutKeyOnly(max_cmd_len=239)
+        sd.put_key(0, 0x40, 1, self.key(239 - self.FRAMING))
+        self.assertEqual(int(sd.sent[0][8:10], 16), 239)
+        with self.assertRaises(ValueError):
+            sd.put_key(0, 0x40, 1, self.key(239 - self.FRAMING + 1))
+
+
 class Install_param_Test(unittest.TestCase):
     def test_gen_install_parameters(self):
         load_parameters = gen_install_parameters(256, 256, '010001001505000000000000000000000000')
@@ -298,5 +538,339 @@ class Install_param_Test(unittest.TestCase):
         load_parameters = gen_install_parameters()
         self.assertEqual(load_parameters, 'c900')
 
+class SCP_Overhead_Test(unittest.TestCase):
+    """SCP.overhead varies according to the current security level:
+    C-MAC + at level >= 3 the worst-case padding!
+    """
+
+    def _scp02(self, security_level):
+        scp = SCP02(card_keys=ck_3des_70)
+        scp.sk = Scp02SessionKeys(0x0001, ck_3des_70)
+        scp.security_level = security_level
+        return scp
+
+    def _scp03(self, security_level, s_mode=8):
+        scp = SCP03(card_keys=KEYSET_AES128, s_mode=s_mode)
+        scp.sk = Scp03SessionKeys(KEYSET_AES128, b'\x00' * s_mode, b'\x11' * s_mode)
+        scp.security_level = security_level
+        return scp
+
+    def test_scp02(self):
+        self.assertEqual(self._scp02(0x00).overhead, 0)   # no wrapping at all
+        self.assertEqual(self._scp02(0x01).overhead, 8)   # C-MAC
+        self.assertEqual(self._scp02(0x03).overhead, 16)  # C-MAC + C-DEC: pad80 to 8, largest fit 239
+
+    def test_scp03_s8(self):
+        self.assertEqual(self._scp03(0x00).overhead, 0)
+        self.assertEqual(self._scp03(0x01).overhead, 8)
+        self.assertEqual(self._scp03(0x03).overhead, 16)  # pad80 to 16 within 247 -> 240, minus pad byte
+        self.assertEqual(self._scp03(0x33).overhead, 16)  # R-MAC/R-ENC add no *command* overhead
+
+    def test_scp03_s16(self):
+        self.assertEqual(self._scp03(0x01, s_mode=16).overhead, 16)
+        self.assertEqual(self._scp03(0x03, s_mode=16).overhead, 32)  # pad80 to 16 within 239 -> 224, minus pad byte
+
+
+class SCP_Lc_Limit_Test_Base(unittest.TestCase):
+    """Test wrap_cmd_apdu() boundary handling: data of (255 - overhead) must produce Lc <= 255 else ValueError"""
+
+    def _load_apdu(self, data_len):
+        return h2b('80E80000') + bytes([data_len]) + b'\xa5' * data_len
+
+    def _check_boundary(self, scp):
+        fits = 255 - scp.overhead
+        wrapped = scp.wrap_cmd_apdu(self._load_apdu(fits))
+        self.assertLessEqual(wrapped[4], 255)
+        self.assertEqual(len(wrapped), 5 + wrapped[4])  # case #3: header + Lc bytes, no Le
+        with self.assertRaises(ValueError) as ctx:
+            scp.wrap_cmd_apdu(self._load_apdu(fits + 1))
+        self.assertIn('Lc', str(ctx.exception))
+
+
+class SCP02_Lc_Limit_Test(SCP_Lc_Limit_Test_Base):
+    """Same session vectors as SCP02_Auth_Test"""
+
+    def setUp(self):
+        self.scp02 = SCP02(card_keys=ck_3des_70)
+        self.scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
+        self.scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
+        self.scp02.gen_ext_auth_apdu()
+
+    def test_cmac_only(self):
+        self.scp02.security_level = 0x01
+        self._check_boundary(self.scp02)  # 247 fits, 248 raises
+
+    def test_cmac_cdec(self):
+        self.scp02.security_level = 0x03
+        self._check_boundary(self.scp02)  # 239 fits (-> Lc 248), 240 raises (would be 256)
+
+    def test_cmac_cdec_wrapped_lc(self):
+        # my actual failing case: 240 bytes at level 3
+        self.scp02.security_level = 0x03
+        wrapped = self.scp02.wrap_cmd_apdu(self._load_apdu(239))
+        self.assertEqual(wrapped[4], 248)  # 239 -> pad80 -> 240 ciphertext + 8 mac
+
+
+class SCP03_Lc_Limit_Test(SCP_Lc_Limit_Test_Base):
+    """Session keys derived directly"""
+
+    def _scp03(self, security_level, s_mode):
+        scp = SCP03(card_keys=KEYSET_AES128, s_mode=s_mode)
+        scp.sk = Scp03SessionKeys(KEYSET_AES128, b'\x00' * s_mode, b'\x11' * s_mode)
+        scp.security_level = security_level
+        return scp
+
+    def test_s8_cmac_only(self):
+        self._check_boundary(self._scp03(0x01, 8))    # 247 fits, 248 raises
+
+    def test_s8_cmac_cdec(self):
+        self._check_boundary(self._scp03(0x03, 8))    # 239 fits, 240 raises
+
+    def test_s16_cmac_only(self):
+        self._check_boundary(self._scp03(0x01, 16))   # 239 fits, 240 raises
+
+    def test_s16_cmac_cdec(self):
+        self._check_boundary(self._scp03(0x03, 16))   # 223 fits, 224 raises
+
+
+class _FakeSccForLoad:
+    """mock lchan.scc: records LOAD APDUs, optionally wrapping them through a real SCP
+    instance first where the Lc overflow used to blow up"""
+
+    def __init__(self, max_cmd_len=255, scp=None):
+        self.max_cmd_len = max_cmd_len
+        self.scp = scp
+        self.sent = []
+        self.wrapped = []
+
+    def send_apdu_checksw(self, apdu, sw='9000'):
+        self.sent.append(apdu.lower())
+        if self.scp:
+            self.wrapped.append(self.scp.wrap_cmd_apdu(h2b(apdu)))
+        return ('', '9000')
+
+
+class Load_ChunkLen_Test(unittest.TestCase):
+    """ADF_SD.load() chunking: block size must use scc.max_cmd_len"""
+
+    payload = b'\xaa' * 500  # actual real world case LOAD TLV: C4 + 8201f4 + 500 = 504 total
+
+    def _sd(self, scc):
+        cmd = type('_Cmd', (), {'lchan': type('_Lchan', (), {'scc': scc})(),
+                                'poutput': lambda self, *args: None})()
+        # cmd2 CommandSet has a r/o _cmd property -> shadow it
+        _SD = type('_SD', (ADF_SD.AddlShellCommands,), {'_cmd': cmd})
+        return _SD.__new__(_SD)
+
+    def _blocks(self, scc):
+        """Get (p1, p2, lc) from LOAD APDU"""
+        for apdu in scc.sent:
+            self.assertEqual(apdu[0:4], '80e8')
+            yield int(apdu[4:6], 16), int(apdu[6:8], 16), int(apdu[8:10], 16)
+
+    def test_default_no_scp(self):
+        """Without SCP the old 240 byte block size is kept, no idea what else might rely on this number"""
+        scc = _FakeSccForLoad(max_cmd_len=255)
+        self._sd(scc).load(self.payload)
+        blocks = list(self._blocks(scc))
+        self.assertEqual([b[2] for b in blocks], [240, 240, 24])
+        self.assertEqual([b[0] for b in blocks], [0x00, 0x00, 0x80])  # P1: last block flagged
+        self.assertEqual([b[1] for b in blocks], [0, 1, 2])           # P2: block num
+
+    def test_default_scp02_level3(self):
+        """max_cmd_len 239 (SCP02 lvl 3) squeezes the blocks"""
+        scc = _FakeSccForLoad(max_cmd_len=239)
+        self._sd(scc).load(self.payload)
+        self.assertEqual([b[2] for b in list(self._blocks(scc))], [239, 239, 26])
+
+    def test_explicit_chunk_len(self):
+        scc = _FakeSccForLoad(max_cmd_len=255)
+        self._sd(scc).load(self.payload, chunk_len=100)
+        self.assertEqual([b[2] for b in list(self._blocks(scc))], [100] * 5 + [4])
+
+    def test_explicit_chunk_len_too_large(self):
+        scc = _FakeSccForLoad(max_cmd_len=239)
+        with self.assertRaises(ValueError):
+            self._sd(scc).load(self.payload, chunk_len=240)
+        self.assertEqual(scc.sent, [])  # nothing sent!
+
+    def test_explicit_chunk_len_zero(self):
+        scc = _FakeSccForLoad(max_cmd_len=255)
+        with self.assertRaises(ValueError):
+            self._sd(scc).load(self.payload, chunk_len=0)
+
+    def test_end_to_end_scp02_level3(self):
+        """original failure: 286 byte CAP + SCP02 lvl 3"""
+        scp02 = SCP02(card_keys=ck_3des_70)
+        scp02.gen_init_update_apdu(host_challenge=h2b('40A62C37FA6304F8'))
+        scp02.parse_init_update_resp(h2b('00000000000000000000700200016B4524ABEE7CF32EA3838BC148F3'))
+        scp02.gen_ext_auth_apdu()
+        scp02.security_level = 0x03
+        scc = _FakeSccForLoad(max_cmd_len=255 - scp02.overhead, scp=scp02)
+        self._sd(scc).load(b'\x5a' * 286)
+        self.assertEqual(len(scc.sent), 2)  # 289 byte TLV in blocks of 239
+        for wrapped in scc.wrapped:
+            self.assertLessEqual(wrapped[4], 255)
+
+# Real Card Data (GET DATA '66'), as returned by sja5 + euicc
+CARD_DATA_V211 = ('6631732f06072a864886fc6b01600c060a2a864886fc6b0202010163090607'
+                  '2a864886fc6b03640b06092a864886fc6b040215')
+CARD_DATA_V22 = ('663b733906072a864886fc6b01600b06092a864886fc6b020202630906072a86'
+                 '4886fc6b03640b06092a864886fc6b040370640b06092a864886fc6b04810400')
+
+
+class _FakeScc:
+    """mock lchan.scc: replays scripted (data, sw) pairs + records the APDUs sent."""
+
+    def __init__(self, responses, card_data=CARD_DATA_V211):
+        self._responses = list(responses)
+        self._card_data = card_data
+        self.sent = []
+
+    def get_data(self, cla, tag):
+        if self._card_data is None:
+            raise SwMatchError('6a88', '9000')
+        return self._card_data, '9000'
+
+    def send_apdu(self, apdu):
+        self.sent.append(apdu.lower())
+        if not self._responses:
+            raise AssertionError('get_status sent unexpected APDU: %s' % apdu)
+        return self._responses.pop(0)
+
+
+class GpVersion_Test(unittest.TestCase):
+    """GP version from Card Recognition Data, which v2.1.1/v2.3.1 section 7.4.1.3
+    require to be present. The OID under tag 60 is {globalPlatform 2 v...}."""
+
+    def test_decode_real_cards(self):
+        self.assertEqual(decode_gp_version(h2b(CARD_DATA_V211)), (2, 1, 1))
+        self.assertEqual(decode_gp_version(h2b(CARD_DATA_V22)), (2, 2))
+
+    def test_unknown_oid_is_none(self):
+        self.assertIsNone(decode_gp_version(h2b('66097307060512345678')))
+
+    def test_tag_lists_follow_the_spec_tables(self):
+        """table 11-36 applications, table 11-37 for load files"""
+        self.assertEqual(b2h(get_status_tag_list('isd')), '5c074f9f70c5cfc4cc')
+        self.assertEqual(b2h(get_status_tag_list('applications')), '5c074f9f70c5cfc4cc')
+        self.assertEqual(b2h(get_status_tag_list('files')), '5c054f9f70cecc')
+        self.assertEqual(b2h(get_status_tag_list('files_and_modules')), '5c064f9f70ce84cc')
+        # C5 never load files, 84 never applications
+        self.assertNotIn('c5', b2h(get_status_tag_list('files')))
+        self.assertNotIn('84', b2h(get_status_tag_list('applications'))[4:])
+
+
+class GetStatus_Pagination_Test(unittest.TestCase):
+    """GPC v2.3.1 section 11.4.3.2 table 11-38 GET STATUS pagination test
+
+    Card answers 6310 when further matches are pending; command reissued with
+    P2 bit 1 "next occurrence" set. Tied to T=0 handling pySim/transport, which
+    used to swallow that 6310 and replied with GET RESPONSE, so page 2 was never fetched."""
+
+    ENTRY_1 = 'e3074f05a000000151'
+    ENTRY_2 = 'e3074f05a000000152'
+
+    def _sd(self, responses, card_data=CARD_DATA_V211):
+        scc = _FakeScc(responses, card_data)
+        cmd = type('_Cmd', (), {'lchan': type('_Lchan', (), {'scc': scc})()})()
+        # cmd2 strikes again, CommandSet exposes _cmd as a read only property, needs shadowing
+        _SD = type('_SD', (ADF_SD.AddlShellCommands,), {'_cmd': cmd})
+        return _SD.__new__(_SD), scc
+
+    def _aids(self, grd_list):
+        return [b2h(grd.to_dict()['gp_registry_related_data'][0]['application_aid']) for grd in grd_list]
+
+    def test_single_page(self):
+        sd, scc = self._sd([(self.ENTRY_1, '9000')])
+        grd_list = sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f24002024f0000'])
+        self.assertEqual(self._aids(grd_list), ['a000000151'])
+
+    def test_two_pages(self):
+        """6310 -> reissue with P2 bit 1 set -> 9000, both pages in result"""
+        sd, scc = self._sd([(self.ENTRY_1, '6310'), (self.ENTRY_2, '9000')])
+        grd_list = sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f24002024f0000',
+                                    '80f24003024f0000'])
+        self.assertEqual(self._aids(grd_list), ['a000000151', 'a000000152'])
+
+    def test_three_pages_keep_p2_next_occurrence(self):
+        sd, scc = self._sd([(self.ENTRY_1, '6310'), (self.ENTRY_2, '6310'), (self.ENTRY_1, '9000')])
+        grd_list = sd.get_status('applications')
+        self.assertEqual([a[6:8] for a in scc.sent], ['02', '03', '03'])
+        self.assertEqual(len(grd_list), 3)
+
+    def test_no_match_returns_empty(self):
+        """6A88 "referenced data not found" is empty result not failure."""
+        sd, _scc = self._sd([('', '6a88')])
+        self.assertEqual(sd.get_status('applications'), [])
+
+    def test_v211_card_gets_no_tag_list(self):
+        """v2.1.1 section 9.4.2.3 has no tag list,not send a tag list"""
+        sd, scc = self._sd([(self.ENTRY_1, '9000')], card_data=CARD_DATA_V211)
+        sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f24002024f0000'])
+        self.assertNotIn('5c', scc.sent[0][8:])
+
+    def test_v22_card_gets_a_tag_list(self):
+        sd, scc = self._sd([(self.ENTRY_1, '9000')], card_data=CARD_DATA_V22)
+        sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f240020b4f005c074f9f70c5cfc4cc00'])
+
+    def test_unknown_version_gets_no_tag_list(self):
+        """If the card will not say, assume the conservative form that works everywhere."""
+        sd, scc = self._sd([(self.ENTRY_1, '9000')], card_data=None)
+        sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f24002024f0000'])
+
+    def test_v22_card_rejecting_tag_list_falls_back(self):
+        """card announcing v2.2+ that still answers 6A80 to the tag list."""
+        sd, scc = self._sd([('', '6a80'), (self.ENTRY_1, '9000')], card_data=CARD_DATA_V22)
+        grd_list = sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f240020b4f005c074f9f70c5cfc4cc00',
+                                    '80f24002024f0000'])
+        self.assertEqual(self._aids(grd_list), ['a000000151'])
+
+    def test_aid_search_qualifier(self):
+        sd, scc = self._sd([(self.ENTRY_1, '9000')])
+        sd.get_status('applications', 'a000000087')
+        self.assertEqual(scc.sent, ['80f24002074f05a00000008700'])
+
+    def test_6a80_is_reported_on_a_v211_card(self):
+        """no tag list -> 6A80 is error"""
+        sd, _scc = self._sd([('', '6a80')], card_data=CARD_DATA_V211)
+        with self.assertRaises(SwMatchError) as ctx:
+            sd.get_status('applications')
+        self.assertEqual(ctx.exception.sw_actual, '6a80')
+
+    def test_unexpected_sw_is_not_silently_truncated(self):
+        """partial is not complete result"""
+        sd, _scc = self._sd([(self.ENTRY_1, '6310'), ('', '6982')])
+        with self.assertRaises(SwMatchError) as ctx:
+            sd.get_status('applications')
+        self.assertEqual(ctx.exception.sw_actual, '6982')
+
+    def test_v22_card_answering_6a88_to_the_tag_list_falls_back(self):
+        """6A88 is the other GET STATUS error condition of table 11-39, section 11.4.2.3
+        says we may get get an error status. 6A88 to the tag-list attempt should be retried
+        without it or we get nothing"""
+        sd, scc = self._sd([('', '6a88'), (self.ENTRY_1, '9000')], card_data=CARD_DATA_V22)
+        grd_list = sd.get_status('applications')
+        self.assertEqual(scc.sent, ['80f240020b4f005c074f9f70c5cfc4cc00',
+                                    '80f24002024f0000'])
+        self.assertEqual(self._aids(grd_list), ['a000000151'])
+
+    def test_v22_card_with_a_genuinely_empty_subset(self):
+        """...and when the retry answers 6A88, the list really is empty."""
+        sd, scc = self._sd([('', '6a88'), ('', '6a88')], card_data=CARD_DATA_V22)
+        self.assertEqual(sd.get_status('applications'), [])
+        self.assertEqual(len(scc.sent), 2)
+
+    def test_6a88_after_a_page_keeps_that_page(self):
+        """6A88 is "no more matches" after we have data, we're done"""
+        sd, _scc = self._sd([(self.ENTRY_1, '6310'), ('', '6a88')], card_data=CARD_DATA_V22)
+        self.assertEqual(self._aids(sd.get_status('applications')), ['a000000151'])
+
 if __name__ == "__main__":
 	unittest.main()

tests/unittests/test_log.py

@@ -20,11 +20,20 @@
 
 import unittest
 import logging
+import cmd2
+from packaging import version
 from pySim.log import PySimLogger
 import io
 import sys
 from inspect import currentframe, getframeinfo
 
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
+    from cmd2 import Color # pylint: disable=no-name-in-module
+    YELLOW = Color.YELLOW
+else: # cmd2>=2.6.2
+    from cmd2 import Fg # pylint: disable=no-name-in-module
+    YELLOW = Fg.YELLOW
+
 log = PySimLogger.get(__name__)
 
 TEST_MSG_DEBUG = "this is a debug message"
@@ -37,6 +46,17 @@ expected_message = None
 
 class PySimLogger_Test(unittest.TestCase):
 
+    def setUp(self):
+        # PySimLogger.setup() is global, so a print callback left installed here fires for
+        # every PySimLogger message emitted by any test module that runs later in the same process
+        # ... where it asserts against a stale 'expected_message' and fails a test that has nothing
+        # to do with logging. Great fun!
+        # Restore before each test.
+        saved = (PySimLogger.print_callback, PySimLogger.verbose)
+        def _restore():
+            PySimLogger.print_callback, PySimLogger.verbose = saved
+        self.addCleanup(_restore)
+
     def __test_01_safe_defaults_one(self, callback, message:str):
         # When log messages are sent to an unconfigured PySimLogger class, we expect the unmodified message being
         # logged to stdout, just as if it were printed via a normal print() statement.
@@ -117,5 +137,18 @@ class PySimLogger_Test(unittest.TestCase):
         expected_message = "CRITICAL: " + TEST_MSG_CRITICAL
         log.critical(TEST_MSG_CRITICAL)
 
+    def test_05_color(self):
+        # A color is either
+        # - raw escape sequence
+        # - cmd2 color object
+        global expected_message
+        expected_message = "\033[33mWARNING: " + TEST_MSG_WARNING + "\033[0m"
+
+        PySimLogger.setup(self._test_print_callback, {logging.WARN: "\033[33m"})
+        log.warning(TEST_MSG_WARNING)
+
+        PySimLogger.setup(self._test_print_callback, {logging.WARN: YELLOW})
+        log.warning(TEST_MSG_WARNING)                   # don't leak cmd2 Color StrEnum
+
 if __name__ == '__main__':
     unittest.main()

tests/unittests/test_ota.py

@@ -300,5 +300,292 @@ class SmsOtaTestCase(OtaTestCase):
                 self.assertEqual(d.last_status_word, t['response']['last_status_word'])
                 self.assertEqual(d.last_response_data, t['response']['last_response_data'])
 
+
+######################################################################
+# Expanded Remote Application data format (ETSI TS 102 226 Section 5.2)
+######################################################################
+
+class BerTlvLengthTestCase(unittest.TestCase):
+    """The definite-length BER-TLV length field (ISO/IEC 8825-1) used by the
+    expanded format, incl. the multi-byte (>127) forms (0x81xx / 0x82xxxx)."""
+    def test_roundtrip(self):
+        # (length value, expected encoded bytes)
+        vectors = [
+            (0,     '00'),
+            (1,     '01'),
+            (127,   '7f'),
+            (128,   '8180'),
+            (198,   '81c6'),   # big ~198 byte GET STATUS registry from a sja5
+            (255,   '81ff'),
+            (256,   '820100'),
+            (65535, '82ffff'),
+        ]
+        for length, encoded in vectors:
+            with self.subTest(length=length):
+                built = BerTlvLen.build(length)
+                self.assertEqual(b2h(built), encoded)
+                self.assertEqual(BerTlvLen.parse(built), length)
+
+
+class ExpandedCmdTestCase(unittest.TestCase):
+    """Command Scripting template TS 102 226 5.2.1"""
+
+    def test_single_capdu_golden(self):
+        # GP GET STATUS, Le=00, TS 102 226 5.2.1.1 R-APDU
+        out = encode_expanded_cmd(h2b('80f24002024f0000'))
+        # aa = TS 101 220 table 7.18 Command Scripting template tag
+        # 0a = length 10
+        # 22 = TS 101 220 table 7.19 C-APDU tag
+        # 08 = length
+        # + C-APDU
+        self.assertEqual(b2h(out), 'aa0a220880f24002024f0000')
+
+    def test_multi_capdu_golden(self):
+        out = encode_expanded_cmd([h2b('80f24002024f0000'), h2b('00a40004023f0000')])
+        self.assertEqual(b2h(out), 'aa14220880f24002024f0000220800a40004023f0000')
+
+    def test_multibyte_length_golden(self):
+        # C-APDU: 4 header + 1 Lc + 195 data = 200 bytes.
+        # 200 byte C-APDU forces long form BER lengths:
+        # C-APDU TLV, 200 -> 81c8 + template 203 -> 81cb
+        capdu = h2b('80f24000') + bytes([195]) + bytes(range(195))
+        self.assertEqual(len(capdu), 200)
+        out = encode_expanded_cmd(capdu)
+        # aa 81 cb | 22 81 c8 | <200 byte capdu>
+        self.assertEqual(b2h(out[:6]), 'aa81cb2281c8')
+        self.assertEqual(out[6:], capdu)
+
+    def test_roundtrip(self):
+        for apdus in [[h2b('80f24002024f0000')],
+                      [h2b('00a40004023f00'), h2b('80f24002024f0000')],
+                      [h2b('00'*250)]]:
+            with self.subTest(n=len(apdus)):
+                out = encode_expanded_cmd(apdus)
+                parsed = ExpandedCmd.parse(out)
+                self.assertEqual([h2b(c.c_apdu) for c in parsed.commands], apdus)
+
+
+class ExpandedRespTestCase(unittest.TestCase):
+    """Decoding of the Response Scripting template (TS 102 226 5.2.2)."""
+
+    def test_registry_golden(self):
+        # real card case: GET STATUS returns a ~198 byte registry TLV + SW 9000
+        # R-APDU = 198 data + 2 SW = 200/81c8
+        # 'number of executed' TLV 80 01 01.
+        registry = bytes(range(198))
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data=b2h(registry), status_word='9000'))])))
+        # ab | 81 ce | 80 01 01 | 23 81 c8 | <198 data> 90 00
+        self.assertEqual(b2h(data[:9]), 'ab81ce8001012381c8')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(dec.number_of_commands, 1)
+        self.assertEqual(len(dec.commands), 1)
+        self.assertEqual(dec.last_status_word, '9000')
+        self.assertEqual(dec.last_response_data, b2h(registry))
+
+    def test_status_only_golden(self):
+        # last command, no response data, SW 6132
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data='', status_word='6132'))])))
+        self.assertEqual(b2h(data), 'ab0780010123026132')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(dec.last_status_word, '6132')
+        self.assertEqual(dec.last_response_data, '')
+
+    def test_multi_command(self):
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=2),
+                    responses=[dict(r_apdu=dict(response_data='6f21', status_word='9000')),
+                               dict(r_apdu=dict(response_data='', status_word='6a82'))])))
+        dec = decode_expanded_resp(data)
+        self.assertEqual(dec.number_of_commands, 2)
+        self.assertEqual([(c.status_word, c.response_data) for c in dec.commands],
+                         [('9000', '6f21'), ('6a82', '')])
+        # last == final R-APDU, error status included
+        self.assertEqual(dec.last_status_word, '6a82')
+        self.assertEqual(dec.last_response_data, '')
+
+    def test_bad_format(self):
+        # ab | 06 | 80 01 01 | 90 01 01
+        data = h2b('ab06800101900101')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(str(dec.bad_format), 'unknown_tag')
+        self.assertIsNone(dec.last_status_word)
+
+    def test_immediate_action_error(self):
+        # ab | 06 | 80 01 01 | 81 01 01
+        data = h2b('ab06800101810101')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(str(dec.immediate_action_response), 'suspension_error')
+
+    def test_script_chaining_error(self):
+        # ab | 06 | 80 01 01 | 83 01 02
+        data = h2b('ab06800101830102')
+        dec = decode_expanded_resp(data)
+        self.assertEqual(str(dec.script_chaining_response), 'not_supported')
+
+    def test_truncation_is_flagged(self):
+        """TS 102 226 5.2.1.1: SW 62F1 means the C-APDU response data was truncated, and
+        "this shall terminate the processing of the command list"
+         halves are invisible in the R-APDU list, truncated + aborted script must not pass as complete"""
+        # second command truncated -> processing stopped at that point
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=2),
+                    responses=[dict(r_apdu=dict(response_data='6f21', status_word='9000')),
+                               dict(r_apdu=dict(response_data='aabb', status_word='62f1'))])))
+        dec = decode_expanded_resp(data)
+        self.assertTrue(dec.truncated)
+        self.assertEqual(dec.last_status_word, '62f1')
+
+    def test_untruncated_response_is_not_flagged(self):
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data='6f21', status_word='9000'))])))
+        self.assertFalse(decode_expanded_resp(data).truncated)
+        # 62xx that is not 62F1 is warning, not truncation
+        data = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data='', status_word='6282'))])))
+        self.assertFalse(decode_expanded_resp(data).truncated)
+
+
+class ExpandedIndefiniteTestCase(unittest.TestCase):
+    """Indef len coding of expanded format TS 102 226 tables
+    5.2a/5.10a; cmd tag AE, resp tag AF.
+    Golden vectors captured from live eUICC over SCP81/HTTPS."""
+
+    def test_cmd_single_golden(self):
+        # RAM GET DATA 80CA00E000 -> AE 80 | 22 05 80ca00e000 | 00 00
+        out = encode_expanded_cmd(h2b('80ca00e000'), length_coding='indefinite')
+        self.assertEqual(b2h(out), 'ae80220580ca00e0000000')
+
+    def test_cmd_multi_golden(self):
+        # RFM: SELECT MF / SELECT EF.ICCID / READ BINARY, each in one C-APDU
+        # TLV, wrapped in indef Command Scripting template
+        out = encode_expanded_cmd([h2b('00a4000c023f00'), h2b('00a4000c022fe2'),
+                                   h2b('00b000000a')], length_coding='indefinite')
+        self.assertEqual(b2h(out),
+                         'ae80220700a4000c023f00220700a4000c022fe2220500b000000a0000')
+
+    def test_cmd_definite_is_default(self):
+        # The default/explicit definite keeps the tag AA
+        self.assertEqual(encode_expanded_cmd(h2b('80ca00e000')),
+                         encode_expanded_cmd(h2b('80ca00e000'), length_coding='definite'))
+        self.assertEqual(b2h(encode_expanded_cmd(h2b('80ca00e000'))), 'aa07220580ca00e000')
+
+    def test_cmd_invalid_length_coding(self):
+        with self.assertRaises(ValueError):
+            encode_expanded_cmd(h2b('80ca00e000'), length_coding='bogus')
+
+    def test_resp_rfm_golden(self):
+        # AF 80 | 23 02 9000 | 23 02 9000 | 23 0c <ICCID> 9000 | 00 00
+        # indef res has no "number of executed" TLV.
+        dec = decode_expanded_resp(h2b(
+            'af80' '23029000' '23029000' '230c988812010000408608149000' '0000'))
+        self.assertEqual(len(dec.commands), 3)
+        self.assertEqual([(c.status_word, c.response_data) for c in dec.commands],
+                         [('9000', ''), ('9000', ''), ('9000', '98881201000040860814')])
+        self.assertEqual(dec.last_status_word, '9000')
+        self.assertEqual(dec.last_response_data, '98881201000040860814')
+        # report the R-APDU count instead
+        self.assertEqual(dec.number_of_commands, 3)
+
+    def test_resp_ram_golden(self):
+        # RAM GET DATA: R-APDU carrying the SD key info TLV + SW.
+        resp = ('af80' '2334e030c00403308810c00402308810c00401308810c00402408810'
+                'c00401408510c00403018810c00402018810c004010188109000' '0000')
+        dec = decode_expanded_resp(h2b(resp))
+        self.assertEqual(len(dec.commands), 1)
+        self.assertEqual(dec.last_status_word, '9000')
+        self.assertEqual(dec.last_response_data,
+                         'e030c00403308810c00402308810c00401308810c00402408810'
+                         'c00401408510c00403018810c00402018810c00401018810')
+
+    def test_resp_truncated_is_rejected(self):
+        # last byte chopped off: the end-of-contents marker is incomplete
+        good = h2b('af80' '23029000' '230c988812010000408608149000' '0000')
+        for cut in (1, 2, 3):
+            with self.subTest(cut=cut):
+                with self.assertRaises(ValueError):
+                    decode_expanded_resp(good[:-cut])
+
+    def test_resp_definite_still_parses(self):
+        # same decoder still handles the definite AB template.
+        dec = decode_expanded_resp(h2b('ab0780010123029000'))
+        self.assertEqual(dec.number_of_commands, 1)
+        self.assertEqual(dec.last_status_word, '9000')
+
+    def test_resp_indefinite_bad_format(self):
+        # AF 80 | 90 01 01 | 00 00 unknown_tag no R-APDU
+        dec = decode_expanded_resp(h2b('af8090010100 00'.replace(' ', '')))
+        self.assertEqual(str(dec.bad_format), 'unknown_tag')
+        self.assertIsNone(dec.last_status_word)
+
+    def test_resp_missing_eoc_raises(self):
+        # AF 80 | 23 02 9000 without end-of-contents.
+        with self.assertRaises(ValueError):
+            decode_expanded_resp(h2b('af8023029000'))
+
+
+class ExpandedSmsPipelineTestCase(unittest.TestCase):
+    """expanded format + TS 102 225 SMS security witj 3DES keyset,
+    to ensure remote_format does not affect the compact path"""
+    def __init__(self, methodName='runTest', **kwargs):
+        super().__init__(methodName, **kwargs)
+        self.od = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3,
+                            kic=h2b('C21DD66ACAC13CB3BC8B331B24AFB57B'),
+                            algo_auth='triple_des_cbc2', kid_idx=3,
+                            kid=h2b('12110C78E678C25408233076AA033615'))
+        self.dialect = OtaDialectSms()
+        self.tar = h2b('000000')
+
+    def test_cmd_expanded_secured_roundtrip(self):
+        spi = SPI_CC_POR_CIPHERED_CC
+        enc = self.dialect.encode_cmd(self.od, self.tar, spi, h2b('80f24002024f0000'),
+                                      remote_format='expanded')
+        # decode_cmd returns opaque 'Command Scripting template'
+        dec_tar, dec_spi, dec_secured = self.dialect.decode_cmd(self.od, enc)
+        self.assertEqual(b2h(dec_tar), b2h(self.tar))
+        self.assertEqual(dec_spi, spi)
+        self.assertEqual(b2h(dec_secured), 'aa0a220880f24002024f0000')
+
+    def test_cmd_expanded_list(self):
+        spi = SPI_CC_POR_CIPHERED_CC
+        enc = self.dialect.encode_cmd(self.od, self.tar, spi,
+                                      [h2b('80f24002024f0000'), h2b('00a40004023f0000')],
+                                      remote_format='expanded')
+        _, _, dec_secured = self.dialect.decode_cmd(self.od, enc)
+        parsed = ExpandedCmd.parse(dec_secured)
+        self.assertEqual([c.c_apdu for c in parsed.commands],
+                         ['80f24002024f0000', '00a40004023f0000'])
+
+    def test_resp_expanded_plaintext(self):
+        # plaintext (u:nciphered + no CC) expanded response SMS
+        # containing a 198 byte GP registry + SW 9000 as above, decode it through decode_resp().
+        spi = SPI_CC_POR_UNCIPHERED_NOCC
+        registry = bytes(range(198))
+        secured = ExpandedRemoteResp.build(dict(body=dict(
+                    num_executed=dict(number_of_commands=1),
+                    responses=[dict(r_apdu=dict(response_data=b2h(registry), status_word='9000'))])))
+        rpl = 1 + 3 + 5 + 1 + 1 + len(secured)   # RHL-STS + secured data
+        resp_body = rpl.to_bytes(2, 'big') + b'\x0a' + self.tar + b'\x00'*5 + b'\x00' + b'\x00' + secured
+        sms = b'\x02\x71\x00' + resp_body
+        r, dec = self.dialect.decode_resp(self.od, spi, sms, remote_format='expanded')
+        self.assertEqual(r.response_status, 'por_ok')
+        self.assertEqual(dec.number_of_commands, 1)
+        self.assertEqual(dec.last_status_word, '9000')
+        self.assertEqual(dec.last_response_data, b2h(registry))
+
+    def test_compact_still_default(self):
+        # no remote_format -> compact default
+        spi = SPI_CC_POR_UNCIPHERED_NOCC
+        r, d = self.dialect.decode_resp(self.od, spi, '027100000e0ab000110000000000000001612f')
+        self.assertEqual(d.number_of_commands, 1)
+        self.assertEqual(d.last_status_word, '612f')
+        self.assertEqual(d.last_response_data, '')
+
+
 if __name__ == "__main__":
 	unittest.main()

tests/unittests/test_param_src.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+
+# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: Neels Hofmeyr
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import math
+from importlib import resources
+import unittest
+from pySim.esim.saip import param_source
+
+import xo
+update_expected_output = False
+
+class D:
+    mandatory = set()
+    optional = set()
+
+    def __init__(self, **kwargs):
+        if (set(kwargs.keys()) - set(self.optional)) != set(self.mandatory):
+            raise RuntimeError(f'{self.__class__.__name__}.__init__():'
+                               f' {set(kwargs.keys())=!r} - {self.optional=!r} != {self.mandatory=!r}')
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+        for k in self.optional:
+            if not hasattr(self, k):
+                setattr(self, k, None)
+
+decimals = '0123456789'
+hexadecimals = '0123456789abcdefABCDEF'
+
+class FakeRandom:
+    vals = b'\xab\xcfm\xf0\x98J_\xcf\x96\x87fp5l\xe7f\xd1\xd6\x97\xc1\xf9]\x8c\x86+\xdb\t^ke\xc1r'
+    i = 0
+
+    @classmethod
+    def next(cls):
+        cls.i = (cls.i + 1) % len(cls.vals)
+        return cls.vals[cls.i]
+
+    @staticmethod
+    def randint(a, b):
+        d = b - a
+        n_bytes = math.ceil(math.log(d, 2))
+        r = int.from_bytes( bytes(FakeRandom.next() for i in range(n_bytes)) )
+        return a + (r % (b - a))
+
+    @staticmethod
+    def randbytes(n):
+        return bytes(FakeRandom.next() for i in range(n))
+
+
+class ParamSourceTest(unittest.TestCase):
+
+    def test_param_source(self):
+
+        class Paramtest(D):
+            mandatory = (
+                    'param_source',
+                    'n',
+                    'expect',
+                    )
+            optional = (
+                    'expect_arg',
+                    'csv_rows',
+                    )
+            param_source: param_source.ParamSource
+            n: int
+            expect: object
+            expect_arg: object
+            csv_rows: object
+
+        def expect_const(t, vals):
+            return tuple(t.expect_arg) == tuple(vals)
+
+        def expect_random(t, vals):
+            chars = t.expect_arg.get('digits')
+            repetitions = (t.n - len(set(vals)))
+            if repetitions:
+                raise RuntimeError(f'expect_random: there are {repetitions} repetitions in the returned values: {vals}')
+            for val_i in range(len(vals)):
+                v = vals[val_i]
+                val_minlen = t.expect_arg.get('val_minlen')
+                val_maxlen = t.expect_arg.get('val_maxlen')
+                if len(v) < val_minlen or len(v) > val_maxlen:
+                    raise RuntimeError(f'expect_random: invalid length {len(v)} for value [{val_i}]: {v!r}, expecting'
+                                       f' {val_minlen}..{val_maxlen}')
+
+                if chars is not None and not all(c in chars for c in v):
+                    raise RuntimeError(f'expect_random: invalid char in value [{val_i}]: {v!r}')
+            return True
+
+        param_source_tests = [
+            Paramtest(param_source=param_source.ConstantSource.from_str('123'),
+                      n=3,
+                      expect=expect_const,
+                      expect_arg=('123', '123', '123')),
+            Paramtest(param_source=param_source.RandomDigitSource.from_str('12345'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': decimals,
+                                  'val_minlen': 5,
+                                  'val_maxlen': 5}),
+            Paramtest(param_source=param_source.RandomDigitSource.from_str('1..999'),
+                      n=10,
+                      expect=expect_random,
+                      expect_arg={'digits': decimals,
+                                  'val_minlen': 1,
+                                  'val_maxlen': 3}),
+            Paramtest(param_source=param_source.RandomDigitSource.from_str('001..999'),
+                      n=10,
+                      expect=expect_random,
+                      expect_arg={'digits': decimals,
+                                  'val_minlen': 3,
+                                  'val_maxlen': 3}),
+            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('12345678'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': hexadecimals,
+                                  'val_minlen': 8,
+                                  'val_maxlen': 8}),
+            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('0*8'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': hexadecimals,
+                                  'val_minlen': 8,
+                                  'val_maxlen': 8}),
+            Paramtest(param_source=param_source.RandomHexDigitSource.from_str('00*4'),
+                      n=3,
+                      expect=expect_random,
+                      expect_arg={'digits': hexadecimals,
+                                  'val_minlen': 8,
+                                  'val_maxlen': 8}),
+            Paramtest(param_source=param_source.IncDigitSource.from_str('10001'),
+                      n=3,
+                      expect=expect_const,
+                      expect_arg=('10001', '10002', '10003')),
+            Paramtest(param_source=param_source.CsvSource('column_name'),
+                      n=3,
+                      expect=expect_const,
+                      expect_arg=('first val', 'second val', 'third val'),
+                      csv_rows=(
+                          {'column_name': 'first val'},
+                          {'column_name': 'second val'},
+                          {'column_name': 'third val'},
+                      )),
+            ]
+
+        outputs = []
+
+        for t in param_source_tests:
+            try:
+                if hasattr(t.param_source, 'random_impl'):
+                    t.param_source.random_impl = FakeRandom
+
+                vals = []
+                for i in range(t.n):
+                    csv_row = None
+                    if t.csv_rows is not None:
+                        csv_row = t.csv_rows[i]
+                    vals.append( t.param_source.get_next(csv_row=csv_row) )
+                if not t.expect(t, vals):
+                    raise RuntimeError(f'invalid values returned: returned {vals}')
+                output = f'ok: {t.param_source.__class__.__name__} {vals=!r}'
+                outputs.append(output)
+                print(output)
+            except RuntimeError as e:
+                raise RuntimeError(f'{t.param_source.__class__.__name__} {t.n=} {t.expect.__name__}({t.expect_arg!r}): {e}') from e
+
+        output = '\n'.join(outputs) + '\n'
+        xo_name = 'test_param_src'
+        if update_expected_output:
+            with resources.path(xo, xo_name) as xo_path:
+                with open(xo_path, 'w', encoding='utf-8') as f:
+                    f.write(output)
+        else:
+            xo_str = resources.read_text(xo, xo_name)
+            if xo_str != output:
+                at = 0
+                while at < len(output):
+                    if output[at] == xo_str[at]:
+                        at += 1
+                        continue
+                    break
+
+                raise RuntimeError(f'output differs from expected output at position {at}: {xo_str[at:at+128]!r}')
+
+if __name__ == "__main__":
+    if '-u' in sys.argv:
+        update_expected_output = True
+        sys.argv.remove('-u')
+    unittest.main()

tests/unittests/test_smpp_ota_tool.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+""" test for smpp-ota-tool SMS handling, specifically the multi part sms OTA response"""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os.path
+import importlib.util
+import unittest
+
+from osmocom.utils import h2b, b2h
+
+from pySim.ota import OtaKeyset, OtaDialectSms, ExpandedRemoteResp
+from pySim.sms import ConcatenatedSmsReassembler, UserDataHeader
+
+# import the hyphenated contrib script as a module to get at SmppHandler
+# why do people name python files like that? why does everything have to be so hard?
+_TOOL_PATH = os.path.join(os.path.dirname(__file__), '..', '..', 'contrib', 'smpp-ota-tool.py')
+_spec = importlib.util.spec_from_file_location('smpp_ota_tool', _TOOL_PATH)
+smpp_ota_tool = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(smpp_ota_tool)
+SmppHandler = smpp_ota_tool.SmppHandler
+
+
+class _FakePdu:
+    """Minimal mock for smpplib deliver_sm pdu."""
+    def __init__(self, short_message):
+        self.short_message = short_message
+
+
+class MultipartRelayTestCase(unittest.TestCase):
+    """message_received_handler must return the reassembled application
+    response and survive POR messages."""
+
+    # 3DES test keyset from tests/unittests/test_ota.py) used to make the
+    # handler happy. responses are plaintext, tests do not depend on keys.
+    def _handler(self, remote_format='expanded'):
+        h = object.__new__(SmppHandler)
+        h.client = None
+        h.ota_dialect = OtaDialectSms()
+        h.ota_keyset = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3,
+                                 kic=h2b('C21DD66ACAC13CB3BC8B331B24AFB57B'),
+                                 algo_auth='triple_des_cbc2', kid_idx=3,
+                                 kid=h2b('12110C78E678C25408233076AA033615'))
+        h.tar = h2b('000000')
+        # unciphered, no CC, PoR required
+        h.spi = {'counter': 'no_counter', 'ciphering': False, 'rc_cc_ds': 'no_rc_cc_ds',
+                 'por_in_submit': False, 'por': 'por_required',
+                 'por_shall_be_ciphered': False, 'por_rc_cc_ds': 'no_rc_cc_ds'}
+        h.remote_format = remote_format
+        h.reassembler = ConcatenatedSmsReassembler()
+        h.response = None
+        return h
+
+    @staticmethod
+    def _plaintext_resp_sms(secured: bytes, sts: int = 0x00) -> bytes:
+        """Build a plaintext (unciphered, no-CC) OTA SMS response packet in the
+        canonical single-part form (UDH 02 71 00 + response packet)."""
+        rpl = 1 + 3 + 5 + 1 + 1 + len(secured)   # RHL-STS + secured data
+        body = (rpl.to_bytes(2, 'big') + b'\x0a' + h2b('000000') + b'\x00' * 5
+                + b'\x00' + bytes([sts]) + secured)
+        return b'\x02\x71\x00' + body
+
+    @staticmethod
+    def _expanded_secured(response_data_hex: str, sw: str = '9000') -> bytes:
+        return ExpandedRemoteResp.build(dict(body=dict(
+            num_executed=dict(number_of_commands=1),
+            responses=[dict(r_apdu=dict(response_data=response_data_hex, status_word=sw))])))
+
+    @staticmethod
+    def _fragment_2(tpud: bytes, ref: int, first_len: int):
+        """Split 02 71 00 + body TP-UD into two SMS parts:
+        - part1 carries the OTA (0x71) IE
+        - part2 only concatenat IE
+        matches sja5 interaction"""
+        assert tpud[:3] == b'\x02\x71\x00'
+        body = tpud[3:]
+        ota_ie = {'iei': 0x71, 'length': 0, 'value': b''}
+
+        def concat(seq):
+            return {'iei': 0x00, 'length': 3, 'value': bytes([ref, 2, seq])}
+        p1 = UserDataHeader([concat(1), ota_ie]).to_bytes() + body[:first_len]
+        p2 = UserDataHeader([concat(2)]).to_bytes() + body[first_len:]
+        return p1, p2
+
+    # ground truth: TP-User-Data captured from a sja5
+    REAL_PART1 = h2b('070003010201710000e412000000df63afe4b06db21e2113be1be09e9b66f1c113ae841cca2d030064ec16b5b80ee5ce824604a4568109d25a82fb74a325df6f911bd0a4f858ece2c770039002c480269fc65953f5fd93ebbe528d97838bac4389a7303db2b073a37a9a1a51890457f41b49fc7905ce337e83449b65560501b8b845fe63339d557a928f2643')
+    REAL_PART2 = h2b('050003010202fd9c4e50ec40fb4427af518e9c08697405d91fbb6e9fa0b0935f48a560e15f2f3f27a2e44ef3a47280acce77f030fb70eb3df863c159177e2c0e3e53052fc7bb7ed171a491ded3ab7921861176a04305bc09fcf526c07bf6bb48a19e67cf18be5bc1')
+    REAL_REASSEMBLED = '02710000e412000000df63afe4b06db21e2113be1be09e9b66f1c113ae841cca2d030064ec16b5b80ee5ce824604a4568109d25a82fb74a325df6f911bd0a4f858ece2c770039002c480269fc65953f5fd93ebbe528d97838bac4389a7303db2b073a37a9a1a51890457f41b49fc7905ce337e83449b65560501b8b845fe63339d557a928f2643fd9c4e50ec40fb4427af518e9c08697405d91fbb6e9fa0b0935f48a560e15f2f3f27a2e44ef3a47280acce77f030fb70eb3df863c159177e2c0e3e53052fc7bb7ed171a491ded3ab7921861176a04305bc09fcf526c07bf6bb48a19e67cf18be5bc1'
+
+    def test_real_card_parts_reassemble(self):
+        """two real card TP-UDs recombine into 233-byte single part packet:
+        UDH 02 71 00 + response packet"""
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self.REAL_PART1))
+        out = r.add(self.REAL_PART2)
+        self.assertEqual(len(out), 233)
+        self.assertEqual(b2h(out), self.REAL_REASSEMBLED)
+
+    def test_multipart_response_not_overwritten_by_por(self):
+        """reassembled application response must survive the ENVELOPE
+        trailing POR which contains no R-APDU"""
+        registry = bytes(range(198))
+        app = self._plaintext_resp_sms(self._expanded_secured(b2h(registry)))
+        part1, part2 = self._fragment_2(app, ref=0x42, first_len=132)
+        # single part form must be too fat -> both parts must be concatenated
+        self.assertGreater(len(app), 140)
+        # ENVELOPE PoR: por_ok, but no app R-APDU
+        inline_por = self._plaintext_resp_sms(b'', sts=0x00)
+
+        h = self._handler()
+        # arrival order
+        self.assertIsNone(h.message_received_handler(_FakePdu(part1)))
+        h.message_received_handler(_FakePdu(part2))
+        h.message_received_handler(_FakePdu(inline_por))
+
+        # self.response must be app response, not the PoR!
+        self.assertIsNotNone(h.response)
+        res, decoded = h.response
+        self.assertEqual(res.response_status, 'por_ok')
+        self.assertIsNotNone(decoded)
+        self.assertEqual(decoded.last_response_data, b2h(registry))
+        self.assertEqual(decoded.last_status_word, '9000')
+
+    def test_undecodable_response_does_not_crash(self):
+        """response the handler can't decode must not escape out of the poll()
+        loop which would kill the tool, it must be ignored"""
+        # por_ok with a not expanded 'secured data' -> expanded parse raises
+        bad = self._plaintext_resp_sms(h2b('01612f'), sts=0x00)
+        h = self._handler(remote_format='expanded')
+        # must NOT raise
+        self.assertIsNone(h.message_received_handler(_FakePdu(bad)))
+        self.assertIsNone(h.response)
+
+    def test_undecodable_por_after_good_response(self):
+        """real app response followed by undecodable PoR:
+        - good response is saved
+        - tool does not crash."""
+        registry = bytes(range(120))
+        app = self._plaintext_resp_sms(self._expanded_secured(b2h(registry)))
+        part1, part2 = self._fragment_2(app, ref=0x07, first_len=110)
+        bad_por = self._plaintext_resp_sms(h2b('deadbeef'), sts=0x00)
+
+        h = self._handler()
+        h.message_received_handler(_FakePdu(part1))
+        h.message_received_handler(_FakePdu(part2))
+        self.assertIsNone(h.message_received_handler(_FakePdu(bad_por)))  # no crash
+        res, decoded = h.response
+        self.assertIsNotNone(decoded)
+        self.assertEqual(decoded.last_response_data, b2h(registry))
+
+    def test_single_part_response_still_works(self):
+        """small response that fits one SMS turns into self.response, handled as before"""
+        h = self._handler()
+        sms = self._plaintext_resp_sms(self._expanded_secured('abcd', sw='9000'))
+        self.assertLessEqual(len(sms), 140)
+        h.message_received_handler(_FakePdu(sms))
+        res, decoded = h.response
+        self.assertIsNotNone(decoded)
+        self.assertEqual(decoded.last_response_data, 'abcd')
+        self.assertEqual(decoded.last_status_word, '9000')
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/unittests/test_sms.py

@@ -103,3 +103,126 @@ class Test_DELIVER(unittest.TestCase):
         self.assertEqual(d.tp_pid, 0x7f)
         self.assertEqual(d.tp_dcs, 0xf6)
         self.assertEqual(d.tp_udl, 8)
+
+
+class Test_ConcatenatedSmsReassembler(unittest.TestCase):
+    """3GPP TS 23.040 9.2.3.24 reassembly of multi-part SMS.
+
+    An OTA response that exceeds a single SHORT MESSAGE is delivered in several parts using
+    the SEND SHORT MESSAGE proactive command. The receiver must recombine the individual
+    parts into a single part before decoding."""
+
+    OTA_IE = {'iei': 0x71, 'length': 0, 'value': b''}
+
+    @staticmethod
+    def _concat8(ref, tot, seq):
+        return {'iei': 0x00, 'length': 3, 'value': bytes([ref, tot, seq])}
+
+    @staticmethod
+    def _concat16(ref, tot, seq):
+        return {'iei': 0x08, 'length': 4, 'value': ref.to_bytes(2, 'big') + bytes([tot, seq])}
+
+    @staticmethod
+    def _part(ies, frag):
+        return UserDataHeader(ies).to_bytes() + frag
+
+    def test_ground_truth_udh(self):
+        # part 1 UDH observed from sja5: 07 00 03 01 02 01 71 00
+        built = self._part([self._concat8(1, 2, 1), self.OTA_IE], b'')
+        self.assertEqual(b2h(built), '0700030102017100')
+
+    def test_ground_truth_udh_16bit(self):
+        # 9.2.3.24.8: 08 | 08 04 <ref16> <total> <seq> | 71 00
+        built = self._part([self._concat16(0x1234, 2, 1), self.OTA_IE], b'')
+        self.assertEqual(b2h(built), '080804123402017100')
+
+    def test_single_part_passthrough(self):
+        r = ConcatenatedSmsReassembler()
+        single = h2b('027100') + bytes(range(20))
+        self.assertEqual(r.add(single), single)
+
+    def test_two_part(self):
+        # second segment contains only the concat IE, no OTA IE
+        pkt = bytes(range(60))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1), self.OTA_IE], pkt[:35])))
+        out = r.add(self._part([self._concat8(1, 2, 2)], pkt[35:]))
+        self.assertEqual(out, h2b('027100') + pkt)
+
+    def test_out_of_order(self):
+        pkt = bytes(range(60))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(5, 2, 2), self.OTA_IE], pkt[35:])))
+        out = r.add(self._part([self._concat8(5, 2, 1), self.OTA_IE], pkt[:35]))
+        self.assertEqual(out, h2b('027100') + pkt)
+
+    def test_three_part_out_of_order(self):
+        pkt = bytes(range(90))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(7, 3, 3)], pkt[60:])))
+        self.assertIsNone(r.add(self._part([self._concat8(7, 3, 1), self.OTA_IE], pkt[:30])))
+        out = r.add(self._part([self._concat8(7, 3, 2)], pkt[30:60]))
+        self.assertEqual(out, h2b('027100') + pkt)
+
+    def test_16bit_reference(self):
+        pkt = bytes(range(40))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat16(0x1234, 2, 1), self.OTA_IE], pkt[:20])))
+        out = r.add(self._part([self._concat16(0x1234, 2, 2)], pkt[20:]))
+        self.assertEqual(out, h2b('027100') + pkt)
+
+    def test_interleaved_references(self):
+        # two concurrent concatenation sets at the same time
+        pkt = bytes(range(60))
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1), self.OTA_IE], pkt[:35])))
+        self.assertIsNone(r.add(self._part([self._concat8(9, 2, 1), self.OTA_IE], b'\xaa')))
+        self.assertEqual(r.add(self._part([self._concat8(1, 2, 2)], pkt[35:])), h2b('027100') + pkt)
+        self.assertEqual(r.add(self._part([self._concat8(9, 2, 2)], b'\xbb')), h2b('027100') + b'\xaa\xbb')
+
+    def test_reserved_concat_ie_is_ignored(self):
+        # TS 23.040 9.2.3.24.1:
+        # - a total of 0
+        # - or a sequence number that is 0 or > total
+        # means "the receiving entity shall ignore the whole Information Element"
+        # the message is handed back unchanged as a single part msg and not rejected
+        # so the caller can handle the problem
+        r = ConcatenatedSmsReassembler()
+        for tot, seq in [(2, 3),    # seq > total
+                         (2, 0),    # seq == 0
+                         (0, 1)]:   # total == 0
+            with self.subTest(total=tot, seq=seq):
+                part = self._part([self._concat8(1, tot, seq)], b'\x00')
+                self.assertEqual(r.add(part), part)
+        # nothing buffered so later valid set still reassembles properly
+        self.assertEqual(r.sets, {})
+        pkt = bytes(range(40))
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1), self.OTA_IE], pkt[:20])))
+        self.assertEqual(r.add(self._part([self._concat8(1, 2, 2)], pkt[20:])), h2b('027100') + pkt)
+
+    def test_inconsistent_totals_do_not_crash(self):
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(1, 3, 3)], b'\x33')))
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1)], b'\x11')))
+        self.assertEqual(r.add(self._part([self._concat8(1, 2, 2)], b'\x22')),
+                         h2b('00') + b'\x11\x22')             # complete total=2 set
+        self.assertIn((0x00, 1, 3), r.sets)                   # total=3 set still waits
+
+    def test_incomplete_sets_are_capped(self):
+        r = ConcatenatedSmsReassembler(max_sets=2)
+        for ref in (1, 2, 3):
+            self.assertIsNone(r.add(self._part([self._concat8(ref, 2, 1)], bytes([ref]))))
+        self.assertEqual(sorted(k[1] for k in r.sets), [2, 3])      # oldest evicted
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 2)], b'\x11')))
+        self.assertEqual(sorted(k[1] for k in r.sets), [1, 3])
+        self.assertEqual(r.add(self._part([self._concat8(3, 2, 2)], b'\x33')), h2b('00') + b'\x03\x33')
+
+    def test_same_reference_in_both_ie_forms(self):
+        # the refno only unique per IE form (9.2.3.24.1 vs .8) -> two sets
+        r = ConcatenatedSmsReassembler()
+        self.assertIsNone(r.add(self._part([self._concat8(1, 2, 1)], b'\x0a')))
+        self.assertIsNone(r.add(self._part([self._concat16(1, 2, 2)], b'\x1b')))
+        self.assertEqual(r.add(self._part([self._concat16(1, 2, 1)], b'\x0b')),
+                         h2b('00') + b'\x0b\x1b')
+        self.assertEqual(r.add(self._part([self._concat8(1, 2, 2)], b'\x1a')),
+                         h2b('00') + b'\x0a\x1a')

tests/unittests/test_tlvs.py

@@ -21,6 +21,7 @@ import logging
 from osmocom.utils import b2h, h2b, all_subclasses
 from osmocom.tlv import *
 
+import pySim.cat
 import pySim.iso7816_4
 import pySim.ts_102_221
 import pySim.ts_102_222

tests/unittests/test_transport.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+
+"""Transport (as in t0/t1) tests"""
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Eric Wild <ewild@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from osmocom.utils import h2b, b2h
+from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
+from pySim.transport import ProactiveHandler, LinkBaseTpdu
+
+
+def _send_short_message_pcmd():
+    """proactive SEND SHORT MESSAGE:
+    D0 | CommandDetails(cmd 1, t 0x13, q 0) | DeviceIdentities(uicc->network)
+       | dummy SMS_TPDU"""
+    body = h2b('8103011300' + '82028183' + '8B04DEADBEEF')
+    pdu = h2b('D0') + bytes([len(body)]) + body
+    pcmd = ProactiveCommand()
+    decoded = pcmd.from_tlv(pdu)
+    return pcmd, decoded
+
+
+class Test_prepare_response(unittest.TestCase):
+    """TERMINAL RESPONSE.
+    multi-part OTA response crash regression test."""
+
+    def setUp(self):
+        self.h = ProactiveHandler.__new__(ProactiveHandler)
+
+    def test_on_decoded_command(self):
+        _pcmd, decoded = _send_short_message_pcmd()
+        til = self.h.prepare_response(decoded)
+        self.assertEqual([type(c).__name__ for c in til],
+                         ['CommandDetails', 'DeviceIdentities', 'Result'])
+        # command details echoed, device id inverted, result OK
+        self.assertEqual(b2h(til[0].to_tlv()), '8103011300')
+        self.assertEqual(b2h(til[1].to_tlv()), '82028381')
+        self.assertEqual(b2h(til[2].to_tlv()), '830100')
+
+    def test_on_collection_resolves_via_decoded(self):
+        # Check that ProactiveCommand collection (empty .children) still works
+        pcmd, _decoded = _send_short_message_pcmd()
+        self.assertEqual(list(getattr(pcmd, 'children', []) or []), [])
+        til = self.h.prepare_response(pcmd)
+        self.assertEqual([type(c).__name__ for c in til],
+                         ['CommandDetails', 'DeviceIdentities', 'Result'])
+        self.assertEqual(b2h(til[0].to_tlv()), '8103011300')
+        self.assertEqual(b2h(til[1].to_tlv()), '82028381')
+        self.assertEqual(b2h(til[2].to_tlv()), '830100')
+
+    def test_missing_command_details_raises_clear_error(self):
+        class _NoChildren:
+            children = []
+        with self.assertRaises(ValueError) as ctx:
+            self.h.prepare_response(_NoChildren())
+        self.assertIn('CommandDetails', str(ctx.exception))
+
+
+class FakeTpduLink(LinkBaseTpdu):
+    """mock LinkBaseTpdu that replays a list of (data, sw) responses + records every TPDU that
+    the T=0 state machine sends. Secretly sending more TPDUs than intended is the error,
+    designed to test "unsolicited GET RESPONSE" mishaps"""
+
+    def __init__(self, responses):
+        super().__init__()
+        self._responses = list(responses)
+        self.sent = []
+
+    def send_tpdu(self, tpdu):
+        self.sent.append(tpdu.lower())
+        if not self._responses:
+            raise AssertionError('T=0 layer sent an unpexpected TPDU: %s (total so far: %s)'
+                                 % (tpdu, self.sent))
+        return self._responses.pop(0)
+
+    def __str__(self):
+        return 'FakeTpduLink'
+
+    def wait_for_card(self, timeout=None, newcardonly=False):
+        pass
+
+    def connect(self):
+        pass
+
+    def get_atr(self):
+        return '3b00'
+
+    def disconnect(self):
+        pass
+
+    def _reset_card(self):
+        pass
+
+
+# GP GET STATUS, wrapped in SCP02 CLA 84, Case #4.
+GET_STATUS = '84f22002094f005c054f9f70c5cc' + '00'
+GET_STATUS_TPDU = '84f22002094f005c054f9f70c5cc'
+
+# generic #4 SELECT by DF name command
+CASE4 = '00a4040c07a0000000871002' + '00'
+CASE4_TPDU = '00a4040c07a0000000871002'
+
+
+class Test_send_apdu_T0(unittest.TestCase):
+    """regression tests for the T=0 state machine in LinkBaseTpdu.__send_apdu_T0()"""
+
+    def _exchange(self, apdu, responses, strict=True, protocol=0):
+        link = FakeTpduLink(responses)
+        link.apdu_strict = strict
+        link.set_tpdu_format(protocol)
+        data, sw = link._send_apdu(apdu)
+        return link, data, sw
+
+    #### TS 102 221 section 7.3.1.1 TPDU construction
+
+    def test_case1_gets_le_appended(self):
+        link, data, sw = self._exchange('00200001', [('', '9000')])
+        self.assertEqual(link.sent, ['0020000100'])
+        self.assertEqual((data, sw), ('', '9000'))
+
+    def test_case3_passed_through_unmodified(self):
+        apdu = '00200001081122334455667788'
+        link, _data, sw = self._exchange(apdu, [('', '9000')])
+        self.assertEqual(link.sent, [apdu])
+        self.assertEqual(sw, '9000')
+
+    def test_case4_le_stripped(self):
+        link, data, sw = self._exchange(CASE4, [('', '9000')])
+        self.assertEqual(link.sent, [CASE4_TPDU])
+        self.assertEqual((data, sw), ('', '9000'))
+
+    #### TS 102 221 7.3.1.1.4 4a GP GET RESPONSE for 61xx / 9fxx
+
+    def test_61xx_fetches_response(self):
+        link, data, sw = self._exchange(CASE4, [('', '6103'), ('a1b2c3', '9000')])
+        self.assertEqual(link.sent, [CASE4_TPDU, '00c0000003'])
+        self.assertEqual((data, sw), ('a1b2c3', '9000'))
+
+    def test_61xx_chained(self):
+        link, data, sw = self._exchange(CASE4,
+                                        [('', '6102'), ('aabb', '6102'), ('ccdd', '9000')])
+        self.assertEqual(link.sent, [CASE4_TPDU, '00c0000002', '00c0000002'])
+        self.assertEqual((data, sw), ('aabbccdd', '9000'))
+
+    def test_9fxx_fetches_response(self):
+        link, data, sw = self._exchange(CASE4, [('', '9f04'), ('deadbeef', '9000')])
+        self.assertEqual(link.sent, [CASE4_TPDU, '00c0000004'])
+        self.assertEqual((data, sw), ('deadbeef', '9000'))
+
+    def test_get_response_inherits_cla(self):
+        """GET RESPONSE must reuse  CLA of command"""
+        link, _data, _sw = self._exchange(GET_STATUS, [('', '6102'), ('aabb', '9000')])
+        self.assertEqual(link.sent, [GET_STATUS_TPDU, '84c0000002'])
+
+    def test_9100_terminates(self):
+        """9100 is final status word, not fetch trigger"""
+        link, data, sw = self._exchange(CASE4, [('', '9100')])
+        self.assertEqual(link.sent, [CASE4_TPDU])
+        self.assertEqual((data, sw), ('', '9100'))
+
+    def test_error_sw_terminates(self):
+        link, data, sw = self._exchange(CASE4, [('', '6982')])
+        self.assertEqual(link.sent, [CASE4_TPDU])
+        self.assertEqual((data, sw), ('', '6982'))
+
+    def test_no_status_word_raises(self):
+        with self.assertRaises(ValueError):
+            self._exchange(CASE4, [('', None)])
+
+    #### TS 102 221 7.3.1.1.4 4b dummy GET RESPONSE
+
+    def test_clause_4b_warning_before_data_bootstraps(self):
+        """warning SW returned for the _command_ TPDU triggers dummy GET RESPONSE (Le=00)"""
+        for warn in ('6200', '6281', '62f1', '6300', '63f1'):
+            with self.subTest(sw=warn):
+                link, data, sw = self._exchange(CASE4,
+                                                [('', warn), ('', '6103'), ('a1b2c3', '9000')])
+                self.assertEqual(link.sent, [CASE4_TPDU, '00c0000000', '00c0000003'])
+                self.assertEqual((data, sw), ('a1b2c3', '9000'))
+
+    def test_warning_after_data_terminates(self):
+        """Once the response has been fetched a warning status word is the final result of the command"""
+        for warn in ('6281', '6283', '63c2', '6300', '62f1', '63f1', '6310'):
+            with self.subTest(sw=warn):
+                link, data, sw = self._exchange(CASE4, [('', '6102'), ('aabb', warn)])
+                self.assertEqual(link.sent, [CASE4_TPDU, '00c0000002'])
+                self.assertEqual((data, sw), ('aabb', warn))
+
+    def test_no_dummy_get_response_when_command_already_returned_data(self):
+        """warning that arrives together with response data (for example 6282 on a case #2 read) is final, too"""
+        link, data, sw = self._exchange('00b0000004', [('01020304', '6282')], strict=False)
+        self.assertEqual(link.sent, ['00b0000004'])
+        self.assertEqual((data, sw), ('01020304', '6282'))
+
+    def test_repeated_warning_does_not_loop(self):
+        """warning -> dummy GET RESPONSE -> warning again must terminate"""
+        link, data, sw = self._exchange(CASE4, [('', '6281'), ('', '6281')])
+        self.assertEqual(link.sent, [CASE4_TPDU, '00c0000000'])
+        self.assertEqual((data, sw), ('', '6281'))
+
+    #### fixed GlobalPlatform GET STATUS pagination
+
+    def test_gp_6310_reaches_the_caller(self):
+        """GET STATUS answers 6310"""
+        link, data, sw = self._exchange(GET_STATUS, [('', '6104'), ('e3024f00', '6310')])
+        self.assertEqual(link.sent, [GET_STATUS_TPDU, '84c0000004'])
+        self.assertEqual((data, sw), ('e3024f00', '6310'))
+
+    def test_gp_get_status_two_pages(self):
+        """Both GET STATUS pages, page 1 6310, reissued with P2 bit 1 set, page 2 9000."""
+        page1 = self._exchange(GET_STATUS, [('', '6104'), ('e3024f00', '6310')])
+        self.assertEqual(page1[1:], ('e3024f00', '6310'))
+        page2 = self._exchange('84f22003094f005c054f9f70c5cc00',
+                               [('', '6104'), ('e3024f01', '9000')])
+        self.assertEqual(page2[0].sent, ['84f22003094f005c054f9f70c5cc', '84c0000004'])
+        self.assertEqual(page2[1:], ('e3024f01', '9000'))
+
+    #### 6cxx and apdu_strict
+
+    def test_6cxx_reissues_command_with_correct_length(self):
+        link, data, sw = self._exchange('00b0000000', [('', '6c04'), ('01020304', '9000')])
+        self.assertEqual(link.sent, ['00b0000000', '00b0000004'])
+        self.assertEqual((data, sw), ('01020304', '9000'))
+
+    def test_strict_mode_does_not_auto_fetch_for_case3(self):
+        apdu = '00200001081122334455667788'
+        link, data, sw = self._exchange(apdu, [('', '6104')], strict=True)
+        self.assertEqual(link.sent, [apdu])
+        self.assertEqual((data, sw), ('', '6104'))
+
+    def test_non_strict_mode_auto_fetches_for_case3(self):
+        apdu = '00200001081122334455667788'
+        link, data, sw = self._exchange(apdu, [('', '6104'), ('aabbccdd', '9000')], strict=False)
+        self.assertEqual(link.sent, [apdu, '00c0000004'])
+        self.assertEqual((data, sw), ('aabbccdd', '9000'))
+
+    #### T=1 briefly
+
+    def test_t1_is_passed_through(self):
+        """T=1 has no GET RESPONSE"""
+        link, data, sw = self._exchange(GET_STATUS, [('e3024f00', '6310')], protocol=1)
+        self.assertEqual(link.sent, [GET_STATUS.lower()])
+        self.assertEqual((data, sw), ('e3024f00', '6310'))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/unittests/xo/test_param_src

@@ -0,0 +1,9 @@
+ok: ConstantSource vals=['123', '123', '123']
+ok: RandomDigitSource vals=['13987', '49298', '55670']
+ok: RandomDigitSource vals=['650', '580', '49', '885', '497', '195', '320', '137', '245', '663']
+ok: RandomDigitSource vals=['638', '025', '232', '779', '826', '972', '650', '580', '049', '885']
+ok: RandomHexDigitSource vals=['6b65c172', 'abcf6df0', '984a5fcf']
+ok: RandomHexDigitSource vals=['96876670', '356ce766', 'd1d697c1']
+ok: RandomHexDigitSource vals=['f95d8c86', '2bdb095e', '6b65c172']
+ok: IncDigitSource vals=['10001', '10002', '10003']
+ok: CsvSource vals=['first val', 'second val', 'third val']