sdkeys kv40 aes

Change-Id: If5b53c840ebd1f224f9bb4706a602b415194f47b

.gitignore

@@ -1,5 +1,5 @@
 *.pyc
-.*.swp
+.*.sw?
 
 /docs/_*
 /docs/generated
@@ -7,3 +7,10 @@
 /.local
 /build
 /pySim.egg-info
+/smdpp-data/sm-dp-sessions*
+dist
+tags
+smdpp-data/certs/DPtls/CERT_S_SM_DP_TLS_NIST.pem
+smdpp-data/certs/DPtls/CERT_S_SM_DP_TLS_BRP.pem
+smdpp-data/generated
+smdpp-data/certs/dhparam2048.pem

README.md

@@ -1,16 +1,29 @@
-pySim - Read, Write and Browse Programmable SIM/USIM/ISIM/HPSIM Cards
-=====================================================================
+pySim - Tools for reading, decoding, browsing SIM/USIM/ISIM/HPSIM/eUICC Cards
+=============================================================================
 
-This repository contains a number of Python programs that can be used
-to read, program (write) and browse all fields/parameters/files on
-SIM/USIM/ISIM/HPSIM cards used in 3GPP cellular networks from 2G to 5G.
+This repository contains a number of Python programs related to working with
+subscriber identity modules of cellular networks, including but not limited
+to SIM, UICC, USIM, ISIM, HPSIMs and eUICCs.
+
+* `pySim-shell.py` can be used to interactively explore, read and decode contents
+  of any of the supported card models / card applications.  Furthermore, if
+  you have the credentials to your card (ADM PIN), you can also write to the card,
+  i.e. edit its contents.
+* `pySim-read.py` and `pySim-prog.py` are _legacy_ tools for batch programming
+  some very common parameters to an entire batch of programmable cards
+* `pySim-trace.py` is a tool to do an in-depth decode of SIM card protocol traces
+  such as those obtained by [Osmocom SIMtrace2](https://osmocom.org/projects/simtrace2/wiki)
+  or [osmo-qcdiag](https://osmocom.org/projects/osmo-qcdiag/wiki).
+* `osmo-smdpp.py` is a proof-of-concept GSMA SGP.22 Consumer eSIM SM-DP+ for lab/research
+* there are more related tools, particularly in the `contrib` directory.
 
 Note that the access control configuration of normal production cards
 issue by operators will restrict significantly which files a normal
 user can read, and particularly write to.
 
 The full functionality of pySim hence can only be used with on so-called
-programmable SIM/USIM/ISIM/HPSIM cards.
+programmable SIM/USIM/ISIM/HPSIM cards, such as the various
+[sysmocom programmable card products](https://shop.sysmocom.de/SIM/).
 
 Such SIM/USIM/ISIM/HPSIM cards are special cards, which - unlike those
 issued by regular commercial operators - come with the kind of keys that
@@ -49,9 +62,9 @@ pySim-shell vs. legacy tools
 ----------------------------
 
 While you will find a lot of online resources still describing the use of
-pySim-prog.py and pySim-read.py, those tools are considered legacy by
+`pySim-prog.py` and `pySim-read.py`, those tools are considered legacy by
 now and have by far been superseded by the much more capable
-pySim-shell.  We strongly encourage users to adopt pySim-shell, unless
+`pySim-shell.py`.  We strongly encourage users to adopt pySim-shell, unless
 they have very specific requirements like batch programming of large
 quantities of cards, which is about the only remaining use case for the
 legacy tools.

contrib/analyze_simaResponse.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+# A tool to analyze the eUICC simaResponse (series of EUICCResponse)
+#
+# (C) 2025 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+from osmocom.utils import h2b, b2h
+from osmocom.tlv import bertlv_parse_one, bertlv_encode_tag, bertlv_encode_len
+from pySim.esim.saip import *
+
+parser = argparse.ArgumentParser(description="""Utility program to analyze the contents of an eUICC simaResponse.""")
+parser.add_argument('SIMA_RESPONSE', help='Hexstring containing the simaResponse as received from the eUICC')
+
+def split_sima_response(sima_response):
+    """split an eUICC simaResponse field into a list of EUICCResponse fields"""
+
+    remainder = sima_response
+    result = []
+    while len(remainder):
+        tdict, l, v, next_remainder = bertlv_parse_one(remainder)
+        rawtag = bertlv_encode_tag(tdict)
+        rawlen = bertlv_encode_len(l)
+        result = result + [remainder[0:len(rawtag) + len(rawlen) + l]]
+        remainder = next_remainder
+    return result
+
+def analyze_status(status):
+    """
+    Convert a status code (integer) into a human readable string
+    (see eUICC Profile Package: Interoperable Format Technical Specification, section 8.11)
+    """
+
+    # SIMA status codes
+    string_values = {0 : 'ok',
+                     1 : 'pe-not-supported',
+                     2 : 'memory-failure',
+                     3 : 'bad-values',
+                     4 : 'not-enough-memory',
+                     5 : 'invalid-request-format',
+                     6 : 'invalid-parameter',
+                     7 : 'runtime-not-supported',
+                     8 : 'lib-not-supported',
+                     9 : 'template-not-supported ',
+                     10 : 'feature-not-supported',
+                     11 : 'pin-code-missing',
+                     31 : 'unsupported-profile-version'}
+
+    string_value = string_values.get(status, None)
+    if string_value is not None:
+        return "%d = %s (SIMA status code)" % (status, string_value)
+
+    # ISO 7816 status words
+    if status >= 24576 and status <= 28671:
+        return "%d = %04x (ISO7816 status word)" % (status, status)
+    elif status >= 36864 and status <= 40959:
+        return "%d = %04x (ISO7816 status word)" % (status, status)
+
+    # Proprietary status codes
+    elif status >= 40960 and status <= 65535:
+        return "%d = %04x (proprietary)" % (status, status)
+
+    # Unknown status codes
+    return "%d (unknown, proprietary?)" % status
+
+def analyze_euicc_response(euicc_response):
+    """Analyze and display the contents of an EUICCResponse"""
+
+    print(" EUICCResponse: %s" % b2h(euicc_response))
+    euicc_response_decoded = asn1.decode('EUICCResponse', euicc_response)
+
+    pe_status = euicc_response_decoded.get('peStatus')
+    print("  peStatus:")
+    for s in pe_status:
+        print("   status: %s" % analyze_status(s.get('status')))
+        print("   identification: %s" % str(s.get('identification', None)))
+        print("   additional-information: %s" % str(s.get('additional-information', None)))
+        print("   offset: %s" % str(s.get('offset', None)))
+
+    if euicc_response_decoded.get('profileInstallationAborted', False) is None:
+        # This type is defined as profileInstallationAborted NULL OPTIONAL, so when it is present it
+        # will have the value None, otherwise it is simply not present.
+        print("  profileInstallationAborted: True")
+    else:
+        print("  profileInstallationAborted: False")
+
+    status_message = euicc_response_decoded.get('statusMessage', None)
+    print("  statusMessage: %s" % str(status_message))
+
+if __name__ == '__main__':
+    opts = parser.parse_args()
+    sima_response = h2b(opts.SIMA_RESPONSE);
+
+    print("simaResponse: %s" % b2h(sima_response))
+    euicc_response_list = split_sima_response(sima_response)
+
+    for euicc_response in euicc_response_list:
+        analyze_euicc_response(euicc_response)

contrib/csv-encrypt-columns.py

@@ -24,20 +24,12 @@ import argparse
 from Cryptodome.Cipher import AES
 from osmocom.utils import h2b, b2h, Hexstr
 
-from pySim.card_key_provider import CardKeyProviderCsv
+from pySim.card_key_provider import CardKeyFieldCryptor
 
-def dict_keys_to_upper(d: dict) -> dict:
-    return {k.upper():v for k,v in d.items()}
-
-class CsvColumnEncryptor:
+class CsvColumnEncryptor(CardKeyFieldCryptor):
     def __init__(self, filename: str, transport_keys: dict):
         self.filename = filename
-        self.transport_keys = dict_keys_to_upper(transport_keys)
-
-    def encrypt_col(self, colname:str, value: str) -> Hexstr:
-        key = self.transport_keys[colname]
-        cipher = AES.new(h2b(key), AES.MODE_CBC, CardKeyProviderCsv.IV)
-        return b2h(cipher.encrypt(h2b(value)))
+        self.crypt = CardKeyFieldCryptor(transport_keys)
 
     def encrypt(self) -> None:
         with open(self.filename, 'r') as infile:
@@ -49,9 +41,8 @@ class CsvColumnEncryptor:
                 cw.writeheader()
 
                 for row in cr:
-                    for key_colname in self.transport_keys:
-                        if key_colname in row:
-                            row[key_colname] = self.encrypt_col(key_colname, row[key_colname])
+                    for fieldname in cr.fieldnames:
+                            row[fieldname] = self.crypt.encrypt_field(fieldname, row[fieldname])
                     cw.writerow(row)
 
 if __name__ == "__main__":
@@ -71,9 +62,5 @@ if __name__ == "__main__":
         print("You must specify at least one key!")
         sys.exit(1)
 
-    csv_column_keys = CardKeyProviderCsv.process_transport_keys(csv_column_keys)
-    for name, key in csv_column_keys.items():
-        print("Encrypting column %s using AES key %s" % (name, key))
-
     cce = CsvColumnEncryptor(opts.CSVFILE, csv_column_keys)
     cce.encrypt()

contrib/csv-to-pgsql.py

@@ -0,0 +1,304 @@
+#!/usr/bin/env python3
+
+# (C) 2025 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Philipp Maier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import logging
+import csv
+import sys
+import os
+import yaml
+import psycopg2
+from psycopg2.sql import Identifier, SQL
+from pathlib import Path
+from pySim.log import PySimLogger
+from packaging import version
+
+log = PySimLogger.get(Path(__file__).stem)
+
+class CardKeyDatabase:
+    def __init__(self, config_filename: str, table_name: str, create_table: bool = False, admin: bool = False):
+        """
+        Initialize database connection and set the table which shall be used as storage for the card key data.
+        In case the specified table does not exist yet it can be created using the create_table_type parameter.
+
+        New tables are always minimal tables which follow a pre-defined table scheme. The user may extend the table
+        with additional columns using the add_cols() later.
+
+        Args:
+                tablename : name of the database table to create.
+                create_table_type : type of the table to create ('UICC' or 'EUICC')
+        """
+
+        def user_from_config_file(config, role: str) -> tuple[str, str]:
+            db_users = config.get('db_users')
+            user = db_users.get(role)
+            if user is None:
+                raise ValueError("user for role '%s' not set up in config file." % role)
+            return user.get('name'), user.get('pass')
+
+        self.table = table_name.lower()
+        self.cols = None
+
+        # Depending on the table type, the table name must contain either the substring "uicc_keys" or "euicc_keys".
+        # This convention will allow us to deduct the table type from the table name.
+        if "euicc_keys" not in table_name and "uicc_keys" not in table_name:
+            raise ValueError("Table name (%s) should contain the substring \"uicc_keys\" or \"euicc_keys\"" % table_name)
+
+        # Read config file
+        log.info("Using config file: %s", config_filename)
+        with open(config_filename, "r") as cfg:
+            config = yaml.load(cfg, Loader=yaml.FullLoader)
+        host = config.get('host')
+        log.info("Database host: %s", host)
+        db_name = config.get('db_name')
+        log.info("Database name: %s", db_name)
+        table_names = config.get('table_names')
+        username_admin, password_admin = user_from_config_file(config, 'admin')
+        username_importer, password_importer = user_from_config_file(config, 'importer')
+        username_reader, _ = user_from_config_file(config, 'reader')
+
+        # Switch between admin and importer user
+        if admin:
+            username, password = username_admin, password_admin
+        else:
+            username, password = username_importer, password_importer
+
+        # Create database connection
+        log.info("Database user: %s", username)
+        self.conn = psycopg2.connect(dbname=db_name, user=username, password=password, host=host)
+        self.cur = self.conn.cursor()
+
+        # In the context of this tool it is not relevant if the table name is present in the config file. However,
+        # pySim-shell.py will require the table name to be configured properly to access the database table.
+        if self.table not in table_names:
+            log.warning("Specified table name (%s) is not yet present in config file (required for access from pySim-shell.py)",
+                        self.table)
+
+        # Create a new minimal database table of the specified table type.
+        if create_table:
+            if not admin:
+                raise ValueError("creation of new table refused, use option --admin and try again.")
+            if "euicc_keys" in self.table:
+                self.__create_table(username_reader, username_importer, ['EID'])
+            elif "uicc_keys" in self.table:
+                self.__create_table(username_reader, username_importer, ['ICCID', 'IMSI'])
+
+        # Ensure a table with the specified name exists
+        log.info("Database table: %s", self.table)
+        if self.get_cols() == []:
+            raise ValueError("Table name (%s) does not exist yet" % self.table)
+        log.info("Database table columns: %s", str(self.get_cols()))
+
+    def __create_table(self, user_reader:str, user_importer:str, cols:list[str]):
+        """
+        Initialize a new table. New tables are always minimal tables with one primary key and additional index columns.
+        Non index-columns may be added later using method _update_cols().
+        """
+
+        # Create table columns with primary key
+        query = SQL("CREATE TABLE {} ({} VARCHAR PRIMARY KEY").format(Identifier(self.table),
+                                                                      Identifier(cols[0].lower()))
+        for c in cols[1:]:
+            query += SQL(", {} VARCHAR").format(Identifier(c.lower()))
+        query += SQL(");")
+        self.cur.execute(query)
+
+        # Create indexes for all other columns
+        for c in cols[1:]:
+            self.cur.execute(query = SQL("CREATE INDEX {} ON {}({});").format(Identifier(c.lower()),
+                                                                              Identifier(self.table),
+                                                                              Identifier(c.lower())))
+
+        # Set permissions
+        self.cur.execute(SQL("GRANT INSERT ON {} TO {};").format(Identifier(self.table),
+                                                                 Identifier(user_importer)))
+        self.cur.execute(SQL("GRANT SELECT ON {} TO {};").format(Identifier(self.table),
+                                                                 Identifier(user_reader)))
+
+        log.info("New database table created: %s", self.table)
+
+    def get_cols(self) -> list[str]:
+        """
+        Get a list of all columns available in the current table scheme.
+
+        Returns:
+                list with column names (in uppercase) of the database table
+        """
+
+        # Return cached col list if present
+        if self.cols:
+            return self.cols
+
+        # Request a list of current cols from the database
+        self.cur.execute("SELECT column_name FROM information_schema.columns where table_name = %s;", (self.table,))
+
+        cols_result = self.cur.fetchall()
+        cols = []
+        for c in cols_result:
+            cols.append(c[0].upper())
+        self.cols = cols
+        return cols
+
+    def get_missing_cols(self, cols_expected:list[str]) -> list[str]:
+        """
+        Check if the current table scheme lacks any of the given expected columns.
+
+        Returns:
+                list with the missing columns.
+        """
+
+        cols_present = self.get_cols()
+        return list(set(cols_expected) - set(cols_present))
+
+    def add_cols(self, cols:list[str]):
+        """
+        Update the current table scheme with additional columns. In case the updated columns are already exist, the
+        table schema is not changed.
+
+        Args:
+                table : name of the database table to alter
+                cols : list with updated colum names to add
+        """
+
+        cols_missing = self.get_missing_cols(cols)
+
+        # Depending on the table type (see constructor), we either have a primary key 'ICCID' (for UICC data), or 'EID'
+        # (for eUICC data). Both table formats different types of data and have rather differen columns also. Let's
+        # prevent the excidentally mixing of both types.
+        if 'ICCID' in cols_missing:
+            raise ValueError("Table %s stores eUCCC key material, refusing to add UICC specific column 'ICCID'" % self.table)
+        if 'EID' in cols_missing:
+            raise ValueError("Table %s stores UCCC key material, refusing to add eUICC specific column 'EID'" % self.table)
+
+        # Add the missing columns to the table
+        self.cols = None
+        for c in cols_missing:
+            self.cur.execute(query = SQL("ALTER TABLE {} ADD {} VARCHAR;").format(Identifier(self.table),
+                                                                                  Identifier(c.lower())))
+
+    def insert_row(self, row:dict[str, str]):
+        """
+        Insert a new row into the database table.
+
+        Args:
+                row : dictionary with the colum names and their designated values
+        """
+
+        # Check if the row is compatible with the current table scheme
+        cols_expected = list(row.keys())
+        cols_missing = self.get_missing_cols(cols_expected)
+        if cols_missing != []:
+            raise ValueError("table %s has incompatible format, the row %s contains unknown cols %s" %
+                             (self.table, str(row), str(cols_missing)))
+
+        # Insert row into datbase table
+        row_keys = list(row.keys())
+        row_values = list(row.values())
+        query = SQL("INSERT INTO {} ").format(Identifier(self.table))
+        query += SQL("({} ").format(Identifier(row_keys[0].lower()))
+        for k in row_keys[1:]:
+            query += SQL(", {}").format(Identifier(k.lower()))
+        query += SQL(") VALUES (%s")
+        for v in row_values[1:]:
+            query += SQL(", %s")
+        query += SQL(");")
+        self.cur.execute(query, row_values)
+
+    def commit(self):
+        self.conn.commit()
+        log.info("Changes to table %s committed!", self.table)
+
+def open_csv(opts: argparse.Namespace):
+    log.info("CSV file: %s", opts.csv)
+    csv_file = open(opts.csv, 'r')
+    cr = csv.DictReader(csv_file)
+    if not cr:
+        raise RuntimeError("could not open DictReader for CSV-File '%s'" % opts.csv)
+    cr.fieldnames = [field.upper() for field in cr.fieldnames]
+    log.info("CSV file columns: %s", str(cr.fieldnames))
+    return cr
+
+def open_db(cr: csv.DictReader, opts: argparse.Namespace) -> CardKeyDatabase:
+    try:
+        db = CardKeyDatabase(os.path.expanduser(opts.pgsql), opts.table_name, opts.create_table, opts.admin)
+
+        # Check CSV format against table schema, add missing columns
+        cols_missing = db.get_missing_cols(cr.fieldnames)
+        if cols_missing != [] and (opts.update_columns or opts.create_table):
+            log.info("Adding missing columns: %s", str(cols_missing))
+            db.add_cols(cols_missing)
+            cols_missing = db.get_missing_cols(cr.fieldnames)
+
+        # Make sure the table schema has no missing columns
+        if cols_missing != []:
+            log.error("Database table lacks CSV file columns: %s -- import aborted!", cols_missing)
+            sys.exit(2)
+    except Exception as e:
+        log.error(str(e).strip())
+        log.error("Database initialization aborted due to error!")
+        sys.exit(2)
+
+    return db
+
+def import_from_csv(db: CardKeyDatabase, cr: csv.DictReader):
+    count = 0
+    for row in cr:
+        try:
+            db.insert_row(row)
+            count+=1
+            if count % 100 == 0:
+                log.info("CSV file import in progress, %d rows imported...", count)
+        except Exception as e:
+            log.error(str(e).strip())
+            log.error("CSV file import aborted due to error, no datasets committed!")
+            sys.exit(2)
+    log.info("CSV file import done, %d rows imported", count)
+
+if __name__ == '__main__':
+    option_parser = argparse.ArgumentParser(description='CSV importer for pySim-shell\'s PostgreSQL Card Key Provider',
+                                            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    option_parser.add_argument("--verbose", help="Enable verbose logging", action='store_true', default=False)
+    option_parser.add_argument('--pgsql', metavar='FILE',
+                               default="~/.osmocom/pysim/card_data_pgsql.cfg",
+                               help='Read card data from PostgreSQL database (config file)')
+    option_parser.add_argument('--csv', metavar='FILE', help='input CSV file with card data', required=True)
+    option_parser.add_argument("--table-name", help="name of the card key table", type=str, required=True)
+    option_parser.add_argument("--update-columns", help="add missing table columns", action='store_true', default=False)
+    option_parser.add_argument("--create-table", action='store_true', help="create new card key table", default=False)
+    option_parser.add_argument("--admin", action='store_true', help="perform action as admin", default=False)
+    opts = option_parser.parse_args()
+
+    PySimLogger.setup(print, {logging.WARN: "\033[33m"})
+    if (opts.verbose):
+        PySimLogger.set_verbose(True)
+        PySimLogger.set_level(logging.DEBUG)
+
+    # Open CSV file
+    cr = open_csv(opts)
+
+    # Open database, create initial table, update column scheme
+    db = open_db(cr, opts)
+
+    # Progress with import
+    if not opts.admin:
+        import_from_csv(db, cr)
+
+    # Commit changes to the database
+    db.commit()

contrib/es2p_client.py

@@ -17,14 +17,14 @@
 
 import copy
 import argparse
-from pySim.esim import es2p
+from pySim.esim import es2p, ActivationCode
 
 EID_HELP='EID of the eUICC for which eSIM shall be made available'
 ICCID_HELP='The ICCID of the eSIM that shall be made available'
 MATCHID_HELP='MatchingID that shall be used by profile download'
 
 parser = argparse.ArgumentParser(description="""
-Utility to manuall issue requests against the ES2+ API of an SM-DP+ according to GSMA SGP.22.""")
+Utility to manually issue requests against the ES2+ API of an SM-DP+ according to GSMA SGP.22.""")
 parser.add_argument('--url', required=True, help='Base URL of ES2+ API endpoint')
 parser.add_argument('--id', required=True, help='Entity identifier passed to SM-DP+')
 parser.add_argument('--client-cert', help='X.509 client certificate used to authenticate to server')
@@ -63,7 +63,7 @@ if __name__ == '__main__':
     data = {}
     for k, v in vars(opts).items():
         if k in ['url', 'id', 'client_cert', 'server_ca_cert', 'command']:
-            # remove keys from dict that shold not end up in JSON...
+            # remove keys from dict that should not end up in JSON...
             continue
         if v is not None:
             data[k] = v
@@ -73,6 +73,11 @@ if __name__ == '__main__':
         res = peer.call_downloadOrder(data)
     elif opts.command == 'confirm-order':
         res = peer.call_confirmOrder(data)
+        matchingId = res.get('matchingId', None)
+        smdpAddress = res.get('smdpAddress', None)
+        if matchingId:
+            ac = ActivationCode(smdpAddress, matchingId, cc_required=bool(opts.confirmationCode))
+            print("Activation Code: '%s'" % ac.to_string())
     elif opts.command == 'cancel-order':
         res = peer.call_cancelOrder(data)
     elif opts.command == 'release-profile':

contrib/es2p_server.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Philipp Maier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import argparse
+import logging
+import json
+import asn1tools
+import asn1tools.codecs.ber
+import asn1tools.codecs.der
+import pySim.esim.rsp as rsp
+import pySim.esim.saip as saip
+from pySim.esim.es2p import param, Es2pApiServerMno, Es2pApiServerHandlerMno
+from osmocom.utils import b2h
+from datetime import datetime
+from analyze_simaResponse import split_sima_response
+from pathlib import Path
+
+logger = logging.getLogger(Path(__file__).stem)
+
+parser = argparse.ArgumentParser(description="""
+Utility to receive and log requests against the ES2+ API of an SM-DP+ according to GSMA SGP.22.""")
+parser.add_argument("--host", help="Host/IP to bind HTTP(S) to", default="localhost")
+parser.add_argument("--port", help="TCP port to bind HTTP(S) to", default=443, type=int)
+parser.add_argument('--server-cert', help='X.509 server certificate used to provide the ES2+ HTTPs service')
+parser.add_argument('--client-ca-cert', help='X.509 CA certificates to authenticate the requesting client(s)')
+parser.add_argument("-v", "--verbose", help="enable debug output", action='store_true', default=False)
+
+def decode_sima_response(sima_response):
+    decoded = []
+    euicc_response_list = split_sima_response(sima_response)
+    for euicc_response in euicc_response_list:
+        decoded.append(saip.asn1.decode('EUICCResponse', euicc_response))
+    return decoded
+
+def decode_result_data(result_data):
+    return rsp.asn1.decode('PendingNotification', result_data)
+
+def decode(data, path="/"):
+    if data is None:
+        return 'none'
+    elif type(data) is datetime:
+        return data.isoformat()
+    elif type(data) is tuple:
+        return {str(data[0]) : decode(data[1], path + str(data[0]) + "/")}
+    elif type(data) is list:
+        new_data = []
+        for item in data:
+            new_data.append(decode(item, path))
+        return new_data
+    elif type(data) is bytes:
+        return b2h(data)
+    elif type(data) is dict:
+        new_data = {}
+        for key, item in data.items():
+            new_key = str(key)
+            if path == '/' and new_key == 'resultData':
+                new_item = decode_result_data(item)
+            elif (path == '/resultData/profileInstallationResult/profileInstallationResultData/finalResult/successResult/' \
+                  or path == '/resultData/profileInstallationResult/profileInstallationResultData/finalResult/errorResult/') \
+                  and new_key == 'simaResponse':
+                new_item = decode_sima_response(item)
+            else:
+                new_item = item
+            new_data[new_key] = decode(new_item, path + new_key + "/")
+        return new_data
+    else:
+        return data
+
+class Es2pApiServerHandlerForLogging(Es2pApiServerHandlerMno):
+    def call_handleDownloadProgressInfo(self, data: dict) -> (dict, str):
+        logging.info("ES2+:handleDownloadProgressInfo: %s" % json.dumps(decode(data)))
+        return {}, None
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARNING,
+                        format='%(asctime)s %(levelname)s %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S')
+
+    Es2pApiServerMno(args.port, args.host, Es2pApiServerHandlerForLogging(), args.server_cert, args.client_ca_cert)
+

contrib/es9p_client.py

@@ -68,7 +68,7 @@ parser_dl.add_argument('--confirmation-code',
 # notification
 parser_ntf = subparsers.add_parser('notification', help='ES9+ (other) notification')
 parser_ntf.add_argument('operation', choices=['enable','disable','delete'],
-                        help='Profile Management Opreation whoise occurrence shall be notififed')
+                        help='Profile Management Operation whoise occurrence shall be notififed')
 parser_ntf.add_argument('--sequence-nr', type=int, required=True,
                         help='eUICC global notification sequence number')
 parser_ntf.add_argument('--notification-address', help='notificationAddress, if different from URL')
@@ -123,17 +123,17 @@ class Es9pClient:
             'profileManagementOperation': PMO(self.opts.operation).to_bitstring(),
             'notificationAddress': self.opts.notification_address or urlparse(self.opts.url).netloc,
         }
-        if opts.iccid:
-            ntf_metadata['iccid'] = h2b(swap_nibbles(opts.iccid))
+        if self.opts.iccid:
+            ntf_metadata['iccid'] = h2b(swap_nibbles(self.opts.iccid))
 
-        if self.opts.operation == 'download':
+        if self.opts.operation == 'install':
             pird = {
-                'transactionId': self.opts.transaction_id,
+                'transactionId': h2b(self.opts.transaction_id),
                 'notificationMetadata': ntf_metadata,
                 'smdpOid': self.opts.smdpp_oid,
                 'finalResult': ('successResult', {
-                    'aid': self.opts.isdp_aid,
-                    'simaResponse': self.opts.sima_response,
+                    'aid': h2b(self.opts.isdp_aid),
+                    'simaResponse': h2b(self.opts.sima_response),
                     }),
             }
             pird_bin = rsp.asn1.encode('ProfileInstallationResultData', pird)

contrib/esim-qrcode-gen.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+# Small command line utility program to encode eSIM QR-Codes
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import argparse
+
+from pySim.esim import ActivationCode
+
+
+option_parser = argparse.ArgumentParser(description="""
+eSIM QR code generator. Will encode the given hostname + activation code
+into the eSIM RSP String format as specified in SGP.22 Section 4.1.  If
+a PNG output file is specified, it will also generate a QR code.""")
+option_parser.add_argument('hostname', help='FQDN of SM-DP+')
+option_parser.add_argument('token', help='MatchingID / Token')
+option_parser.add_argument('--oid', help='SM-DP+ OID in CERT.DPauth.ECDSA')
+option_parser.add_argument('--confirmation-code-required', action='store_true',
+                           help='Whether a Confirmation Code is required')
+option_parser.add_argument('--png', help='Output PNG file name (no PNG is written if omitted)')
+
+
+
+if __name__ == '__main__':
+    opts = option_parser.parse_args()
+
+    ac = ActivationCode(opts.hostname, opts.token, opts.oid, opts.confirmation_code_required)
+    print(ac.to_string())
+    if opts.png:
+        with open(opts.png, 'wb') as f:
+            img = ac.to_qrcode()
+            img.save(f)
+            print("# generated QR code stored to '%s'" % (opts.png))

contrib/esim_gen_metadata.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+# (C) 2025 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+from osmocom.utils import h2b, swap_nibbles
+from pySim.esim.es8p import ProfileMetadata
+
+parser = argparse.ArgumentParser(description="""Utility program to generate profile metadata in the
+StoreMetadataRequest format based on input values from the command line.""")
+parser.add_argument('--iccid', required=True, help="ICCID of eSIM profile");
+parser.add_argument('--spn', required=True, help="Service Provider Name");
+parser.add_argument('--profile-name', required=True, help="eSIM Profile Name");
+parser.add_argument('--profile-class', choices=['test', 'operational', 'provisioning'],
+                    default='operational', help="Profile Class");
+parser.add_argument('--outfile', required=True, help="Output File Name");
+
+if __name__ == '__main__':
+    opts = parser.parse_args()
+
+    iccid_bin = h2b(swap_nibbles(opts.iccid))
+    pmd = ProfileMetadata(iccid_bin, spn=opts.spn, profile_name=opts.profile_name,
+                          profile_class=opts.profile_class)
+
+    with open(opts.outfile, 'wb') as f:
+        f.write(pmd.gen_store_metadata_request())
+    print("Written StoreMetadataRequest to '%s'" % opts.outfile)

contrib/generate_smdpp_certs.py

@@ -0,0 +1,661 @@
+#!/usr/bin/env python3
+
+"""
+Faithfully reproduces the smdpp certs contained in SGP.26_v1.5_Certificates_18_07_2024.zip
+available at https://www.gsma.com/solutions-and-impact/technologies/esim/gsma_resources/sgp-26-test-certificate-definition-v1-5/
+Only usable for testing, it obviously uses a different CI key.
+"""
+
+import os
+import binascii
+from datetime import datetime
+from cryptography import x509
+from cryptography.x509.oid import NameOID, ExtensionOID
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.backends import default_backend
+
+# Custom OIDs used in certificates
+OID_CERTIFICATE_POLICIES_CI = "2.23.146.1.2.1.0"  # CI cert policy
+OID_CERTIFICATE_POLICIES_TLS = "2.23.146.1.2.1.3"  # DPtls cert policy
+OID_CERTIFICATE_POLICIES_AUTH = "2.23.146.1.2.1.4"  # DPauth cert policy
+OID_CERTIFICATE_POLICIES_PB = "2.23.146.1.2.1.5"  # DPpb cert policy
+
+# Subject Alternative Name OIDs
+OID_CI_RID = "2.999.1"  # CI Registered ID
+OID_DP_RID = "2.999.10"  # DP+ Registered ID
+OID_DP2_RID = "2.999.12"  # DP+2 Registered ID
+OID_DP4_RID = "2.999.14"  # DP+4 Registered ID
+OID_DP8_RID = "2.999.18"  # DP+8 Registered ID
+
+
+class SimplifiedCertificateGenerator:
+    def __init__(self):
+        self.backend = default_backend()
+        # Store generated CI keys to sign other certs
+        self.ci_certs = {}  # {"BRP": cert, "NIST": cert}
+        self.ci_keys = {}   # {"BRP": key, "NIST": key}
+
+    def get_curve(self, curve_type):
+        """Get the appropriate curve object."""
+        if curve_type == "BRP":
+            return ec.BrainpoolP256R1()
+        else:
+            return ec.SECP256R1()
+
+    def generate_key_pair(self, curve):
+        """Generate a new EC key pair."""
+        private_key = ec.generate_private_key(curve, self.backend)
+        return private_key
+
+    def load_private_key_from_hex(self, hex_key, curve):
+        """Load EC private key from hex string."""
+        key_bytes = binascii.unhexlify(hex_key.replace(":", "").replace(" ", "").replace("\n", ""))
+        key_int = int.from_bytes(key_bytes, 'big')
+        return ec.derive_private_key(key_int, curve, self.backend)
+
+    def generate_ci_cert(self, curve_type):
+        """Generate CI certificate for either BRP or NIST curve."""
+        curve = self.get_curve(curve_type)
+        private_key = self.generate_key_pair(curve)
+
+        # Build subject and issuer (self-signed) - same for both
+        subject = issuer = x509.Name([
+            x509.NameAttribute(NameOID.COMMON_NAME, "Test CI"),
+            x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, "TESTCERT"),
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "RSPTEST"),
+            x509.NameAttribute(NameOID.COUNTRY_NAME, "IT"),
+        ])
+
+        # Build certificate - all parameters same for both
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(issuer)
+        builder = builder.not_valid_before(datetime(2020, 4, 1, 8, 27, 51))
+        builder = builder.not_valid_after(datetime(2055, 4, 1, 8, 27, 51))
+        builder = builder.serial_number(0xb874f3abfa6c44d3)
+        builder = builder.public_key(private_key.public_key())
+
+        # Add extensions - all same for both
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.BasicConstraints(ca=True, path_length=None),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier(OID_CERTIFICATE_POLICIES_CI),
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=False,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=True,
+                crl_sign=True,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectAlternativeName([
+                x509.RegisteredID(x509.ObjectIdentifier(OID_CI_RID))
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.CRLDistributionPoints([
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-A.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                ),
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-B.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                )
+            ]),
+            critical=False
+        )
+
+        certificate = builder.sign(private_key, hashes.SHA256(), self.backend)
+
+        self.ci_keys[curve_type] = private_key
+        self.ci_certs[curve_type] = certificate
+
+        return certificate, private_key
+
+    def generate_dp_cert(self, curve_type, subject_cn, serial, key_hex,
+                        cert_policy_oid, rid_oid, validity_start, validity_end):
+        """Generate a DP certificate signed by CI - works for both BRP and NIST."""
+        curve = self.get_curve(curve_type)
+        private_key = self.load_private_key_from_hex(key_hex, curve)
+
+        ci_cert = self.ci_certs[curve_type]
+        ci_key = self.ci_keys[curve_type]
+
+        subject = x509.Name([
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "ACME"),
+            x509.NameAttribute(NameOID.COMMON_NAME, subject_cn),
+        ])
+
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(ci_cert.subject)
+        builder = builder.not_valid_before(validity_start)
+        builder = builder.not_valid_after(validity_end)
+        builder = builder.serial_number(serial)
+        builder = builder.public_key(private_key.public_key())
+
+        builder = builder.add_extension(
+            x509.AuthorityKeyIdentifier.from_issuer_public_key(ci_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectAlternativeName([
+                x509.RegisteredID(x509.ObjectIdentifier(rid_oid))
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=True,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=False,
+                crl_sign=False,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier(cert_policy_oid),
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CRLDistributionPoints([
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-A.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                ),
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-B.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                )
+            ]),
+            critical=False
+        )
+
+        certificate = builder.sign(ci_key, hashes.SHA256(), self.backend)
+
+        return certificate, private_key
+
+    def generate_tls_cert(self, curve_type, subject_cn, dns_name, serial, key_hex,
+                         rid_oid, validity_start, validity_end):
+        """Generate a TLS certificate signed by CI."""
+        curve = self.get_curve(curve_type)
+        private_key = self.load_private_key_from_hex(key_hex, curve)
+
+        ci_cert = self.ci_certs[curve_type]
+        ci_key = self.ci_keys[curve_type]
+
+        subject = x509.Name([
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "ACME"),
+            x509.NameAttribute(NameOID.COMMON_NAME, subject_cn),
+        ])
+
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(ci_cert.subject)
+        builder = builder.not_valid_before(validity_start)
+        builder = builder.not_valid_after(validity_end)
+        builder = builder.serial_number(serial)
+        builder = builder.public_key(private_key.public_key())
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=True,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=False,
+                crl_sign=False,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.ExtendedKeyUsage([
+                x509.oid.ExtendedKeyUsageOID.SERVER_AUTH,
+                x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH
+            ]),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier(OID_CERTIFICATE_POLICIES_TLS),
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.AuthorityKeyIdentifier.from_issuer_public_key(ci_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectAlternativeName([
+                x509.DNSName(dns_name),
+                x509.RegisteredID(x509.ObjectIdentifier(rid_oid))
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.CRLDistributionPoints([
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-A.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                ),
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-B.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                )
+            ]),
+            critical=False
+        )
+
+        certificate = builder.sign(ci_key, hashes.SHA256(), self.backend)
+
+        return certificate, private_key
+
+    def generate_eum_cert(self, curve_type, key_hex):
+        """Generate EUM certificate signed by CI."""
+        curve = self.get_curve(curve_type)
+        private_key = self.load_private_key_from_hex(key_hex, curve)
+
+        ci_cert = self.ci_certs[curve_type]
+        ci_key = self.ci_keys[curve_type]
+
+        subject = x509.Name([
+            x509.NameAttribute(NameOID.COUNTRY_NAME, "ES"),
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "RSP Test EUM"),
+            x509.NameAttribute(NameOID.COMMON_NAME, "EUM Test"),
+        ])
+
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(ci_cert.subject)
+        builder = builder.not_valid_before(datetime(2020, 4, 1, 9, 28, 37))
+        builder = builder.not_valid_after(datetime(2054, 3, 24, 9, 28, 37))
+        builder = builder.serial_number(0x12345678)
+        builder = builder.public_key(private_key.public_key())
+
+        builder = builder.add_extension(
+            x509.AuthorityKeyIdentifier.from_issuer_public_key(ci_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=False,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=True,
+                crl_sign=False,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier("2.23.146.1.2.1.2"),  # EUM policy
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectAlternativeName([
+                x509.RegisteredID(x509.ObjectIdentifier("2.999.5"))
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.BasicConstraints(ca=True, path_length=0),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CRLDistributionPoints([
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-B.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                )
+            ]),
+            critical=False
+        )
+
+        # Name Constraints
+        constrained_name = x509.Name([
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "RSP Test EUM"),
+            x509.NameAttribute(NameOID.SERIAL_NUMBER, "89049032"),
+        ])
+
+        name_constraints = x509.NameConstraints(
+            permitted_subtrees=[
+                x509.DirectoryName(constrained_name)
+            ],
+            excluded_subtrees=None
+        )
+
+        builder = builder.add_extension(
+            name_constraints,
+            critical=True
+        )
+
+        certificate = builder.sign(ci_key, hashes.SHA256(), self.backend)
+
+        return certificate, private_key
+
+    def generate_euicc_cert(self, curve_type, eum_cert, eum_key, key_hex):
+        """Generate eUICC certificate signed by EUM."""
+        curve = self.get_curve(curve_type)
+        private_key = self.load_private_key_from_hex(key_hex, curve)
+
+        subject = x509.Name([
+            x509.NameAttribute(NameOID.COUNTRY_NAME, "ES"),
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "RSP Test EUM"),
+            x509.NameAttribute(NameOID.SERIAL_NUMBER, "89049032123451234512345678901235"),
+            x509.NameAttribute(NameOID.COMMON_NAME, "Test eUICC"),
+        ])
+
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(eum_cert.subject)
+        builder = builder.not_valid_before(datetime(2020, 4, 1, 9, 48, 58))
+        builder = builder.not_valid_after(datetime(7496, 1, 24, 9, 48, 58))
+        builder = builder.serial_number(0x0200000000000001)
+        builder = builder.public_key(private_key.public_key())
+
+        builder = builder.add_extension(
+            x509.AuthorityKeyIdentifier.from_issuer_public_key(eum_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=True,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=False,
+                crl_sign=False,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier("2.23.146.1.2.1.1"),  # eUICC policy
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=True
+        )
+
+        certificate = builder.sign(eum_key, hashes.SHA256(), self.backend)
+
+        return certificate, private_key
+
+    def save_cert_and_key(self, cert, key, cert_path_der, cert_path_pem, key_path_sk, key_path_pk):
+        """Save certificate and key in various formats."""
+        # Create directories if needed
+        os.makedirs(os.path.dirname(cert_path_der), exist_ok=True)
+
+        with open(cert_path_der, "wb") as f:
+            f.write(cert.public_bytes(serialization.Encoding.DER))
+
+        if cert_path_pem:
+            with open(cert_path_pem, "wb") as f:
+                f.write(cert.public_bytes(serialization.Encoding.PEM))
+
+        if key and key_path_sk:
+            with open(key_path_sk, "wb") as f:
+                f.write(key.private_bytes(
+                    encoding=serialization.Encoding.PEM,
+                    format=serialization.PrivateFormat.TraditionalOpenSSL,
+                    encryption_algorithm=serialization.NoEncryption()
+                ))
+
+        if key and key_path_pk:
+            with open(key_path_pk, "wb") as f:
+                f.write(key.public_key().public_bytes(
+                    encoding=serialization.Encoding.PEM,
+                    format=serialization.PublicFormat.SubjectPublicKeyInfo
+                ))
+
+
+def main():
+    gen = SimplifiedCertificateGenerator()
+
+    output_dir = "smdpp-data/generated"
+    os.makedirs(output_dir, exist_ok=True)
+
+    print("=== Generating CI Certificates ===")
+
+    for curve_type in ["BRP", "NIST"]:
+        ci_cert, ci_key = gen.generate_ci_cert(curve_type)
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            ci_cert, ci_key,
+            f"{output_dir}/CertificateIssuer/CERT_CI{suffix}.der",
+            f"{output_dir}/CertificateIssuer/CERT_CI{suffix}.pem",
+            None, None
+        )
+        print(f"Generated CI {curve_type} certificate")
+
+    print("\n=== Generating DPauth Certificates ===")
+
+    dpauth_configs = [
+        ("BRP", "TEST SM-DP+", 256, "93:fb:33:d0:58:4f:34:9b:07:f8:b5:d2:af:93:d7:c3:e3:54:b3:49:a3:b9:13:50:2e:6a:bc:07:0e:4d:49:29", OID_DP_RID, "DPauth"),
+        ("NIST", "TEST SM-DP+", 256, "0a:7c:c1:c2:44:e6:0c:52:cd:5b:78:07:ab:8c:36:0c:26:52:46:01:50:7d:ca:bc:5d:d5:98:b5:a6:16:d5:d5", OID_DP_RID, "DPauth"),
+        ("BRP", "TEST SM-DP+2", 512, "0c:17:35:5c:01:1d:0f:e8:d7:da:dd:63:f1:97:85:cf:6c:51:cb:cd:46:6a:e8:8b:e8:f8:1b:c1:05:88:46:f6", OID_DP2_RID, "DP2auth"),
+        ("NIST", "TEST SM-DP+2", 512, "9c:32:a0:95:d4:88:42:d9:ff:a4:04:f7:12:51:2a:a2:c5:42:5a:1a:26:38:6a:b6:a1:45:d5:81:1e:03:91:41", OID_DP2_RID, "DP2auth"),
+    ]
+
+    for curve_type, cn, serial, key_hex, rid_oid, name_prefix in dpauth_configs:
+        cert, key = gen.generate_dp_cert(
+            curve_type, cn, serial, key_hex,
+            OID_CERTIFICATE_POLICIES_AUTH, rid_oid,
+            datetime(2020, 4, 1, 8, 31, 30),
+            datetime(2030, 3, 30, 8, 31, 30)
+        )
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/DPauth/CERT_S_SM_{name_prefix}{suffix}.der",
+            None,
+            f"{output_dir}/DPauth/SK_S_SM_{name_prefix}{suffix}.pem",
+            f"{output_dir}/DPauth/PK_S_SM_{name_prefix}{suffix}.pem"
+        )
+        print(f"Generated {name_prefix} {curve_type} certificate")
+
+    print("\n=== Generating DPpb Certificates ===")
+
+    dppb_configs = [
+        ("BRP", "TEST SM-DP+", 257, "75:ff:32:2f:41:66:16:da:e1:a4:84:ef:71:d4:87:4f:b0:df:32:95:fd:35:c2:cb:a4:89:fb:b2:bb:9c:7b:f6", OID_DP_RID, "DPpb"),
+        ("NIST", "TEST SM-DP+", 257, "dc:d6:94:b7:78:95:7e:8e:9a:dd:bd:d9:44:33:e9:ef:8f:73:d1:1e:49:1c:48:d4:25:a3:8a:94:91:bd:3b:ed", OID_DP_RID, "DPpb"),
+        ("BRP", "TEST SM-DP+2", 513, "9c:ae:2e:1a:56:07:a9:d5:78:38:2e:ee:93:2e:25:1f:52:30:4f:86:ee:b1:f1:70:8c:db:d3:c0:7b:e2:cd:3d", OID_DP2_RID, "DP2pb"),
+        ("NIST", "TEST SM-DP+2", 513, "66:93:11:49:63:9d:ba:ac:1d:c3:d3:06:c5:8b:d2:df:d2:2f:73:bf:63:ac:86:31:98:32:90:b5:7f:90:93:45", OID_DP2_RID, "DP2pb"),
+    ]
+
+    for curve_type, cn, serial, key_hex, rid_oid, name_prefix in dppb_configs:
+        cert, key = gen.generate_dp_cert(
+            curve_type, cn, serial, key_hex,
+            OID_CERTIFICATE_POLICIES_PB, rid_oid,
+            datetime(2020, 4, 1, 8, 34, 46),
+            datetime(2030, 3, 30, 8, 34, 46)
+        )
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/DPpb/CERT_S_SM_{name_prefix}{suffix}.der",
+            None,
+            f"{output_dir}/DPpb/SK_S_SM_{name_prefix}{suffix}.pem",
+            f"{output_dir}/DPpb/PK_S_SM_{name_prefix}{suffix}.pem"
+        )
+        print(f"Generated {name_prefix} {curve_type} certificate")
+
+    print("\n=== Generating DPtls Certificates ===")
+
+    dptls_configs = [
+        ("BRP", "testsmdpplus1.example.com", "testsmdpplus1.example.com", 9, "3f:67:15:28:02:b3:f4:c7:fa:e6:79:58:55:f6:82:54:1e:45:e3:5e:ff:f4:e8:a0:55:65:a0:f1:91:2a:78:2e", OID_DP_RID, "DP_TLS_BRP"),
+        ("NIST", "testsmdpplus1.example.com", "testsmdpplus1.example.com", 9, "a0:3e:7c:e4:55:04:74:be:a4:b7:a8:73:99:ce:5a:8c:9f:66:1b:68:0f:94:01:39:ff:f8:4e:9d:ec:6a:4d:8c", OID_DP_RID, "DP_TLS_NIST"),
+        ("NIST", "testsmdpplus2.example.com", "testsmdpplus2.example.com", 12, "4e:65:61:c6:40:88:f6:69:90:7a:db:e3:94:b1:1a:84:24:2e:03:3a:82:a8:84:02:31:63:6d:c9:1b:4e:e3:f5", OID_DP2_RID, "DP2_TLS"),
+        ("NIST", "testsmdpplus4.example.com", "testsmdpplus4.example.com", 14, "f2:65:9d:2f:52:8f:4b:11:37:40:d5:8a:0d:2a:f3:eb:2b:48:e1:22:c2:b6:0a:6a:f6:fc:96:ad:86:be:6f:a4", OID_DP4_RID, "DP4_TLS"),
+        ("NIST", "testsmdpplus8.example.com", "testsmdpplus8.example.com", 18, "ff:6e:4a:50:9b:ad:db:38:10:88:31:c2:3c:cc:2d:44:30:7a:f2:81:e9:25:96:7f:8c:df:1d:95:54:a0:28:8d", OID_DP8_RID, "DP8_TLS"),
+    ]
+
+    for curve_type, cn, dns, serial, key_hex, rid_oid, name_prefix in dptls_configs:
+        cert, key = gen.generate_tls_cert(
+            curve_type, cn, dns, serial, key_hex, rid_oid,
+            datetime(2024, 7, 9, 15, 29, 36),
+            datetime(2025, 8, 11, 15, 29, 36)
+        )
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/DPtls/CERT_S_SM_{name_prefix}.der",
+            None,
+            f"{output_dir}/DPtls/SK_S_SM_{name_prefix.replace('_BRP', '_BRP').replace('_NIST', '_NIST')}.pem",
+            f"{output_dir}/DPtls/PK_S_SM_{name_prefix.replace('_BRP', '_BRP').replace('_NIST', '_NIST')}.pem"
+        )
+        print(f"Generated {name_prefix} certificate")
+
+    print("\n=== Generating EUM Certificates ===")
+
+    eum_configs = [
+        ("BRP", "12:9b:0a:b1:3f:17:e1:4a:40:b6:fa:4e:d8:23:e0:cf:46:5b:7b:3d:73:24:05:e6:29:5d:3b:23:b0:45:c9:9a"),
+        ("NIST", "25:e6:75:77:28:e1:e9:51:13:51:9c:dc:34:55:5c:29:ba:ed:23:77:3a:c5:af:dd:dc:da:d9:84:89:8a:52:f0"),
+    ]
+
+    eum_certs = {}
+    eum_keys = {}
+
+    for curve_type, key_hex in eum_configs:
+        cert, key = gen.generate_eum_cert(curve_type, key_hex)
+        eum_certs[curve_type] = cert
+        eum_keys[curve_type] = key
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/EUM/CERT_EUM{suffix}.der",
+            None,
+            f"{output_dir}/EUM/SK_EUM{suffix}.pem",
+            f"{output_dir}/EUM/PK_EUM{suffix}.pem"
+        )
+        print(f"Generated EUM {curve_type} certificate")
+
+    print("\n=== Generating eUICC Certificates ===")
+
+    euicc_configs = [
+        ("BRP", "8d:c3:47:a7:6d:b7:bd:d6:22:2d:d7:5e:a1:a1:68:8a:ca:81:1e:4c:bc:6a:7f:6a:ef:a4:b2:64:19:62:0b:90"),
+        ("NIST", "11:e1:54:67:dc:19:4f:33:71:83:e4:60:c9:f6:32:60:09:1e:12:e8:10:26:cd:65:61:e1:7c:6d:85:39:cc:9c"),
+    ]
+
+    for curve_type, key_hex in euicc_configs:
+        cert, key = gen.generate_euicc_cert(curve_type, eum_certs[curve_type], eum_keys[curve_type], key_hex)
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/eUICC/CERT_EUICC{suffix}.der",
+            None,
+            f"{output_dir}/eUICC/SK_EUICC{suffix}.pem",
+            f"{output_dir}/eUICC/PK_EUICC{suffix}.pem"
+        )
+        print(f"Generated eUICC {curve_type} certificate")
+
+    print("\n=== Certificate generation complete! ===")
+    print(f"All certificates saved to: {output_dir}/")
+
+if __name__ == "__main__":
+    main()

contrib/jenkins.sh

@@ -42,6 +42,9 @@ case "$JOB_TYPE" in
 
 	# Run pySim-shell integration tests (requires physical cards)
 	python3 -m unittest discover -v -s ./tests/pySim-shell_test/
+
+	# Run pySim-smpp2sim test
+	tests/pySim-smpp2sim_test/pySim-smpp2sim_test.sh
 	;;
 "distcheck")
 	virtualenv -p python3 venv --system-site-packages

contrib/saip-tool.py

@@ -19,14 +19,13 @@ import os
 import sys
 import argparse
 import logging
-import zipfile
 from pathlib import Path as PlPath
 from typing import List
 from osmocom.utils import h2b, b2h, swap_nibbles
+from osmocom.construct import GreedyBytes, StripHeaderAdapter
 
 from pySim.esim.saip import *
 from pySim.esim.saip.validation import CheckBasicStructure
-from pySim import javacard
 from pySim.pprint import HexBytesPrettyPrinter
 
 pp = HexBytesPrettyPrinter(indent=4,width=500)
@@ -48,22 +47,73 @@ parser_dump.add_argument('--dump-decoded', action='store_true', help='Dump decod
 
 parser_check = subparsers.add_parser('check', help='Run constraint checkers on PE-Sequence')
 
+parser_rpe = subparsers.add_parser('extract-pe', help='Extract specified PE to (DER encoded) file')
+parser_rpe.add_argument('--pe-file', required=True, help='PE file name')
+parser_rpe.add_argument('--identification', type=int, help='Extract PE matching specified identification')
+
 parser_rpe = subparsers.add_parser('remove-pe', help='Remove specified PEs from PE-Sequence')
 parser_rpe.add_argument('--output-file', required=True, help='Output file name')
-parser_rpe.add_argument('--identification', type=int, action='append', help='Remove PEs matching specified identification')
+parser_rpe.add_argument('--identification', default=[], type=int, action='append', help='Remove PEs matching specified identification')
+parser_rpe.add_argument('--type', default=[], action='append', help='Remove PEs matching specified type')
 
-parser_rn = subparsers.add_parser('remove-naa', help='Remove speciifed NAAs from PE-Sequence')
+parser_rn = subparsers.add_parser('remove-naa', help='Remove specified NAAs from PE-Sequence')
 parser_rn.add_argument('--output-file', required=True, help='Output file name')
 parser_rn.add_argument('--naa-type', required=True, choices=NAAs.keys(), help='Network Access Application type to remove')
 # TODO: add an --naa-index or the like, so only one given instance can be removed
 
 parser_info = subparsers.add_parser('info', help='Display information about the profile')
+parser_info.add_argument('--apps', action='store_true', help='List applications and their related instances')
 
 parser_eapp = subparsers.add_parser('extract-apps', help='Extract applications as loadblock file')
 parser_eapp.add_argument('--output-dir', default='.', help='Output directory (where to store files)')
 parser_eapp.add_argument('--format', default='cap', choices=['ijc', 'cap'], help='Data format of output files')
 
-parser_info = subparsers.add_parser('tree', help='Display the filesystem tree')
+parser_aapp = subparsers.add_parser('add-app', help='Add application to PE-Sequence')
+parser_aapp.add_argument('--output-file', required=True, help='Output file name')
+parser_aapp.add_argument('--applet-file', required=True, help='Applet file name')
+parser_aapp.add_argument('--aid', required=True, help='Load package AID')
+parser_aapp.add_argument('--sd-aid', default=None, help='Security Domain AID')
+parser_aapp.add_argument('--non-volatile-code-limit', default=None, type=int, help='Non volatile code limit (C6)')
+parser_aapp.add_argument('--volatile-data-limit', default=None, type=int, help='Volatile data limit (C7)')
+parser_aapp.add_argument('--non-volatile-data-limit', default=None, type=int, help='Non volatile data limit (C8)')
+parser_aapp.add_argument('--hash-value', default=None, help='Hash value')
+
+parser_rapp = subparsers.add_parser('remove-app', help='Remove application from PE-Sequence')
+parser_rapp.add_argument('--output-file', required=True, help='Output file name')
+parser_rapp.add_argument('--aid', required=True, help='Load package AID')
+
+parser_aappi = subparsers.add_parser('add-app-inst', help='Add application instance to Application PE')
+parser_aappi.add_argument('--output-file', required=True, help='Output file name')
+parser_aappi.add_argument('--aid', required=True, help='Load package AID')
+parser_aappi.add_argument('--class-aid', required=True, help='Class AID')
+parser_aappi.add_argument('--inst-aid', required=True, help='Instance AID (must match Load package AID)')
+parser_aappi.add_argument('--app-privileges', default='000000', help='Application privileges')
+parser_aappi.add_argument('--volatile-memory-quota', default=None, type=int, help='Volatile memory quota (C7)')
+parser_aappi.add_argument('--non-volatile-memory-quota', default=None, type=int, help='Non volatile memory quota (C8)')
+parser_aappi.add_argument('--app-spec-pars', default='00', help='Application specific parameters (C9)')
+parser_aappi.add_argument('--uicc-toolkit-app-spec-pars', help='UICC toolkit application specific parameters field')
+parser_aappi.add_argument('--uicc-access-app-spec-pars', help='UICC Access application specific parameters field')
+parser_aappi.add_argument('--uicc-adm-access-app-spec-pars', help='UICC Administrative access application specific parameters field')
+parser_aappi.add_argument('--process-data', default=[], action='append', help='Process personalization APDUs')
+
+parser_rappi = subparsers.add_parser('remove-app-inst', help='Remove application instance from Application PE')
+parser_rappi.add_argument('--output-file', required=True, help='Output file name')
+parser_rappi.add_argument('--aid', required=True, help='Load package AID')
+parser_rappi.add_argument('--inst-aid', required=True, help='Instance AID')
+
+esrv_flag_choices = [t.name for t in asn1.types['ServicesList'].type.root_members]
+parser_esrv = subparsers.add_parser('edit-mand-srv-list', help='Add/Remove service flag from/to mandatory services list')
+parser_esrv.add_argument('--output-file', required=True, help='Output file name')
+parser_esrv.add_argument('--add-flag', default=[], choices=esrv_flag_choices, action='append', help='Add flag to mandatory services list')
+parser_esrv.add_argument('--remove-flag', default=[], choices=esrv_flag_choices, action='append', help='Remove flag from mandatory services list')
+
+parser_tree = subparsers.add_parser('tree', help='Display the filesystem tree')
+
+def write_pes(pes: ProfileElementSequence, output_file:str):
+    """write the PE sequence to a file"""
+    print("Writing %u PEs to file '%s'..." % (len(pes.pe_list), output_file))
+    with open(output_file, 'wb') as f:
+        f.write(pes.to_der())
 
 def do_split(pes: ProfileElementSequence, opts):
     i = 0
@@ -120,6 +170,14 @@ def do_check(pes: ProfileElementSequence, opts):
     checker.check(pes)
     print("All good!")
 
+def do_extract_pe(pes: ProfileElementSequence, opts):
+    new_pe_list = []
+    for pe in pes.pe_list:
+        if pe.identification == opts.identification:
+            print("Extracting PE %s (id=%u) to file %s..." % (pe, pe.identification, opts.pe_file))
+            with open(opts.pe_file, 'wb') as f:
+                f.write(pe.to_der())
+
 def do_remove_pe(pes: ProfileElementSequence, opts):
     new_pe_list = []
     for pe in pes.pe_list:
@@ -128,13 +186,14 @@ def do_remove_pe(pes: ProfileElementSequence, opts):
             if identification in opts.identification:
                 print("Removing PE %s (id=%u) from Sequence..." % (pe, identification))
                 continue
+        if pe.type in opts.type:
+            print("Removing PE %s (type=%s) from Sequence..." % (pe, pe.type))
+            continue
         new_pe_list.append(pe)
 
     pes.pe_list = new_pe_list
     pes._process_pelist()
-    print("Writing %u PEs to file '%s'..." % (len(pes.pe_list), opts.output_file))
-    with open(opts.output_file, 'wb') as f:
-        f.write(pes.to_der())
+    write_pes(pes, opts.output_file)
 
 def do_remove_naa(pes: ProfileElementSequence, opts):
     if not opts.naa_type in NAAs:
@@ -142,9 +201,83 @@ def do_remove_naa(pes: ProfileElementSequence, opts):
     naa = NAAs[opts.naa_type]
     print("Removing NAAs of type '%s' from Sequence..." % opts.naa_type)
     pes.remove_naas_of_type(naa)
-    print("Writing %u PEs to file '%s'..." % (len(pes.pe_list), opts.output_file))
-    with open(opts.output_file, 'wb') as f:
-        f.write(pes.to_der())
+    write_pes(pes, opts.output_file)
+
+def info_apps(pes:ProfileElementSequence):
+    def show_member(dictionary:Optional[dict], member:str, indent:str="\t", mandatory:bool = False, limit:bool = False):
+        if dictionary is None:
+            return
+        value = dictionary.get(member, None)
+        if value is None and mandatory == True:
+            print("%s%s: (missing!)" % (indent, member))
+            return
+        elif value is None:
+            return
+
+        if limit and len(value) > 40:
+            print("%s%s: '%s...%s' (%u bytes)" % (indent, member, b2h(value[:20]), b2h(value[-20:]), len(value)))
+        else:
+            print("%s%s: '%s' (%u bytes)" % (indent, member, b2h(value), len(value)))
+
+    apps = pes.pe_by_type.get('application', [])
+    if len(apps) == 0:
+        print("No Application PE present!")
+        return;
+
+    for app_pe in enumerate(apps):
+        print("Application #%u:" % app_pe[0])
+        print("\tloadBlock:")
+        load_block = app_pe[1].decoded['loadBlock']
+        show_member(load_block, 'loadPackageAID', "\t\t", True)
+        show_member(load_block, 'securityDomainAID', "\t\t")
+        show_member(load_block, 'nonVolatileCodeLimitC6', "\t\t")
+        show_member(load_block, 'volatileDataLimitC7', "\t\t")
+        show_member(load_block, 'nonVolatileDataLimitC8', "\t\t")
+        show_member(load_block, 'hashValue', "\t\t")
+        show_member(load_block, 'loadBlockObject', "\t\t", True, True)
+        for inst in enumerate(app_pe[1].decoded.get('instanceList', [])):
+            print("\tinstanceList[%u]:" % inst[0])
+            show_member(inst[1], 'applicationLoadPackageAID', "\t\t", True)
+            if inst[1].get('applicationLoadPackageAID', None) != load_block.get('loadPackageAID', None):
+                print("\t\t(applicationLoadPackageAID should be the same as loadPackageAID!)")
+            show_member(inst[1], 'classAID', "\t\t", True)
+            show_member(inst[1], 'instanceAID', "\t\t", True)
+            show_member(inst[1], 'extraditeSecurityDomainAID', "\t\t")
+            show_member(inst[1], 'applicationPrivileges', "\t\t", True)
+            show_member(inst[1], 'lifeCycleState', "\t\t", True)
+            show_member(inst[1], 'applicationSpecificParametersC9', "\t\t", True)
+            sys_specific_pars = inst[1].get('systemSpecificParameters', None)
+            if sys_specific_pars:
+                print("\t\tsystemSpecificParameters:")
+                show_member(sys_specific_pars, 'volatileMemoryQuotaC7', "\t\t\t")
+                show_member(sys_specific_pars, 'nonVolatileMemoryQuotaC8', "\t\t\t")
+                show_member(sys_specific_pars, 'globalServiceParameters', "\t\t\t")
+                show_member(sys_specific_pars, 'implicitSelectionParameter', "\t\t\t")
+                show_member(sys_specific_pars, 'volatileReservedMemory', "\t\t\t")
+                show_member(sys_specific_pars, 'nonVolatileReservedMemory', "\t\t\t")
+                show_member(sys_specific_pars, 'ts102226SIMFileAccessToolkitParameter', "\t\t\t")
+                additional_cl_pars = inst.get('ts102226AdditionalContactlessParameters', None)
+                if additional_cl_pars:
+                    print("\t\t\tts102226AdditionalContactlessParameters:")
+                    show_member(additional_cl_pars, 'protocolParameterData', "\t\t\t\t")
+                show_member(sys_specific_pars, 'userInteractionContactlessParameters', "\t\t\t")
+                show_member(sys_specific_pars, 'cumulativeGrantedVolatileMemory', "\t\t\t")
+                show_member(sys_specific_pars, 'cumulativeGrantedNonVolatileMemory', "\t\t\t")
+            app_pars = inst[1].get('applicationParameters', None)
+            if app_pars:
+                print("\t\tapplicationParameters:")
+                show_member(app_pars, 'uiccToolkitApplicationSpecificParametersField', "\t\t\t")
+                show_member(app_pars, 'uiccAccessApplicationSpecificParametersField', "\t\t\t")
+                show_member(app_pars, 'uiccAdministrativeAccessApplicationSpecificParametersField', "\t\t\t")
+            ctrl_ref_tp = inst[1].get('controlReferenceTemplate', None)
+            if ctrl_ref_tp:
+                print("\t\tcontrolReferenceTemplate:")
+                show_member(ctrl_ref_tp, 'applicationProviderIdentifier', "\t\t\t", True)
+            process_data = inst[1].get('processData', None)
+            if process_data:
+                print("\t\tprocessData:")
+                for proc in process_data:
+                    print("\t\t\t" + b2h(proc))
 
 def do_info(pes: ProfileElementSequence, opts):
     def get_naa_count(pes: ProfileElementSequence) -> dict:
@@ -154,6 +287,10 @@ def do_info(pes: ProfileElementSequence, opts):
             ret[naa_type] = len(pes.pes_by_naa[naa_type])
         return ret
 
+    if opts.apps:
+        info_apps(pes)
+        return;
+
     pe_hdr_dec = pes.pe_by_type['header'][0].decoded
     print()
     print("SAIP Profile Version: %u.%u" % (pe_hdr_dec['major-version'], pe_hdr_dec['minor-version']))
@@ -192,7 +329,7 @@ def do_info(pes: ProfileElementSequence, opts):
         print("Security domain Instance AID: %s" % b2h(sd.decoded['instance']['instanceAID']))
         # FIXME: 'applicationSpecificParametersC9' parsing to figure out enabled SCP
         for key in sd.keys:
-            print("\tKVN=0x%02x, KID=0x%02x, %s" % (key.key_version_number, key.key_identifier, key.key_components))
+            print("\t%s" % repr(key))
 
     # RFM
     print()
@@ -213,16 +350,98 @@ def do_extract_apps(pes:ProfileElementSequence, opts):
     apps = pes.pe_by_type.get('application', [])
     for app_pe in apps:
         package_aid = b2h(app_pe.decoded['loadBlock']['loadPackageAID'])
-
         fname = os.path.join(opts.output_dir, '%s-%s.%s' % (pes.iccid, package_aid, opts.format))
-        load_block_obj = app_pe.decoded['loadBlock']['loadBlockObject']
         print("Writing Load Package AID: %s to file %s" % (package_aid, fname))
-        if opts.format == 'ijc':
-            with open(fname, 'wb') as f:
-                f.write(load_block_obj)
+        app_pe.to_file(fname)
+
+def do_add_app(pes:ProfileElementSequence, opts):
+    print("Applying applet file: '%s'..." % opts.applet_file)
+    app_pe = ProfileElementApplication.from_file(opts.applet_file,
+                                                 opts.aid,
+                                                 opts.sd_aid,
+                                                 opts.non_volatile_code_limit,
+                                                 opts.volatile_data_limit,
+                                                 opts.non_volatile_data_limit,
+                                                 opts.hash_value)
+
+    security_domain = pes.pe_by_type.get('securityDomain', [])
+    if len(security_domain) == 0:
+        print("profile package does not contain a securityDomain, please add a securityDomain PE first!")
+    elif len(security_domain) > 1:
+        print("adding an application PE to profiles with multiple securityDomain is not supported yet!")
+    else:
+        pes.insert_after_pe(security_domain[0], app_pe)
+        print("application PE inserted into PE Sequence after securityDomain PE AID: %s" %
+              b2h(security_domain[0].decoded['instance']['instanceAID']))
+        write_pes(pes, opts.output_file)
+
+def do_remove_app(pes:ProfileElementSequence, opts):
+    apps = pes.pe_by_type.get('application', [])
+    for app_pe in apps:
+        package_aid = b2h(app_pe.decoded['loadBlock']['loadPackageAID'])
+        if opts.aid == package_aid:
+            identification = app_pe.identification
+            opts_remove_pe = argparse.Namespace()
+            opts_remove_pe.identification = [app_pe.identification]
+            opts_remove_pe.type = []
+            opts_remove_pe.output_file = opts.output_file
+            print("Found Load Package AID: %s, removing related PE (id=%u) from Sequence..." %
+                  (package_aid, identification))
+            do_remove_pe(pes, opts_remove_pe)
+            return
+    print("Load Package AID: %s not found in PE Sequence" % opts.aid)
+
+def do_add_app_inst(pes:ProfileElementSequence, opts):
+    apps = pes.pe_by_type.get('application', [])
+    for app_pe in apps:
+        package_aid = b2h(app_pe.decoded['loadBlock']['loadPackageAID'])
+        if opts.aid == package_aid:
+            print("Found Load Package AID: %s, adding new instance AID: %s to Application PE..." %
+                  (opts.aid, opts.inst_aid))
+            app_pe.add_instance(opts.aid,
+                                opts.class_aid,
+                                opts.inst_aid,
+                                opts.app_privileges,
+                                opts.app_spec_pars,
+                                opts.uicc_toolkit_app_spec_pars,
+                                opts.uicc_access_app_spec_pars,
+                                opts.uicc_adm_access_app_spec_pars,
+                                opts.volatile_memory_quota,
+                                opts.non_volatile_memory_quota,
+                                opts.process_data)
+            write_pes(pes, opts.output_file)
+            return
+    print("Load Package AID: %s not found in PE Sequence" % opts.aid)
+
+def do_remove_app_inst(pes:ProfileElementSequence, opts):
+    apps = pes.pe_by_type.get('application', [])
+    for app_pe in apps:
+        if opts.aid == b2h(app_pe.decoded['loadBlock']['loadPackageAID']):
+            print("Found Load Package AID: %s, removing instance AID: %s from Application PE..." %
+                  (opts.aid, opts.inst_aid))
+            app_pe.remove_instance(opts.inst_aid)
+            write_pes(pes, opts.output_file)
+            return
+    print("Load Package AID: %s not found in PE Sequence" % opts.aid)
+
+def do_edit_mand_srv_list(pes: ProfileElementSequence, opts):
+    header = pes.pe_by_type.get('header', [])[0]
+
+    for s in opts.add_flag:
+        print("Adding service '%s' to mandatory services list..." % s)
+        header.mandatory_service_add(s)
+    for s in opts.remove_flag:
+        if s in header.decoded['eUICC-Mandatory-services'].keys():
+            print("Removing service '%s' from mandatory services list..." % s)
+            header.mandatory_service_remove(s)
         else:
-            with io.BytesIO(load_block_obj) as f, zipfile.ZipFile(fname, 'w') as z:
-                javacard.ijc_to_cap(f, z, package_aid)
+            print("Service '%s' not present in mandatory services list, cannot remove!" % s)
+
+    print("The following services are now set mandatory:")
+    for s in header.decoded['eUICC-Mandatory-services'].keys():
+        print("\t%s" % s)
+
+    write_pes(pes, opts.output_file)
 
 def do_tree(pes:ProfileElementSequence, opts):
     pes.mf.print_tree()
@@ -246,6 +465,8 @@ if __name__ == '__main__':
         do_dump(pes, opts)
     elif opts.command == 'check':
         do_check(pes, opts)
+    elif opts.command == 'extract-pe':
+        do_extract_pe(pes, opts)
     elif opts.command == 'remove-pe':
         do_remove_pe(pes, opts)
     elif opts.command == 'remove-naa':
@@ -254,5 +475,15 @@ if __name__ == '__main__':
         do_info(pes, opts)
     elif opts.command == 'extract-apps':
         do_extract_apps(pes, opts)
+    elif opts.command == 'add-app':
+        do_add_app(pes, opts)
+    elif opts.command == 'remove-app':
+        do_remove_app(pes, opts)
+    elif opts.command == 'add-app-inst':
+        do_add_app_inst(pes, opts)
+    elif opts.command == 'remove-app-inst':
+        do_remove_app_inst(pes, opts)
+    elif opts.command == 'edit-mand-srv-list':
+        do_edit_mand_srv_list(pes, opts)
     elif opts.command == 'tree':
         do_tree(pes, opts)

contrib/saip-tool_example_add-app.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# This is an example script to illustrate how to add JAVA card applets to an existing eUICC profile package.
+
+PYSIMPATH=../
+INPATH=../smdpp-data/upp/TS48V1-A-UNIQUE.der
+OUTPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello.der
+APPPATH=./HelloSTK_09122024.cap
+
+# Download example applet (see also https://gitea.osmocom.org/sim-card/hello-stk):
+if ! [ -f $APPPATH ]; then
+    wget https://osmocom.org/attachments/download/8931/HelloSTK_09122024.cap
+fi
+
+# Step #1: Create the application PE and load the ijc contents from the .cap file:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $INPATH add-app \
+		      --output-file $OUTPATH --applet-file $APPPATH --aid 'D07002CA44'
+
+# Step #2: Create the application instance inside the application PE created in step #1:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $OUTPATH add-app-inst --output-file $OUTPATH \
+		      --aid 'D07002CA44' \
+		      --class-aid 'D07002CA44900101' \
+		      --inst-aid 'D07002CA44900101' \
+		      --app-privileges '00' \
+		      --app-spec-pars '00' \
+		      --uicc-toolkit-app-spec-pars '01001505000000000000000000000000'
+
+# Display the contents of the resulting application PE:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $OUTPATH info --apps
+
+# For an explanation of --uicc-toolkit-app-spec-pars, see:
+# ETSI TS 102 226, section 8.2.1.3.2.2.1

contrib/saip-tool_example_extract-apps.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# This is an example script to illustrate how to extract JAVA card applets from an existing eUICC profile package.
+
+PYSIMPATH=../
+INPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello.der
+OUTPATH=./
+
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $INPATH extract-apps --output-dir ./ --format ijc

contrib/saip-tool_example_remove-app-inst.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# This is an example script to illustrate how to remove a JAVA card applet instance from an application PE inside an
+# existing eUICC profile package.
+
+PYSIMPATH=../
+INPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello.der
+OUTPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello-no-inst.der
+
+# Remove application PE entirely
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $INPATH remove-app-inst \
+		      --output-file $OUTPATH --aid 'd07002ca44' --inst-aid 'd07002ca44900101'
+
+# Display the contents of the resulting application PE:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $OUTPATH info --apps

contrib/saip-tool_example_remove-app.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# This is an example script to illustrate how to remove a JAVA card applet from an existing eUICC profile package.
+
+PYSIMPATH=../
+INPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello.der
+OUTPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-no-hello.der
+
+# Remove application PE entirely
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $INPATH remove-app \
+		      --output-file $OUTPATH --aid 'D07002CA44'
+
+# Display the contents of the resulting application PE:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $OUTPATH info --apps

contrib/smpp-ota-tool.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+
+# (C) 2026 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Harald Welte, Philipp Maier
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import logging
+import smpplib.gsm
+import smpplib.client
+import smpplib.consts
+import time
+from pySim.ota import OtaKeyset, OtaDialectSms, OtaAlgoCrypt, OtaAlgoAuth, CNTR_REQ, RC_CC_DS, POR_REQ
+from pySim.utils import b2h, h2b, is_hexstr
+from pathlib import Path
+
+logger = logging.getLogger(Path(__file__).stem)
+
+option_parser = argparse.ArgumentParser(description='Tool to send OTA SMS RFM/RAM messages via SMPP',
+                                        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+option_parser.add_argument("--host", help="Host/IP of the SMPP server", default="localhost")
+option_parser.add_argument("--port", help="TCP port of the SMPP server", default=2775, type=int)
+option_parser.add_argument("--system-id", help="System ID to use to bind to the SMPP server", default="test")
+option_parser.add_argument("--password", help="Password to use to bind to the SMPP server", default="test")
+option_parser.add_argument("--verbose", help="Enable verbose logging", action='store_true', default=False)
+algo_crypt_choices = []
+algo_crypt_classes = OtaAlgoCrypt.__subclasses__()
+for cls in algo_crypt_classes:
+    algo_crypt_choices.append(cls.enum_name)
+option_parser.add_argument("--algo-crypt", choices=algo_crypt_choices, default='triple_des_cbc2',
+                           help="OTA crypt algorithm")
+algo_auth_choices = []
+algo_auth_classes = OtaAlgoAuth.__subclasses__()
+for cls in algo_auth_classes:
+    algo_auth_choices.append(cls.enum_name)
+option_parser.add_argument("--algo-auth", choices=algo_auth_choices, default='triple_des_cbc2',
+                           help="OTA auth algorithm")
+option_parser.add_argument('--kic', required=True, type=is_hexstr, help='OTA key (KIC)')
+option_parser.add_argument('--kic-idx', default=1, type=int, help='OTA key index (KIC)')
+option_parser.add_argument('--kid', required=True, type=is_hexstr, help='OTA key (KID)')
+option_parser.add_argument('--kid-idx', default=1, type=int, help='OTA key index (KID)')
+option_parser.add_argument('--cntr', default=0, type=int, help='replay protection counter')
+option_parser.add_argument('--tar', required=True, type=is_hexstr, help='Toolkit Application Reference')
+option_parser.add_argument("--cntr-req", choices=CNTR_REQ.decmapping.values(), default='no_counter',
+                           help="Counter requirement")
+option_parser.add_argument('--no-ciphering', action='store_true', default=False, help='Disable ciphering')
+option_parser.add_argument("--rc-cc-ds", choices=RC_CC_DS.decmapping.values(), default='cc',
+                           help="message check (rc=redundency check, cc=crypt. checksum, ds=digital signature)")
+option_parser.add_argument('--por-in-submit', action='store_true', default=False,
+                           help='require PoR to be sent via SMS-SUBMIT')
+option_parser.add_argument('--por-no-ciphering', action='store_true', default=False, help='Disable ciphering (PoR)')
+option_parser.add_argument("--por-rc-cc-ds", choices=RC_CC_DS.decmapping.values(), default='cc',
+                           help="PoR check (rc=redundency check, cc=crypt. checksum, ds=digital signature)")
+option_parser.add_argument("--por-req", choices=POR_REQ.decmapping.values(), default='por_required',
+                           help="Proof of Receipt requirements")
+option_parser.add_argument('--src-addr', default='12', type=str, help='SMS source address (MSISDN)')
+option_parser.add_argument('--dest-addr', default='23', type=str, help='SMS destination address (MSISDN)')
+option_parser.add_argument('--timeout', default=10, type=int, help='Maximum response waiting time')
+option_parser.add_argument('-a', '--apdu', action='append', required=True, type=is_hexstr, help='C-APDU to send')
+
+class SmppHandler:
+    client = None
+
+    def __init__(self, host: str, port: int,
+                 system_id: str, password: str,
+                 ota_keyset: OtaKeyset, spi: dict, tar: bytes):
+        """
+        Initialize connection to SMPP server and set static OTA SMS-TPDU ciphering parameters
+        Args:
+                host : Hostname or IPv4/IPv6 address of the SMPP server
+                port : TCP Port of the SMPP server
+                system_id: SMPP System-ID used by ESME (client) to bind
+                password: SMPP Password used by ESME (client) to bind
+                ota_keyset: OTA keyset to be used for SMS-TPDU ciphering
+                spi: Security Parameter Indicator (SPI) to be used for SMS-TPDU ciphering
+                tar: Toolkit Application Reference (TAR) of the targeted card application
+        """
+
+        # Create and connect SMPP client
+        client = smpplib.client.Client(host, port, allow_unknown_opt_params=True)
+        client.set_message_sent_handler(self.message_sent_handler)
+        client.set_message_received_handler(self.message_received_handler)
+        client.connect()
+        client.bind_transceiver(system_id=system_id, password=password)
+        self.client = client
+
+        # Setup static OTA parameters
+        self.ota_dialect = OtaDialectSms()
+        self.ota_keyset = ota_keyset
+        self.tar = tar
+        self.spi = spi
+
+    def __del__(self):
+        if self.client:
+            self.client.unbind()
+            self.client.disconnect()
+
+    def message_received_handler(self, pdu):
+        if pdu.short_message:
+            logger.info("SMS-TPDU received: %s", b2h(pdu.short_message))
+            try:
+                dec = self.ota_dialect.decode_resp(self.ota_keyset, self.spi, pdu.short_message)
+            except ValueError:
+                # Retry to decoding with ciphering disabled (in case the card has problems to decode the SMS-TDPU
+                # we have sent, the response will contain an unencrypted error message)
+                spi = self.spi.copy()
+                spi['por_shall_be_ciphered'] = False
+                spi['por_rc_cc_ds'] = 'no_rc_cc_ds'
+                dec = self.ota_dialect.decode_resp(self.ota_keyset, spi, pdu.short_message)
+            logger.info("SMS-TPDU decoded: %s", dec)
+            self.response = dec
+        return None
+
+    def message_sent_handler(self, pdu):
+            logger.debug("SMS-TPDU sent: pdu_sequence=%s pdu_message_id=%s", pdu.sequence, pdu.message_id)
+
+    def transceive_sms_tpdu(self, tpdu: bytes, src_addr: str, dest_addr: str, timeout: int) -> tuple:
+        """
+        Transceive SMS-TPDU. This method sends the SMS-TPDU to the SMPP server, and waits for a response. The method
+        returns when the response is received.
+
+        Args:
+                tpdu : short message content (plaintext)
+                src_addr : short message source address
+                dest_addr : short message destination address
+                timeout : timeout after which this method should give up waiting for a response
+        Returns:
+                tuple containing the response (plaintext)
+        """
+
+        logger.info("SMS-TPDU sending: %s...", b2h(tpdu))
+
+        self.client.send_message(
+            # TODO: add parameters to switch source_addr_ton and dest_addr_ton between SMPP_TON_INTL and SMPP_NPI_ISDN
+            source_addr_ton=smpplib.consts.SMPP_TON_INTL,
+            source_addr=src_addr,
+            dest_addr_ton=smpplib.consts.SMPP_TON_INTL,
+            destination_addr=dest_addr,
+            short_message=tpdu,
+            # TODO: add parameters to set data_coding and esm_class
+            data_coding=smpplib.consts.SMPP_ENCODING_BINARY,
+            esm_class=smpplib.consts.SMPP_GSMFEAT_UDHI,
+            protocol_id=0x7f,
+            # TODO: add parameter to use registered delivery
+            # registered_delivery=True,
+        )
+
+        logger.info("SMS-TPDU sent, waiting for response...")
+        timestamp_sent=int(time.time())
+        self.response = None
+        while self.response is None:
+            self.client.poll()
+            if int(time.time()) - timestamp_sent > timeout:
+                raise ValueError("Timeout reached, no response SMS-TPDU received!")
+        return self.response
+
+    def transceive_apdu(self, apdu: bytes, src_addr: str, dest_addr: str, timeout: int) -> tuple[bytes, bytes]:
+        """
+        Transceive APDU. This method wraps the given APDU into an SMS-TPDU, sends it to the SMPP server and waits for
+        the response. When the response is received, the last response data and the last status word is extracted from
+        the response and returned to the caller.
+
+        Args:
+                apdu : one or more concatenated APDUs
+                src_addr : short message source address
+                dest_addr : short message destination address
+                timeout : timeout after which this method should give up waiting for a response
+        Returns:
+                tuple containing the last response data and the last status word as byte strings
+        """
+
+        logger.info("C-APDU sending: %s...", b2h(apdu))
+
+        # translate to Secured OTA RFM
+        secured = self.ota_dialect.encode_cmd(self.ota_keyset, self.tar, self.spi, apdu=apdu)
+        # add user data header
+        tpdu = b'\x02\x70\x00' + secured
+        # send via SMPP
+        response =  self.transceive_sms_tpdu(tpdu, src_addr, dest_addr, timeout)
+
+        # Extract last_response_data and last_status_word from the response
+        sw = None
+        resp = None
+        for container in response:
+            if container:
+                container_dict = dict(container)
+                resp = container_dict.get('last_response_data')
+                sw = container_dict.get('last_status_word')
+        if resp is None:
+            raise ValueError("Response does not contain any last_response_data, no R-APDU received!")
+        if sw is None:
+            raise ValueError("Response does not contain any last_status_word, no R-APDU received!")
+
+        logger.info("R-APDU received: %s %s", resp, sw)
+        return h2b(resp), h2b(sw)
+
+if __name__ == '__main__':
+    opts = option_parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if opts.verbose else logging.INFO,
+                        format='%(asctime)s %(levelname)s %(message)s',
+                        datefmt='%Y-%m-%d %H:%M:%S')
+
+    if opts.kic_idx != opts.kid_idx:
+        logger.warning("KIC index (%s) and KID index (%s) are different (security violation, card should reject message)",
+                       opts.kic_idx, opts.kid_idx)
+
+    ota_keyset = OtaKeyset(algo_crypt=opts.algo_crypt,
+                           kic_idx=opts.kic_idx,
+                           kic=h2b(opts.kic),
+                           algo_auth=opts.algo_auth,
+                           kid_idx=opts.kid_idx,
+                           kid=h2b(opts.kid),
+                           cntr=opts.cntr)
+    spi = {'counter' : opts.cntr_req,
+           'ciphering' : not opts.no_ciphering,
+           'rc_cc_ds': opts.rc_cc_ds,
+           'por_in_submit': opts.por_in_submit,
+           'por_shall_be_ciphered': not opts.por_no_ciphering,
+           'por_rc_cc_ds': opts.por_rc_cc_ds,
+           'por': opts.por_req}
+    apdu = h2b("".join(opts.apdu))
+
+    smpp_handler = SmppHandler(opts.host, opts.port, opts.system_id, opts.password, ota_keyset, spi, h2b(opts.tar))
+    resp, sw = smpp_handler.transceive_apdu(apdu, opts.src_addr, opts.dest_addr, opts.timeout)
+    print("%s %s" % (b2h(resp), b2h(sw)))

contrib/suci-keytool.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+# small utility program to deal with 5G SUCI key material, at least for the ECIES Protection Scheme
+# Profile A (curve25519) and B (secp256r1)
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+# SPDX-License-Identifier: GPL-2.0+
+
+import argparse
+
+from osmocom.utils import b2h
+from Cryptodome.PublicKey import ECC
+# if used with pycryptodome < v3.21.0 you will get the following error when using curve25519:
+# "Cryptodome.PublicKey.ECC.UnsupportedEccFeature: Unsupported ECC purpose (OID: 1.3.101.110)"
+
+def gen_key(opts):
+    # FIXME: avoid overwriting key files
+    mykey = ECC.generate(curve=opts.curve)
+    data = mykey.export_key(format='PEM')
+    with open(opts.key_file, "wt") as f:
+        f.write(data)
+
+def dump_pkey(opts):
+
+    #with open("curve25519-1.key", "r") as f:
+
+    with open(opts.key_file, "r") as f:
+        data = f.read()
+        mykey = ECC.import_key(data)
+
+        der = mykey.public_key().export_key(format='raw', compress=opts.compressed)
+        print(b2h(der))
+
+arg_parser = argparse.ArgumentParser(description="""Generate or export SUCI keys for 5G SA networks""")
+arg_parser.add_argument('--key-file', help='The key file to use', required=True)
+
+subparsers = arg_parser.add_subparsers(dest='command', help="The command to perform", required=True)
+
+parser_genkey = subparsers.add_parser('generate-key', help='Generate a new key pair')
+parser_genkey.add_argument('--curve', help='The ECC curve to use', choices=['secp256r1','curve25519'], required=True)
+
+parser_dump_pkey = subparsers.add_parser('dump-pub-key', help='Dump the public key')
+parser_dump_pkey.add_argument('--compressed', help='Use point compression', action='store_true')
+
+if __name__ == '__main__':
+
+    opts = arg_parser.parse_args()
+
+    if opts.command == 'generate-key':
+        gen_key(opts)
+    elif opts.command == 'dump-pub-key':
+        dump_pkey(opts)

contrib/unber.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# A more useful verion of the 'unber' tool provided with asn1c:
+# A more useful version of the 'unber' tool provided with asn1c:
 # Give a hierarchical decode of BER/DER-encoded ASN.1 TLVs
 
 import sys

contrib/wsrc_card_client.py

@@ -1,112 +0,0 @@
-#!/usr/bin/env python3
-
-"""Connect smartcard to a remote server, so the remote server can take control and
-perform commands on it."""
-
-import sys
-import json
-import logging
-import argparse
-from osmocom.utils import b2h
-import websockets
-
-from pySim.transport import init_reader, argparse_add_reader_args, LinkBase
-from pySim.commands import SimCardCommands
-from pySim.wsrc.client_blocking import WsClientBlocking
-from pySim.exceptions import NoCardError
-from pySim.wsrc import WSRC_DEFAULT_PORT_CARD
-
-logging.basicConfig(format="[%(levelname)s] %(asctime)s %(message)s", level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-class CardWsClientBlocking(WsClientBlocking):
-    """Implementation of the card (reader) client of the WSRC (WebSocket Remote Card) protocol"""
-
-    def __init__(self, ws_uri, tp: LinkBase):
-        super().__init__('card', ws_uri)
-        self.tp = tp
-
-    def perform_outbound_hello(self):
-        hello_data = {
-            'atr': b2h(self.tp.get_atr()),
-            # TODO: include various card information in the HELLO message
-        }
-        super().perform_outbound_hello(hello_data)
-
-    def handle_rx_c_apdu(self, rx: dict):
-        """handle an inbound APDU transceive command"""
-        data, sw = self.tp.send_apdu(rx['command'])
-        tx = {
-            'response': data,
-            'sw': sw,
-        }
-        self.tx_json('r_apdu', tx)
-
-    def handle_rx_disconnect(self, rx: dict):
-        """server tells us to disconnect"""
-        self.tx_json('disconnect_ack')
-        # FIXME: tear down connection and/or terminate entire program
-
-    def handle_rx_state_notification(self, rx: dict):
-        logger.info("State Notification: %s" % rx['new_state'])
-
-    def handle_rx_print(self, rx: dict):
-        """print a message (text) given by server to the local console/log"""
-        logger.info("SERVER MSG: %s" % rx['message'])
-        # no response
-
-    def handle_rx_reset_req(self, rx: dict):
-        """server tells us to reset the card"""
-        self.tp.reset_card()
-        self.tx_json('reset_resp', {'atr': b2h(self.tp.get_atr())})
-
-
-parser = argparse.ArgumentParser()
-argparse_add_reader_args(parser)
-parser.add_argument("--uri", default="ws://localhost:%u/" % (WSRC_DEFAULT_PORT_CARD),
-                    help="URI of the sever to which to connect")
-
-
-if __name__ == '__main__':
-    opts = parser.parse_args()
-
-    # open the card reader / slot
-    logger.info("Initializing Card Reader...")
-    try:
-        tp = init_reader(opts)
-    except Exception as e:
-        logger.fatal("Error opening reader: %s" % e)
-        sys.exit(1)
-
-    logger.info("Connecting to Card...")
-    try:
-        tp.connect()
-    except NoCardError as e:
-        logger.fatal("Error opening card! Is a card inserted in the reader?")
-        sys.exit(1)
-
-    scc = SimCardCommands(transport=tp)
-    logger.info("Detected Card with ATR: %s" % b2h(tp.get_atr()))
-
-    # TODO: gather various information about the card; print it
-
-    # create + connect the client to the server
-    cl = CardWsClientBlocking(opts.uri, tp)
-    logger.info("Connecting to remote server...")
-    try:
-        cl.connect()
-        logger.info("Successfully connected to Server")
-    except ConnectionRefusedError as e:
-        logger.fatal(e)
-        sys.exit(1)
-
-    try:
-        while True:
-            # endless loop: wait for inbound command from server + execute it
-            cl.rx_and_execute_cmd()
-    except websockets.exceptions.ConnectionClosedOK as e:
-        print(e)
-        sys.exit(1)
-    except KeyboardInterrupt as e:
-        print(e.__class__.__name__)
-        sys.exit(2)

contrib/wsrc_server.py

@@ -1,354 +0,0 @@
-#!/usr/bin/env python
-
-import json
-import asyncio
-import logging
-import argparse
-from typing import Optional, Tuple
-from websockets.asyncio.server import serve
-from websockets.exceptions import ConnectionClosedError
-from osmocom.utils import Hexstr, swap_nibbles
-
-from pySim.utils import SwMatchstr, ResTuple, sw_match, dec_iccid
-from pySim.exceptions import SwMatchError
-from pySim.wsrc import WSRC_DEFAULT_PORT_USER, WSRC_DEFAULT_PORT_CARD
-
-logging.basicConfig(format="[%(levelname)s] %(asctime)s %(message)s", level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-card_clients = set()
-user_clients = set()
-
-class WsClientLogAdapter(logging.LoggerAdapter):
-    """LoggerAdapter adding context (for now: remote IP/Port of client) to log"""
-    def process(self, msg, kwargs):
-        return '%s([%s]:%u) %s' % (self.extra['type'], self.extra['remote_addr'][0],
-                                   self.extra['remote_addr'][1], msg), kwargs
-
-class WsClient:
-    def __init__(self, websocket, hello: dict):
-        self.websocket = websocket
-        self.hello = hello
-        self.identity = {}
-        self.logger = WsClientLogAdapter(logger, {'type': self.__class__.__name__,
-                                                  'remote_addr': websocket.remote_address})
-
-    def __str__(self):
-        return '%s([%s]:%u)' % (self.__class__.__name__, self.websocket.remote_address[0],
-                                self.websocket.remote_address[1])
-
-    async def rx_json(self):
-        rx = await self.websocket.recv()
-        rx_js = json.loads(rx)
-        self.logger.debug("Rx: %s", rx_js)
-        assert 'msg_type' in rx
-        return rx_js
-
-    async def tx_json(self, msg_type:str, d: dict = {}):
-        """Transmit a json-serializable dict to the peer"""
-        d['msg_type'] = msg_type
-        d_js = json.dumps(d)
-        self.logger.debug("Tx: %s", d_js)
-        await self.websocket.send(d_js)
-
-    async def tx_hello_ack(self):
-        await self.tx_json('hello_ack')
-
-    async def xceive_json(self, msg_type:str, d:dict = {}, exp_msg_type:Optional[str] = None) -> dict:
-        await self.tx_json(msg_type, d)
-        rx = await self.rx_json()
-        if exp_msg_type:
-            assert rx['msg_type'] == exp_msg_type
-        return rx;
-
-    async def tx_error(self, message: str):
-        """Transmit an error message to the peer"""
-        event = {
-            "message": message,
-        }
-        self.logger.error("Transmitting error message: '%s'" % message)
-        await self.tx_json('error', event)
-
-#    async def ws_hdlr(self):
-#        """kind of a 'main' function for the websocket client: wait for incoming message,
-#        and handle it."""
-#        try:
-#            async for message in self.websocket:
-#                method = getattr(self, 'handle_rx_%s' % message['msg_type'], None)
-#                if not method:
-#                    await self.tx_error("Unknonw msg_type: %s" % message['msg_type'])
-#                else:
-#                    method(message)
-#        except ConnectionClosedError:
-#            # we handle this in the outer loop
-#            pass
-
-class CardClient(WsClient):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.identity['ATR'] = self.hello['atr']
-        self.state = 'init'
-
-    def __str__(self):
-        eid = self.identity.get('EID', None)
-        if eid:
-            return '%s(EID=%s)' % (self.__class__.__name__, eid)
-        iccid = self.identity.get('ICCID', None)
-        if iccid:
-            return '%s(ICCID=%s)' % (self.__class__.__name__, iccid)
-        return super().__str__()
-
-    def listing_entry(self) -> dict:
-        return {'remote_addr': self.websocket.remote_address,
-                'identities': self.identity,
-                'state': self.state}
-
-    """A websocket client that represents a reader/card. This is what we use to talk to a card"""
-    async def xceive_apdu_raw(self, cmd: Hexstr) -> ResTuple:
-        """transceive a single APDU with the card"""
-        message = await self.xceive_json('c_apdu', {'command': cmd}, 'r_apdu')
-        return message['response'], message['sw']
-
-    async def xceive_apdu(self, pdu: Hexstr) -> ResTuple:
-        """transceive an APDU with the card, handling T=0 GET_RESPONSE cases"""
-        prev_pdu = pdu
-        data, sw = await self.xceive_apdu_raw(pdu)
-
-        if sw is not None:
-            while (sw[0:2] in ['9f', '61', '62', '63']):
-                # SW1=9F: 3GPP TS 51.011 9.4.1, Responses to commands which are correctly executed
-                # SW1=61: ISO/IEC 7816-4, Table 5 — General meaning of the interindustry values of SW1-SW2
-                # SW1=62: ETSI TS 102 221 7.3.1.1.4 Clause 4b): 62xx, 63xx, 9xxx != 9000
-                pdu_gr = pdu[0:2] + 'c00000' + sw[2:4]
-                prev_pdu = pdu_gr
-                d, sw = await self.xceive_apdu_raw(pdu_gr)
-                data += d
-            if sw[0:2] == '6c':
-                # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
-                pdu_gr = prev_pdu[0:8] + sw[2:4]
-                data, sw = await self.xceive_apdu_raw(pdu_gr)
-
-        return data, sw
-
-    async def xceive_apdu_checksw(self, pdu: Hexstr, sw: SwMatchstr = "9000") -> ResTuple:
-        """like xceive_apdu, but checking the status word matches the expected pattern"""
-        rv = await self.xceive_apdu(pdu)
-        last_sw = rv[1]
-
-        if not sw_match(rv[1], sw):
-            raise SwMatchError(rv[1], sw.lower())
-        return rv
-
-    async def card_reset(self):
-        """reset the card"""
-        rx = await self.xceive_json('reset_req', exp_msg_type='reset_resp')
-        self.identity['ATR'] = rx['atr']
-
-    async def notify_state(self):
-        """notify the card client of a state [change]"""
-        await self.tx_json('state_notification', {'new_state': self.state})
-
-    async def get_iccid(self):
-        """high-level method to obtain the ICCID of the card"""
-        await self.xceive_apdu_checksw('00a40000023f00') # SELECT MF
-        await self.xceive_apdu_checksw('00a40000022fe2') # SELECT EF.ICCID
-        res, sw = await self.xceive_apdu_checksw('00b0000000') # READ BINARY
-        return dec_iccid(res)
-
-    async def get_eid_sgp22(self):
-        """high-level method to obtain the EID of a SGP.22 consumer eUICC"""
-        await self.xceive_apdu_checksw('00a4040410a0000005591010ffffffff8900000100')
-        res, sw = await self.xceive_apdu_checksw('80e2910006bf3e035c015a')
-        return res[-32:]
-
-    async def set_state(self, new_state:str):
-        self.logger.info("Card now in '%s' state" % new_state)
-        self.state = new_state
-        await self.notify_state()
-
-    async def identify(self):
-        # identify the card by asking for its EID and/or ICCID
-        try:
-            eid = await self.get_eid_sgp22()
-            self.logger.debug("EID: %s", eid)
-            self.identity['EID'] = eid
-        except SwMatchError:
-            pass
-        try:
-            iccid = await self.get_iccid()
-            self.logger.debug("ICCID: %s", iccid)
-            self.identity['ICCID'] = iccid
-        except SwMatchError:
-            pass
-        await self.set_state('ready')
-
-    async def associate_user(self, user):
-        assert self.state == 'ready'
-        self.user = user
-        await self.set_state('associated')
-
-    async def disassociate_user(self):
-        assert self.user
-        assert self.state == 'associated'
-        await self.set_state('ready')
-
-    @staticmethod
-    def find_client_for_id(id_type: str, id_str: str) -> Optional['CardClient']:
-        for c in card_clients:
-            if c.state != 'ready':
-                continue
-            c_id = c.identity.get(id_type.upper(), None)
-            if c_id and c_id.lower() == id_str.lower():
-                return c
-        return None
-
-class UserClient(WsClient):
-    """A websocket client representing a user application like pySim-shell."""
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.state = 'init'
-        self.card = None
-
-    async def associate_card(self, card: CardClient):
-        assert self.state == 'init'
-        self.card = card
-        self.state = 'associated'
-
-    async def disassociate_card(self):
-        assert self.state == 'associated'
-        self.card = None
-        self.state = 'init'
-
-    async def state_init(self):
-        """Wait for incoming 'select_card' and process it."""
-        while True:
-            rx = await self.rx_json()
-            if rx['msg_type'] == 'select_card':
-                # look-up if the card can be found
-                card = CardClient.find_client_for_id(rx['id_type'], rx['id_str'])
-                if not card:
-                    await self.tx_error('No CardClient found for %s == %s' % (rx['id_type'], rx['id_str']))
-                    continue
-                # transition to next statee
-                await card.associate_user(self)
-                await self.associate_card(card)
-                await self.tx_json('select_card_ack', {'identities': card.identity})
-                break
-            elif rx['msg_type'] == 'list_cards':
-                res = [x.listing_entry() for x in card_clients]
-                await self.tx_json('list_cards_ack', {'cards': res})
-            else:
-                self.tx_error('Unknown message type %s' % rx['msg_type'])
-
-    async def state_selected(self):
-        while True:
-            rx = await self.rx_json()
-            if rx['msg_type'] == 'c_apdu':
-                rsp, sw = await self.card.xceive_apdu_raw(rx['command'])
-                await self.tx_json('r_apdu', {'response': rsp, 'sw': sw})
-            elif rx['msg_type'] == 'reset_req':
-                await self.card.card_reset()
-                await self.tx_json('reset_resp')
-            else:
-                self.logger.warning("Unknown/unsupported command '%s' received" % rx['msg_type'])
-
-
-
-async def card_conn_hdlr(websocket):
-    """Handler for incoming connection to 'card' port."""
-    # receive first message, which should be a 'hello'
-    rx_raw = await websocket.recv()
-    rx = json.loads(rx_raw)
-    assert rx['msg_type'] == 'hello'
-    client_type = rx['client_type']
-
-    if client_type != 'card':
-        logger.error("Rejecting client (unknown type %s) connection", client_type)
-        raise ValueError("client_type '%s' != expected 'card'" % client_type)
-
-    card = CardClient(websocket, rx)
-    card.logger.info("connection accepted")
-    # first go through identity phase
-    async with asyncio.timeout(30):
-        await card.tx_hello_ack()
-        card.logger.info("hello-handshake completed")
-        card_clients.add(card)
-        # first obtain the identity of the card
-        await card.identify()
-    # then go into the "main loop"
-    try:
-        # wait 'indefinitely'. We cannot call websocket.recv() here, as we will call another
-        # recv() while waiting for the R-APDU after forwarding one from the user client.
-        await websocket.wait_closed()
-    finally:
-        card.logger.info("connection closed")
-        card_clients.remove(card)
-
-
-async def user_conn_hdlr(websocket):
-    """Handler for incoming connection to 'user' port."""
-    # receive first message, which should be a 'hello'
-    rx_raw = await websocket.recv()
-    rx = json.loads(rx_raw)
-    assert rx['msg_type'] == 'hello'
-    client_type = rx['client_type']
-
-    if client_type != 'user':
-        logger.error("Rejecting client (unknown type %s) connection", client_type)
-        raise ValueError("client_type '%s' != expected 'card'" % client_type)
-
-    user = UserClient(websocket, rx)
-    user.logger.info("connection accepted")
-    # first go through hello phase
-    async with asyncio.timeout(10):
-        await user.tx_hello_ack()
-        user.logger.info("hello-handshake completed")
-        user_clients.add(user)
-        # first wait for the user to specify the select the card
-        try:
-            await user.state_init()
-        except ConnectionClosedError:
-            user.logger.info("connection closed")
-            user_clients.remove(user)
-            return
-        except asyncio.exceptions.CancelledError: # direct cause of TimeoutError
-            user.logger.error("User failed to transition to 'associated' state within 10s")
-            user_clients.remove(user)
-            return
-        except Exception as e:
-            user.logger.error("Unknown exception %s", e)
-            raise e
-            user_clients.remove(user)
-            return
-    # then perform APDU exchanges without any time limit
-    try:
-        await user.state_selected()
-    except ConnectionClosedError:
-        pass
-    finally:
-        user.logger.info("connection closed")
-        await user.card.disassociate_user()
-        await user.disassociate_card()
-        user_clients.remove(user)
-
-
-parser = argparse.ArgumentParser(description="""Osmocom WebSocket Remote Card server""")
-parser.add_argument('--user-bind-port', type=int, default=WSRC_DEFAULT_PORT_USER,
-                    help="port number to bind for connections from users")
-parser.add_argument('--user-bind-host', type=str, default='localhost',
-                    help="local Host/IP to bind for connections from users")
-parser.add_argument('--card-bind-port', type=int, default=WSRC_DEFAULT_PORT_CARD,
-                    help="port number to bind for connections from cards")
-parser.add_argument('--card-bind-host', type=str, default='localhost',
-                    help="local Host/IP to bind for connections from cards")
-
-if __name__ == '__main__':
-    opts = parser.parse_args()
-
-    async def main():
-        # we use different ports for user + card connections to ensure they can
-        # have different packet filter rules apply to them
-        async with serve(card_conn_hdlr, opts.card_bind_host, opts.card_bind_port), serve(user_conn_hdlr, opts.user_bind_host, opts.user_bind_port):
-            await asyncio.get_running_loop().create_future() # run forever
-
-    asyncio.run(main())

docs/cap-tutorial.rst

@@ -0,0 +1,103 @@
+
+Guide: Installing JAVA-card applets
+===================================
+
+Almost all modern-day UICC cards have some form of JAVA-card / Sim-Toolkit support, which allows the installation
+of customer specific JAVA-card applets. The installation of JAVA-card applets is usually done via the standardized
+GlobalPlatform (GPC_SPE_034) ISD (Issuer Security Domain) application interface during the card provisioning process.
+(it is also possible to load JAVA-card applets in field via OTA-SMS, but that is beyond the scope of this guide). In
+this guide we will go through the individual steps that are required to load JAVA-card applet onto an UICC card.
+
+
+Preparation
+~~~~~~~~~~~
+
+In this example we will install the CAP file HelloSTK_09122024.cap [1] on an sysmoISIM-SJA2 card. Since the interface
+is standardized, the exact card model does not matter.
+
+The example applet makes use of the STK (Sim-Toolkit), so we must supply STK installation parameters. Those
+parameters are supplied in the form of a hexstring and should be provided by the applet manufacturer. The available
+parameters and their exact encoding is specified in ETSI TS 102 226, section 8.2.1.3.2.1. The installation of
+HelloSTK_09122024.cap [1], will require the following STK installation parameters: "010001001505000000000000000000000000"
+
+During the installation, we also have to set a memory quota for the volatile and for the non volatile card memory.
+Those values also should be provided by the applet manufacturer. In this example, we will allow 255 bytes of volatile
+memory and 255 bytes of non volatile memory to be consumed by the applet.
+
+To install JAVA-card applets, one must be in the possession of the key material belonging to the card. The keys are
+usually provided by the card manufacturer. The following example will use the following keyset:
+
++---------+----------------------------------+
+| Keyname | Keyvalue                         |
++=========+==================================+
+| DEK/KIK | 5524F4BECFE96FB63FC29D6BAAC6058B |
++---------+----------------------------------+
+| ENC/KIC | 542C37A6043679F2F9F71116418B1CD5 |
++---------+----------------------------------+
+| MAC/KID | 34F11BAC8E5390B57F4E601372339E3C |
++---------+----------------------------------+
+
+[1] https://osmocom.org/projects/cellular-infrastructure/wiki/HelloSTK
+
+
+Applet Installation
+~~~~~~~~~~~~~~~~~~~
+
+To prepare the installation, a secure channel to the ISD must be established first:
+
+::
+
+    pySIM-shell (00:MF)> select ADF.ISD
+    {
+        "application_id": "a000000003000000",
+	"proprietary_data": {
+	    "maximum_length_of_data_field_in_command_message": 255
+        }
+    }
+    pySIM-shell (00:MF/ADF.ISD)> establish_scp02 --key-dek 5524F4BECFE96FB63FC29D6BAAC6058B --key-enc 542C37A6043679F2F9F71116418B1CD5 --key-mac 34F11BAC8E5390B57F4E601372339E3C --security-level 1
+    Successfully established a SCP02[01] secure channel
+
+.. warning:: In case you get an "EXCEPTION of type 'ValueError' occurred with message: card cryptogram doesn't match" error message, it is very likely that there is a problem with the key material. The card may lock the ISD access after a certain amount of failed tries. Carefully check the key material any try again.
+
+
+When the secure channel is established, we are ready to install the applet. The installation normally is a multi step
+procedure, where the loading of an executable load file is announced first, then loaded and then installed in a final
+step. The pySim-shell command ``install_cap`` automatically takes care of those three steps.
+
+::
+
+    pySIM-shell (SCP02[01]:00:MF/ADF.ISD)> install_cap /home/user/HelloSTK_09122024.cap --install-parameters-non-volatile-memory-quota 255 --install-parameters-volatile-memory-quota 255 --install-parameters-stk 010001001505000000000000000000000000
+    loading cap file: /home/user/HelloSTK_09122024.cap ...
+    parameters:
+     security-domain-aid: a000000003000000
+     load-file: 569 bytes
+     load-file-aid: d07002ca44
+     module-aid: d07002ca44900101
+     application-aid: d07002ca44900101
+     install-parameters: c900ef1cc80200ffc70200ffca12010001001505000000000000000000000000
+    step #1: install for load...
+    step #2: load...
+    Loaded a total of 573 bytes in 3 blocks. Don't forget install_for_install (and make selectable) now!
+    step #3: install_for_install (and make selectable)...
+    done.
+
+The applet is now installed on the card. We can now quit pySim-shell and remove the card from the reader and test the
+applet in a mobile phone. There should be a new STK application with one menu entry shown, that will greet the user
+when pressed.
+
+
+Applet Removal
+~~~~~~~~~~~~~~
+
+To remove the applet, we must establish a secure channel to the ISD (see above). Then we can delete the applet using the
+``delete_card_content`` command.
+
+::
+
+    pySIM-shell (SCP02[01]:00:MF/ADF.ISD)> delete_card_content D07002CA44 --delete-related-objects
+
+The parameter "D07002CA44" is the load-file-AID of the applet. The load-file-AID is encoded in the .cap file and also
+displayed during the installation process. It is also important to note that when the applet is installed, it cannot
+be installed (under the same AID) again until it is removed.
+
+

docs/card-key-provider.rst

@@ -1,6 +1,5 @@
-
-Retrieving card-individual keys via CardKeyProvider
-==================================================
+Retrieving card-individual keys via CardKeyProvider
+===================================================
 
 When working with a batch of cards, or more than one card in general, it
 is a lot of effort to manually retrieve the card-specific PIN (like
@@ -21,9 +20,11 @@ example develop your own CardKeyProvider that queries some kind of
 database for the key material, or that uses a key derivation function to
 derive card-specific key material from a global master key.
 
-The only actual CardKeyProvider implementation included in pySim is the
-`CardKeyProviderCsv` which retrieves the key material from a
-[potentially encrypted] CSV file.
+pySim already includes two CardKeyProvider implementations. One to retrieve
+key material from a CSV file (`CardKeyProviderCsv`) and a second one that allows
+to retrieve the key material from a PostgreSQL database (`CardKeyProviderPgsql`).
+Both implementations equally implement a column encryption scheme that allows
+to protect sensitive columns using a *transport key*
 
 
 The CardKeyProviderCsv
@@ -41,11 +42,224 @@ of pySim-shell.  If you do not specify a CSV file, pySim will attempt to
 open a CSV file from the default location at
 `~/.osmocom/pysim/card_data.csv`, and use that, if it exists.
 
+The `CardKeyProviderCsv` is suitable to manage small amounts of key material
+locally. However, if your card inventory is very large and the key material
+must be made available on multiple sites, the `CardKeyProviderPgsql` is the
+better option.
+
+
+The CardKeyProviderPgsql
+------------------------
+
+With the `CardKeyProviderPgsql` you can use a PostgreSQL database as storage
+medium. The implementation comes with a CSV importer tool that consumes the
+same CSV files you would normally use with the `CardKeyProviderCsv`, so you
+can just use your existing CSV files and import them into the database.
+
+
+Requirements
+^^^^^^^^^^^^
+
+The `CardKeyProviderPgsql` uses the `Psycopg` PostgreSQL database adapter
+(https://www.psycopg.org). `Psycopg` is not part of the default requirements
+of pySim-shell and must be installed separately. `Psycopg` is available as
+Python package under the name `psycopg2-binary`.
+
+
+Setting up the database
+^^^^^^^^^^^^^^^^^^^^^^^
+
+From the perspective of the database, the `CardKeyProviderPgsql` has only
+minimal requirements. You do not have to create any tables in advance. An empty
+database and at least one user that may create, alter and insert into tables is
+sufficient. However, for increased reliability and as a protection against
+incorrect operation, the `CardKeyProviderPgsql` supports a hierarchical model
+with three users (or roles):
+
+* **admin**:
+  This should be the owner of the database. It is intended to be used for
+  administrative tasks like adding new tables or adding new columns to existing
+  tables. This user should not be used to insert new data into tables or to access
+  data from within pySim-shell using the `CardKeyProviderPgsql`
+
+* **importer**:
+  This user is used when feeding new data into an existing table. It should only
+  be able to insert new rows into existing tables. It should not be used for
+  administrative tasks or to access data from within pySim-shell using the
+  `CardKeyProviderPgsql`
+
+* **reader**:
+  To access data from within pySim shell using the `CardKeyProviderPgsql` the
+  reader user is the correct one to use. This user should have no write access
+  to the database or any of the tables.
+
+
+Creating a config file
+^^^^^^^^^^^^^^^^^^^^^^
+
+The default location for the config file is `~/.osmocom/pysim/card_data_pgsql.cfg`
+The file uses `yaml` syntax and should look like the example below:
+
+::
+
+   host: "127.0.0.1"
+   db_name: "my_database"
+   table_names:
+   - "uicc_keys"
+   - "euicc_keys"
+   db_users:
+      admin:
+         name: "my_admin_user"
+         pass: "my_admin_password"
+      importer:
+         name: "my_importer_user"
+         pass: "my_importer_password"
+      reader:
+         name: "my_reader_user"
+         pass: "my_reader_password"
+
+This file is used by pySim-shell and by the importer tool. Both expect the file
+in the aforementioned location. In case you want to store the file in a
+different location you may use the `--pgsql` commandline option to provide a
+custom config file path.
+
+The hostname and the database name for the PostgreSQL database is set with the
+`host` and `db_name` fields. The field `db_users` sets the user names and
+passwords for each of the aforementioned users (or roles). In case only a single
+admin user is used, all three entries may be populated with the same user name
+and password (not recommended)
+
+The field `table_names` sets the tables that the `CardKeyProviderPgsql` shall
+use to query to locate card key data. You can set up as many tables as you
+want, `CardKeyProviderPgsql` will query them in order, one by one until a
+matching entry is found.
+
+NOTE: In case you do not want to disclose the admin and the importer credentials
+to pySim-shell you may remove those lines. pySim-shell will only require the
+`reader` entry under `db_users`.
+
+
+Using the Importer
+^^^^^^^^^^^^^^^^^^
+
+Before data can be imported, you must first create a database table. Tables
+are created with the provided importer tool, which can be found under
+`contrib/csv-to-pgsql.py`. This tool is used to create the database table and
+read the data from the provided CSV file into the database.
+
+As mentioned before, all CSV file formats that work with `CardKeyProviderCsv`
+may be used. To demonstrate how the import process works, let's assume you want
+to import a CSV file format that looks like the following example. Let's also
+assume that you didn't get the Global Platform keys from your card vendor for
+this batch of UICC cards, so your CSV file lacks the columns for those fields.
+
+::
+
+   "id","imsi","iccid","acc","pin1","puk1","pin2","puk2","ki","opc","adm1"
+   "card1","999700000000001","8900000000000000001","0001","1111","11111111","0101","01010101","11111111111111111111111111111111","11111111111111111111111111111111","11111111"
+   "card2","999700000000002","8900000000000000002","0002","2222","22222222","0202","02020202","22222222222222222222222222222222","22222222222222222222222222222222","22222222"
+   "card3","999700000000003","8900000000000000003","0003","3333","22222222","0303","03030303","33333333333333333333333333333333","33333333333333333333333333333333","33333333"
+
+Since this is your first import, the database still lacks the table. To
+instruct the importer to create a new table, you may use the `--create-table`
+option. You also have to pick an appropriate name for the table. Any name may
+be chosen as long as it contains the string `uicc_keys` or `euicc_keys`,
+depending on the type of data (`UICC` or `eUICC`) you intend to store in the
+table. The creation of the table is an administrative task and can only be done
+with the `admin` user. The `admin` user is selected using the `--admin` switch.
+
+::
+
+   $ PYTHONPATH=../ ./csv-to-pgsql.py --csv ./csv-to-pgsql_example_01.csv --table-name uicc_keys --create-table --admin
+   INFO: CSV file: ./csv-to-pgsql_example_01.csv
+   INFO: CSV file columns: ['ID', 'IMSI', 'ICCID', 'ACC', 'PIN1', 'PUK1', 'PIN2', 'PUK2', 'KI', 'OPC', 'ADM1']
+   INFO: Using config file: /home/user/.osmocom/pysim/card_data_pgsql.cfg
+   INFO: Database host: 127.0.0.1
+   INFO: Database name: my_database
+   INFO: Database user: my_admin_user
+   INFO: New database table created: uicc_keys
+   INFO: Database table: uicc_keys
+   INFO: Database table columns: ['ICCID', 'IMSI']
+   INFO: Adding missing columns: ['PIN2', 'PUK1', 'PUK2', 'ACC', 'ID', 'PIN1', 'ADM1', 'KI', 'OPC']
+   INFO: Changes to table uicc_keys committed!
+
+The importer has created a new table with the name `uicc_keys`. The table is
+now ready to be filled with data.
+
+::
+
+   $ PYTHONPATH=../ ./csv-to-pgsql.py --csv ./csv-to-pgsql_example_01.csv --table-name uicc_keys
+   INFO: CSV file: ./csv-to-pgsql_example_01.csv
+   INFO: CSV file columns: ['ID', 'IMSI', 'ICCID', 'ACC', 'PIN1', 'PUK1', 'PIN2', 'PUK2', 'KI', 'OPC', 'ADM1']
+   INFO: Using config file: /home/user/.osmocom/pysim/card_data_pgsql.cfg
+   INFO: Database host: 127.0.0.1
+   INFO: Database name: my_database
+   INFO: Database user: my_importer_user
+   INFO: Database table: uicc_keys
+   INFO: Database table columns: ['ICCID', 'IMSI', 'PIN2', 'PUK1', 'PUK2', 'ACC', 'ID', 'PIN1', 'ADM1', 'KI', 'OPC']
+   INFO: CSV file import done, 3 rows imported
+   INFO: Changes to table uicc_keys committed!
+
+A quick `SELECT * FROM uicc_keys;` at the PostgreSQL console should now display
+the contents of the CSV file you have fed into the importer.
+
+Let's now assume that with your next batch of UICC cards your vendor includes
+the Global Platform keys so your CSV format changes. It may now look like this:
+
+::
+
+   "id","imsi","iccid","acc","pin1","puk1","pin2","puk2","ki","opc","adm1","scp02_dek_1","scp02_enc_1","scp02_mac_1"
+   "card4","999700000000004","8900000000000000004","0004","4444","44444444","0404","04040404","44444444444444444444444444444444","44444444444444444444444444444444","44444444","44444444444444444444444444444444","44444444444444444444444444444444","44444444444444444444444444444444"
+   "card5","999700000000005","8900000000000000005","0005","4444","55555555","0505","05050505","55555555555555555555555555555555","55555555555555555555555555555555","55555555","55555555555555555555555555555555","55555555555555555555555555555555","55555555555555555555555555555555"
+   "card6","999700000000006","8900000000000000006","0006","4444","66666666","0606","06060606","66666666666666666666666666666666","66666666666666666666666666666666","66666666","66666666666666666666666666666666","66666666666666666666666666666666","66666666666666666666666666666666"
+
+When importing data from an updated CSV format the database table also has
+to be updated. This is done using the `--update-columns` switch. Like when
+creating new tables, this operation also requires admin privileges, so the
+`--admin` switch is required again.
+
+::
+
+   $ PYTHONPATH=../ ./csv-to-pgsql.py --csv ./csv-to-pgsql_example_02.csv --table-name uicc_keys --update-columns --admin
+   INFO: CSV file: ./csv-to-pgsql_example_02.csv
+   INFO: CSV file columns: ['ID', 'IMSI', 'ICCID', 'ACC', 'PIN1', 'PUK1', 'PIN2', 'PUK2', 'KI', 'OPC', 'ADM1', 'SCP02_DEK_1', 'SCP02_ENC_1', 'SCP02_MAC_1']
+   INFO: Using config file: /home/user/.osmocom/pysim/card_data_pgsql.cfg
+   INFO: Database host: 127.0.0.1
+   INFO: Database name: my_database
+   INFO: Database user: my_admin_user
+   INFO: Database table: uicc_keys
+   INFO: Database table columns: ['ICCID', 'IMSI', 'PIN2', 'PUK1', 'PUK2', 'ACC', 'ID', 'PIN1', 'ADM1', 'KI', 'OPC']
+   INFO: Adding missing columns: ['SCP02_ENC_1', 'SCP02_MAC_1', 'SCP02_DEK_1']
+   INFO: Changes to table uicc_keys committed!
+
+When the new table columns are added, the import may be continued like the
+first one:
+
+::
+
+   $ PYTHONPATH=../ ./csv-to-pgsql.py --csv ./csv-to-pgsql_example_02.csv --table-name uicc_keys
+   INFO: CSV file: ./csv-to-pgsql_example_02.csv
+   INFO: CSV file columns: ['ID', 'IMSI', 'ICCID', 'ACC', 'PIN1', 'PUK1', 'PIN2', 'PUK2', 'KI', 'OPC', 'ADM1', 'SCP02_DEK_1', 'SCP02_ENC_1', 'SCP02_MAC_1']
+   INFO: Using config file: /home/user/.osmocom/pysim/card_data_pgsql.cfg
+   INFO: Database host: 127.0.0.1
+   INFO: Database name: my_database
+   INFO: Database user: my_importer_user
+   INFO: Database table: uicc_keys
+   INFO: Database table columns: ['ICCID', 'IMSI', 'PIN2', 'PUK1', 'PUK2', 'ACC', 'ID', 'PIN1', 'ADM1', 'KI', 'OPC', 'SCP02_ENC_1', 'SCP02_MAC_1', 'SCP02_DEK_1']
+   INFO: CSV file import done, 3 rows imported
+   INFO: Changes to table uicc_keys committed!
+
+On the PostgreSQL console a `SELECT * FROM uicc_keys;` should now show the
+imported data with the added columns. All important data should now also be
+available from within pySim-shell via the `CardKeyProviderPgsql`.
+
+
 Column-Level CSV encryption
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+---------------------------
 
 pySim supports column-level CSV encryption.  This feature will make sure
-that your key material is not stored in plaintext in the CSV file.
+that your key material is not stored in plaintext in the CSV file (or
+database).
 
 The encryption mechanism uses AES in CBC mode.  You can use any key
 length permitted by AES (128/192/256 bit).
@@ -73,6 +287,8 @@ by all columns of the set:
 * `SCP03_ISDA` is a group alias for `SCP03_ENC_ISDA`, `SCP03_MAC_ISDA`, `SCP03_DEK_ISDA`
 * `SCP03_ISDR` is a group alias for `SCP03_ENC_ISDR`, `SCP03_MAC_ISDR`, `SCP03_DEK_ISDR`
 
+NOTE: When using `CardKeyProviderPqsl`, the input CSV files must be encrypted
+before import.
 
 Field naming
 ------------
@@ -83,9 +299,9 @@ Field naming
 * For look-up of eUICC specific key material (like SCP03 keys for the
   ISD-R, ECASD), pySim uses the `EID` field as lookup key.
 
-As soon as the CardKeyProviderCsv finds a line (row) in your CSV where
-the ICCID or EID match, it looks for the column containing the requested
-data.
+As soon as the CardKeyProvider finds a line (row) in your CSV file
+(or database) where the ICCID or EID match, it looks for the column containing
+the requested data.
 
 
 ADM PIN

docs/conf.py

@@ -18,9 +18,17 @@ sys.path.insert(0, os.path.abspath('..'))
 # -- Project information -----------------------------------------------------
 
 project = 'osmopysim-usermanual'
-copyright = '2009-2023 by Sylvain Munaut, Harald Welte, Philipp Maier, Supreeth Herle, Merlin Chlosta'
+copyright = '2009-2025 by Sylvain Munaut, Harald Welte, Philipp Maier, Supreeth Herle, Merlin Chlosta'
 author = 'Sylvain Munaut, Harald Welte, Philipp Maier, Supreeth Herle, Merlin Chlosta'
 
+# PDF: Avoid that the authors list exceeds the page by inserting '\and'
+# manually as line break (https://github.com/sphinx-doc/sphinx/issues/6875)
+latex_elements = {
+    "maketitle":
+        r"""\author{Sylvain Munaut, Harald Welte, Philipp Maier, \and Supreeth Herle, Merlin Chlosta}
+\sphinxmaketitle
+"""
+}
 
 # -- General configuration ---------------------------------------------------
 

docs/index.rst

@@ -41,9 +41,14 @@ pySim consists of several parts:
    shell
    trace
    legacy
+   smpp2sim
    library
+   library-esim
    osmo-smdpp
-   wsrc
+   sim-rest
+   suci-keytool
+   saip-tool
+   smpp-ota-tool
 
 
 Indices and tables

docs/legacy.rst

@@ -49,7 +49,7 @@ Two modes are possible:
     Ki and OPc will be generated during each programming cycle. This means fresh keys are generated, even when the
     ``--num`` remains unchanged.
 
-    The parameter ``--num`` specifies a card individual number. This number will be manged into the random seed so that
+    The parameter ``--num`` specifies a card individual number. This number will be managed into the random seed so that
     it serves as an identifier for a particular set of randomly generated parameters.
 
     In the example above the parameters ``--mcc``, and ``--mnc`` are specified as well, since they identify the GSM
@@ -77,7 +77,7 @@ the parameter ``--type``. The following card types are supported:
 
 Specifying the card reader:
 
-It is most common to use ``pySim-prog`` together whith a PCSC reader. The PCSC reader number is specified via the
+It is most common to use ``pySim-prog`` together with a PCSC reader. The PCSC reader number is specified via the
 ``--pcsc-device`` or ``-p`` option. However, other reader types (such as serial readers and modems) are supported. Use
 the ``--help`` option of ``pySim-prog`` for more information.
 

docs/library-esim.rst

@@ -0,0 +1,95 @@
+pySim eSIM libraries
+====================
+
+The pySim eSIM libraries implement a variety of functionality related to the GSMA eSIM universe,
+including the various interfaces of SGP.21 + SGP.22, as well as Interoperable Profile decioding,
+validation, personalization and encoding.
+
+.. automodule:: pySim.esim
+   :members:
+
+
+GSMA SGP.21/22 Remote SIM Provisioning (RSP) - High Level
+---------------------------------------------------------
+
+pySim.esim.rsp
+~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.rsp
+   :members:
+
+pySim.esim.es2p
+~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.es2p
+   :members:
+
+
+pySim.esim.es8p
+~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.es8p
+   :members:
+
+
+pySim.esim.es9p
+~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.es9p
+   :members:
+
+
+GSMA SGP.21/22 Remote SIM Provisioning (RSP) - Low Level
+--------------------------------------------------------
+
+pySim.esim.bsp
+~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.bsp
+   :members:
+
+pySim.esim.http_json_api
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.http_json_api
+   :members:
+
+pySim.esim.x509_cert
+~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.x509_cert
+   :members:
+
+SIMalliance / TCA Interoperable Profile
+---------------------------------------
+
+
+pySim.esim.saip
+~~~~~~~~~~~~~~~
+.. automodule:: pySim.esim.saip
+   :members:
+
+
+pySim.esim.saip.oid
+~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.saip.oid
+   :members:
+
+pySim.esim.saip.personalization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.saip.personalization
+   :members:
+
+pySim.esim.saip.templates
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.saip.templates
+   :members:
+
+pySim.esim.saip.validation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.saip.validation
+   :members:

docs/osmo-smdpp.rst

@@ -21,9 +21,9 @@ osmo-smdpp currently
 
 * [by default] uses test certificates copied from GSMA SGP.26 into `./smdpp-data/certs`, assuming that your
   osmo-smdpp would be running at the host name `testsmdpplus1.example.com`. You can of course replace those
-  certificates with your own, whether SGP.26 derived or part of a *private root CA* setup with mathcing eUICCs.
+  certificates with your own, whether SGP.26 derived or part of a *private root CA* setup with matching eUICCs.
 * doesn't understand profile state. Any profile can always be downloaded any number of times, irrespective
-  of the EID or whether it was donwloaded before.  This is actually very useful for R&D and testing, as it
+  of the EID or whether it was downloaded before.  This is actually very useful for R&D and testing, as it
   doesn't require you to generate new profiles all the time.  This logic of course is unsuitable for
   production usage.
 * doesn't perform any personalization, so the IMSI/ICCID etc. are always identical (the ones that are stored in
@@ -40,16 +40,21 @@ osmo-smdpp currently
 Running osmo-smdpp
 ------------------
 
-osmo-smdpp does not have built-in TLS support as the used *twisted* framework appears to have
-problems when using the example elliptic curve certificates (both NIST and Brainpool) from GSMA.
+osmo-smdpp comes with built-in TLS support which is enabled by default. However, it is always possible to
+disable the built-in TLS support if needed.
+
+In order to use osmo-smdpp without the built-in TLS support, it has to be put behind a TLS reverse proxy,
+which terminates the ES9+ HTTPS traffic from the LPA, and then forwards it as plain HTTP to osmo-smdpp.
+
+NOTE: The built in TLS support in osmo-smdpp makes use of the python *twisted* framework. Older versions
+of this framework appear to have problems when using the example elliptic curve certificates (both NIST and
+Brainpool) from GSMA.
 
-So in order to use it, you have to put it behind a TLS reverse proxy, which terminates the ES9+
-HTTPS from the LPA, and then forwards it as plain HTTP to osmo-smdpp.
 
 nginx as TLS proxy
 ~~~~~~~~~~~~~~~~~~
 
-If you use `nginx` as web server, you can use the following configuration snippet::
+If you chose to use `nginx` as TLS reverse proxy, you can use the following configuration snippet::
 
   upstream smdpp {
           server localhost:8000;
@@ -82,7 +87,7 @@ software.
 supplementary files
 ~~~~~~~~~~~~~~~~~~~
 
-The `smdpp-data/certs`` directory contains the DPtls, DPauth and DPpb as well as CI certificates
+The `smdpp-data/certs` directory contains the DPtls, DPauth and DPpb as well as CI certificates
 used; they are copied from GSMA SGP.26 v2.  You can of course replace them with custom certificates
 if you're operating eSIM with a *private root CA*.
 
@@ -92,22 +97,43 @@ The `smdpp-data/upp` directory contains the UPP (Unprotected Profile Package) us
 commandline options
 ~~~~~~~~~~~~~~~~~~~
 
-osmo-smdpp currently doesn't have any configuration file or command line options.  You just run it,
-and it will bind its plain-HTTP ES9+ interface to local TCP port 8000.
+Typically, you just run osmo-smdpp without any arguments, and it will bind its built-in HTTPS ES9+ interface to
+`localhost` TCP port 443. In this case an external TLS reverse proxy is not needed.
 
+osmo-smdpp currently doesn't have any configuration file.
 
+There are command line options for binding:
+
+Bind the HTTPS ES9+ to a port other than 443::
+
+  ./osmo-smdpp.py -p 8443
+
+Disable the built-in TLS support and bind the plain-HTTP ES9+ to a port 8000::
+
+  ./osmo-smdpp.py -p 8000 --nossl
+
+Bind the HTTP ES9+ to a different local interface::
+
+  ./osmo-smdpp.py -H 127.0.0.2
 
 DNS setup for your LPA
 ~~~~~~~~~~~~~~~~~~~~~~
 
 The LPA must resolve `testsmdpplus1.example.com` to the IP address of your TLS proxy.
 
-It must also accept the TLS certificates used by your TLS proxy.
+It must also accept the TLS certificates used by your TLS proxy. In case osmo-smdpp is used with built-in TLS support,
+it will use the certificates provided in smdpp-data.
+
+NOTE: The HTTPS ES9+ interface cannot be addressed by the LPA directly via its IP address. The reason for this is that
+the included SGP.26 (DPtls) test certificates explicitly restrict the hostname to `testsmdpplus1.example.com` in the
+`X509v3 Subject Alternative Name` extension. Using a bare IP address as hostname may cause the certificate to be
+rejected by the LPA.
+
 
 Supported eUICC
 ~~~~~~~~~~~~~~~
 
-If you run osmo-smdpp with the included SGP.26 certificates, you must use an eUICC with matching SGP.26
+If you run osmo-smdpp with the included SGP.26 (DPauth, DPpb) certificates, you must use an eUICC with matching SGP.26
 certificates, i.e. the EUM certificate must be signed by a SGP.26 test root CA and the eUICC certificate
 in turn must be signed by that SGP.26 EUM certificate.
 

docs/saip-tool.rst

@@ -0,0 +1,137 @@
+saip-tool
+=========
+
+eSIM profiles are stored as a sequence of profile element (PE) objects in an ASN.1 DER encoded binary file. To inspect,
+verify or make changes to those files, the `saip-tool.py` utility can be used.
+
+NOTE: The file format, eSIM SAIP (SimAlliance Interoperable Profile) is specified in `TCA eUICC Profile Package:
+Interoperable Format Technical Specification`
+
+
+Profile Package Examples
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+pySim ships with a set of TS48 profile package examples. Those examples can be found in `pysim/smdpp-data/upp`. The
+files can be used as input for `saip-tool.py`. (see also GSMA TS.48 - Generic eUICC Test Profile for Device Testing)
+
+See also: https://github.com/GSMATerminals/Generic-eUICC-Test-Profile-for-Device-Testing-Public
+
+JAVA card applets
+~~~~~~~~~~~~~~~~~
+
+The `saip-tool.py` can also be used to manage JAVA-card applets (Application PE) inside a profile package. The user has
+the option to add, remove and inspect applications and their instances. In the following we will discuss a few JAVA-card
+related use-cases of `saip-tool.py`
+
+NOTE: see also `contrib` folder for script examples (`saip-tool_example_*.sh`)
+
+Inserting applications
+----------------------
+
+An application is usually inserted in two steps. In the first step, the application PE is created and populated with
+the executable code from a provided `.cap` or `.ijc` file. The user also has to pick a suitable load block AID.
+
+The application instance, which exists inside the application PE, is created in a second step. Here the user must
+reference the load block AID and pick, among other application related parameters, a suitable class and instance AID.
+
+Example: Adding a JAVA-card applet to an existing profile package
+::
+
+  # Step #1: Create the application PE and load the ijc contents from the .cap file:
+  $ ./contrib/saip-tool.py upp.der add-app --output-file upp_with_app.der --applet-file app.cap --aid '1122334455'
+  Read 28 PEs from file 'upp.der'
+  Applying applet file: 'app.cap'...
+  application PE inserted into PE Sequence after securityDomain PE AID: a000000151000000
+  Writing 29 PEs to file 'upp_with_app.der'...
+
+  # Step #2: Create the application instance inside the application PE created in step #1:
+  $ ./contrib/saip-tool.py upp_with_app.der add-app-inst --output-file upp_with_app_and_instance.der \
+          --aid '1122334455' \
+          --class-aid '112233445501' \
+          --inst-aid '112233445501' \
+          --app-privileges '00' \
+          --app-spec-pars '00' \
+          --uicc-toolkit-app-spec-pars '01001505000000000000000000000000'
+  Read 29 PEs from file 'upp_with_app.der'
+  Found Load Package AID: 1122334455, adding new instance AID: 112233445501 to Application PE...
+  Writing 29 PEs to file 'upp_with_app_and_instance.der'...
+
+NOTE: The parameters of the sub-commands `add-app` and `add-app-inst` are application specific. It is up to the application
+developer to pick parameters that suit the application correctly. For an exact command reference see section
+`saip-tool syntax`. For parameter details see `TCA eUICC Profile Package: Interoperable Format Technical Specification`,
+section 8.7 and ETSI TS 102 226, section 8.2.1.3.2
+
+
+Inspecting applications
+-----------------------
+
+To inspect the application PE contents of an existing profile package, sub-command `info` with parameter '--apps' can
+be used. This command lists out all application and their parameters in detail. This allows an application developer
+to check if the applet insertaion was carried out as expected.
+
+Example: Listing applications and their parameters
+::
+
+  $ ./contrib/saip-tool.py upp_with_app_and_instance.der info --apps
+  Read 29 PEs from file 'upp_with_app_and_instance.der'
+  Application #0:
+          loadBlock:
+                  loadPackageAID: '1122334455' (5 bytes)
+                  loadBlockObject: '01000fdecaffed010204000105d07002ca440200...681080056810a00633b44104b431066800a10231' (569 bytes)
+          instanceList[0]:
+                  applicationLoadPackageAID: '1122334455' (5 bytes)
+                  classAID: '112233445501' (8 bytes)
+                  instanceAID: '112233445501' (8 bytes)
+                  applicationPrivileges: '00' (1 bytes)
+                  lifeCycleState: '07' (1 bytes)
+                  applicationSpecificParametersC9: '00' (1 bytes)
+                  applicationParameters:
+                          uiccToolkitApplicationSpecificParametersField: '01001505000000000000000000000000' (16 bytes)
+
+In case further analysis with external tools or transfer of applications from one profile package to another is
+necessary, the executable code in the `loadBlockObject` field can be extracted to an `.ijc` or an `.cap` file.
+
+Example: Extracting applications from a profile package
+::
+
+  $ ./contrib/saip-tool.py upp_with_app_and_instance.der extract-apps --output-dir ./apps --format ijc
+  Read 29 PEs from file 'upp_with_app_and_instance.der'
+  Writing Load Package AID: 1122334455 to file ./apps/8949449999999990023f-1122334455.ijc
+
+
+Removing applications
+---------------------
+
+An application PE can be removed using sub-command `remove-app`. The user passes the load package AID as parameter. Then
+`saip-tool.py` will search for the related application PE and delete it from the PE sequence.
+
+Example: Remove an application from a profile package
+::
+
+  $ ./contrib/saip-tool.py upp_with_app_and_instance.der remove-app --output-file upp_without_app.der --aid '1122334455'
+  Read 29 PEs from file 'upp_with_app_and_instance.der'
+  Found Load Package AID: 1122334455, removing related PE (id=23) from Sequence...
+  Removing PE application (id=23) from Sequence...
+  Writing 28 PEs to file 'upp_without_app.der'...
+
+In some cases it is useful to remove only an instance from an existing application PE. This may be the case when the
+an application developer wants to modify parameters of an application by removing and re-adding the instance. The
+operation basically rolls the state back to step 1 explained in section :ref:`Inserting applications`
+
+Example: Remove an application instance from an application PE
+::
+
+  $ ./contrib/saip-tool.py upp_with_app_and_instance.der remove-app-inst --output-file upp_without_app.der --aid '1122334455' --inst-aid '112233445501'
+  Read 29 PEs from file 'upp_with_app_and_instance.der'
+  Found Load Package AID: 1122334455, removing instance AID: 112233445501 from Application PE...
+  Removing instance from Application PE...
+  Writing 29 PEs to file 'upp_with_app.der'...
+
+
+saip-tool syntax
+~~~~~~~~~~~~~~~~
+
+.. argparse::
+   :module: contrib.saip-tool
+   :func: parser
+   :prog: contrib/saip-tool.py

docs/shell.rst

@@ -1,4 +1,4 @@
-pySim-shell
+pySim-shell
 ===========
 
 pySim-shell is an interactive command line shell for all kind of interactions with SIM cards,
@@ -67,6 +67,7 @@ Usage Examples
    :caption: Tutorials for pySIM-shell:
 
    suci-tutorial
+   cap-tutorial
 
 
 Advanced Topics
@@ -1005,6 +1006,24 @@ ARA-M applet.  Use it with caution, there is no undo.  Any rules later
 intended must be manually inserted again using :ref:`aram_store_ref_ar_do`
 
 
+aram_lock
+~~~~~~~~~
+This command allows to lock the access to the STORE DATA command. This renders
+all access rules stored within the ARA-M applet effectively read-only. The lock
+can only be removed via a secure channel to the security domain and is therefore
+suitable to prevent unauthorized changes to ARA-M rules.
+
+Removal of the lock:
+::
+
+  pySIM-shell (SCP02[01]:00:MF/ADF.ISD)> install_for_personalization A00000015141434C00
+  pySIM-shell (SCP02[01]:00:MF/ADF.ISD)> apdu --expect-sw 9000 80E2900001A2
+
+NOTE: ARA-M Locking is a proprietary feature that is specific to sysmocom's
+fork of Bertrand Martel's ARA-M implementation. ARA-M Locking is supported in
+newer (2025) applet versions from v0.1.0 onward.
+
+
 GlobalPlatform commands
 -----------------------
 
@@ -1047,6 +1066,18 @@ delete_key
    :module: pySim.global_platform
    :func: ADF_SD.AddlShellCommands.del_key_parser
 
+load
+~~~~
+.. argparse::
+   :module: pySim.global_platform
+   :func: ADF_SD.AddlShellCommands.load_parser
+
+install_cap
+~~~~~~~~~~~
+.. argparse::
+   :module: pySim.global_platform
+   :func: ADF_SD.AddlShellCommands.install_cap_parser
+
 install_for_personalization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. argparse::
@@ -1059,6 +1090,12 @@ install_for_install
    :module: pySim.global_platform
    :func: ADF_SD.AddlShellCommands.inst_inst_parser
 
+install_for_load
+~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.global_platform
+   :func: ADF_SD.AddlShellCommands.inst_load_parser
+
 delete_card_content
 ~~~~~~~~~~~~~~~~~~~
 .. argparse::
@@ -1118,7 +1155,7 @@ es10x_store_data
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.es10x_store_data_parser
+   :func: CardApplicationISDR.AddlShellCommands.es10x_store_data_parser
 
 get_euicc_configured_addresses
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1137,7 +1174,7 @@ set_default_dp_address
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.set_def_dp_addr_parser
+   :func: CardApplicationISDR.AddlShellCommands.set_def_dp_addr_parser
 
 get_euicc_challenge
 ~~~~~~~~~~~~~~~~~~~
@@ -1280,7 +1317,7 @@ remove_notification_from_list
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.rem_notif_parser
+   :func: CardApplicationISDR.AddlShellCommands.rem_notif_parser
 
 Example::
 
@@ -1329,7 +1366,7 @@ enable_profile
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.en_prof_parser
+   :func: CardApplicationISDR.AddlShellCommands.en_prof_parser
 
 Example (successful)::
 
@@ -1351,7 +1388,7 @@ disable_profile
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.dis_prof_parser
+   :func: CardApplicationISDR.AddlShellCommands.dis_prof_parser
 
 Example (successful)::
 
@@ -1365,7 +1402,7 @@ delete_profile
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.del_prof_parser
+   :func: CardApplicationISDR.AddlShellCommands.del_prof_parser
 
 Example::
 
@@ -1374,6 +1411,13 @@ Example::
       "delete_result": "ok"
   }
 
+euicc_memory_reset
+~~~~~~~~~~~~~~~~~~
+
+.. argparse::
+   :module: pySim.euicc
+   :func: CardApplicationISDR.AddlShellCommands.mem_res_parser
+
 
 get_eid
 ~~~~~~~
@@ -1392,7 +1436,7 @@ set_nickname
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.set_nickname_parser
+   :func: CardApplicationISDR.AddlShellCommands.set_nickname_parser
 
 Example::
 

docs/sim-rest.rst

@@ -0,0 +1,118 @@
+sim-rest-server
+===============
+
+Sometimes there are use cases where a [remote] application will need
+access to a USIM for authentication purposes.  This is, for example, in
+case an IMS test client needs to perform USIM based authentication
+against an IMS core.
+
+The pysim repository contains two programs: `sim-rest-server.py` and
+`sim-rest-client.py` that implement a simple approach to achieve the
+above:
+
+`sim-rest-server.py` speaks to a [usually local] USIM via the PC/SC
+API and provides a high-level REST API towards [local or remote]
+applications that wish to perform UMTS AKA using the USIM.
+
+`sim-rest-client.py` implements a small example client program to
+illustrate how the REST API provided by `sim-rest-server.py` can be
+used.
+
+REST API Calls
+--------------
+
+POST /sim-auth-api/v1/slot/SLOT_NR
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+where SLOT_NR is the integer-encoded slot number (corresponds to PC/SC
+reader number).  When using a single sysmoOCTSIM board, this is in the range of 0..7
+
+Example: `/sim-auth-api/v1/slot/0` for the first slot.
+
+Request Body
+############
+
+The request body is a JSON document, comprising of
+    1. the RAND and AUTN parameters as hex-encoded string
+    2. the application against which to authenticate (USIM, ISIM)
+
+Example:
+::
+
+        {
+            "rand": "bb685a4b2fc4d697b9d6a129dd09a091",
+            "autn": "eea7906f8210000004faf4a7df279b56"
+        }
+
+HTTP Status Codes
+#################
+
+HTTP status codes are used to represent errors within the REST server
+and the SIM reader hardware.  They are not used to communicate protocol
+level errors reported by the SIM Card.  An unsuccessful authentication
+will hence have a `200 OK` HTTP Status code and then encode the SIM
+specific error information in the Response Body.
+
+======  =========== ================================
+Status  Code        Description
+------  ----------- --------------------------------
+200     OK          Successful execution
+400     Bad Request Request body is malformed
+404     Not Found   Specified SIM Slot doesn't exist
+410     Gone        No SIM card inserted in slot
+======  =========== ================================
+
+Response Body
+#############
+
+The response body is a JSON document, either
+
+#. a successful outcome; encoding RES, CK, IK as hex-encoded string
+#. a sync failure; encoding AUTS as hex-encoded string
+#. errors
+   #. authentication error (incorrect MAC)
+   #. authentication error (security context not supported)
+   #. key freshness failure
+   #. unspecified card error
+
+Example (success):
+::
+
+        {
+           "successful_3g_authentication": {
+                "res": "b15379540ec93985",
+                "ck": "713fde72c28cbd282a4cd4565f3d6381",
+                "ik": "2e641727c95781f1020d319a0594f31a",
+                "kc": "771a2c995172ac42"
+            }
+        }
+
+Example (re-sync case):
+::
+
+        {
+            "synchronisation_failure": {
+                "auts": "dc2a591fe072c92d7c46ecfe97e5"
+            }
+        }
+
+Concrete example using the included sysmoISIM-SJA2
+--------------------------------------------------
+
+This was tested using SIMs ending in IMSI numbers 45890...45899
+
+The following command were executed successfully:
+
+Slot 0
+::
+
+        $ /usr/local/src/pysim/contrib/sim-rest-client.py -c 1 -n 0 -k 841EAD87BC9D974ECA1C167409357601 -o 3211CACDD64F51C3FD3013ECD9A582A0
+        -> {'rand': 'fb195c7873b20affa278887920b9dd57', 'autn': 'd420895a6aa2000089cd016f8d8ae67c'}
+        <- {'successful_3g_authentication': {'res': '131004db2ff1ce8e', 'ck': 'd42eb5aa085307903271b2422b698bad', 'ik': '485f81e6fd957fe3cad374adf12fe1ca', 'kc': '64d3f2a32f801214'}}
+
+Slot 1
+::
+
+        $ /usr/local/src/pysim/contrib/sim-rest-client.py -c 1 -n 1 -k 5C2CE9633FF9B502B519A4EACD16D9DF -o 9834D619E71A02CD76F00CC7AA34FB32
+        -> {'rand': '433dc5553db95588f1d8b93870930b66', 'autn': '126bafdcbe9e00000026a208da61075d'}
+        <- {'successful_3g_authentication': {'res': '026d7ac42d379207', 'ck': '83a90ba331f47a95c27a550b174c4a1f', 'ik': '31e1d10329ffaf0ca1684a1bf0b0a14a', 'kc': 'd15ac5b0fff73ecc'}}

docs/smpp-ota-tool.rst

@@ -0,0 +1,179 @@
+smpp-ota-tool
+=============
+
+The `smpp-ota-tool` allows users to send OTA SMS messages containing APDU scripts (RFM, RAM) via an SMPP server. The
+intended audience are developers who want to test/evaluate the OTA SMS interface of a SIM/UICC/eUICC. `smpp-ota-tool`
+is intended to be used as a companion tool for :ref:`pySim-smpp2sim`, however it should be usable on any other SMPP
+server (such as a production SMSC of a live cellular network) as well.
+
+From the technical perspective `smpp-ota-tool` takes the role of an SMPP ESME. It takes care of the encoding, encryption
+and checksumming (signing) of the RFM/RAM OTA SMS and eventually submits it to the SMPP server. The program then waits
+for a response. The response is automatically parsed and printed on stdout. This makes the program also suitable to be
+called from shell scripts.
+
+.. note:: In the following we will we will refer to `SIM` as one of the following: `SIM`, `USIM`, `ISIM`, `UICC`,
+	  `eUICC`, `eSIM`.
+
+Applying OTA keys
+~~~~~~~~~~~~~~~~~
+
+Depending on the `SIM` type you will receive one or more sets of keys which you can use to communicate with the `SIM`
+through a secure channel protocol. When using the OTA SMS method, the SCP80 protocol is used and it therefore crucial
+to use a keyset that is actually suitable for SCP80.
+
+A keyset usually consists of three keys:
+
+#. KIC: the key used for ciphering (encryption/decryption)
+#. KID: the key used to compute a cryptographic checksum (signing)
+#. KIK: the key used to encrypt/decrypt key material (key rotation, adding of new keys)
+
+From the transport security perspective, only KIC and KID are relevant. The KIK (also referenced as "Data Encryption
+Key", DEK) is only used when keys are rotated or new keys are added (see also ETSI TS 102 226, section 8.2.1.5).
+
+When the keyset is programmed into the security domain of the `SIM`, it is tied to a specific cryptographic algorithm
+(3DES, AES128 or AES256) and a so called Key Version Number (KVN). The term "Key Version Number" is misleading, since
+it is actually not a version number. It is a unique identifier of a certain keyset which also identifies for which
+secure channel protocol the keyset may be used. Keysets with a KVN from 1-15 (``0x01``-``0x0F``) are suitable for SCP80.
+This means that it is not only important to know just the KIC/KID/KIK keys. Also the related algorithms and the KVN
+numbers must be known.
+
+.. note:: SCP80 keysets typically start counting from 1 upwards. Typical configurations use a set of 3 keysets with
+	  KVN numbers 1-3.
+
+Addressing an Application
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When communicating with a specific application on a `SIM` via SCP80, it is important to address that application with
+the correct parameters. The following two parameters must be known in advance:
+
+#. TAR: The Toolkit Application Reference (TAR) number is a three byte value that uniquely addresses an application
+   on the `SIM`. The exact values may vary (see also ETSI TS 101 220, Table D.1).
+#. MSL: The Minimum Security Level (MSL) is a bit-field that dictates which of the security measures encoded in the
+   SPI are mandatory (see also ETSI TS 102 225, section 5.1.1).
+
+A practical example
+~~~~~~~~~~~~~~~~~~~
+
+.. note:: This tutorial assumes that pySim-smpp2sim is running on the local machine with its default parameters.
+	  See also :ref:`pySim-smpp2sim`.
+
+Let's assume that an OTA SMS shall be sent to the SIM RFM application of an sysmoISIM-SJA2. What we want to do is to
+select DF.GSM and to get the select response back.
+
+We have received the following key material from the `SIM` vendor:
+
+::
+
+   KIC1: F09C43EE1A0391665CC9F05AF4E0BD10
+   KID1: 01981F4A20999F62AF99988007BAF6CA
+   KIK1: 8F8AEE5CDCC5D361368BC45673D99195
+   KIC2: 01022916E945B656FDE03F806A105FA2
+   KID2: D326CB69F160333CC5BD1495D448EFD6
+   KIK2: 08037E0590DFE049D4975FFB8652F625
+   KIC3: 2B22824D0D27A3A1CEEC512B312082B4
+   KID3: F1697766925A11F4458295590137B672
+   KIK3: C7EE69B2C5A1C8E160DD36A38EB517B3
+
+Those are three keysets. The enumeration is directly equal to the KVN used. All three keysets are 3DES keys, which
+means triple_des_cbc2 is the correct algorithm to use.
+
+.. note:: The key set configuration can be confirmed by retrieving the key configuration using
+	  `get_data key_information` from within an SCP02 session on ADF.ISD.
+
+In this example we intend to address the SIM RFM application on the `SIM`. Which according to the manual has TAR ``B00010``
+and MSL ``0x06``. When we hold ``0x06`` = ``0b00000110`` against the SPI coding chart (see also ETSI TS 102 225,
+section 5.1.1). We can deduct that Ciphering and Cryptographic Checksum are mandatory.
+
+.. note:: The MSL (see also ETSI TS 102 226, section 6.1) is assigned to an application by the `SIM` issuer. It is a
+	  custom decision and may vary with different `SIM` types/profiles. In the case of sysmoISIM-SJS1/SJA2/SJA5 the
+	  counter requirement has been waived to simplify lab/research type use. In productive environments, `SIM`
+	  applications should ideally use an MSL that makes the counter mandatory.
+
+In order to select DF.GSM (``0x7F20``) and to retrieve the select response, two APDUs are needed. The first APDU is the
+select command ``A0A40000027F20`` and the second is the related get-response command ``A0C0000016``. Those APDUs will be
+concatenated and are sent in a single message. The message containing the concatenated APDUs works as a script that
+is received by the SIM RFM application and then executed. This method poses some limitations that have to be taken into
+account when making requests like this (see also ETSI TS 102 226, section 5).
+
+With this information we may now construct a commandline for `smpp-ota-tool.py`. We will pass the KVN as kid_idx and
+kic_idx (see also ETSI TS 102 225, Table 2, fields `KIc` and `KID`). Both index values should refer to the same
+keyset/KVN as keysets should not be mixed. (`smpp-ota-tool` still provides separate parameters anyway to allow testing
+with invalid keyset combinations)
+
+::
+
+   $ PYTHONPATH=./ ./contrib/smpp-ota-tool.py --kic F09C43EE1A0391665CC9F05AF4E0BD10 --kid 01981F4A20999F62AF99988107BAF6CA --kid_idx 1 --kic_idx 1 --algo-crypt triple_des_cbc2 --algo-auth triple_des_cbc2 --tar B00010 --apdu A0A40000027F20 --apdu A0C0000016
+   2026-02-26 17:13:56 INFO Connecting to localhost:2775...
+   2026-02-26 17:13:56 INFO C-APDU sending: a0a40000027f20a0c0000016...
+   2026-02-26 17:13:56 INFO SMS-TPDU sending: 02700000281506191515b00010da1d6cbbd0d11ce4330d844c7408340943e843f67a6d7b0674730881605fd62d...
+   2026-02-26 17:13:56 INFO SMS-TPDU sent, waiting for response...
+   2026-02-26 17:13:56 INFO SMS-TPDU received: 027100002c12b000107ddf58d1780f771638b3975759f4296cf5c31efc87a16a1b61921426baa16da1b5ba1a9951d59a39
+   2026-02-26 17:13:56 INFO SMS-TPDU decoded: (Container(rpl=44, rhl=18, tar=b'\xb0\x00\x10', cntr=b'\x00\x00\x00\x00\x00', pcntr=0, response_status=uEnumIntegerString.new(0, 'por_ok'), cc_rc=b'\x8f\xea\xf5.\xf4\x0e\xc2\x14', secured_data=b'\x02\x90\x00\x00\x00\xff\xff\x7f \x02\x00\x00\x00\x00\x00\t\xb1\x065\x04\x00\x83\x8a\x83\x8a'), Container(number_of_commands=2, last_status_word=u'9000', last_response_data=u'0000ffff7f2002000000000009b106350400838a838a'))
+   2026-02-26 17:13:56 INFO R-APDU received: 0000ffff7f2002000000000009b106350400838a838a 9000
+   0000ffff7f2002000000000009b106350400838a838a 9000
+   2026-02-26 17:13:56 INFO Disconnecting...
+
+The result we see is the select response of DF.GSM and a status word indicating that the last command has been
+processed normally.
+
+As we can see, this mechanism now allows us to perform small administrative tasks remotely. We can read the contents of
+files remotely or make changes to files. Depending on the changes we make, there may be security issues arising from
+replay attacks. With the commandline above, the communication is encrypted and protected by a cryptographic checksum,
+so an adversary can neither read, nor alter the message. However, an adversary could still replay an intercepted
+message and the `SIM` would happily execute the contained APDUs again.
+
+To prevent this, we may include a replay protection counter within the message. In this case, the MSL indicates that a
+replay protection counter is not required. However, to extended the security of our messages, we may chose to use a
+counter anyway. In the following example, we will encode a counter value of 100. We will instruct the `SIM` to make sure
+that the value we send is higher than the counter value that is currently stored in the `SIM`.
+
+To add a replay connection counter we add the commandline arguments `--cntr-req` to set the counter requirement and
+`--cntr` to pass the counter value.
+
+::
+
+   $ PYTHONPATH=./ ./contrib/smpp-ota-tool.py --kic F09C43EE1A0391665CC9F05AF4E0BD10 --kid 01981F4A20999F62AF99988107BAF6CA --kid_idx 1 --kic_idx 1 --algo-crypt triple_des_cbc2 --algo-auth triple_des_cbc2 --tar B00010 --apdu A0A40000027F20 --apdu A0C0000016 --cntr-req counter_must_be_higher --cntr 100
+   2026-02-26 17:16:39 INFO Connecting to localhost:2775...
+   2026-02-26 17:16:39 INFO C-APDU sending: a0a40000027f20a0c0000016...
+   2026-02-26 17:16:39 INFO SMS-TPDU sending: 02700000281516191515b000103a4f599e94f2b5dcfbbda984761b7977df6514c57a580fb4844787c436d2eade...
+   2026-02-26 17:16:39 INFO SMS-TPDU sent, waiting for response...
+   2026-02-26 17:16:39 INFO SMS-TPDU received: 027100002c12b0001049fb0315f6c6401b553867f412cefaf9355b38271178edb342a3bc9cc7e670cdc1f45eea6ffcbb39
+   2026-02-26 17:16:39 INFO SMS-TPDU decoded: (Container(rpl=44, rhl=18, tar=b'\xb0\x00\x10', cntr=b'\x00\x00\x00\x00d', pcntr=0, response_status=uEnumIntegerString.new(0, 'por_ok'), cc_rc=b'\xa9/\xc7\xc9\x00"\xab5', secured_data=b'\x02\x90\x00\x00\x00\xff\xff\x7f \x02\x00\x00\x00\x00\x00\t\xb1\x065\x04\x00\x83\x8a\x83\x8a'), Container(number_of_commands=2, last_status_word=u'9000', last_response_data=u'0000ffff7f2002000000000009b106350400838a838a'))
+   2026-02-26 17:16:39 INFO R-APDU received: 0000ffff7f2002000000000009b106350400838a838a 9000
+   0000ffff7f2002000000000009b106350400838a838a 9000
+   2026-02-26 17:16:39 INFO Disconnecting...
+
+The `SIM` has accepted the message. The message got processed and the `SIM` has set its internal to 100. As an experiment,
+we may try to re-use the counter value:
+
+::
+
+   $ PYTHONPATH=./ ./contrib/smpp-ota-tool.py --kic F09C43EE1A0391665CC9F05AF4E0BD10 --kid 01981F4A20999F62AF99988107BAF6CA --kid_idx 1 --kic_idx 1 --algo-crypt triple_des_cbc2 --algo-auth triple_des_cbc2 --tar B00010 --apdu A0A40000027F20 --apdu A0C0000016 --cntr-req counter_must_be_higher --cntr 100
+   2026-02-26 17:16:43 INFO Connecting to localhost:2775...
+   2026-02-26 17:16:43 INFO C-APDU sending: a0a40000027f20a0c0000016...
+   2026-02-26 17:16:43 INFO SMS-TPDU sending: 02700000281516191515b000103a4f599e94f2b5dcfbbda984761b7977df6514c57a580fb4844787c436d2eade...
+   2026-02-26 17:16:43 INFO SMS-TPDU sent, waiting for response...
+   2026-02-26 17:16:43 INFO SMS-TPDU received: 027100000b0ab0001000000000000006
+   2026-02-26 17:16:43 INFO SMS-TPDU decoded: (Container(rpl=11, rhl=10, tar=b'\xb0\x00\x10', cntr=b'\x00\x00\x00\x00\x00', pcntr=0, response_status=uEnumIntegerString.new(6, 'undefined_security_error'), cc_rc=b'', secured_data=b''), None)
+   Traceback (most recent call last):
+   File "/home/user/work/git_master/pysim/./contrib/smpp-ota-tool.py", line 238, in <module>
+    resp, sw = smpp_handler.transceive_apdu(apdu, opts.src_addr, opts.dest_addr, opts.timeout)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   File "/home/user/work/git_master/pysim/./contrib/smpp-ota-tool.py", line 162, in transceive_apdu
+     raise ValueError("Response does not contain any last_response_data, no R-APDU received!")
+   ValueError: Response does not contain any last_response_data, no R-APDU received!
+   2026-02-26 17:16:43 INFO Disconnecting...
+
+As we can see, the `SIM` has rejected the message with an `undefined_security_error`. The replay-protection-counter
+ensures that a message can only be sent once.
+
+.. note:: The replay-protection-counter is implemented as a 5 byte integer value (see also ETSI TS 102 225, Table 3).
+	  When the counter has reached its maximum, it will not overflow nor can it be reset.
+
+smpp-ota-tool syntax
+~~~~~~~~~~~~~~~~~~~~
+
+.. argparse::
+   :module: contrib.smpp-ota-tool
+   :func: option_parser
+   :prog: contrib/smpp-ota-tool.py

docs/smpp2sim.rst

@@ -0,0 +1,59 @@
+pySim-smpp2sim
+==============
+
+This is a program to emulate the entire communication path SMSC-CN-RAN-ME
+that is usually between an OTA backend and the SIM card.  This allows
+to play with SIM OTA technology without using a mobile network or even
+a mobile phone.
+
+An external application can act as SMPP ESME and must encode (and
+encrypt/sign) the OTA SMS and submit them via SMPP to this program, just
+like it would submit it normally to a SMSC (SMS Service Centre).  The
+program then re-formats the SMPP-SUBMIT into a SMS DELIVER TPDU and
+passes it via an ENVELOPE APDU to the SIM card that is locally inserted
+into a smart card reader.
+
+The path from SIM to external OTA application works the opposite way.
+
+The default SMPP system_id is `test`.  Likewise, the default SMPP
+password is `test`
+
+Running pySim-smpp2sim
+----------------------
+
+The command accepts the same command line arguments for smart card interface device selection as pySim-shell,
+as well as a few SMPP specific arguments:
+
+.. argparse::
+   :module: pySim-smpp2sim
+   :func: option_parser
+   :prog: pySim-smpp2sim.py
+
+
+Example execution with sample output
+------------------------------------
+
+So for a simple system with a single PC/SC device, you would typically use something like
+`./pySim-smpp2sim.py -p0` to start the program.  You will see output like this at start-up
+::
+
+  Using reader PCSC[HID Global OMNIKEY 3x21 Smart Card Reader [OMNIKEY 3x21 Smart Card Reader] 00 00]
+  INFO     root: Binding Virtual SMSC to TCP Port 2775 at ::
+
+The application has hence bound to local TCP port 2775 and expects your SMS-sending applications to send their
+SMS there.  Once you do, you will see log output like below:
+::
+
+  WARNING  smpp.twisted.protocol: SMPP connection established from ::ffff:127.0.0.1 to port 2775
+  INFO     smpp.twisted.server: Added CommandId.bind_transceiver bind for 'test'. Active binds: CommandId.bind_transceiver: 1, CommandId.bind_transmitter: 0, CommandId.bind_receiver: 0. Max binds: 2
+  INFO     smpp.twisted.protocol: Bind request succeeded for test. 1 active binds
+
+And once your external program is sending SMS to the simulated SMSC, it will log something like
+::
+
+  INFO     root: SMS_DELIVER(MTI=0, MMS=False, LP=False, RP=False, UDHI=True, SRI=False, OA=AddressField(TON=international, NPI=unknown, 12), PID=7f, DCS=f6, SCTS=bytearray(b'"pR\x00\x00\x00\x00'), UDL=45, UD=b"\x02p\x00\x00(\x15\x16\x19\x12\x12\xb0\x00\x01'\xfa(\xa5\xba\xc6\x9d<^\x9d\xf2\xc7\x15]\xfd\xdeD\x9c\x82k#b\x15Ve0x{0\xe8\xbe]")
+  SMSPPDownload(DeviceIdentities({'source_dev_id': 'network', 'dest_dev_id': 'uicc'}),Address({'ton_npi': 0, 'call_number': '0123456'}),SMS_TPDU({'tpdu': '400290217ff6227052000000002d02700000281516191212b0000127fa28a5bac69d3c5e9df2c7155dfdde449c826b236215566530787b30e8be5d'}))
+  INFO     root: ENVELOPE: d147820283818604001032548b3b400290217ff6227052000000002d02700000281516191212b0000127fa28a5bac69d3c5e9df2c7155dfdde449c826b236215566530787b30e8be5d
+  INFO     root: SW 9000: 027100002412b000019a551bb7c28183652de0ace6170d0e563c5e949a3ba56747fe4c1dbbef16642c
+
+.. note:: for sending OTA SMS messages :ref:`smpp-ota-tool` may be used.

docs/suci-keytool.rst

@@ -0,0 +1,58 @@
+suci-keytool
+============
+
+Subscriber concealment is an important feature of the 5G SA architecture: It avoids the many privacy
+issues associated with having a permanent identifier (SUPI, traditionally the IMSI) transmitted in plain text
+over the air interface.  Using SUCI solves this issue not just for the air interface; it even ensures the SUPI/IMSI
+is not known to the visited network (VPLMN) at all.
+
+In principle, the SUCI mechanism works by encrypting the SUPI by asymmetric (public key) cryptography:
+Only the HPLMN is in possession of the private key and hence can decrypt the SUCI to the SUPI, while
+each subscriber has the public key in order to encrypt their SUPI into the SUCI.  In reality, the
+details are more complex, as there are ephemeral keys and cryptographic MAC involved.
+
+In any case, in order to operate a SUCI-enabled 5G SA network, you will have to
+
+#. generate a ECC key pair of public + private key
+#. deploy the public key on your USIMs
+#. deploy the private key on your 5GC, specifically the UDM function
+
+pysim contains (in its `contrib` directory) a small utility program that can make it easy to generate
+such keys: `suci-keytool.py`
+
+Generating keys
+~~~~~~~~~~~~~~~
+
+Example: Generating a *secp256r1* ECC public key pair and storing it to `/tmp/suci.key`:
+::
+
+        $ ./contrib/suci-keytool.py --key-file /tmp/suci.key generate-key --curve secp256r1
+
+Dumping public keys
+~~~~~~~~~~~~~~~~~~~
+
+In order to store the key to SIM cards as part of `ADF.USIM/DF.5GS/EF.SUCI_Calc_Info`, you will need
+a hexadecimal representation of the public key.  You can achieve that using the `dump-pub-key` operation
+of suci-keytool:
+
+Example: Dumping the public key part from a previously generated key file:
+::
+
+        $ ./contrib/suci-keytool.py --key-file /tmp/suci.key dump-pub-key
+        0473152f32523725f5175d255da2bd909de97b1d06449a9277bc629fe42112f8643e6b69aa6dce6c86714ccbe6f2e0f4f4898d102e2b3f0c18ce26626f052539bb
+
+If you want the point-compressed representation, you can use the `--compressed` option:
+::
+
+        $ ./contrib/suci-keytool.py --key-file /tmp/suci.key dump-pub-key --compressed
+        0373152f32523725f5175d255da2bd909de97b1d06449a9277bc629fe42112f864
+
+
+
+suci-keytool syntax
+~~~~~~~~~~~~~~~~~~~
+
+.. argparse::
+   :module: contrib.suci-keytool
+   :func: arg_parser
+   :prog: contrib/suci-keytool.py

docs/suci-tutorial.rst

@@ -30,7 +30,7 @@ This guide covers the basic workflow of provisioning SIM cards with the 5G SUCI
 
 For specific information on sysmocom SIM cards, refer to
 
-* the `sysmoISIM-SJA5 User Manual <https://sysmocom.de/manuals/sysmoisim-sja5-manual.pdf>`__ for the curent
+* the `sysmoISIM-SJA5 User Manual <https://sysmocom.de/manuals/sysmoisim-sja5-manual.pdf>`__ for the current
   sysmoISIM-SJA5 product
 * the `sysmoISIM-SJA2 User Manual <https://sysmocom.de/manuals/sysmousim-manual.pdf>`__ for the older
   sysmoISIM-SJA2 product

docs/wsrc.rst

@@ -1,57 +0,0 @@
-WebSocket Remote Card (WSRC)
-============================
-
-WSRC (*Web Socket Remote Card*) is a mechanism by which card readers can be made remotely available
-via a computer network.  The transport mechanism is (as the name implies) a WebSocket.  This transport
-method was chosen to be as firewall/NAT friendly as possible.
-
-WSRC Network Architecture
--------------------------
-
-In a WSRC network, there are three major elements:
-
-* The **WSRC Card Client** which exposes a locally attached smart card (usually via a Smart Card Reader)
-  to a remote *WSRC Server*
-* The **WSRC Server** manges incoming connections from both *WSRC Card Clients* as well as *WSRC User Clients*
-* The **WSRC User Client** is a user application, like for example pySim-shell, which is accessing a remote
-  card by connecting to the *WSRC Server* which relays the information to the selected *WSRC Card Client*
-
-WSRC Protocol
--------------
-
-The WSRC protocl consits of JSON objects being sent over a websocket.  The websocket communication itself
-is based on HTTP and should usually operate via TLS for security reasons.
-
-The detailed protocol is currently still WIP. The plan is to document it here.
-
-
-pySim implementations
----------------------
-
-
-wsrc_server
-~~~~~~~~~~~~~~~~
-.. argparse::
-   :filename: ../contrib/wsrc_server.py
-   :func: parser
-   :prog: contrib/wsrc_server.py
-
-
-wsrc_card_client
-~~~~~~~~~~~~~~~~
-.. argparse::
-   :filename: ../contrib/wsrc_card_client.py
-   :func: parser
-   :prog: contrib/wsrc_card_client.py
-
-pySim-shell
-~~~~~~~~~~~
-
-pySim-shell can talk to a remote card via WSRC if you use the *wsrc transport*, for example like this:
-
-::
-
-    ./pySim-shell.py --wsrc-eid 89882119900000000000000000007280 --wsrc-serer-url ws://localhost:4220/
-
-
-You can specify `--wsrc-eid` or `--wsrc-iccid` to identify the remote eUICC or UICC you would like to select.

osmo-smdpp.py

@@ -17,28 +17,124 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import json
-import sys
-import argparse
-import uuid
-import os
-import functools
-from typing import Optional, Dict, List
-from pprint import pprint as pp
-
-import base64
-from base64 import b64decode
-from klein import Klein
-from twisted.web.iweb import IRequest
+# asn1tools issue https://github.com/eerimoq/asn1tools/issues/194
+# must be first here
 import asn1tools
+import asn1tools.codecs.ber
+import asn1tools.codecs.der
+# do not move the code
+def fix_asn1_oid_decoding():
+    fix_asn1_schema = """
+    TestModule DEFINITIONS ::= BEGIN
+        TestOid ::= SEQUENCE {
+            oid OBJECT IDENTIFIER
+        }
+    END
+    """
 
-from osmocom.utils import h2b, b2h, swap_nibbles
+    fix_asn1_asn1 = asn1tools.compile_string(fix_asn1_schema, codec='der')
+    fix_asn1_oid_string = '2.999.10'
+    fix_asn1_encoded = fix_asn1_asn1.encode('TestOid', {'oid': fix_asn1_oid_string})
+    fix_asn1_decoded = fix_asn1_asn1.decode('TestOid', fix_asn1_encoded)
 
-import pySim.esim.rsp as rsp
-from pySim.esim import saip, PMO
-from pySim.esim.es8p import *
-from pySim.esim.x509_cert import oid, cert_policy_has_oid, cert_get_auth_key_id
-from pySim.esim.x509_cert import CertAndPrivkey, CertificateSet, cert_get_subject_key_id, VerifyError
+    if (fix_asn1_decoded['oid'] != fix_asn1_oid_string):
+        # ASN.1 OBJECT IDENTIFIER Decoding Issue:
+        #
+        # In ASN.1 BER/DER encoding, the first two arcs of an OBJECT IDENTIFIER are
+        # combined into a single value: (40 * arc0) + arc1. This is encoded as a base-128
+        # variable-length quantity (and commonly known as VLQ or base-128 encoding)
+        # as specified in ITU-T X.690 §8.19, it can span multiple bytes if
+        # the value is large.
+        #
+        # For arc0 = 0 or 1, arc1 must be in [0, 39]. For arc0 = 2, arc1 can be any non-negative integer.
+        # All subsequent arcs (arc2, arc3, ...) are each encoded as a separate base-128 VLQ.
+        #
+        # The decoding bug occurs when the decoder does not properly split the first
+        # subidentifier for arc0 = 2 and arc1 >= 40. Instead of decoding:
+        #   - arc0 = 2
+        #   - arc1 = (first_subidentifier - 80)
+        # it may incorrectly interpret the first_subidentifier as arc0 = (first_subidentifier // 40),
+        # arc1 = (first_subidentifier % 40), which is only valid for arc1 < 40.
+        #
+        # This patch handles it properly for all valid OBJECT IDENTIFIERs
+        # with large second arcs, by applying the ASN.1 rules:
+        #   - if first_subidentifier < 40: arc0 = 0, arc1 = first_subidentifier
+        #   - elif first_subidentifier < 80: arc0 = 1, arc1 = first_subidentifier - 40
+        #   - else: arc0 = 2, arc1 = first_subidentifier - 80
+        #
+        # This problem is not uncommon, see for example https://github.com/randombit/botan/issues/4023
+
+        def fixed_decode_object_identifier(data, offset, end_offset):
+            """Decode ASN.1 OBJECT IDENTIFIER from bytes to dotted string, fixing large second arc handling."""
+            def read_subidentifier(data, offset):
+                value = 0
+                while True:
+                    b = data[offset]
+                    value = (value << 7) | (b & 0x7F)
+                    offset += 1
+                    if not (b & 0x80):
+                        break
+                return value, offset
+
+            subid, offset = read_subidentifier(data, offset)
+            if subid < 40:
+                first = 0
+                second = subid
+            elif subid < 80:
+                first = 1
+                second = subid - 40
+            else:
+                first = 2
+                second = subid - 80
+            arcs = [first, second]
+
+            while offset < end_offset:
+                subid, offset = read_subidentifier(data, offset)
+                arcs.append(subid)
+
+            return '.'.join(str(x) for x in arcs)
+
+        asn1tools.codecs.ber.decode_object_identifier = fixed_decode_object_identifier
+        asn1tools.codecs.der.decode_object_identifier = fixed_decode_object_identifier
+
+        # test our patch
+        asn1 = asn1tools.compile_string(fix_asn1_schema, codec='der')
+        decoded = asn1.decode('TestOid', fix_asn1_encoded)['oid']
+        assert fix_asn1_oid_string == str(decoded)
+
+fix_asn1_oid_decoding()
+
+from cryptography.hazmat.primitives.asymmetric.utils import encode_dss_signature # noqa: E402
+from cryptography import x509 # noqa: E402
+from cryptography.exceptions import InvalidSignature # noqa: E402
+from cryptography.hazmat.primitives import hashes # noqa: E402
+from cryptography.hazmat.primitives.asymmetric import ec, dh # noqa: E402
+from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat, PrivateFormat, NoEncryption, ParameterFormat # noqa: E402
+from pathlib import Path # noqa: E402
+import json # noqa: E402
+import sys # noqa: E402
+import argparse # noqa: E402
+import uuid # noqa: E402
+import os # noqa: E402
+import functools # noqa: E402
+from typing import Optional, Dict, List # noqa: E402
+from pprint import pprint as pp # noqa: E402
+
+import base64 # noqa: E402
+from base64 import b64decode # noqa: E402
+from klein import Klein # noqa: E402
+from twisted.web.iweb import IRequest # noqa: E402
+
+from osmocom.utils import h2b, b2h, swap_nibbles # noqa: E402
+
+import pySim.esim.rsp as rsp # noqa: E402
+from pySim.esim import saip, PMO # noqa: E402
+from pySim.esim.es8p import ProfileMetadata,UnprotectedProfilePackage,ProtectedProfilePackage,BoundProfilePackage,BspInstance # noqa: E402
+from pySim.esim.x509_cert import oid, cert_policy_has_oid, cert_get_auth_key_id # noqa: E402
+from pySim.esim.x509_cert import CertAndPrivkey, CertificateSet, cert_get_subject_key_id, VerifyError # noqa: E402
+
+import logging # noqa: E402
+logger = logging.getLogger(__name__)
 
 # HACK: make this configurable
 DATA_DIR = './smdpp-data'
@@ -54,6 +150,173 @@ def set_headers(request: IRequest):
     request.setHeader('Content-Type', 'application/json;charset=UTF-8')
     request.setHeader('X-Admin-Protocol', 'gsma/rsp/v2.1.0')
 
+def validate_request_headers(request: IRequest):
+    """Validate mandatory HTTP headers according to SGP.22."""
+    content_type = request.getHeader('Content-Type')
+    if not content_type or not content_type.startswith('application/json'):
+        raise ApiError('1.2.1', '2.1', 'Invalid Content-Type header')
+
+    admin_protocol = request.getHeader('X-Admin-Protocol')
+    if admin_protocol and not admin_protocol.startswith('gsma/rsp/v'):
+        raise ApiError('1.2.2', '2.1', 'Unsupported X-Admin-Protocol version')
+
+def get_eum_certificate_variant(eum_cert) -> str:
+    """Determine EUM certificate variant by checking Certificate Policies extension.
+    Returns 'O' for old variant, or 'NEW' for Ov3/A/B/C variants."""
+
+    try:
+        cert_policies_ext = eum_cert.extensions.get_extension_for_oid(
+            x509.oid.ExtensionOID.CERTIFICATE_POLICIES
+        )
+
+        for policy in cert_policies_ext.value:
+            policy_oid = policy.policy_identifier.dotted_string
+            logger.debug(f"Found certificate policy: {policy_oid}")
+
+            if policy_oid == '2.23.146.1.2.1.2':
+                logger.debug("Detected EUM certificate variant: O (old)")
+                return 'O'
+            elif policy_oid == '2.23.146.1.2.1.0.0.0':
+                logger.debug("Detected EUM certificate variant: Ov3/A/B/C (new)")
+                return 'NEW'
+    except x509.ExtensionNotFound:
+        logger.debug("No Certificate Policies extension found")
+    except Exception as e:
+        logger.debug(f"Error checking certificate policies: {e}")
+
+def parse_permitted_eins_from_cert(eum_cert) -> List[str]:
+    """Extract permitted IINs from EUM certificate using the appropriate method
+    based on certificate variant (O vs Ov3/A/B/C).
+    Returns list of permitted IINs (basically prefixes that valid EIDs must start with)."""
+
+    # Determine certificate variant first
+    cert_variant = get_eum_certificate_variant(eum_cert)
+    permitted_iins = []
+
+    if cert_variant == 'O':
+        # Old variant - use nameConstraints extension
+        permitted_iins.extend(_parse_name_constraints_eins(eum_cert))
+
+    else:
+        # New variants (Ov3, A, B, C) - use GSMA permittedEins extension
+        permitted_iins.extend(_parse_gsma_permitted_eins(eum_cert))
+
+    unique_iins = list(set(permitted_iins))
+
+    logger.debug(f"Total unique permitted IINs found: {len(unique_iins)}")
+    return unique_iins
+
+def _parse_gsma_permitted_eins(eum_cert) -> List[str]:
+    """Parse the GSMA permittedEins extension using correct ASN.1 structure.
+    PermittedEins ::= SEQUENCE OF PrintableString
+    Each string contains an IIN (Issuer Identification Number) - a prefix of valid EIDs."""
+    permitted_iins = []
+
+    try:
+        permitted_eins_oid = x509.ObjectIdentifier('2.23.146.1.2.2.0')  # sgp26: 2.23.146.1.2.2.0 = ASN1:SEQUENCE:permittedEins
+
+        for ext in eum_cert.extensions:
+            if ext.oid == permitted_eins_oid:
+                logger.debug(f"Found GSMA permittedEins extension: {ext.oid}")
+
+                # Get the DER-encoded extension value
+                ext_der = ext.value.value if hasattr(ext.value, 'value') else ext.value
+
+                if isinstance(ext_der, bytes):
+                    try:
+                        permitted_eins_schema = """
+                        PermittedEins DEFINITIONS ::= BEGIN
+                            PermittedEins ::= SEQUENCE OF PrintableString
+                        END
+                        """
+                        decoder = asn1tools.compile_string(permitted_eins_schema)
+                        decoded_strings = decoder.decode('PermittedEins', ext_der)
+
+                        for iin_string in decoded_strings:
+                            # Each string contains an IIN -> prefix of euicc EID
+                            iin_clean = iin_string.strip().upper()
+
+                            # IINs is 8 chars per sgp22, var len according to sgp29, fortunately we don't care
+                            if (len(iin_clean) == 8 and
+                                all(c in '0123456789ABCDEF' for c in iin_clean) and
+                                    len(iin_clean) % 2 == 0):
+                                permitted_iins.append(iin_clean)
+                                logger.debug(f"Found permitted IIN (GSMA): {iin_clean}")
+                            else:
+                                logger.debug(f"Invalid IIN format: {iin_string} (cleaned: {iin_clean})")
+                    except Exception as e:
+                        logger.debug(f"Error parsing GSMA permittedEins extension: {e}")
+
+    except Exception as e:
+        logger.debug(f"Error accessing GSMA certificate extensions: {e}")
+
+    return permitted_iins
+
+
+def _parse_name_constraints_eins(eum_cert) -> List[str]:
+    """Parse permitted IINs from nameConstraints extension (variant O)."""
+    permitted_iins = []
+
+    try:
+        # Look for nameConstraints extension
+        name_constraints_ext = eum_cert.extensions.get_extension_for_oid(
+            x509.oid.ExtensionOID.NAME_CONSTRAINTS
+        )
+
+        name_constraints = name_constraints_ext.value
+
+        # Check permittedSubtrees for IIN constraints
+        if name_constraints.permitted_subtrees:
+            for subtree in name_constraints.permitted_subtrees:
+
+                if isinstance(subtree, x509.DirectoryName):
+                    for attribute in subtree.value:
+                        # IINs for O in serialNumber
+                        if attribute.oid == x509.oid.NameOID.SERIAL_NUMBER:
+                            serial_value = attribute.value.upper()
+                            # sgp22 8, sgp29 var len, fortunately we don't care
+                            if (len(serial_value) == 8 and
+                                all(c in '0123456789ABCDEF' for c in serial_value) and
+                                    len(serial_value) % 2 == 0):
+                                permitted_iins.append(serial_value)
+                                logger.debug(f"Found permitted IIN (nameConstraints/DN): {serial_value}")
+
+    except x509.ExtensionNotFound:
+        logger.debug("No nameConstraints extension found")
+    except Exception as e:
+        logger.debug(f"Error parsing nameConstraints: {e}")
+
+    return permitted_iins
+
+
+def validate_eid_range(eid: str, eum_cert) -> bool:
+    """Validate that EID is within the permitted EINs of the EUM certificate."""
+    if not eid or len(eid) != 32:
+        logger.debug(f"Invalid EID format: {eid}")
+        return False
+
+    try:
+        permitted_eins = parse_permitted_eins_from_cert(eum_cert)
+
+        if not permitted_eins:
+            logger.debug("Warning: No permitted EINs found in EUM certificate")
+            return False
+
+        eid_normalized = eid.upper()
+        logger.debug(f"Validating EID {eid_normalized} against {len(permitted_eins)} permitted EINs")
+
+        for permitted_ein in permitted_eins:
+                if eid_normalized.startswith(permitted_ein):
+                    logger.debug(f"EID {eid_normalized} matches permitted EIN {permitted_ein}")
+                    return True
+
+        logger.debug(f"EID {eid_normalized} is not in any permitted EIN list")
+        return False
+
+    except Exception as e:
+        logger.debug(f"Error validating EID: {e}")
+        return False
+
 def build_status_code(subject_code: str, reason_code: str, subject_id: Optional[str], message: Optional[str]) -> Dict:
     r = {'subjectCode': subject_code, 'reasonCode': reason_code }
     if subject_id:
@@ -72,12 +335,6 @@ def build_resp_header(js: dict, status: str = 'Executed-Success', status_code_da
     if status_code_data:
         js['header']['functionExecutionStatus']['statusCodeData'] = status_code_data
 
-from cryptography.hazmat.primitives.asymmetric.utils import decode_dss_signature, encode_dss_signature
-from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat, PrivateFormat, NoEncryption
-from cryptography.hazmat.primitives.asymmetric import ec
-from cryptography.hazmat.primitives import hashes
-from cryptography.exceptions import InvalidSignature
-from cryptography import x509
 
 def ecdsa_tr03111_to_dss(sig: bytes) -> bytes:
     """convert an ECDSA signature from BSI TR-03111 format to DER: first get long integers; then encode those."""
@@ -125,22 +382,36 @@ class SmDppHttpServer:
     def ci_get_cert_for_pkid(self, ci_pkid: bytes) -> Optional[x509.Certificate]:
         """Find CI certificate for given key identifier."""
         for cert in self.ci_certs:
-            print("cert: %s" % cert)
+            logger.debug("cert: %s" % cert)
             subject_exts = list(filter(lambda x: isinstance(x.value, x509.SubjectKeyIdentifier), cert.extensions))
-            print(subject_exts)
+            logger.debug(subject_exts)
             subject_pkid = subject_exts[0].value
-            print(subject_pkid)
+            logger.debug(subject_pkid)
             if subject_pkid and subject_pkid.key_identifier == ci_pkid:
                 return cert
         return None
 
-    def __init__(self, server_hostname: str, ci_certs_path: str, use_brainpool: bool = False):
+    def validate_certificate_chain_for_verification(self, euicc_ci_pkid_list: List[bytes]) -> bool:
+        """Validate that SM-DP+ has valid certificate chains for the given CI PKIDs."""
+        for ci_pkid in euicc_ci_pkid_list:
+            ci_cert = self.ci_get_cert_for_pkid(ci_pkid)
+            if ci_cert:
+                # Check if our DPauth certificate chains to this CI
+                try:
+                    cs = CertificateSet(ci_cert)
+                    cs.verify_cert_chain(self.dp_auth.cert)
+                    return True
+                except VerifyError:
+                    continue
+        return False
+
+    def __init__(self, server_hostname: str, ci_certs_path: str, common_cert_path: str, use_brainpool: bool = False, in_memory: bool = False):
         self.server_hostname = server_hostname
         self.upp_dir = os.path.realpath(os.path.join(DATA_DIR, 'upp'))
         self.ci_certs = self.load_certs_from_path(ci_certs_path)
         # load DPauth cert + key
         self.dp_auth = CertAndPrivkey(oid.id_rspRole_dp_auth_v2)
-        cert_dir = os.path.join(DATA_DIR, 'certs')
+        cert_dir = common_cert_path
         if use_brainpool:
             self.dp_auth.cert_from_der_file(os.path.join(cert_dir, 'DPauth', 'CERT_S_SM_DPauth_ECDSA_BRP.der'))
             self.dp_auth.privkey_from_pem_file(os.path.join(cert_dir, 'DPauth', 'SK_S_SM_DPauth_ECDSA_BRP.pem'))
@@ -155,7 +426,15 @@ class SmDppHttpServer:
         else:
             self.dp_pb.cert_from_der_file(os.path.join(cert_dir, 'DPpb', 'CERT_S_SM_DPpb_ECDSA_NIST.der'))
             self.dp_pb.privkey_from_pem_file(os.path.join(cert_dir, 'DPpb', 'SK_S_SM_DPpb_ECDSA_NIST.pem'))
-        self.rss = rsp.RspSessionStore(os.path.join(DATA_DIR, "sm-dp-sessions"))
+        if in_memory:
+            self.rss = rsp.RspSessionStore(in_memory=True)
+            logger.info("Using in-memory session storage")
+        else:
+            # Use different session database files for BRP and NIST to avoid file locking during concurrent runs
+            session_db_suffix = "BRP" if use_brainpool else "NIST"
+            db_path = os.path.join(DATA_DIR, f"sm-dp-sessions-{session_db_suffix}")
+            self.rss = rsp.RspSessionStore(filename=db_path, in_memory=False)
+            logger.info(f"Using file-based session storage: {db_path}")
 
     @app.handle_errors(ApiError)
     def handle_apierror(self, request: IRequest, failure):
@@ -179,11 +458,10 @@ class SmDppHttpServer:
         functionality, such as JSON decoding/encoding and debug-printing."""
         @functools.wraps(func)
         def _api_wrapper(self, request: IRequest):
-            # TODO: evaluate User-Agent + X-Admin-Protocol header
-            # TODO: reject any non-JSON Content-type
+            validate_request_headers(request)
 
             content = json.loads(request.content.read())
-            print("Rx JSON: %s" % json.dumps(content))
+            logger.debug("Rx JSON: %s" % json.dumps(content))
             set_headers(request)
 
             output = func(self, request, content)
@@ -191,7 +469,7 @@ class SmDppHttpServer:
                 return ''
 
             build_resp_header(output)
-            print("Tx JSON: %s" % json.dumps(output))
+            logger.debug("Tx JSON: %s" % json.dumps(output))
             return json.dumps(output)
         return _api_wrapper
 
@@ -210,7 +488,7 @@ class SmDppHttpServer:
 
         euiccInfo1_bin = b64decode(content['euiccInfo1'])
         euiccInfo1 = rsp.asn1.decode('EUICCInfo1', euiccInfo1_bin)
-        print("Rx euiccInfo1: %s" % euiccInfo1)
+        logger.debug("Rx euiccInfo1: %s" % euiccInfo1)
         #euiccInfo1['svn']
 
         # TODO: If euiccCiPKIdListForSigningV3 is present ...
@@ -218,6 +496,12 @@ class SmDppHttpServer:
         pkid_list = euiccInfo1['euiccCiPKIdListForSigning']
         if 'euiccCiPKIdListForSigningV3' in euiccInfo1:
             pkid_list = pkid_list + euiccInfo1['euiccCiPKIdListForSigningV3']
+
+        # Validate that SM-DP+ supports certificate chains for verification
+        verification_pkid_list = euiccInfo1.get('euiccCiPKIdListForVerification', [])
+        if verification_pkid_list and not self.validate_certificate_chain_for_verification(verification_pkid_list):
+            raise ApiError('8.8.4', '3.7', 'The SM-DP+ has no CERT.DPauth.SIG which chains to one of the eSIM CA Root CA Certificate with a Public Key supported by the eUICC')
+
         # verify it supports one of the keys indicated by euiccCiPKIdListForSigning
         ci_cert = None
         for x in pkid_list:
@@ -232,13 +516,6 @@ class SmDppHttpServer:
         if not ci_cert:
            raise ApiError('8.8.2', '3.1', 'None of the proposed Public Key Identifiers is supported by the SM-DP+')
 
-        # TODO: Determine the set of CERT.DPauth.SIG that satisfy the following criteria:
-        # * Part of a certificate chain ending at one of the eSIM CA RootCA Certificate, whose Public Keys is
-        #   supported by the eUICC (indicated by euiccCiPKIdListForVerification).
-        # * Using a certificate chain that the eUICC and the LPA both support:
-        #euiccInfo1['euiccCiPKIdListForVerification']
-        #   raise ApiError('8.8.4', '3.7', 'The SM-DP+ has no CERT.DPauth.SIG which chains to one of the eSIM CA Root CA CErtificate with a Public Key supported by the eUICC')
-
         # Generate a TransactionID which is used to identify the ongoing RSP session. The TransactionID
         # SHALL be unique within the scope and lifetime of each SM-DP+.
         transactionId = uuid.uuid4().hex.upper()
@@ -254,9 +531,9 @@ class SmDppHttpServer:
             'serverAddress': self.server_hostname,
             'serverChallenge': serverChallenge,
             }
-        print("Tx serverSigned1: %s" % serverSigned1)
+        logger.debug("Tx serverSigned1: %s" % serverSigned1)
         serverSigned1_bin = rsp.asn1.encode('ServerSigned1', serverSigned1)
-        print("Tx serverSigned1: %s" % rsp.asn1.decode('ServerSigned1', serverSigned1_bin))
+        logger.debug("Tx serverSigned1: %s" % rsp.asn1.decode('ServerSigned1', serverSigned1_bin))
         output = {}
         output['serverSigned1'] = b64encode2str(serverSigned1_bin)
 
@@ -285,7 +562,7 @@ class SmDppHttpServer:
 
         authenticateServerResp_bin = b64decode(content['authenticateServerResponse'])
         authenticateServerResp = rsp.asn1.decode('AuthenticateServerResponse', authenticateServerResp_bin)
-        print("Rx %s: %s" % authenticateServerResp)
+        logger.debug("Rx %s: %s" % authenticateServerResp)
         if authenticateServerResp[0] == 'authenticateResponseError':
             r_err = authenticateServerResp[1]
             #r_err['transactionId']
@@ -336,10 +613,12 @@ class SmDppHttpServer:
         if not self._ecdsa_verify(euicc_cert, euiccSignature1_bin, euiccSigned1_bin):
             raise ApiError('8.1', '6.1', 'Verification failed (euiccSignature1 over euiccSigned1)')
 
-        # TODO: verify EID of eUICC cert is  within permitted range of EUM cert
-
         ss.eid = ss.euicc_cert.subject.get_attributes_for_oid(x509.oid.NameOID.SERIAL_NUMBER)[0].value
-        print("EID (from eUICC cert): %s" % ss.eid)
+        logger.debug("EID (from eUICC cert): %s" % ss.eid)
+
+        # Verify EID is within permitted range of EUM certificate
+        if not validate_eid_range(ss.eid, eum_cert):
+            raise ApiError('8.1.4', '6.1', 'EID is not within the permitted range of the EUM certificate')
 
         # Verify that the serverChallenge attached to the ongoing RSP session matches the
         # serverChallenge returned by the eUICC. Otherwise, the SM-DP+ SHALL return a status code "eUICC -
@@ -350,6 +629,7 @@ class SmDppHttpServer:
         # If ctxParams1 contains a ctxParamsForCommonAuthentication data object, the SM-DP+ Shall [...]
         # TODO: We really do a very simplistic job here, this needs to be properly implemented later,
         # considering all the various cases, profile state, etc.
+        iccid_str = None
         if euiccSigned1['ctxParams1'][0] == 'ctxParamsForCommonAuthentication':
             cpca = euiccSigned1['ctxParams1'][1]
             matchingId = cpca.get('matchingId', None)
@@ -372,7 +652,7 @@ class SmDppHttpServer:
             # there's currently no other option in the ctxParams1 choice, so this cannot happen
             raise ApiError('1.3.1', '2.2', 'ctxParams1 missing mandatory ctxParamsForCommonAuthentication')
 
-        # FIXME: we actually want to perform the profile binding herr, and read the profile metadat from the profile
+        # FIXME: we actually want to perform the profile binding herr, and read the profile metadata from the profile
 
         # Put together profileMetadata + _bin
         ss.profileMetadata = ProfileMetadata(iccid_bin=h2b(swap_nibbles(iccid_str)), spn="OsmocomSPN", profile_name=matchingId)
@@ -414,7 +694,7 @@ class SmDppHttpServer:
 
         prepDownloadResp_bin = b64decode(content['prepareDownloadResponse'])
         prepDownloadResp = rsp.asn1.decode('PrepareDownloadResponse', prepDownloadResp_bin)
-        print("Rx %s: %s" % prepDownloadResp)
+        logger.debug("Rx %s: %s" % prepDownloadResp)
 
         if prepDownloadResp[0] == 'downloadResponseError':
             r_err = prepDownloadResp[1]
@@ -436,37 +716,37 @@ class SmDppHttpServer:
 
         # store otPK.EUICC.ECKA in session state
         ss.euicc_otpk = euiccSigned2['euiccOtpk']
-        print("euiccOtpk: %s" % (b2h(ss.euicc_otpk)))
+        logger.debug("euiccOtpk: %s" % (b2h(ss.euicc_otpk)))
 
         # Generate a one-time ECKA key pair (ot{PK,SK}.DP.ECKA) using the curve indicated by the Key Parameter
         # Reference value of CERT.DPpb.ECDDSA
-        print("curve = %s" % self.dp_pb.get_curve())
+        logger.debug("curve = %s" % self.dp_pb.get_curve())
         ss.smdp_ot = ec.generate_private_key(self.dp_pb.get_curve())
         # extract the public key in (hopefully) the right format for the ES8+ interface
         ss.smdp_otpk = ss.smdp_ot.public_key().public_bytes(Encoding.X962, PublicFormat.UncompressedPoint)
-        print("smdpOtpk: %s" % b2h(ss.smdp_otpk))
-        print("smdpOtsk: %s" % b2h(ss.smdp_ot.private_bytes(Encoding.DER, PrivateFormat.PKCS8, NoEncryption())))
+        logger.debug("smdpOtpk: %s" % b2h(ss.smdp_otpk))
+        logger.debug("smdpOtsk: %s" % b2h(ss.smdp_ot.private_bytes(Encoding.DER, PrivateFormat.PKCS8, NoEncryption())))
 
         ss.host_id = b'mahlzeit'
 
         # Generate Session Keys using the CRT, otPK.eUICC.ECKA and otSK.DP.ECKA according to annex G
         euicc_public_key = ec.EllipticCurvePublicKey.from_encoded_point(ss.smdp_ot.curve, ss.euicc_otpk)
         ss.shared_secret = ss.smdp_ot.exchange(ec.ECDH(), euicc_public_key)
-        print("shared_secret: %s" % b2h(ss.shared_secret))
+        logger.debug("shared_secret: %s" % b2h(ss.shared_secret))
 
         # TODO: Check if this order requires a Confirmation Code verification
 
         #  Perform actual protection + binding of profile package (or return  pre-bound one)
         with open(os.path.join(self.upp_dir, ss.matchingId)+'.der', 'rb') as f:
             upp = UnprotectedProfilePackage.from_der(f.read(), metadata=ss.profileMetadata)
-            # HACK: Use empty PPP as we're still debuggin the configureISDP step, and we want to avoid
+            # HACK: Use empty PPP as we're still debugging the configureISDP step, and we want to avoid
             # cluttering the log with stuff happening after the failure
             #upp = UnprotectedProfilePackage.from_der(b'', metadata=ss.profileMetadata)
         if False:
             # Use random keys
             bpp = BoundProfilePackage.from_upp(upp)
         else:
-            # Use sesssion keys
+            # Use session keys
             ppp = ProtectedProfilePackage.from_upp(upp, BspInstance(b'\x00'*16, b'\x11'*16, b'\x22'*16))
             bpp = BoundProfilePackage.from_ppp(ppp)
 
@@ -486,22 +766,22 @@ class SmDppHttpServer:
         request.setResponseCode(204)
         pendingNotification_bin = b64decode(content['pendingNotification'])
         pendingNotification = rsp.asn1.decode('PendingNotification', pendingNotification_bin)
-        print("Rx %s: %s" % pendingNotification)
+        logger.debug("Rx %s: %s" % pendingNotification)
         if pendingNotification[0] == 'profileInstallationResult':
             profileInstallRes = pendingNotification[1]
             pird = profileInstallRes['profileInstallationResultData']
             transactionId = b2h(pird['transactionId'])
             ss = self.rss.get(transactionId, None)
             if ss is None:
-                print("Unable to find session for transactionId")
-                return
+                logger.warning(f"Unable to find session for transactionId: {transactionId}")
+                return None  # Will return HTTP 204 with empty body
             profileInstallRes['euiccSignPIR']
             # TODO: use original data, don't re-encode?
             pird_bin = rsp.asn1.encode('ProfileInstallationResultData', pird)
             # verify eUICC signature
             if not self._ecdsa_verify(ss.euicc_cert, profileInstallRes['euiccSignPIR'], pird_bin):
                 raise Exception('ECDSA signature verification failed on notification')
-            print("Profile Installation Final Result: ", pird['finalResult'])
+            logger.debug("Profile Installation Final Result: %s", pird['finalResult'])
             # remove session state
             del self.rss[transactionId]
         elif pendingNotification[0] == 'otherSignedNotification':
@@ -526,7 +806,7 @@ class SmDppHttpServer:
             iccid = other_notif.get('iccid', None)
             if iccid:
                 iccid = swap_nibbles(b2h(iccid))
-            print("handleNotification: EID %s: %s of %s" % (eid, pmo, iccid))
+            logger.debug("handleNotification: EID %s: %s of %s" % (eid, pmo, iccid))
         else:
             raise ValueError(pendingNotification)
 
@@ -539,7 +819,7 @@ class SmDppHttpServer:
     @rsp_api_wrapper
     def cancelSession(self, request: IRequest, content: dict) -> dict:
         """See ES9+ CancelSession in SGP.22 Section 5.6.5"""
-        print("Rx JSON: %s" % content)
+        logger.debug("Rx JSON: %s" % content)
         transactionId = content['transactionId']
 
         # Verify that the received transactionId is known and relates to an ongoing RSP session
@@ -549,7 +829,7 @@ class SmDppHttpServer:
 
         cancelSessionResponse_bin = b64decode(content['cancelSessionResponse'])
         cancelSessionResponse = rsp.asn1.decode('CancelSessionResponse', cancelSessionResponse_bin)
-        print("Rx %s: %s" % cancelSessionResponse)
+        logger.debug("Rx %s: %s" % cancelSessionResponse)
 
         if cancelSessionResponse[0] == 'cancelSessionResponseError':
             # FIXME: print some error
@@ -581,15 +861,53 @@ class SmDppHttpServer:
 
 def main(argv):
     parser = argparse.ArgumentParser()
-    #parser.add_argument("-H", "--host", help="Host/IP to bind HTTP to", default="localhost")
-    #parser.add_argument("-p", "--port", help="TCP port to bind HTTP to", default=8000)
-    #parser.add_argument("-v", "--verbose", help="increase output verbosity", action='count', default=0)
-
+    parser.add_argument("-H", "--host", help="Host/IP to bind HTTP(S) to", default="localhost")
+    parser.add_argument("-p", "--port", help="TCP port to bind HTTP(S) to", default=443)
+    parser.add_argument("-c", "--certdir", help=f"cert subdir relative to {DATA_DIR}", default="certs")
+    parser.add_argument("-s", "--nossl", help="disable built in SSL/TLS support", action='store_true', default=False)
+    parser.add_argument("-v", "--verbose", help="dump more raw info", action='store_true', default=False)
+    parser.add_argument("-b", "--brainpool", help="Use Brainpool curves instead of NIST",
+                        action='store_true', default=False)
+    parser.add_argument("-m", "--in-memory", help="Use ephermal in-memory session storage (for concurrent runs)",
+                        action='store_true', default=False)
     args = parser.parse_args()
 
-    hs = SmDppHttpServer(HOSTNAME, os.path.join(DATA_DIR, 'certs', 'CertificateIssuer'), use_brainpool=False)
-    #hs.app.run(endpoint_description="ssl:port=8000:dhParameters=dh_param_2048.pem")
-    hs.app.run("localhost", 8000)
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARNING)
+
+    common_cert_path = os.path.join(DATA_DIR, args.certdir)
+    hs = SmDppHttpServer(server_hostname=HOSTNAME, ci_certs_path=os.path.join(common_cert_path, 'CertificateIssuer'), common_cert_path=common_cert_path, use_brainpool=args.brainpool)
+    if(args.nossl):
+        hs.app.run(args.host, args.port)
+    else:
+        curve_type = 'BRP' if args.brainpool else 'NIST'
+        cert_derpath = Path(common_cert_path) / 'DPtls' / f'CERT_S_SM_DP_TLS_{curve_type}.der'
+        cert_pempath = Path(common_cert_path) / 'DPtls' / f'CERT_S_SM_DP_TLS_{curve_type}.pem'
+        cert_skpath = Path(common_cert_path) / 'DPtls' / f'SK_S_SM_DP_TLS_{curve_type}.pem'
+        dhparam_path = Path(common_cert_path) / "dhparam2048.pem"
+        if not dhparam_path.exists():
+            print("Generating dh params, this takes a few seconds..")
+            # Generate DH parameters with 2048-bit key size and generator 2
+            parameters = dh.generate_parameters(generator=2, key_size=2048)
+            pem_data = parameters.parameter_bytes(encoding=Encoding.PEM,format=ParameterFormat.PKCS3)
+            with open(dhparam_path, 'wb') as file:
+                file.write(pem_data)
+            print("DH params created successfully")
+
+        if not cert_pempath.exists():
+            print("Translating tls server cert from DER to PEM..")
+            with open(cert_derpath, 'rb') as der_file:
+                der_cert_data = der_file.read()
+
+            cert = x509.load_der_x509_certificate(der_cert_data)
+            pem_cert = cert.public_bytes(Encoding.PEM) #.decode('utf-8')
+
+            with open(cert_pempath, 'wb') as pem_file:
+                pem_file.write(pem_cert)
+
+        SERVER_STRING = f'ssl:{args.port}:privateKey={cert_skpath}:certKey={cert_pempath}:dhParameters={dhparam_path}'
+        print(SERVER_STRING)
+
+        hs.app.run(host=HOSTNAME, port=args.port, endpoint_description=SERVER_STRING)
 
 if __name__ == "__main__":
     main(sys.argv)

pySim-prog.py

@@ -39,7 +39,7 @@ from pySim.commands import SimCardCommands
 from pySim.transport import init_reader, argparse_add_reader_args
 from pySim.legacy.cards import _cards_classes, card_detect
 from pySim.utils import derive_milenage_opc, calculate_luhn, dec_iccid
-from pySim.profile.ts_51_011 import EF_AD
+from pySim.ts_51_011 import EF_AD
 from pySim.legacy.ts_51_011 import EF
 from pySim.card_handler import *
 from pySim.utils import *
@@ -586,7 +586,7 @@ def read_params_csv(opts, imsi=None, iccid=None):
         else:
             row['mnc'] = row.get('mnc', mnc_from_imsi(row.get('imsi'), False))
 
-	# NOTE: We might concider to specify a new CSV field "mnclen" in our
+	# NOTE: We might consider to specify a new CSV field "mnclen" in our
 	# CSV files for a better automatization. However, this only makes sense
 	# when the tools and databases we export our files from will also add
 	# such a field.

pySim-read.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 #
-# Utility to display some informations about a SIM card
+# Utility to display some information about a SIM card
 #
 #
 # Copyright (C) 2009  Sylvain Munaut <tnt@246tNt.com>
@@ -31,7 +31,7 @@ import sys
 
 from osmocom.utils import h2b, h2s, swap_nibbles, rpad
 
-from pySim.profile.ts_51_011 import EF_SST_map, EF_AD
+from pySim.ts_51_011 import EF_SST_map, EF_AD
 from pySim.legacy.ts_51_011 import EF, DF
 from pySim.ts_31_102 import EF_UST_map
 from pySim.legacy.ts_31_102 import EF_USIM_ADF_map
@@ -44,6 +44,7 @@ from pySim.exceptions import SwMatchError
 from pySim.legacy.cards import card_detect, SimCard, UsimCard, IsimCard
 from pySim.utils import dec_imsi, dec_iccid
 from pySim.legacy.utils import format_xplmn_w_act, dec_st, dec_msisdn
+from pySim.ts_51_011 import EF_SMSP
 
 option_parser = argparse.ArgumentParser(description='Legacy tool for reading some parts of a SIM card',
                                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -141,6 +142,15 @@ if __name__ == '__main__':
     (res, sw) = card.read_record('SMSP', 1)
     if sw == '9000':
         print("SMSP: %s" % (res,))
+        ef_smsp = EF_SMSP()
+        smsc_a = ef_smsp.decode_record_bin(h2b(res), 1).get('tp_sc_addr', {})
+        smsc_n = smsc_a.get('call_number', None)
+        if smsc_a.get('ton_npi', {}).get('type_of_number', None) == 'international' and smsc_n is not None:
+            smsc = '+' + smsc_n
+        else:
+            smsc = smsc_n
+        if smsc is not None:
+            print("SMSC: %s" % (smsc,))
     else:
         print("SMSP: Can't read, response code = %s" % (sw,))
 

pySim-shell.py

@@ -22,19 +22,25 @@ from typing import List, Optional
 import json
 import traceback
 import re
-
 import cmd2
 from packaging import version
 from cmd2 import style
+
+import logging
+from pySim.log import PySimLogger
+from osmocom.utils import auto_uint8
+
 # cmd2 >= 2.3.0 has deprecated the bg/fg in favor of Bg/Fg :(
 if version.parse(cmd2.__version__) < version.parse("2.3.0"):
     from cmd2 import fg, bg # pylint: disable=no-name-in-module
     RED = fg.red
+    YELLOW = fg.yellow
     LIGHT_RED = fg.bright_red
     LIGHT_GREEN = fg.bright_green
 else:
     from cmd2 import Fg, Bg # pylint: disable=no-name-in-module
     RED = Fg.RED
+    YELLOW = Fg.YELLOW
     LIGHT_RED = Fg.LIGHT_RED
     LIGHT_GREEN = Fg.LIGHT_GREEN
 from cmd2 import CommandSet, with_default_category, with_argparser
@@ -60,12 +66,15 @@ from pySim.card_handler import CardHandler, CardHandlerAuto
 from pySim.filesystem import CardMF, CardEF, CardDF, CardADF, LinFixedEF, TransparentEF, BerTlvEF
 from pySim.ts_102_221 import pin_names
 from pySim.ts_102_222 import Ts102222Commands
+from pySim.gsm_r import DF_EIRENE
 from pySim.cat import ProactiveCommand
 
-from pySim.card_key_provider import CardKeyProviderCsv, card_key_provider_register, card_key_provider_get_field
+from pySim.card_key_provider import CardKeyProviderCsv, CardKeyProviderPgsql
+from pySim.card_key_provider import card_key_provider_register, card_key_provider_get_field, card_key_provider_get
 
 from pySim.app import init_card
 
+log = PySimLogger.get(Path(__file__).stem)
 
 class Cmd2Compat(cmd2.Cmd):
     """Backwards-compatibility wrapper around cmd2.Cmd to support older and newer
@@ -91,15 +100,19 @@ class PysimApp(Cmd2Compat):
 (C) 2021-2023 by Harald Welte, sysmocom - s.f.m.c. GmbH and contributors
 Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/shell.html """
 
-    def __init__(self, card, rs, sl, ch, script=None):
+    def __init__(self, verbose, card, rs, sl, ch, script=None):
         if version.parse(cmd2.__version__) < version.parse("2.0.0"):
             kwargs = {'use_ipython': True}
         else:
             kwargs = {'include_ipy': True}
 
+        self.verbose = verbose
+        self._onchange_verbose('verbose', False, self.verbose);
+
         # pylint: disable=unexpected-keyword-arg
         super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
                          auto_load_commands=False, startup_script=script, **kwargs)
+        PySimLogger.setup(self.poutput, {logging.WARN: YELLOW})
         self.intro = style(self.BANNER, fg=RED)
         self.default_category = 'pySim-shell built-in commands'
         self.card = None
@@ -125,6 +138,9 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         self.add_settable(Settable2Compat('apdu_strict', bool,
                                           'Enforce APDU responses according to ISO/IEC 7816-3, table 12', self,
                                           onchange_cb=self._onchange_apdu_strict))
+        self.add_settable(Settable2Compat('verbose', bool,
+                                          'Enable/disable verbose logging', self,
+                                          onchange_cb=self._onchange_verbose))
         self.equip(card, rs)
 
     def equip(self, card, rs):
@@ -154,6 +170,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         # When a card object and a runtime state is present, (re)equip pySim-shell with everything that is
         # needed to operate on cards.
         if self.card and self.rs:
+            self.rs.reset()
             self.lchan = self.rs.lchan[0]
             self._onchange_conserve_write(
                 'conserve_write', False, self.conserve_write)
@@ -208,6 +225,13 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
             else:
                 self.card._scc._tp.apdu_strict = False
 
+    def _onchange_verbose(self, param_name, old, new):
+        PySimLogger.set_verbose(new)
+        if new == True:
+            PySimLogger.set_level(logging.DEBUG)
+        else:
+            PySimLogger.set_level(logging.INFO)
+
     class Cmd2ApduTracer(ApduTracer):
         def __init__(self, cmd2_app):
             self.cmd2 = cmd2_app
@@ -217,18 +241,23 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
             self.cmd2.poutput("<- %s: %s" % (sw, resp))
 
     def update_prompt(self):
+        if self.rs and self.rs.adm_verified:
+            prompt_char = '#'
+        else:
+            prompt_char = '>'
+
         if self.lchan:
             path_str = self.lchan.selected_file.fully_qualified_path_str(not self.numeric_path)
             scp = self.lchan.scc.scp
             if scp:
-                self.prompt = 'pySIM-shell (%s:%02u:%s)> ' % (str(scp), self.lchan.lchan_nr, path_str)
+                self.prompt = 'pySIM-shell (%s:%02u:%s)%c ' % (str(scp), self.lchan.lchan_nr, path_str, prompt_char)
             else:
-                self.prompt = 'pySIM-shell (%02u:%s)> ' % (self.lchan.lchan_nr, path_str)
+                self.prompt = 'pySIM-shell (%02u:%s)%c ' % (self.lchan.lchan_nr, path_str, prompt_char)
         else:
             if self.card:
-                self.prompt = 'pySIM-shell (no card profile)> '
+                self.prompt = 'pySIM-shell (no card profile)%c ' % prompt_char
             else:
-                self.prompt = 'pySIM-shell (no card)> '
+                self.prompt = 'pySIM-shell (no card)%c ' % prompt_char
 
     @cmd2.with_category(CUSTOM_CATEGORY)
     def do_intro(self, _):
@@ -258,7 +287,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
     def do_apdu(self, opts):
         """Send a raw APDU to the card, and print SW + Response.
         CAUTION: this command bypasses the logical channel handling of pySim-shell and card state changes are not
-        tracked. Dpending on the raw APDU sent, pySim-shell may not continue to work as expected if you e.g. select
+        tracked. Depending on the raw APDU sent, pySim-shell may not continue to work as expected if you e.g. select
         a different file."""
 
         # When sending raw APDUs we access the scc object through _scc member of the card object. It should also be
@@ -287,7 +316,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         if self.rs is None:
             # In case no runtime state is available we go the direct route
             self.card._scc.reset_card()
-            atr = b2h(self.card._scc.get_atr())
+            atr = self.card._scc.get_atr()
         else:
             atr = self.rs.reset(self)
         self.poutput('Card ATR: %s' % atr)
@@ -329,7 +358,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
 
     def _process_card(self, first, script_path):
 
-        # Early phase of card initialzation (this part may fail with an exception)
+        # Early phase of card initialization (this part may fail with an exception)
         try:
             rs, card = init_card(self.sl)
             rc = self.equip(card, rs)
@@ -370,7 +399,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
 
     bulk_script_parser = argparse.ArgumentParser()
     bulk_script_parser.add_argument('SCRIPT_PATH', help="path to the script file")
-    bulk_script_parser.add_argument('--halt_on_error', help='stop card handling if an exeption occurs',
+    bulk_script_parser.add_argument('--halt_on_error', help='stop card handling if an exception occurs',
                                     action='store_true')
     bulk_script_parser.add_argument('--tries', type=int, default=2,
                                     help='how many tries before trying the next card')
@@ -470,11 +499,37 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         """Echo (print) a string on the console"""
         self.poutput(' '.join(opts.STRING))
 
+    query_card_key_parser = argparse.ArgumentParser()
+    query_card_key_parser.add_argument('FIELDS', help="fields to query", type=str, nargs='+')
+    query_card_key_parser.add_argument('--key', help='lookup key (typically \'ICCID\' or \'EID\')',
+                                       type=str, required=True)
+    query_card_key_parser.add_argument('--value', help='lookup key match value (e.g \'8988211000000123456\')',
+                                       type=str, required=True)
+    @cmd2.with_argparser(query_card_key_parser)
+    @cmd2.with_category(CUSTOM_CATEGORY)
+    def do_query_card_key(self, opts):
+        """Manually query the Card Key Provider"""
+        result = card_key_provider_get(opts.FIELDS, opts.key, opts.value)
+        self.poutput("Result:")
+        if result == {}:
+            self.poutput(" (none)")
+        for k in result.keys():
+            self.poutput(" %s: %s" % (str(k), str(result.get(k))))
+
     @cmd2.with_category(CUSTOM_CATEGORY)
     def do_version(self, opts):
         """Print the pySim software version."""
-        import pkg_resources
-        self.poutput(pkg_resources.get_distribution('pySim'))
+        from importlib.metadata import version as vsn
+        self.poutput("pyosmocom " + vsn('pyosmocom'))
+        import os
+        cwd = os.path.dirname(os.path.realpath(__file__))
+        if os.path.isdir(os.path.join(cwd, ".git")):
+            import subprocess
+            url = subprocess.check_output(['git', 'config', '--get', 'remote.origin.url']).decode('ascii').strip()
+            version = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=cwd).decode('ascii').strip()
+            self.poutput(os.path.basename(url) + " " + version)
+        else:
+            self.poutput("pySim " + vsn('pySim'))
 
 @with_default_category('pySim Commands')
 class PySimCommands(CommandSet):
@@ -724,7 +779,7 @@ class PySimCommands(CommandSet):
                     body = {}
                     for t in tags:
                         result = self._cmd.lchan.retrieve_data(t)
-                        (tag, l, val, remainer) = bertlv_parse_one(h2b(result[0]))
+                        (tag, l, val, remainder) = bertlv_parse_one(h2b(result[0]))
                         body[t] = b2h(val)
                 else:
                     raise RuntimeError('Unsupported structure "%s" of file "%s"' % (structure, filename))
@@ -853,6 +908,8 @@ class PySimCommands(CommandSet):
             self._cmd.lchan.scc.verify_chv(adm_chv_num, h2b(pin_adm))
         else:
             raise ValueError("error: cannot authenticate, no adm-pin!")
+        self._cmd.rs.adm_verified = True
+        self._cmd.update_prompt()
 
     def do_cardinfo(self, opts):
         """Display information about the currently inserted card"""
@@ -906,36 +963,53 @@ class Iso7816Commands(CommandSet):
             raise RuntimeError("cannot find %s for ICCID '%s'" % (field, iccid))
         return result
 
+    @staticmethod
+    def __select_pin_nr(pin_type:str, pin_nr:int) -> int:
+        if pin_type:
+            # pylint: disable=unsubscriptable-object
+            return pin_names.inverse[pin_type]
+        return pin_nr
+
+    @staticmethod
+    def __add_pin_nr_to_ArgumentParser(chv_parser):
+        group = chv_parser.add_mutually_exclusive_group()
+        group.add_argument('--pin-type',
+                           choices=[x for x in pin_names.values()
+                                    if (x.startswith('PIN') or x.startswith('2PIN'))],
+                           help='Specifiy pin type (default is PIN1)')
+        group.add_argument('--pin-nr', type=auto_uint8, default=0x01,
+                           help='PIN Number, 1=PIN1, 0x81=2PIN1 or custom value (see also TS 102 221, Table 9.3")')
+
     verify_chv_parser = argparse.ArgumentParser()
-    verify_chv_parser.add_argument(
-        '--pin-nr', type=int, default=1, help='PIN Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
     verify_chv_parser.add_argument('PIN', nargs='?', type=is_decimal,
                                    help='PIN code value. If none given, CSV file will be queried')
+    __add_pin_nr_to_ArgumentParser(verify_chv_parser)
 
     @cmd2.with_argparser(verify_chv_parser)
     def do_verify_chv(self, opts):
         """Verify (authenticate) using specified CHV (PIN) code, which is how the specifications
         call it if you authenticate yourself using the specified PIN.  There usually is at least PIN1 and
-        PIN2."""
-        pin = self.get_code(opts.PIN, "PIN" + str(opts.pin_nr))
-        (data, sw) = self._cmd.lchan.scc.verify_chv(opts.pin_nr, h2b(pin))
+        2PIN1 (see also TS 102 221 Section 9.5.1 / Table 9.3)."""
+        pin_nr = self.__select_pin_nr(opts.pin_type, opts.pin_nr)
+        pin = self.get_code(opts.PIN, "PIN" + str(pin_nr))
+        (data, sw) = self._cmd.lchan.scc.verify_chv(pin_nr, h2b(pin))
         self._cmd.poutput("CHV verification successful")
 
     unblock_chv_parser = argparse.ArgumentParser()
-    unblock_chv_parser.add_argument(
-        '--pin-nr', type=int, default=1, help='PUK Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
     unblock_chv_parser.add_argument('PUK', nargs='?', type=is_decimal,
                                    help='PUK code value. If none given, CSV file will be queried')
     unblock_chv_parser.add_argument('NEWPIN', nargs='?', type=is_decimal,
                                    help='PIN code value. If none given, CSV file will be queried')
+    __add_pin_nr_to_ArgumentParser(unblock_chv_parser)
 
     @cmd2.with_argparser(unblock_chv_parser)
     def do_unblock_chv(self, opts):
         """Unblock PIN code using specified PUK code"""
-        new_pin = self.get_code(opts.NEWPIN, "PIN" + str(opts.pin_nr))
-        puk = self.get_code(opts.PUK, "PUK" + str(opts.pin_nr))
+        pin_nr = self.__select_pin_nr(opts.pin_type, opts.pin_nr)
+        new_pin = self.get_code(opts.NEWPIN, "PIN" + str(pin_nr))
+        puk = self.get_code(opts.PUK, "PUK" + str(pin_nr))
         (data, sw) = self._cmd.lchan.scc.unblock_chv(
-            opts.pin_nr, h2b(puk), h2b(new_pin))
+            pin_nr, h2b(puk), h2b(new_pin))
         self._cmd.poutput("CHV unblock successful")
 
     change_chv_parser = argparse.ArgumentParser()
@@ -943,42 +1017,42 @@ class Iso7816Commands(CommandSet):
                                    help='PIN code value. If none given, CSV file will be queried')
     change_chv_parser.add_argument('PIN', nargs='?', type=is_decimal,
                                    help='PIN code value. If none given, CSV file will be queried')
-    change_chv_parser.add_argument(
-        '--pin-nr', type=int, default=1, help='PUK Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
+    __add_pin_nr_to_ArgumentParser(change_chv_parser)
 
     @cmd2.with_argparser(change_chv_parser)
     def do_change_chv(self, opts):
         """Change PIN code to a new PIN code"""
-        new_pin = self.get_code(opts.NEWPIN, "PIN" + str(opts.pin_nr))
-        pin = self.get_code(opts.PIN, "PIN" + str(opts.pin_nr))
+        pin_nr = self.__select_pin_nr(opts.pin_type, opts.pin_nr)
+        new_pin = self.get_code(opts.NEWPIN, "PIN" + str(pin_nr))
+        pin = self.get_code(opts.PIN, "PIN" + str(pin_nr))
         (data, sw) = self._cmd.lchan.scc.change_chv(
-            opts.pin_nr, h2b(pin), h2b(new_pin))
+            pin_nr, h2b(pin), h2b(new_pin))
         self._cmd.poutput("CHV change successful")
 
     disable_chv_parser = argparse.ArgumentParser()
-    disable_chv_parser.add_argument(
-        '--pin-nr', type=int, default=1, help='PIN Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
     disable_chv_parser.add_argument('PIN', nargs='?', type=is_decimal,
                                    help='PIN code value. If none given, CSV file will be queried')
+    __add_pin_nr_to_ArgumentParser(disable_chv_parser)
 
     @cmd2.with_argparser(disable_chv_parser)
     def do_disable_chv(self, opts):
         """Disable PIN code using specified PIN code"""
-        pin = self.get_code(opts.PIN, "PIN" + str(opts.pin_nr))
-        (data, sw) = self._cmd.lchan.scc.disable_chv(opts.pin_nr, h2b(pin))
+        pin_nr = self.__select_pin_nr(opts.pin_type, opts.pin_nr)
+        pin = self.get_code(opts.PIN, "PIN" + str(pin_nr))
+        (data, sw) = self._cmd.lchan.scc.disable_chv(pin_nr, h2b(pin))
         self._cmd.poutput("CHV disable successful")
 
     enable_chv_parser = argparse.ArgumentParser()
-    enable_chv_parser.add_argument(
-        '--pin-nr', type=int, default=1, help='PIN Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
+    __add_pin_nr_to_ArgumentParser(enable_chv_parser)
     enable_chv_parser.add_argument('PIN', nargs='?', type=is_decimal,
                                    help='PIN code value. If none given, CSV file will be queried')
 
     @cmd2.with_argparser(enable_chv_parser)
     def do_enable_chv(self, opts):
         """Enable PIN code using specified PIN code"""
-        pin = self.get_code(opts.PIN, "PIN" + str(opts.pin_nr))
-        (data, sw) = self._cmd.lchan.scc.enable_chv(opts.pin_nr, h2b(pin))
+        pin_nr = self.__select_pin_nr(opts.pin_type, opts.pin_nr)
+        pin = self.get_code(opts.PIN, "PIN" + str(pin_nr))
+        (data, sw) = self._cmd.lchan.scc.enable_chv(pin_nr, h2b(pin))
         self._cmd.poutput("CHV enable successful")
 
     def do_deactivate_file(self, opts):
@@ -1062,16 +1136,26 @@ argparse_add_reader_args(option_parser)
 global_group = option_parser.add_argument_group('General Options')
 global_group.add_argument('--script', metavar='PATH', default=None,
                           help='script with pySim-shell commands to be executed automatically at start-up')
-global_group.add_argument('--csv', metavar='FILE',
-                          default=None, help='Read card data from CSV file')
-global_group.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
-                          help='per-CSV-column AES transport key')
 global_group.add_argument("--card_handler", dest="card_handler_config", metavar="FILE",
                           help="Use automatic card handling machine")
 global_group.add_argument("--noprompt", help="Run in non interactive mode",
                           action='store_true', default=False)
 global_group.add_argument("--skip-card-init", help="Skip all card/profile initialization",
                           action='store_true', default=False)
+global_group.add_argument("--verbose", help="Enable verbose logging",
+                          action='store_true', default=False)
+
+card_key_group = option_parser.add_argument_group('Card Key Provider Options')
+card_key_group.add_argument('--csv', metavar='FILE',
+                            default="~/.osmocom/pysim/card_data.csv",
+                            help='Read card data from CSV file')
+card_key_group.add_argument('--pgsql', metavar='FILE',
+                            default="~/.osmocom/pysim/card_data_pgsql.cfg",
+                            help='Read card data from PostgreSQL database (config file)')
+card_key_group.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                            help=argparse.SUPPRESS, dest='column_key')
+card_key_group.add_argument('--column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                            help='per-column AES transport key', dest='column_key')
 
 adm_group = global_group.add_mutually_exclusive_group()
 adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM1', dest='pin_adm', default=None,
@@ -1086,23 +1170,29 @@ option_parser.add_argument("command", nargs='?',
 option_parser.add_argument('command_args', nargs=argparse.REMAINDER,
                            help="Optional Arguments for command")
 
-
 if __name__ == '__main__':
-
     startup_errors = False
     opts = option_parser.parse_args()
 
+    # Ensure that we are able to print formatted warnings from the beginning.
+    PySimLogger.setup(print, {logging.WARN: YELLOW})
+    if opts.verbose:
+        PySimLogger.set_verbose(True)
+        PySimLogger.set_level(logging.DEBUG)
+    else:
+        PySimLogger.set_verbose(False)
+        PySimLogger.set_level(logging.INFO)
+
     # Register csv-file as card data provider, either from specified CSV
     # or from CSV file in home directory
-    csv_column_keys = {}
-    for par in opts.csv_column_key:
+    column_keys = {}
+    for par in opts.column_key:
         name, key = par.split(':')
-        csv_column_keys[name] = key
-    csv_default = str(Path.home()) + "/.osmocom/pysim/card_data.csv"
-    if opts.csv:
-        card_key_provider_register(CardKeyProviderCsv(opts.csv, csv_column_keys))
-    if os.path.isfile(csv_default):
-        card_key_provider_register(CardKeyProviderCsv(csv_default, csv_column_keys))
+        column_keys[name] = key
+    if os.path.isfile(os.path.expanduser(opts.csv)):
+        card_key_provider_register(CardKeyProviderCsv(os.path.expanduser(opts.csv), column_keys))
+    if os.path.isfile(os.path.expanduser(opts.pgsql)):
+        card_key_provider_register(CardKeyProviderPgsql(os.path.expanduser(opts.pgsql), column_keys))
 
     # Init card reader driver
     sl = init_reader(opts, proactive_handler = Proact())
@@ -1118,7 +1208,7 @@ if __name__ == '__main__':
     # able to tolerate and recover from that.
     try:
         rs, card = init_card(sl, opts.skip_card_init)
-        app = PysimApp(card, rs, sl, ch)
+        app = PysimApp(opts.verbose, card, rs, sl, ch)
     except:
         startup_errors = True
         print("Card initialization (%s) failed with an exception:" % str(sl))
@@ -1130,7 +1220,7 @@ if __name__ == '__main__':
             print(" it should also be noted that some readers may behave strangely when no card")
             print(" is inserted.)")
             print("")
-        app = PysimApp(None, None, sl, ch)
+        app = PysimApp(opts.verbose, None, None, sl, ch)
 
     # If the user supplies an ADM PIN at via commandline args authenticate
     # immediately so that the user does not have to use the shell commands
@@ -1150,14 +1240,18 @@ if __name__ == '__main__':
     # Run optional commands
     for c in opts.execute_command:
         if not startup_errors:
-            app.onecmd_plus_hooks(c)
+            stop = app.onecmd_plus_hooks(c)
+            if stop == True:
+                sys.exit(0)
         else:
             print("Errors during startup, refusing to execute command (%s)" % c)
 
     # Run optional command
     if opts.command:
         if not startup_errors:
-            app.onecmd_plus_hooks('{} {}'.format(opts.command, ' '.join(opts.command_args)))
+            stop = app.onecmd_plus_hooks('{} {}'.format(opts.command, ' '.join(opts.command_args)))
+            if stop == True:
+                sys.exit(0)
         else:
             print("Errors during startup, refusing to execute command (%s)" % opts.command)
 
@@ -1168,7 +1262,9 @@ if __name__ == '__main__':
                 print("Error: script file (%s) not readable!" % opts.script)
                 startup_errors = True
             else:
-                app.onecmd_plus_hooks('{} {}'.format('run_script', opts.script), add_to_history = False)
+                stop = app.onecmd_plus_hooks('{} {}'.format('run_script', opts.script), add_to_history = False)
+                if stop == True:
+                    sys.exit(0)
         else:
             print("Errors during startup, refusing to execute script (%s)" % opts.script)
 

pySim-smpp2sim.py

@@ -0,0 +1,428 @@
+#!/usr/bin/env python3
+#
+# Program to emulate the entire communication path SMSC-MSC-BSC-BTS-ME
+# that is usually between an OTA backend and the SIM card.  This allows
+# to play with SIM OTA technology without using a mobile network or even
+# a mobile phone.
+#
+# An external application must encode (and encrypt/sign) the OTA SMS
+# and submit them via SMPP to this program, just like it would submit
+# it normally to a SMSC (SMS Service Centre).  The program then re-formats
+# the SMPP-SUBMIT into a SMS DELIVER TPDU and passes it via an ENVELOPE
+# APDU to the SIM card that is locally inserted into a smart card reader.
+#
+# The path from SIM to external OTA application works the opposite way.
+
+# (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import logging
+import colorlog
+
+from twisted.protocols import basic
+from twisted.internet import defer, endpoints, protocol, reactor, task
+from twisted.cred.portal import IRealm
+from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
+from twisted.cred.portal import Portal
+from zope.interface import implementer
+
+from smpp.twisted.config import SMPPServerConfig
+from smpp.twisted.server import SMPPServerFactory, SMPPBindManager
+from smpp.twisted.protocol import SMPPSessionStates, DataHandlerResponse
+
+from smpp.pdu import pdu_types, operations, pdu_encoding
+
+from pySim.sms import SMS_DELIVER, SMS_SUBMIT, AddressField
+
+from pySim.transport import LinkBase, ProactiveHandler, argparse_add_reader_args, init_reader, ApduTracer
+from pySim.commands import SimCardCommands
+from pySim.cards import UiccCardBase
+from pySim.exceptions import *
+from pySim.cat import ProactiveCommand, SendShortMessage, SMS_TPDU, SMSPPDownload, BearerDescription
+from pySim.cat import DeviceIdentities, Address, OtherAddress, UiccTransportLevel, BufferSize
+from pySim.cat import ChannelStatus, ChannelData, ChannelDataLength
+from pySim.utils import b2h, h2b
+
+logger = logging.getLogger(__name__)
+
+# MSISDNs to use when generating proactive SMS messages
+SIM_MSISDN='23'
+ESME_MSISDN='12'
+
+# HACK: we need some kind of mapping table between system_id and card-reader
+# or actually route based on MSISDNs
+hackish_global_smpp = None
+
+class MyApduTracer(ApduTracer):
+    def trace_response(self, cmd, sw, resp):
+        print("-> %s %s" % (cmd[:10], cmd[10:]))
+        print("<- %s: %s" % (sw, resp))
+
+class TcpProtocol(protocol.Protocol):
+    def dataReceived(self, data):
+        pass
+
+    def connectionLost(self, reason):
+        pass
+
+
+def tcp_connected_callback(p: protocol.Protocol):
+    """called by twisted TCP client."""
+    logger.error("%s: connected!" % p)
+
+class ProactChannel:
+    """Representation of a single protective channel."""
+    def __init__(self, channels: 'ProactChannels', chan_nr: int):
+        self.channels = channels
+        self.chan_nr = chan_nr
+        self.ep = None
+
+    def close(self):
+        """Close the channel."""
+        if self.ep:
+            self.ep.disconnect()
+        self.channels.channel_delete(self.chan_nr)
+
+class ProactChannels:
+    """Wrapper class for maintaining state of proactive channels."""
+    def __init__(self):
+        self.channels = {}
+
+    def channel_create(self) -> ProactChannel:
+        """Create a new proactive channel, allocating its integer number."""
+        for i in range(1, 9):
+            if not i in self.channels:
+                self.channels[i] = ProactChannel(self, i)
+                return self.channels[i]
+        raise ValueError('Cannot allocate another channel: All channels active')
+
+    def channel_delete(self, chan_nr: int):
+        del self.channels[chan_nr]
+
+class Proact(ProactiveHandler):
+    #def __init__(self, smpp_factory):
+    #    self.smpp_factory = smpp_factory
+    def __init__(self):
+        self.channels = ProactChannels()
+
+    @staticmethod
+    def _find_first_element_of_type(instlist, cls):
+        for i in instlist:
+            if isinstance(i, cls):
+                return i
+        return None
+
+    """Call-back which the pySim transport core calls whenever it receives a
+    proactive command from the SIM."""
+    def handle_SendShortMessage(self, pcmd: ProactiveCommand):
+        # {'smspp_download': [{'device_identities': {'source_dev_id': 'network',
+        #                                            'dest_dev_id': 'uicc'}},
+        #                     {'address': {'ton_npi': {'ext': True,
+        #                                              'type_of_number': 'international',
+        #                                              'numbering_plan_id': 'isdn_e164'},
+        #                                  'call_number': '79'}},
+        #                     {'sms_tpdu': {'tpdu': '40048111227ff6407070611535004d02700000481516011212000001fe4c0943aea42e45021c078ae06c66afc09303608874b72f58bacadb0dcf665c29349c799fbb522e61709c9baf1890015e8e8e196e36153106c8b92f95153774'}}
+        #                    ]}
+        """Card requests sending a SMS. We need to pass it on to the ESME via SMPP."""
+        logger.info("SendShortMessage")
+        logger.info(pcmd)
+        # Relevant parts in pcmd: Address, SMS_TPDU
+        addr_ie = Proact._find_first_element_of_type(pcmd.children, Address)
+        sms_tpdu_ie = Proact._find_first_element_of_type(pcmd.children, SMS_TPDU)
+        raw_tpdu = sms_tpdu_ie.decoded['tpdu']
+        submit = SMS_SUBMIT.from_bytes(raw_tpdu)
+        submit.tp_da = AddressField(addr_ie.decoded['call_number'], addr_ie.decoded['ton_npi']['type_of_number'],
+                                    addr_ie.decoded['ton_npi']['numbering_plan_id'])
+        logger.info(submit)
+        self.send_sms_via_smpp(submit)
+
+    def handle_OpenChannel(self, pcmd: ProactiveCommand):
+        """Card requests opening a new channel via a UDP/TCP socket."""
+        # {'open_channel': [{'command_details': {'command_number': 1,
+        #                                        'type_of_command': 'open_channel',
+        #                                        'command_qualifier': 3}},
+        #                   {'device_identities': {'source_dev_id': 'uicc',
+        #                                          'dest_dev_id': 'terminal'}},
+        #                   {'bearer_description': {'bearer_type': 'default',
+        #                                           'bearer_parameters': ''}},
+        #                   {'buffer_size': 1024},
+        #                   {'uicc_transport_level': {'protocol_type': 'tcp_uicc_client_remote',
+        #                                             'port_number': 32768}},
+        #                   {'other_address': {'type_of_address': 'ipv4',
+        #                                      'address': '01020304'}}
+        #                  ]}
+        logger.info("OpenChannel")
+        logger.info(pcmd)
+        transp_lvl_ie = Proact._find_first_element_of_type(pcmd.children, UiccTransportLevel)
+        other_addr_ie = Proact._find_first_element_of_type(pcmd.children, OtherAddress)
+        bearer_desc_ie = Proact._find_first_element_of_type(pcmd.children, BearerDescription)
+        buffer_size_ie = Proact._find_first_element_of_type(pcmd.children, BufferSize)
+        if transp_lvl_ie.decoded['protocol_type'] != 'tcp_uicc_client_remote':
+            raise ValueError('Unsupported protocol_type')
+        if other_addr_ie.decoded.get('type_of_address', None) != 'ipv4':
+            raise ValueError('Unsupported type_of_address')
+        ipv4_bytes = h2b(other_addr_ie.decoded['address'])
+        ipv4_str = '%u.%u.%u.%u' % (ipv4_bytes[0], ipv4_bytes[1], ipv4_bytes[2], ipv4_bytes[3])
+        port_nr = transp_lvl_ie.decoded['port_number']
+        print("%s:%u" % (ipv4_str, port_nr))
+        channel = self.channels.channel_create()
+        channel.ep = endpoints.TCP4ClientEndpoint(reactor, ipv4_str, port_nr)
+        channel.prot = TcpProtocol()
+        d = endpoints.connectProtocol(channel.ep, channel.prot)
+        # FIXME: why is this never called despite the client showing the inbound connection?
+        d.addCallback(tcp_connected_callback)
+
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'open_channel',
+        #                       'command_qualifier': 3}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_status': '8100'},
+        #  {'bearer_description': {'bearer_type': 'default', 'bearer_parameters': ''}},
+        #  {'buffer_size': 1024}
+        # ]
+        return self.prepare_response(pcmd) + [ChannelStatus(decoded='8100'), bearer_desc_ie, buffer_size_ie]
+
+    def handle_CloseChannel(self, pcmd: ProactiveCommand):
+        """Close a channel."""
+        logger.info("CloseChannel")
+        logger.info(pcmd)
+
+    def handle_ReceiveData(self, pcmd: ProactiveCommand):
+        """Receive/read data from the socket."""
+        # {'receive_data': [{'command_details': {'command_number': 1,
+        #                                        'type_of_command': 'receive_data',
+        #                                        'command_qualifier': 0}},
+        #                   {'device_identities': {'source_dev_id': 'uicc',
+        #                                          'dest_dev_id': 'channel_1'}},
+        #                   {'channel_data_length': 9}
+        #                  ]}
+        logger.info("ReceiveData")
+        logger.info(pcmd)
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'receive_data',
+        #                       'command_qualifier': 0}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_data': '16030100040e000000'},
+        #  {'channel_data_length': 0}
+        # ]
+        return self.prepare_response(pcmd) + []
+
+    def handle_SendData(self, pcmd: ProactiveCommand):
+        """Send/write data received from the SIM to the socket."""
+        # {'send_data': [{'command_details': {'command_number': 1,
+        #                                     'type_of_command': 'send_data',
+        #                                     'command_qualifier': 1}},
+        #                {'device_identities': {'source_dev_id': 'uicc',
+        #                                       'dest_dev_id': 'channel_1'}},
+        #                {'channel_data': '160301003c010000380303d0f45e12b52ce5bb522750dd037738195334c87a46a847fe2b6886cada9ea6bf00000a00ae008c008b00b0002c010000050001000101'}
+        #               ]}
+        logger.info("SendData")
+        logger.info(pcmd)
+        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
+        chan_data_ie = Proact._find_first_element_of_type(pcmd.children, ChannelData)
+        chan_str = dev_id_ie.decoded['dest_dev_id']
+        chan_nr = 1 # FIXME
+        chan = self.channels.channels.get(chan_nr, None)
+        # FIXME chan.prot.transport.write(h2b(chan_data_ie.decoded))
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'send_data',
+        #                       'command_qualifier': 1}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_data_length': 255}
+        # ]
+        return self.prepare_response(pcmd) + [ChannelDataLength(decoded=255)]
+
+    def handle_SetUpEventList(self, pcmd: ProactiveCommand):
+        # {'set_up_event_list': [{'command_details': {'command_number': 1,
+        #                                             'type_of_command': 'set_up_event_list',
+        #                                             'command_qualifier': 0}},
+        #                        {'device_identities': {'source_dev_id': 'uicc',
+        #                                               'dest_dev_id': 'terminal'}},
+        #                        {'event_list': ['data_available', 'channel_status']}
+        #                       ]}
+        logger.info("SetUpEventList")
+        logger.info(pcmd)
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'set_up_event_list',
+        #                       'command_qualifier': 0}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}}
+        # ]
+        return self.prepare_response(pcmd)
+
+    def getChannelStatus(self, pcmd: ProactiveCommand):
+        logger.info("GetChannelStatus")
+        logger.info(pcmd)
+        return self.prepare_response(pcmd) + []
+
+    def send_sms_via_smpp(self, submit: SMS_SUBMIT):
+        # while in a normal network the phone/ME would *submit* a message to the SMSC,
+        # we are actually emulating the SMSC itself, so we must *deliver* the message
+        # to the ESME
+        deliver = SMS_DELIVER.from_submit(submit)
+        deliver_smpp = deliver.to_smpp()
+
+        hackish_global_smpp.sendDataRequest(deliver_smpp)
+#       # obtain the connection/binding of system_id to be used for delivering MO-SMS to the ESME
+#       connection = smpp_server.getBoundConnections[system_id].getNextBindingForDelivery()
+#       connection.sendDataRequest(deliver_smpp)
+
+
+
+def dcs_is_8bit(dcs):
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED):
+        return True
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED_COMMON):
+        return True
+    # pySim-smpp2sim.py:150:21: E1101: Instance of 'DataCodingScheme' has no 'GSM_MESSAGE_CLASS' member (no-member)
+    # pylint: disable=no-member
+    if dcs.scheme == pdu_types.DataCodingScheme.GSM_MESSAGE_CLASS and dcs.schemeData['msgCoding'] == pdu_types.DataCodingGsmMsgCoding.DATA_8BIT:
+        return True
+    else:
+        return False
+
+
+class MyServer:
+
+    @implementer(IRealm)
+    class SmppRealm:
+        def requestAvatar(self, avatarId, mind, *interfaces):
+            return ('SMPP', avatarId, lambda: None)
+
+    def __init__(self, tcp_port:int = 2775, bind_ip = '::', system_id:str = 'test', password:str = 'test'):
+        smpp_config = SMPPServerConfig(msgHandler=self._msgHandler,
+                                       systems={system_id: {'max_bindings': 2}})
+        portal = Portal(self.SmppRealm())
+        credential_checker = InMemoryUsernamePasswordDatabaseDontUse()
+        credential_checker.addUser(system_id, password)
+        portal.registerChecker(credential_checker)
+        self.factory = SMPPServerFactory(smpp_config, auth_portal=portal)
+        logger.info('Binding Virtual SMSC to TCP Port %u at %s' % (tcp_port, bind_ip))
+        smppEndpoint = endpoints.TCP6ServerEndpoint(reactor, tcp_port, interface=bind_ip)
+        smppEndpoint.listen(self.factory)
+        self.tp = self.scc = self.card = None
+
+    def connect_to_card(self, tp: LinkBase):
+        self.tp = tp
+        self.scc = SimCardCommands(self.tp)
+        self.card = UiccCardBase(self.scc)
+        # this should be part of UiccCardBase, but FairewavesSIM breaks with that :/
+        self.scc.cla_byte = "00"
+        self.scc.sel_ctrl = "0004"
+        self.card.read_aids()
+        self.card.select_adf_by_aid(adf='usim')
+        # FIXME: create a more realistic profile than ffffff
+        self.scc.terminal_profile('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff')
+
+    def _msgHandler(self, system_id, smpp, pdu):
+        """Handler for incoming messages received via SMPP from ESME."""
+        # HACK: we need some kind of mapping table between system_id and card-reader
+        # or actually route based on MSISDNs
+        global hackish_global_smpp
+        hackish_global_smpp = smpp
+        if pdu.id == pdu_types.CommandId.submit_sm:
+            return self.handle_submit_sm(system_id, smpp, pdu)
+        else:
+            logger.warning('Rejecting non-SUBMIT commandID')
+            return pdu_types.CommandStatus.ESME_RINVCMDID
+
+    def handle_submit_sm(self, system_id, smpp, pdu):
+        """SUBMIT-SM was received via SMPP from ESME. We need to deliver it to the SIM."""
+        # check for valid data coding scheme + PID
+        if not dcs_is_8bit(pdu.params['data_coding']):
+            logger.warning('Rejecting non-8bit DCS')
+            return pdu_types.CommandStatus.ESME_RINVDCS
+        if pdu.params['protocol_id'] != 0x7f:
+            logger.warning('Rejecting non-SIM PID')
+            return pdu_types.CommandStatus.ESME_RINVDCS
+
+        # 1) build a SMS-DELIVER (!) from the SMPP-SUBMIT
+        tpdu = SMS_DELIVER.from_smpp_submit(pdu)
+        logger.info(tpdu)
+        # 2) wrap into the CAT ENVELOPE for SMS-PP-Download
+        tpdu_ie = SMS_TPDU(decoded={'tpdu': b2h(tpdu.to_bytes())})
+        addr_ie = Address(decoded={'ton_npi': {'ext':False, 'type_of_number':'unknown', 'numbering_plan_id':'unknown'}, 'call_number': '0123456'})
+        dev_ids = DeviceIdentities(decoded={'source_dev_id': 'network', 'dest_dev_id': 'uicc'})
+        sms_dl = SMSPPDownload(children=[dev_ids, addr_ie, tpdu_ie])
+        # 3) send to the card
+        envelope_hex = b2h(sms_dl.to_tlv())
+        logger.info("ENVELOPE: %s" % envelope_hex)
+        (data, sw) = self.scc.envelope(envelope_hex)
+        logger.info("SW %s: %s" % (sw, data))
+        if sw in ['9200', '9300']:
+            # TODO send back RP-ERROR message with TP-FCS == 'SIM Application Toolkit Busy'
+            return pdu_types.CommandStatus.ESME_RSUBMITFAIL
+        elif sw == '9000' or sw[0:2] in ['6f', '62', '63'] and len(data):
+            # data something like 027100000e0ab000110000000000000001612f or
+            # 027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c
+            # which is the user-data portion of the SMS starting with the UDH (027100)
+            # TODO: return the response back to the sender in an RP-ACK; PID/DCS like in CMD
+            deliver = operations.DeliverSM(service_type=pdu.params['service_type'],
+                                           source_addr_ton=pdu.params['dest_addr_ton'],
+                                           source_addr_npi=pdu.params['dest_addr_npi'],
+                                           source_addr=pdu.params['destination_addr'],
+                                           dest_addr_ton=pdu.params['source_addr_ton'],
+                                           dest_addr_npi=pdu.params['source_addr_npi'],
+                                           destination_addr=pdu.params['source_addr'],
+                                           esm_class=pdu.params['esm_class'],
+                                           protocol_id=pdu.params['protocol_id'],
+                                           priority_flag=pdu.params['priority_flag'],
+                                           data_coding=pdu.params['data_coding'],
+                                           short_message=h2b(data))
+            smpp.sendDataRequest(deliver)
+            return pdu_types.CommandStatus.ESME_ROK
+        else:
+            return pdu_types.CommandStatus.ESME_RSUBMITFAIL
+
+
+option_parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+argparse_add_reader_args(option_parser)
+smpp_group = option_parser.add_argument_group('SMPP Options')
+smpp_group.add_argument('--smpp-bind-port', type=int, default=2775,
+                        help='TCP Port to bind the SMPP socket to')
+smpp_group.add_argument('--smpp-bind-ip', default='::',
+                        help='IPv4/IPv6 address to bind the SMPP socket to')
+smpp_group.add_argument('--smpp-system-id', default='test',
+                        help='SMPP System-ID used by ESME to bind')
+smpp_group.add_argument('--smpp-password', default='test',
+                        help='SMPP Password used by ESME to bind')
+
+if __name__ == '__main__':
+    log_format='%(log_color)s%(levelname)-8s%(reset)s %(name)s: %(message)s'
+    colorlog.basicConfig(level=logging.INFO, format = log_format)
+    logger = colorlog.getLogger()
+
+    opts = option_parser.parse_args()
+
+    tp = init_reader(opts, proactive_handler = Proact())
+    if tp is None:
+        exit(1)
+    tp.connect()
+
+    global g_ms
+    g_ms = MyServer(opts.smpp_bind_port, opts.smpp_bind_ip, opts.smpp_system_id, opts.smpp_password)
+    g_ms.connect_to_card(tp)
+    reactor.run()
+

pySim-trace.py

@@ -13,7 +13,7 @@ from osmocom.utils import JsonEncoder
 from pySim.cards import UiccCardBase
 from pySim.commands import SimCardCommands
 from pySim.profile import CardProfile
-from pySim.profile.ts_102_221 import CardProfileUICC
+from pySim.ts_102_221 import CardProfileUICC
 from pySim.ts_31_102 import CardApplicationUSIM
 from pySim.ts_31_103 import CardApplicationISIM
 from pySim.euicc import CardApplicationISDR, CardApplicationECASD
@@ -23,6 +23,7 @@ from pySim.apdu_source.gsmtap import GsmtapApduSource
 from pySim.apdu_source.pyshark_rspro import PysharkRsproPcap, PysharkRsproLive
 from pySim.apdu_source.pyshark_gsmtap import PysharkGsmtapPcap
 from pySim.apdu_source.tca_loader_log import TcaLoaderLogApduSource
+from pySim.apdu_source.stdin_hex import StdinHexApduSource
 
 from pySim.apdu.ts_102_221 import UiccSelect, UiccStatus
 
@@ -151,7 +152,7 @@ global_group.add_argument('--no-suppress-select', action='store_false', dest='su
 global_group.add_argument('--no-suppress-status', action='store_false', dest='suppress_status',
                           help="""
     Don't suppress displaying STATUS APDUs. We normally suppress them as they don't provide any
-    information that was not already received in resposne to the most recent SEELCT.""")
+    information that was not already received in response to the most recent SEELCT.""")
 global_group.add_argument('--show-raw-apdu', action='store_true', dest='show_raw_apdu',
                           help="""Show the raw APDU in addition to its parsed form.""")
 
@@ -188,7 +189,11 @@ parser_rspro_pyshark_live.add_argument('-i', '--interface', required=True,
 parser_tcaloader_log = subparsers.add_parser('tca-loader-log', help="""
     Read APDUs from a TCA Loader log file.""")
 parser_tcaloader_log.add_argument('-f', '--log-file', required=True,
-                                  help='Name of te log file to be read')
+                                  help='Name of the log file to be read')
+
+parser_stdin_hex = subparsers.add_parser('stdin-hex', help="""
+    Read APDUs as hex-string from stdin.""")
+
 
 if __name__ == '__main__':
 
@@ -205,6 +210,8 @@ if __name__ == '__main__':
         s = PysharkGsmtapPcap(opts.pcap_file)
     elif opts.source == 'tca-loader-log':
         s = TcaLoaderLogApduSource(opts.log_file)
+    elif opts.source == 'stdin-hex':
+        s = StdinHexApduSource()
     else:
         raise ValueError("unsupported source %s", opts.source)
 

pySim/apdu/__init__.py

@@ -29,7 +29,7 @@ import abc
 import typing
 from typing import List, Dict, Optional
 from termcolor import colored
-from construct import Byte, GreedyBytes
+from construct import Byte
 from construct import Optional as COptional
 from osmocom.construct import *
 from osmocom.utils import *

pySim/apdu/ts_31_102.py

@@ -9,7 +9,7 @@ APDU commands of 3GPP TS 31.102 V16.6.0
 """
 from typing import Dict
 
-from construct import BitStruct, Enum, BitsInteger, Int8ub, Bytes, this, Struct, If, Switch, Const
+from construct import BitStruct, Enum, BitsInteger, Int8ub, this, Struct, If, Switch, Const
 from construct import Optional as COptional
 from osmocom.construct import *
 

pySim/apdu_source/pyshark_gsmtap.py

@@ -84,5 +84,5 @@ class PysharkGsmtapPcap(_PysharkGsmtap):
         Args:
             pcap_filename: File name of the pcap file to be opened
         """
-        pyshark_inst = pyshark.FileCapture(pcap_filename, display_filter='gsm_sim', use_json=True, keep_packets=False)
+        pyshark_inst = pyshark.FileCapture(pcap_filename, display_filter='gsm_sim || iso7816.atr', use_json=True, keep_packets=False)
         super().__init__(pyshark_inst)

pySim/apdu_source/stdin_hex.py

@@ -0,0 +1,39 @@
+# coding=utf-8
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from pySim.utils import h2b
+
+from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_102_222 import ApduCommands as UiccAdmApduCommands
+from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
+from pySim.apdu.global_platform import ApduCommands as GpApduCommands
+
+from . import ApduSource, PacketType, CardReset
+
+ApduCommands = UiccApduCommands + UiccAdmApduCommands + UsimApduCommands + GpApduCommands
+
+class StdinHexApduSource(ApduSource):
+    """ApduSource for reading apdu hex-strings from stdin."""
+
+    def read_packet(self) -> PacketType:
+        while True:
+            command = input("C-APDU >")
+            if len(command) == 0:
+                continue
+            response = '9000'
+            return ApduCommands.parse_cmd_bytes(h2b(command) + h2b(response))

pySim/app.py

@@ -22,8 +22,8 @@ from pySim.filesystem import CardModel, CardApplication
 from pySim.cards import card_detect, SimCardBase, UiccCardBase, CardBase
 from pySim.runtime import RuntimeState
 from pySim.profile import CardProfile
-from pySim.profile.cdma_ruim import CardProfileRUIM
-from pySim.profile.ts_102_221 import CardProfileUICC
+from pySim.cdma_ruim import CardProfileRUIM
+from pySim.ts_102_221 import CardProfileUICC
 from pySim.utils import all_subclasses
 from pySim.exceptions import SwMatchError
 
@@ -40,7 +40,7 @@ import pySim.ts_31_103
 import pySim.ts_31_104
 import pySim.ara_m
 import pySim.global_platform
-import pySim.profile.euicc
+import pySim.euicc
 
 def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState, SimCardBase]:
     """

pySim/ara_m.py

@@ -26,7 +26,7 @@ Support for the Secure Element Access Control, specifically the ARA-M inside an
 #
 
 
-from construct import GreedyBytes, GreedyString, Struct, Enum, Int8ub, Int16ub
+from construct import GreedyString, Struct, Enum, Int8ub, Int16ub
 from construct import Optional as COptional
 from osmocom.construct import *
 from osmocom.tlv import *
@@ -38,17 +38,17 @@ import pySim.global_platform
 
 
 class AidRefDO(BER_TLV_IE, tag=0x4f):
-    # SEID v1.1 Table 6-3
+    # GPD_SPE_013 v1.1 Table 6-3
     _construct = HexAdapter(GreedyBytes)
 
 
 class AidRefEmptyDO(BER_TLV_IE, tag=0xc0):
-    # SEID v1.1 Table 6-3
+    # GPD_SPE_013 v1.1 Table 6-3
     pass
 
 
 class DevAppIdRefDO(BER_TLV_IE, tag=0xc1):
-    # SEID v1.1 Table 6-4
+    # GPD_SPE_013 v1.1 Table 6-4
     _construct = HexAdapter(GreedyBytes)
 
 
@@ -58,12 +58,12 @@ class PkgRefDO(BER_TLV_IE, tag=0xca):
 
 
 class RefDO(BER_TLV_IE, tag=0xe1, nested=[AidRefDO, AidRefEmptyDO, DevAppIdRefDO, PkgRefDO]):
-    # SEID v1.1 Table 6-5
+    # GPD_SPE_013 v1.1 Table 6-5
     pass
 
 
 class ApduArDO(BER_TLV_IE, tag=0xd0):
-    # SEID v1.1 Table 6-8
+    # GPD_SPE_013 v1.1 Table 6-8
     def _from_bytes(self, do: bytes):
         if len(do) == 1:
             if do[0] == 0x00:
@@ -108,7 +108,7 @@ class ApduArDO(BER_TLV_IE, tag=0xd0):
 
 
 class NfcArDO(BER_TLV_IE, tag=0xd1):
-    # SEID v1.1 Table 6-9
+    # GPD_SPE_013 v1.1 Table 6-9
     _construct = Struct('nfc_event_access_rule' /
                         Enum(Int8ub, never=0, always=1))
 
@@ -120,122 +120,122 @@ class PermArDO(BER_TLV_IE, tag=0xdb):
 
 
 class ArDO(BER_TLV_IE, tag=0xe3, nested=[ApduArDO, NfcArDO, PermArDO]):
-    # SEID v1.1 Table 6-7
+    # GPD_SPE_013 v1.1 Table 6-7
     pass
 
 
 class RefArDO(BER_TLV_IE, tag=0xe2, nested=[RefDO, ArDO]):
-    # SEID v1.1 Table 6-6
+    # GPD_SPE_013 v1.1 Table 6-6
     pass
 
 
 class ResponseAllRefArDO(BER_TLV_IE, tag=0xff40, nested=[RefArDO]):
-    # SEID v1.1 Table 4-2
+    # GPD_SPE_013 v1.1 Table 4-2
     pass
 
 
 class ResponseArDO(BER_TLV_IE, tag=0xff50, nested=[ArDO]):
-    # SEID v1.1 Table 4-3
+    # GPD_SPE_013 v1.1 Table 4-3
     pass
 
 
 class ResponseRefreshTagDO(BER_TLV_IE, tag=0xdf20):
-    # SEID v1.1 Table 4-4
+    # GPD_SPE_013 v1.1 Table 4-4
     _construct = Struct('refresh_tag'/HexAdapter(Bytes(8)))
 
 
 class DeviceInterfaceVersionDO(BER_TLV_IE, tag=0xe6):
-    # SEID v1.1 Table 6-12
+    # GPD_SPE_013 v1.1 Table 6-12
     _construct = Struct('major'/Int8ub, 'minor'/Int8ub, 'patch'/Int8ub)
 
 
 class DeviceConfigDO(BER_TLV_IE, tag=0xe4, nested=[DeviceInterfaceVersionDO]):
-    # SEID v1.1 Table 6-10
+    # GPD_SPE_013 v1.1 Table 6-10
     pass
 
 
 class ResponseDeviceConfigDO(BER_TLV_IE, tag=0xff7f, nested=[DeviceConfigDO]):
-    # SEID v1.1 Table 5-14
+    # GPD_SPE_013 v1.1 Table 5-14
     pass
 
 
 class AramConfigDO(BER_TLV_IE, tag=0xe5, nested=[DeviceInterfaceVersionDO]):
-    # SEID v1.1 Table 6-11
+    # GPD_SPE_013 v1.1 Table 6-11
     pass
 
 
 class ResponseAramConfigDO(BER_TLV_IE, tag=0xdf21, nested=[AramConfigDO]):
-    # SEID v1.1 Table 4-5
+    # GPD_SPE_013 v1.1 Table 4-5
     pass
 
 
 class CommandStoreRefArDO(BER_TLV_IE, tag=0xf0, nested=[RefArDO]):
-    # SEID v1.1 Table 5-2
+    # GPD_SPE_013 v1.1 Table 5-2
     pass
 
 
 class CommandDelete(BER_TLV_IE, tag=0xf1, nested=[AidRefDO, AidRefEmptyDO, RefDO, RefArDO]):
-    # SEID v1.1 Table 5-4
+    # GPD_SPE_013 v1.1 Table 5-4
     pass
 
 
 class CommandUpdateRefreshTagDO(BER_TLV_IE, tag=0xf2):
-    # SEID V1.1 Table 5-6
+    # GPD_SPE_013 V1.1 Table 5-6
     pass
 
 
 class CommandRegisterClientAidsDO(BER_TLV_IE, tag=0xf7, nested=[AidRefDO, AidRefEmptyDO]):
-    # SEID v1.1 Table 5-7
+    # GPD_SPE_013 v1.1 Table 5-7
     pass
 
 
 class CommandGet(BER_TLV_IE, tag=0xf3, nested=[AidRefDO, AidRefEmptyDO]):
-    # SEID v1.1 Table 5-8
+    # GPD_SPE_013 v1.1 Table 5-8
     pass
 
 
 class CommandGetAll(BER_TLV_IE, tag=0xf4):
-    # SEID v1.1 Table 5-9
+    # GPD_SPE_013 v1.1 Table 5-9
     pass
 
 
 class CommandGetClientAidsDO(BER_TLV_IE, tag=0xf6):
-    # SEID v1.1 Table 5-10
+    # GPD_SPE_013 v1.1 Table 5-10
     pass
 
 
 class CommandGetNext(BER_TLV_IE, tag=0xf5):
-    # SEID v1.1 Table 5-11
+    # GPD_SPE_013 v1.1 Table 5-11
     pass
 
 
 class CommandGetDeviceConfigDO(BER_TLV_IE, tag=0xf8):
-    # SEID v1.1 Table 5-12
+    # GPD_SPE_013 v1.1 Table 5-12
     pass
 
 
 class ResponseAracAidDO(BER_TLV_IE, tag=0xff70, nested=[AidRefDO, AidRefEmptyDO]):
-    # SEID v1.1 Table 5-13
+    # GPD_SPE_013 v1.1 Table 5-13
     pass
 
 
 class BlockDO(BER_TLV_IE, tag=0xe7):
-    # SEID v1.1 Table 6-13
+    # GPD_SPE_013 v1.1 Table 6-13
     _construct = Struct('offset'/Int16ub, 'length'/Int8ub)
 
 
-# SEID v1.1 Table 4-1
+# GPD_SPE_013 v1.1 Table 4-1
 class GetCommandDoCollection(TLV_IE_Collection, nested=[RefDO, DeviceConfigDO]):
     pass
 
 
-# SEID v1.1 Table 4-2
+# GPD_SPE_013 v1.1 Table 4-2
 class GetResponseDoCollection(TLV_IE_Collection, nested=[ResponseAllRefArDO, ResponseArDO,
                                                          ResponseRefreshTagDO, ResponseAramConfigDO]):
     pass
 
 
-# SEID v1.1 Table 5-1
+# GPD_SPE_013 v1.1 Table 5-1
 class StoreCommandDoCollection(TLV_IE_Collection,
                                nested=[BlockDO, CommandStoreRefArDO, CommandDelete,
                                        CommandUpdateRefreshTagDO, CommandRegisterClientAidsDO,
@@ -244,7 +244,7 @@ class StoreCommandDoCollection(TLV_IE_Collection,
     pass
 
 
-# SEID v1.1 Section 5.1.2
+# GPD_SPE_013 v1.1 Section 5.1.2
 class StoreResponseDoCollection(TLV_IE_Collection,
                                 nested=[ResponseAllRefArDO, ResponseAracAidDO, ResponseDeviceConfigDO]):
     pass
@@ -317,12 +317,12 @@ class ADF_ARAM(CardADF):
         store_ref_ar_do_parse = argparse.ArgumentParser()
         # REF-DO
         store_ref_ar_do_parse.add_argument(
-            '--device-app-id', required=True, help='Identifies the specific device application that the rule appplies to. Hash of Certificate of Application Provider, or UUID. (20/32 hex bytes)')
+            '--device-app-id', required=True, help='Identifies the specific device application that the rule applies to. Hash of Certificate of Application Provider, or UUID. (20/32 hex bytes)')
         aid_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
         aid_grp.add_argument(
-            '--aid', help='Identifies the specific SE application for which rules are to be stored. Can be a partial AID, containing for example only the RID.  (5-16 hex bytes)')
+            '--aid', help='Identifies the specific SE application for which rules are to be stored. Can be a partial AID, containing for example only the RID. (5-16 or 0 hex bytes)')
         aid_grp.add_argument('--aid-empty', action='store_true',
-                             help='No specific SE application, applies to all applications')
+                             help='No specific SE application, applies to implicitly selected application (all channels)')
         store_ref_ar_do_parse.add_argument(
             '--pkg-ref', help='Full Android Java package name (up to 127 chars ASCII)')
         # AR-DO
@@ -389,12 +389,17 @@ class ADF_ARAM(CardADF):
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
+        def do_aram_lock(self, opts):
+            """Lock STORE DATA command to prevent unauthorized changes
+            (Proprietary feature that is specific to sysmocom's fork of Bertrand Martel’s ARA-M implementation.)"""
+            self._cmd.lchan.scc.send_apdu_checksw('80e2900001A1', '9000')
+
 
 # SEAC v1.1 Section 4.1.2.2 + 5.1.2.2
 sw_aram = {
     'ARA-M': {
         '6381': 'Rule successfully stored but an access rule already exists',
-        '6382': 'Rule successfully stored bu contained at least one unknown (discarded) BER-TLV',
+        '6382': 'Rule successfully stored but contained at least one unknown (discarded) BER-TLV',
         '6581': 'Memory Problem',
         '6700': 'Wrong Length in Lc',
         '6981': 'DO is not supported by the ARA-M/ARA-C',
@@ -423,10 +428,13 @@ class CardApplicationARAM(CardApplication):
         # matching key.
         if dictlist is None:
             return None
-        obj = None
         for d in dictlist:
-            obj = d.get(key, obj)
-        return obj
+            if key in d:
+                obj = d.get(key)
+                if obj is None:
+                    return ""
+                return obj
+        return None
 
     @staticmethod
     def __export_ref_ar_do_list(ref_ar_do_list):
@@ -437,6 +445,7 @@ class CardApplicationARAM(CardApplication):
         if ref_do_list and ar_do_list:
             # Get ref_do parameters
             aid_ref_do = CardApplicationARAM.__export_get_from_dictlist('aid_ref_do', ref_do_list)
+            aid_ref_empty_do = CardApplicationARAM.__export_get_from_dictlist('aid_ref_empty_do', ref_do_list)
             dev_app_id_ref_do = CardApplicationARAM.__export_get_from_dictlist('dev_app_id_ref_do', ref_do_list)
             pkg_ref_do = CardApplicationARAM.__export_get_from_dictlist('pkg_ref_do', ref_do_list)
 
@@ -447,9 +456,11 @@ class CardApplicationARAM(CardApplication):
 
             # Write command-line
             export_str += "aram_store_ref_ar_do"
-            if aid_ref_do:
+            if aid_ref_do is not None and len(aid_ref_do) > 0:
                 export_str += (" --aid %s" % aid_ref_do)
-            else:
+            elif aid_ref_do is not None:
+                export_str += " --aid \"\""
+            if aid_ref_empty_do is not None:
                 export_str += " --aid-empty"
             if dev_app_id_ref_do:
                 export_str += (" --device-app-id %s" % dev_app_id_ref_do)

pySim/card_handler.py

@@ -7,7 +7,7 @@ there are also automatic card feeders.
 """
 
 #
-# (C) 2019 by Sysmocom s.f.m.c. GmbH
+# (C) 2019 by sysmocom - s.f.m.c. GmbH
 # All Rights Reserved
 #
 # This program is free software: you can redistribute it and/or modify

pySim/card_key_provider.py

@@ -10,7 +10,7 @@ the need of manually entering the related card-individual data on every
 operation with pySim-shell.
 """
 
-# (C) 2021-2024 by Sysmocom s.f.m.c. GmbH
+# (C) 2021-2025 by sysmocom - s.f.m.c. GmbH
 # All Rights Reserved
 #
 # Author: Philipp Maier, Harald Welte
@@ -31,128 +31,226 @@ operation with pySim-shell.
 from typing import List, Dict, Optional
 from Cryptodome.Cipher import AES
 from osmocom.utils import h2b, b2h
+from pySim.log import PySimLogger
 
 import abc
 import csv
+import logging
+import yaml
+
+log = PySimLogger.get(__name__)
 
 card_key_providers = []  # type: List['CardKeyProvider']
 
-# well-known groups of columns relate to a given functionality.  This avoids having
-# to specify the same transport key N number of times, if the same key is used for multiple
-# fields of one group, like KIC+KID+KID of one SD.
-CRYPT_GROUPS = {
-    'UICC_SCP02': ['UICC_SCP02_KIC1', 'UICC_SCP02_KID1', 'UICC_SCP02_KIK1'],
-    'UICC_SCP03': ['UICC_SCP03_KIC1', 'UICC_SCP03_KID1', 'UICC_SCP03_KIK1'],
-    'SCP03_ISDR': ['SCP03_ENC_ISDR', 'SCP03_MAC_ISDR', 'SCP03_DEK_ISDR'],
-    'SCP03_ISDA': ['SCP03_ENC_ISDR', 'SCP03_MAC_ISDA', 'SCP03_DEK_ISDA'],
-    'SCP03_ECASD': ['SCP03_ENC_ECASD', 'SCP03_MAC_ECASD', 'SCP03_DEK_ECASD'],
+class CardKeyFieldCryptor:
+    """
+    A Card key field encryption class that may be used by Card key provider implementations to add support for
+    a column-based encryption to protect sensitive material (cryptographic key material, ADM keys, etc.).
+    The sensitive material is encrypted using a "key-encryption key", occasionally also known as "transport key"
+    before it is stored into a file or database (see also GSMA FS.28). The "transport key" is then used to decrypt
+    the key material on demand.
+    """
+
+    # well-known groups of columns relate to a given functionality.  This avoids having
+    # to specify the same transport key N number of times, if the same key is used for multiple
+    # fields of one group, like KIC+KID+KID of one SD.
+    __CRYPT_GROUPS = {
+            'UICC_SCP02': ['UICC_SCP02_KIC1', 'UICC_SCP02_KID1', 'UICC_SCP02_KIK1'],
+            'UICC_SCP03': ['UICC_SCP03_KIC1', 'UICC_SCP03_KID1', 'UICC_SCP03_KIK1'],
+            'SCP03_ISDR': ['SCP03_ENC_ISDR', 'SCP03_MAC_ISDR', 'SCP03_DEK_ISDR'],
+            'SCP03_ISDA': ['SCP03_ENC_ISDA', 'SCP03_MAC_ISDA', 'SCP03_DEK_ISDA'],
+            'SCP03_ECASD': ['SCP03_ENC_ECASD', 'SCP03_MAC_ECASD', 'SCP03_DEK_ECASD'],
     }
 
-class CardKeyProvider(abc.ABC):
-    """Base class, not containing any concrete implementation."""
-
-    VALID_KEY_FIELD_NAMES = ['ICCID', 'EID', 'IMSI' ]
-
-    # check input parameters, but do nothing concrete yet
-    def _verify_get_data(self, fields: List[str] = [], key: str = 'ICCID', value: str = "") -> Dict[str, str]:
-        """Verify multiple fields for identified card.
-
-        Args:
-                fields : list of valid field names such as 'ADM1', 'PIN1', ... which are to be obtained
-                key : look-up key to identify card data, such as 'ICCID'
-                value : value for look-up key to identify card data
-        Returns:
-                dictionary of {field, value} strings for each requested field from 'fields'
-        """
-
-        if key not in self.VALID_KEY_FIELD_NAMES:
-            raise ValueError("Key field name '%s' is not a valid field name, valid field names are: %s" %
-                             (key, str(self.VALID_KEY_FIELD_NAMES)))
-
-        return {}
-
-    def get_field(self, field: str, key: str = 'ICCID', value: str = "") -> Optional[str]:
-        """get a single field from CSV file using a specified key/value pair"""
-        fields = [field]
-        result = self.get(fields, key, value)
-        return result.get(field)
-
-    @abc.abstractmethod
-    def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
-        """Get multiple card-individual fields for identified card.
-
-        Args:
-                fields : list of valid field names such as 'ADM1', 'PIN1', ... which are to be obtained
-                key : look-up key to identify card data, such as 'ICCID'
-                value : value for look-up key to identify card data
-        Returns:
-                dictionary of {field, value} strings for each requested field from 'fields'
-        """
-
-
-class CardKeyProviderCsv(CardKeyProvider):
-    """Card key provider implementation that allows to query against a specified CSV file.
-    Supports column-based encryption as it is generally a bad idea to store cryptographic key material in
-    plaintext.  Instead, the key material should be encrypted by a "key-encryption key", occasionally also
-    known as "transport key" (see GSMA FS.28)."""
-    IV = b'\x23' * 16
-    csv_file = None
-    filename = None
-
-    def __init__(self, filename: str, transport_keys: dict):
-        """
-        Args:
-                filename : file name (path) of CSV file containing card-individual key/data
-                transport_keys : a dict indexed by field name, whose values are hex-encoded AES keys for the
-                                 respective field (column) of the CSV.  This is done so that different fields
-                                 (columns) can use different transport keys, which is strongly recommended by
-                                 GSMA FS.28
-        """
-        self.csv_file = open(filename, 'r')
-        if not self.csv_file:
-            raise RuntimeError("Could not open CSV file '%s'" % filename)
-        self.filename = filename
-        self.transport_keys = self.process_transport_keys(transport_keys)
+    __IV = b'\x23' * 16
 
     @staticmethod
-    def process_transport_keys(transport_keys: dict):
+    def __dict_keys_to_upper(d: dict) -> dict:
+            return {k.upper():v for k,v in d.items()}
+
+    @staticmethod
+    def __process_transport_keys(transport_keys: dict, crypt_groups: dict):
         """Apply a single transport key to multiple fields/columns, if the name is a group."""
         new_dict = {}
         for name, key in transport_keys.items():
-            if name in CRYPT_GROUPS:
-                for field in CRYPT_GROUPS[name]:
+            if name in crypt_groups:
+                for field in crypt_groups[name]:
                     new_dict[field] = key
             else:
                 new_dict[name] = key
         return new_dict
 
-    def _decrypt_field(self, field_name: str, encrypted_val: str) -> str:
-        """decrypt a single field, if we have a transport key for the field of that name."""
-        if not field_name in self.transport_keys:
+    def __init__(self, transport_keys: dict):
+        """
+        Create new field encryptor/decryptor object and set transport keys, usually one for each column. In some cases
+        it is also possible to use a single key for multiple columns (see also __CRYPT_GROUPS)
+
+        Args:
+                transport_keys : a dict indexed by field name, whose values are hex-encoded AES keys for the
+                                 respective field (column) of the CSV. This is done so that different fields
+                                 (columns) can use different transport keys, which is strongly recommended by
+                                 GSMA FS.28
+        """
+        self.transport_keys = self.__process_transport_keys(self.__dict_keys_to_upper(transport_keys),
+                                                            self.__CRYPT_GROUPS)
+        for name, key in self.transport_keys.items():
+                log.debug("Encrypting/decrypting field %s using AES key %s" % (name, key))
+
+    def decrypt_field(self, field_name: str, encrypted_val: str) -> str:
+        """
+        Decrypt a single field. The decryption is only applied if we have a transport key is known under the provided
+        field name, otherwise the field is treated as plaintext and passed through as it is.
+
+        Args:
+                field_name : name of the field to decrypt (used to identify which key to use)
+                encrypted_val : encrypted field value
+
+        Returns:
+                plaintext field value
+        """
+        if not field_name.upper() in self.transport_keys:
             return encrypted_val
-        cipher = AES.new(h2b(self.transport_keys[field_name]), AES.MODE_CBC, self.IV)
+        cipher = AES.new(h2b(self.transport_keys[field_name.upper()]), AES.MODE_CBC, self.__IV)
         return b2h(cipher.decrypt(h2b(encrypted_val)))
 
-    def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
-        super()._verify_get_data(fields, key, value)
+    def encrypt_field(self, field_name: str, plaintext_val: str) -> str:
+        """
+        Encrypt a single field. The encryption is only applied if we have a transport key is known under the provided
+        field name, otherwise the field is treated as non sensitive and passed through as it is.
 
+        Args:
+                field_name : name of the field to decrypt (used to identify which key to use)
+                encrypted_val : encrypted field value
+
+        Returns:
+                plaintext field value
+        """
+        if not field_name.upper() in self.transport_keys:
+            return plaintext_val
+        cipher = AES.new(h2b(self.transport_keys[field_name.upper()]), AES.MODE_CBC, self.__IV)
+        return b2h(cipher.encrypt(h2b(plaintext_val)))
+
+class CardKeyProvider(abc.ABC):
+    """Base class, not containing any concrete implementation."""
+
+    @abc.abstractmethod
+    def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
+        """
+        Get multiple card-individual fields for identified card. This method should not fail with an exception in
+        case the entry, columns or even the key column itsself is not found.
+
+        Args:
+                fields : list of valid field names such as 'ADM1', 'PIN1', ... which are to be obtained
+                key : look-up key to identify card data, such as 'ICCID'
+                value : value for look-up key to identify card data
+        Returns:
+                dictionary of {field : value, ...} strings for each requested field from 'fields'. In case nothing is
+                fond None shall be returned.
+        """
+
+    def __str__(self):
+        return type(self).__name__
+
+class CardKeyProviderCsv(CardKeyProvider):
+    """Card key provider implementation that allows to query against a specified CSV file."""
+
+    def __init__(self, csv_filename: str, transport_keys: dict):
+        """
+        Args:
+                csv_filename : file name (path) of CSV file containing card-individual key/data
+                transport_keys : (see class CardKeyFieldCryptor)
+        """
+        log.info("Using CSV file as card key data source: %s" % csv_filename)
+        self.csv_file = open(csv_filename, 'r')
+        if not self.csv_file:
+            raise RuntimeError("Could not open CSV file '%s'" % csv_filename)
+        self.csv_filename = csv_filename
+        self.crypt = CardKeyFieldCryptor(transport_keys)
+
+    def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         self.csv_file.seek(0)
         cr = csv.DictReader(self.csv_file)
         if not cr:
-            raise RuntimeError(
-                "Could not open DictReader for CSV-File '%s'" % self.filename)
+            raise RuntimeError("Could not open DictReader for CSV-File '%s'" % self.csv_filename)
         cr.fieldnames = [field.upper() for field in cr.fieldnames]
 
-        rc = {}
+        if key not in cr.fieldnames:
+            return None
+        return_dict = {}
         for row in cr:
             if row[key] == value:
                 for f in fields:
                     if f in row:
-                        rc.update({f: self._decrypt_field(f, row[f])})
+                        return_dict.update({f: self.crypt.decrypt_field(f, row[f])})
                     else:
-                        raise RuntimeError("CSV-File '%s' lacks column '%s'" %
-                                           (self.filename, f))
-        return rc
+                        raise RuntimeError("CSV-File '%s' lacks column '%s'" % (self.csv_filename, f))
+        if return_dict == {}:
+            return None
+        return return_dict
+
+class CardKeyProviderPgsql(CardKeyProvider):
+    """Card key provider implementation that allows to query against a specified PostgreSQL database table."""
+
+    def __init__(self, config_filename: str, transport_keys: dict):
+        """
+        Args:
+                config_filename : file name (path) of CSV file containing card-individual key/data
+                transport_keys : (see class CardKeyFieldCryptor)
+        """
+        import psycopg2
+        log.info("Using SQL database as card key data source: %s" % config_filename)
+        with open(config_filename, "r") as cfg:
+            config = yaml.load(cfg, Loader=yaml.FullLoader)
+            log.info("Card key database name: %s" % config.get('db_name'))
+            db_users = config.get('db_users')
+            user = db_users.get('reader')
+            if user is None:
+                raise ValueError("user for role 'reader' not set up in config file.")
+            self.conn = psycopg2.connect(dbname=config.get('db_name'),
+                                         user=user.get('name'),
+                                         password=user.get('pass'),
+                                         host=config.get('host'))
+            self.tables = config.get('table_names')
+            log.info("Card key database tables: %s" % str(self.tables))
+            self.crypt = CardKeyFieldCryptor(transport_keys)
+
+    def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
+        import psycopg2
+        from psycopg2.sql import Identifier, SQL
+        db_result = None
+        for t in self.tables:
+            self.conn.rollback()
+            cur = self.conn.cursor()
+
+            # Make sure that the database table and the key column actually exists. If not, move on to the next table
+            cur.execute("SELECT column_name FROM information_schema.columns where table_name = %s;", (t,))
+            cols_result = cur.fetchall()
+            if cols_result == []:
+                log.warning("Card Key database seems to lack table %s, check config file!" % t)
+                continue
+            if (key.lower(),) not in cols_result:
+                continue
+
+            # Query requested columns from database table
+            query = SQL("SELECT {}").format(Identifier(fields[0].lower()))
+            for f in fields[1:]:
+                query += SQL(", {}").format(Identifier(f.lower()))
+            query += SQL(" FROM {} WHERE {} = %s LIMIT 1;").format(Identifier(t.lower()),
+                                                                  Identifier(key.lower()))
+            cur.execute(query, (value,))
+            db_result = cur.fetchone()
+            cur.close()
+
+            if db_result:
+                break
+
+        if db_result is None:
+            return None
+        result = dict(zip(fields, db_result))
+
+        for k in result.keys():
+            result[k] = self.crypt.decrypt_field(k, result.get(k))
+        return result
 
 
 def card_key_provider_register(provider: CardKeyProvider, provider_list=card_key_providers):
@@ -163,11 +261,11 @@ def card_key_provider_register(provider: CardKeyProvider, provider_list=card_key
             provider_list : override the list of providers from the global default
     """
     if not isinstance(provider, CardKeyProvider):
-        raise ValueError("provider is not a card data provier")
+        raise ValueError("provider is not a card data provider")
     provider_list.append(provider)
 
 
-def card_key_provider_get(fields, key: str, value: str, provider_list=card_key_providers) -> Dict[str, str]:
+def card_key_provider_get(fields: list[str], key: str, value: str, provider_list=card_key_providers) -> Dict[str, str]:
     """Query all registered card data providers for card-individual [key] data.
 
     Args:
@@ -178,17 +276,21 @@ def card_key_provider_get(fields, key: str, value: str, provider_list=card_key_p
     Returns:
             dictionary of {field, value} strings for each requested field from 'fields'
     """
+    key = key.upper()
+    fields = [f.upper() for f in fields]
     for p in provider_list:
         if not isinstance(p, CardKeyProvider):
-            raise ValueError(
-                "provider list contains element which is not a card data provier")
+            raise ValueError("Provider list contains element which is not a card data provider")
+        log.debug("Searching for card key data (key=%s, value=%s, provider=%s)" % (key, value, str(p)))
         result = p.get(fields, key, value)
         if result:
+            log.debug("Found card data: %s" % (str(result)))
             return result
-    return {}
+
+    raise ValueError("Unable to find card key data (key=%s, value=%s, fields=%s)" % (key, value, str(fields)))
 
 
-def card_key_provider_get_field(field: str, key: str, value: str, provider_list=card_key_providers) -> Optional[str]:
+def card_key_provider_get_field(field: str, key: str, value: str, provider_list=card_key_providers) -> str:
     """Query all registered card data providers for a single field.
 
     Args:
@@ -199,11 +301,7 @@ def card_key_provider_get_field(field: str, key: str, value: str, provider_list=
     Returns:
             dictionary of {field, value} strings for the requested field
     """
-    for p in provider_list:
-        if not isinstance(p, CardKeyProvider):
-            raise ValueError(
-                "provider list contains element which is not a card data provier")
-        result = p.get_field(field, key, value)
-        if result:
-            return result
-    return None
+
+    fields = [field]
+    result = card_key_provider_get(fields, key, value, card_key_providers)
+    return result.get(field.upper())

pySim/cards.py

@@ -25,8 +25,8 @@
 from typing import Optional, Tuple
 from osmocom.utils import *
 
-from pySim.profile.ts_102_221 import EF_DIR, CardProfileUICC
-from pySim.profile.ts_51_011 import DF_GSM
+from pySim.ts_102_221 import EF_DIR, CardProfileUICC
+from pySim.ts_51_011 import DF_GSM
 from pySim.utils import SwHexstr
 from pySim.commands import Path, SimCardCommands
 
@@ -73,6 +73,16 @@ class CardBase:
         # callers having to do hasattr('read_aids') ahead of every call.
         return []
 
+    def adf_present(self, adf: str = "usim") -> bool:
+        # a non-UICC doesn't have any applications. Convenience helper to avoid
+        # callers having to do hasattr('adf_present') ahead of every call.
+        return False
+
+    def select_adf_by_aid(self, adf: str = "usim", scc: Optional[SimCardCommands] = None) -> Tuple[Optional[Hexstr], Optional[SwHexstr]]:
+        # a non-UICC doesn't have any applications. Convenience helper to avoid
+        # callers having to do hasattr('select_adf_by_aid') ahead of every call.
+        return (None, None)
+
 
 class SimCardBase(CardBase):
     """Here we only add methods for commands specified in TS 51.011, without
@@ -102,6 +112,8 @@ class UiccCardBase(SimCardBase):
     def probe(self) -> bool:
         # EF.DIR is a mandatory EF on all ICCIDs; however it *may* also exist on a TS 51.011 SIM
         ef_dir = EF_DIR()
+        # select MF first
+        self.file_exists("3f00")
         return self.file_exists(ef_dir.fid)
 
     def read_aids(self) -> List[Hexstr]:

pySim/cat.py

@@ -20,19 +20,19 @@ as described in 3GPP TS 31.111."""
 
 from typing import List
 from bidict import bidict
-from construct import Int8ub, Int16ub, Byte, Bytes, BitsInteger
+from construct import Int8ub, Int16ub, Byte, BitsInteger
 from construct import Struct, Enum, BitStruct, this
-from construct import GreedyBytes, Switch, GreedyRange, FlagsEnum
+from construct import Switch, GreedyRange, FlagsEnum
 from osmocom.tlv import TLV_IE, COMPR_TLV_IE, BER_TLV_IE, TLV_IE_Collection
-from osmocom.construct import PlmnAdapter, BcdAdapter, HexAdapter, GsmStringAdapter, TonNpi, GsmString
-from osmocom.utils import b2h
+from osmocom.construct import PlmnAdapter, BcdAdapter, GsmStringAdapter, TonNpi, GsmString, Bytes, GreedyBytes
+from osmocom.utils import b2h, h2b
 from pySim.utils import dec_xplmn_w_act
 
 # Tag values as per TS 101 220 Table 7.23
 
 # TS 102 223 Section 8.1
 class Address(COMPR_TLV_IE, tag=0x86):
-    _construct = Struct('ton_npi'/Int8ub,
+    _construct = Struct('ton_npi'/TonNpi,
                         'call_number'/BcdAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.2
@@ -255,24 +255,24 @@ class Result(COMPR_TLV_IE, tag=0x83):
                             'launch_browser_generic_error': AddlInfoLaunchBrowser,
                             'bearer_independent_protocol_error': AddlInfoBip,
                             'frames_error': AddlInfoFrames
-                        }, default=HexAdapter(GreedyBytes)))
+                        }, default=GreedyBytes))
 
 # TS 102 223 Section 8.13  + TS 31.111 Section 8.13
 class SMS_TPDU(COMPR_TLV_IE, tag=0x8B):
-    _construct = Struct('tpdu'/HexAdapter(GreedyBytes))
+    _construct = Struct('tpdu'/GreedyBytes)
 
 # TS 31.111 Section 8.14
 class SsString(COMPR_TLV_IE, tag=0x89):
-    _construct = Struct('ton_npi'/TonNpi, 'ss_string'/HexAdapter(GreedyBytes))
+    _construct = Struct('ton_npi'/TonNpi, 'ss_string'/GreedyBytes)
 
 
 # TS 102 223 Section 8.15
 class TextString(COMPR_TLV_IE, tag=0x8D):
     _test_de_encode = [
-        ( '8d090470617373776f7264', {'dcs': 4, 'text_string': '70617373776f7264'} ),
+        ( '8d090470617373776f7264', {'dcs': 4, 'text_string': b'password'} )
     ]
     _construct = Struct('dcs'/Int8ub, # TS 03.38
-                        'text_string'/HexAdapter(GreedyBytes))
+                        'text_string'/GreedyBytes)
 
 # TS 102 223 Section 8.16
 class Tone(COMPR_TLV_IE, tag=0x8E):
@@ -308,34 +308,34 @@ class Tone(COMPR_TLV_IE, tag=0x8E):
 # TS 31 111 Section 8.17
 class USSDString(COMPR_TLV_IE, tag=0x8A):
     _construct = Struct('dcs'/Int8ub,
-                        'ussd_string'/HexAdapter(GreedyBytes))
+                        'ussd_string'/GreedyBytes)
 
 # TS 102 223 Section 8.18
 class FileList(COMPR_TLV_IE, tag=0x92):
-    FileId=HexAdapter(Bytes(2))
+    FileId=Bytes(2)
     _construct = Struct('number_of_files'/Int8ub,
                         'files'/GreedyRange(FileId))
 
-# TS 102 223 Secton 8.19
+# TS 102 223 Section 8.19
 class LocationInformation(COMPR_TLV_IE, tag=0x93):
     pass
 
-# TS 102 223 Secton 8.20
+# TS 102 223 Section 8.20
 class IMEI(COMPR_TLV_IE, tag=0x94):
     _construct = BcdAdapter(GreedyBytes)
 
-# TS 102 223 Secton 8.21
+# TS 102 223 Section 8.21
 class HelpRequest(COMPR_TLV_IE, tag=0x95):
     pass
 
-# TS 102 223 Secton 8.22
+# TS 102 223 Section 8.22
 class NetworkMeasurementResults(COMPR_TLV_IE, tag=0x96):
     _construct = BcdAdapter(GreedyBytes)
 
 # TS 102 223 Section 8.23
 class DefaultText(COMPR_TLV_IE, tag=0x97):
     _construct = Struct('dcs'/Int8ub,
-                        'text_string'/HexAdapter(GreedyBytes))
+                        'text_string'/GreedyBytes)
 
 # TS 102 223 Section 8.24
 class ItemsNextActionIndicator(COMPR_TLV_IE, tag=0x98):
@@ -394,7 +394,7 @@ class ItemIconIdentifierList(COMPR_TLV_IE, tag=0x9f):
 
 # TS 102 223 Section 8.35
 class CApdu(COMPR_TLV_IE, tag=0xA2):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.37
 class TimerIdentifier(COMPR_TLV_IE, tag=0xA4):
@@ -406,7 +406,7 @@ class TimerValue(COMPR_TLV_IE, tag=0xA5):
 
 # TS 102 223 Section 8.40
 class AtCommand(COMPR_TLV_IE, tag=0xA8):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.43
 class ImmediateResponse(COMPR_TLV_IE, tag=0xAB):
@@ -418,7 +418,7 @@ class DtmfString(COMPR_TLV_IE, tag=0xAC):
 
 # TS 102 223 Section 8.45
 class Language(COMPR_TLV_IE, tag=0xAD):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 31.111 Section 8.46
 class TimingAdvance(COMPR_TLV_IE, tag=0xC6):
@@ -440,7 +440,7 @@ class Bearer(COMPR_TLV_IE, tag=0xB2):
 
 # TS 102 223 Section 8.50
 class ProvisioningFileReference(COMPR_TLV_IE, tag=0xB3):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.51
 class BrowserTerminationCause(COMPR_TLV_IE, tag=0xB4):
@@ -449,7 +449,7 @@ class BrowserTerminationCause(COMPR_TLV_IE, tag=0xB4):
 # TS 102 223 Section 8.52
 class BearerDescription(COMPR_TLV_IE, tag=0xB5):
     _test_de_encode = [
-        ( 'b50103', {'bearer_parameters': '', 'bearer_type': 'default'} ),
+        ( 'b50103', {'bearer_parameters': b'', 'bearer_type': 'default'} ),
     ]
     # TS 31.111 Section 8.52.1
     BearerParsCs = Struct('data_rate'/Int8ub,
@@ -492,11 +492,11 @@ class BearerDescription(COMPR_TLV_IE, tag=0xB5):
                                                         'packet_grps_utran_eutran': BearerParsPacket,
                                                         'packet_with_extd_params': BearerParsPacketExt,
                                                         'ng_ran': BearerParsNgRan,
-                                                    }, default=HexAdapter(GreedyBytes)))
+                                                    }, default=GreedyBytes))
 
 # TS 102 223 Section 8.53
 class ChannelData(COMPR_TLV_IE, tag = 0xB6):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.54
 class ChannelDataLength(COMPR_TLV_IE, tag = 0xB7):
@@ -510,15 +510,15 @@ class BufferSize(COMPR_TLV_IE, tag = 0xB9):
 class ChannelStatus(COMPR_TLV_IE, tag = 0xB8):
     # complex decoding, depends on out-of-band context/knowledge :(
     # for default / TCP Client mode: bit 8 of first byte indicates connected, 3 LSB indicate channel nr
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.58
 class OtherAddress(COMPR_TLV_IE, tag = 0xBE):
     _test_de_encode = [
-        ( 'be052101020304', {'address': '01020304', 'type_of_address': 'ipv4'} ),
+        ( 'be052101020304', {'address': h2b('01020304'), 'type_of_address': 'ipv4'} ),
     ]
     _construct = Struct('type_of_address'/Enum(Int8ub, ipv4=0x21, ipv6=0x57),
-                        'address'/HexAdapter(GreedyBytes))
+                        'address'/GreedyBytes)
 
 # TS 102 223 Section 8.59
 class UiccTransportLevel(COMPR_TLV_IE, tag = 0xBC):
@@ -532,7 +532,7 @@ class UiccTransportLevel(COMPR_TLV_IE, tag = 0xBC):
 
 # TS 102 223 Section 8.60
 class Aid(COMPR_TLV_IE, tag=0xAF):
-    _construct = Struct('aid'/HexAdapter(GreedyBytes))
+    _construct = Struct('aid'/GreedyBytes)
 
 # TS 102 223 Section 8.61
 class AccessTechnology(COMPR_TLV_IE, tag=0xBF):
@@ -546,35 +546,35 @@ class ServiceRecord(COMPR_TLV_IE, tag=0xC1):
     BearerTechId = Enum(Int8ub, technology_independent=0, bluetooth=1, irda=2, rs232=3, usb=4)
     _construct = Struct('local_bearer_technology'/BearerTechId,
                         'service_identifier'/Int8ub,
-                        'service_record'/HexAdapter(GreedyBytes))
+                        'service_record'/GreedyBytes)
 
 # TS 102 223 Section 8.64
 class DeviceFilter(COMPR_TLV_IE, tag=0xC2):
     _construct = Struct('local_bearer_technology'/ServiceRecord.BearerTechId,
-                        'device_filter'/HexAdapter(GreedyBytes))
+                        'device_filter'/GreedyBytes)
 
 # TS 102 223 Section 8.65
 class ServiceSearchIE(COMPR_TLV_IE, tag=0xC3):
     _construct = Struct('local_bearer_technology'/ServiceRecord.BearerTechId,
-                        'service_search'/HexAdapter(GreedyBytes))
+                        'service_search'/GreedyBytes)
 
 # TS 102 223 Section 8.66
 class AttributeInformation(COMPR_TLV_IE, tag=0xC4):
     _construct = Struct('local_bearer_technology'/ServiceRecord.BearerTechId,
-                        'attribute_information'/HexAdapter(GreedyBytes))
+                        'attribute_information'/GreedyBytes)
 
 
 # TS 102 223 Section 8.68
 class RemoteEntityAddress(COMPR_TLV_IE, tag=0xC9):
     _construct = Struct('coding_type'/Enum(Int8ub, ieee802_16=0, irda=1),
-                        'address'/HexAdapter(GreedyBytes))
+                        'address'/GreedyBytes)
 
 # TS 102 223 Section 8.70
 class NetworkAccessName(COMPR_TLV_IE, tag=0xC7):
     _test_de_encode = [
-        ( 'c704036e6161', '036e6161' ),
+        ( 'c704036e6161', h2b('036e6161') ),
     ]
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.72
 class TextAttribute(COMPR_TLV_IE, tag=0xD0):
@@ -618,15 +618,15 @@ class FrameIdentifier(COMPR_TLV_IE, tag=0xE8):
 
 # TS 102 223 Section 8.82
 class MultimediaMessageReference(COMPR_TLV_IE, tag=0xEA):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.83
 class MultimediaMessageIdentifier(COMPR_TLV_IE, tag=0xEB):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.85
 class MmContentIdentifier(COMPR_TLV_IE, tag=0xEE):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.89
 class ActivateDescriptor(COMPR_TLV_IE, tag=0xFB):
@@ -649,7 +649,7 @@ class ContactlessFunctionalityState(COMPR_TLV_IE, tag=0xD4):
 # TS 31.111 Section 8.91
 class RoutingAreaIdentification(COMPR_TLV_IE, tag=0xF3):
     _construct = Struct('mcc_mnc'/PlmnAdapter(Bytes(3)),
-                        'lac'/HexAdapter(Bytes(2)),
+                        'lac'/Bytes(2),
                         'rac'/Int8ub)
 
 # TS 31.111 Section 8.92
@@ -709,15 +709,15 @@ class EcatSequenceNumber(COMPR_TLV_IE, tag=0xA1):
 
 # TS 102 223 Section 8.99
 class EncryptedTlvList(COMPR_TLV_IE, tag=0xA2):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.100
 class Mac(COMPR_TLV_IE, tag=0xE0):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.101
 class SaTemplate(COMPR_TLV_IE, tag=0xA3):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.103
 class RefreshEnforcementPolicy(COMPR_TLV_IE, tag=0xBA):
@@ -725,7 +725,7 @@ class RefreshEnforcementPolicy(COMPR_TLV_IE, tag=0xBA):
 
 # TS 102 223 Section 8.104
 class DnsServerAddress(COMPR_TLV_IE, tag=0xC0):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.105
 class SupportedRadioAccessTechnologies(COMPR_TLV_IE, tag=0xB4):

pySim/cdma_ruim.py

@@ -25,9 +25,9 @@ from osmocom.construct import *
 
 from pySim.filesystem import *
 from pySim.profile import CardProfile, CardProfileAddon
-from pySim.profile.ts_51_011 import CardProfileSIM
-from pySim.profile.ts_51_011 import DF_TELECOM, DF_GSM
-from pySim.profile.ts_51_011 import EF_ServiceTable
+from pySim.ts_51_011 import CardProfileSIM
+from pySim.ts_51_011 import DF_TELECOM, DF_GSM
+from pySim.ts_51_011 import EF_ServiceTable
 
 
 # Mapping between CDMA Service Number and its description
@@ -115,7 +115,7 @@ class EF_AD(TransparentEF):
     '''3.4.33 Administrative Data'''
 
     _test_de_encode = [
-        ( "000000", { 'ms_operation_mode' : 'normal', 'additional_info' : '0000', 'rfu' : '' } ),
+        ( "000000", { 'ms_operation_mode' : 'normal', 'additional_info' : b'\x00\x00', 'rfu' : b'' } ),
     ]
     _test_no_pad = True
 
@@ -128,15 +128,15 @@ class EF_AD(TransparentEF):
         cell_test = 0x04
 
     def __init__(self, fid='6f43', sfid=None, name='EF.AD',
-                 desc='Service Provider Name', size=(3, None), **kwargs):
+                 desc='Administrative Data', size=(3, None), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct(
             # Byte 1: Display Condition
             'ms_operation_mode'/Enum(Byte, self.OP_MODE),
             # Bytes 2-3: Additional information
-            'additional_info'/HexAdapter(Bytes(2)),
+            'additional_info'/Bytes(2),
             # Bytes 4..: RFU
-            'rfu'/HexAdapter(GreedyBytesRFU),
+            'rfu'/GreedyBytesRFU,
         )
 
 

pySim/commands.py

@@ -32,7 +32,6 @@ from osmocom.tlv import bertlv_encode_len
 from pySim.utils import sw_match, expand_hex, SwHexstr, ResTuple, SwMatchstr
 from pySim.exceptions import SwMatchError
 from pySim.transport import LinkBase
-from pySim.ts_102_221 import decode_select_response
 
 # A path can be either just a FID or a list of FID
 Path = typing.Union[Hexstr, List[Hexstr]]
@@ -142,7 +141,7 @@ class SimCardCommands:
         Returns:
                 Tuple of (decoded_data, sw)
         """
-        cmd = cmd_constr.build(cmd_data) if cmd_data else ''
+        cmd = cmd_constr.build(cmd_data) if cmd_data else b''
         lc = i2h([len(cmd)]) if cmd_data else ''
         le = '00' if resp_constr else ''
         pdu = ''.join([cla, ins, p1, p2, lc, b2h(cmd), le])
@@ -177,6 +176,40 @@ class SimCardCommands:
             raise SwMatchError(sw, sw_exp.lower(), self._tp.sw_interpreter)
         return (rsp, sw)
 
+    # Extract a single FCP item from TLV
+    def __parse_fcp(self, fcp: Hexstr):
+        # see also: ETSI TS 102 221, chapter 11.1.1.3.1 Response for MF,
+        # DF or ADF
+        from pytlv.TLV import TLV
+        tlvparser = TLV(['82', '83', '84', 'a5', '8a', '8b',
+                        '8c', '80', 'ab', 'c6', '81', '88'])
+
+        # pytlv is case sensitive!
+        fcp = fcp.lower()
+
+        if fcp[0:2] != '62':
+            raise ValueError(
+                'Tag of the FCP template does not match, expected 62 but got %s' % fcp[0:2])
+
+        # Unfortunately the spec is not very clear if the FCP length is
+        # coded as one or two byte vale, so we have to try it out by
+        # checking if the length of the remaining TLV string matches
+        # what we get in the length field.
+        # See also ETSI TS 102 221, chapter 11.1.1.3.0 Base coding.
+        # TODO: this likely just is normal BER-TLV ("All data objects are BER-TLV except if otherwise # defined.")
+        exp_tlv_len = int(fcp[2:4], 16)
+        if len(fcp[4:]) // 2 == exp_tlv_len:
+            skip = 4
+        else:
+            exp_tlv_len = int(fcp[2:6], 16)
+            if len(fcp[4:]) // 2 == exp_tlv_len:
+                skip = 6
+            raise ValueError('Cannot determine length of TLV-length')
+
+        # Skip FCP tag and length
+        tlv = fcp[skip:]
+        return tlvparser.parse(tlv)
+
     # Tell the length of a record by the card response
     # USIMs respond with an FCP template, which is different
     # from what SIMs responds. See also:
@@ -184,8 +217,10 @@ class SimCardCommands:
     # SIM: GSM 11.11, chapter 9.2.1 SELECT
     def __record_len(self, r) -> int:
         if self.sel_ctrl == "0004":
-            fcp_parsed = decode_select_response(r[-1])
-            return fcp_parsed['file_descriptor']['record_len']
+            tlv_parsed = self.__parse_fcp(r[-1])
+            file_descriptor = tlv_parsed['82']
+            # See also ETSI TS 102 221, chapter 11.1.1.4.3 File Descriptor
+            return int(file_descriptor[4:8], 16)
         else:
             return int(r[-1][28:30], 16)
 
@@ -193,8 +228,8 @@ class SimCardCommands:
     # above.
     def __len(self, r) -> int:
         if self.sel_ctrl == "0004":
-            fcp_parsed = decode_select_response(r[-1])
-            return fcp_parsed['file_size']
+            tlv_parsed = self.__parse_fcp(r[-1])
+            return int(tlv_parsed['80'], 16)
         else:
             return int(r[-1][4:8], 16)
 
@@ -250,7 +285,7 @@ class SimCardCommands:
         return self.send_apdu_checksw(self.cla_byte + "a40304")
 
     def select_adf(self, aid: Hexstr) -> ResTuple:
-        """Execute SELECT a given Applicaiton ADF.
+        """Execute SELECT a given Application ADF.
 
         Args:
                 aid : application identifier as hex string
@@ -542,7 +577,7 @@ class SimCardCommands:
 
         Args:
                 rand : 16 byte random data as hex string (RAND)
-                autn : 8 byte Autentication Token (AUTN)
+                autn : 8 byte Authentication Token (AUTN)
                 context : 16 byte random data ('3g' or 'gsm')
         """
         # 3GPP TS 31.102 Section 7.1.2.1
@@ -700,7 +735,7 @@ class SimCardCommands:
         Args:
                 payload : payload as hex string
         """
-        return self.send_apdu_checksw('80c20000%02x%s' % (len(payload)//2, payload), apply_lchan = False)
+        return self.send_apdu_checksw('80c20000%02x%s' % (len(payload)//2, payload) + "00", apply_lchan = False)
 
     def terminal_profile(self, payload: Hexstr) -> ResTuple:
         """Send TERMINAL PROFILE to card

pySim/esim/__init__.py

@@ -54,6 +54,8 @@ def compile_asn1_subdir(subdir_name:str, codec='der'):
     __ver = sys.version_info
     if (__ver.major, __ver.minor) >= (3, 9):
         for i in resources.files('pySim.esim').joinpath('asn1').joinpath(subdir_name).iterdir():
+            if not i.name.endswith('.asn'):
+                continue
             asn_txt += i.read_text()
             asn_txt += "\n"
     #else:
@@ -61,8 +63,8 @@ def compile_asn1_subdir(subdir_name:str, codec='der'):
     return asn1tools.compile_string(asn_txt, codec=codec)
 
 
-# SGP.22 section 4.1 Activation Code
 class ActivationCode:
+    """SGP.22 section 4.1 Activation Code"""
     def __init__(self, hostname:str, token:str, oid: Optional[str] = None, cc_required: Optional[bool] = False):
         if '$' in hostname:
             raise ValueError('$ sign not permitted in hostname')
@@ -78,6 +80,7 @@ class ActivationCode:
 
     @staticmethod
     def decode_str(ac: str) -> dict:
+        """decode an activation code from its string representation."""
         if ac[0] != '1':
             raise ValueError("Unsupported AC_Format '%s'!" % ac[0])
         ac_elements = ac.split('$')

pySim/esim/bsp.py

@@ -1,11 +1,9 @@
-# Early proof-of-concept implementation of
-# GSMA eSIM RSP (Remote SIM Provisioning BSP (BPP Protection Protocol),
-# where BPP is the Bound  Profile Package.  So the full expansion is the
-# "GSMA eSIM Remote SIM Provisioning Bound Profile Packate Protection Protocol"
-#
-# Originally (SGP.22 v2.x) this was called SCP03t, but it has since been
-# renamed to BSP.
-#
+"""Implementation of GSMA eSIM RSP (Remote SIM Provisioning BSP (BPP Protection Protocol),
+where BPP is the Bound  Profile Package.  So the full expansion is the
+"GSMA eSIM Remote SIM Provisioning Bound Profile Packate Protection Protocol"
+
+Originally (SGP.22 v2.x) this was called SCP03t, but it has since been renamed to BSP."""
+
 # (C) 2023 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -45,6 +43,7 @@ logger.addHandler(logging.NullHandler())
 MAX_SEGMENT_SIZE = 1020
 
 class BspAlgo(abc.ABC):
+    """Base class representing a cryptographic algorithm within the BSP (BPP Security Protocol)."""
     blocksize: int
 
     def _get_padding(self, in_len: int, multiple: int, padding: int = 0) -> bytes:
@@ -62,6 +61,7 @@ class BspAlgo(abc.ABC):
         return self.__class__.__name__
 
 class BspAlgoCrypt(BspAlgo, abc.ABC):
+    """Base class representing an encryption/decryption algorithm within the BSP (BPP Security Protocol)."""
 
     def __init__(self, s_enc: bytes):
         self.s_enc = s_enc
@@ -73,7 +73,7 @@ class BspAlgoCrypt(BspAlgo, abc.ABC):
         block_nr = self.block_nr
         ciphertext = self._encrypt(padded_data)
         logger.debug("encrypt(block_nr=%u, s_enc=%s, plaintext=%s, padded=%s) -> %s",
-                     block_nr, b2h(self.s_enc), b2h(data), b2h(padded_data), b2h(ciphertext))
+                     block_nr, b2h(self.s_enc)[:20], b2h(data)[:20], b2h(padded_data)[:20], b2h(ciphertext)[:20])
         return ciphertext
 
     def decrypt(self, data:bytes) -> bytes:
@@ -93,6 +93,7 @@ class BspAlgoCrypt(BspAlgo, abc.ABC):
         """Actual implementation, to be implemented by derived class."""
 
 class BspAlgoCryptAES128(BspAlgoCrypt):
+    """AES-CBC-128 implementation of the BPP Security Protocol for GSMA SGP.22 eSIM."""
     name = 'AES-CBC-128'
     blocksize = 16
 
@@ -133,6 +134,7 @@ class BspAlgoCryptAES128(BspAlgoCrypt):
 
 
 class BspAlgoMac(BspAlgo, abc.ABC):
+    """Base class representing a message authentication code algorithm within the BSP (BPP Security Protocol)."""
     l_mac = 0 # must be overridden by derived class
 
     def __init__(self, s_mac: bytes, initial_mac_chaining_value: bytes):
@@ -147,10 +149,20 @@ class BspAlgoMac(BspAlgo, abc.ABC):
         temp_data = self.mac_chain + tag_and_length + data
         old_mcv = self.mac_chain
         c_mac = self._auth(temp_data)
+
+        # DEBUG: Show MAC computation details
+        logger.debug(f"MAC_DEBUG: tag=0x{tag:02x}, lcc={lcc}")
+        logger.debug(f"MAC_DEBUG: tag_and_length: {tag_and_length.hex()}")
+        logger.debug(f"MAC_DEBUG: mac_chain[:20]: {old_mcv[:20].hex()}")
+        logger.debug(f"MAC_DEBUG: temp_data[:20]: {temp_data[:20].hex()}")
+        logger.debug(f"MAC_DEBUG: c_mac: {c_mac.hex()}")
+
         # The output data is computed by concatenating the following data: the tag, the final length, the result of step 2 and the C-MAC value.
         ret = tag_and_length + data + c_mac
+        logger.debug(f"MAC_DEBUG: final_output[:20]: {ret[:20].hex()}")
+
         logger.debug("auth(tag=0x%x, mcv=%s, s_mac=%s, plaintext=%s, temp=%s) -> %s",
-                     tag, b2h(old_mcv), b2h(self.s_mac), b2h(data), b2h(temp_data), b2h(ret))
+                     tag, b2h(old_mcv)[:20], b2h(self.s_mac)[:20], b2h(data)[:20], b2h(temp_data)[:20], b2h(ret)[:20])
         return ret
 
     def verify(self, ciphertext: bytes) -> bool:
@@ -167,6 +179,7 @@ class BspAlgoMac(BspAlgo, abc.ABC):
         """To be implemented by algorithm specific derived class."""
 
 class BspAlgoMacAES128(BspAlgoMac):
+    """AES-CMAC-128 implementation of the BPP Security Protocol for GSMA SGP.22 eSIM."""
     name = 'AES-CMAC-128'
     l_mac = 8
 
@@ -201,6 +214,11 @@ def bsp_key_derivation(shared_secret: bytes, key_type: int, key_length: int, hos
     s_enc = out[l:2*l]
     s_mac = out[l*2:3*l]
 
+    logger.debug(f"BSP_KDF_DEBUG: kdf_out = {b2h(out)}")
+    logger.debug(f"BSP_KDF_DEBUG: initial_mcv = {b2h(initial_mac_chaining_value)}")
+    logger.debug(f"BSP_KDF_DEBUG: s_enc = {b2h(s_enc)}")
+    logger.debug(f"BSP_KDF_DEBUG: s_mac = {b2h(s_mac)}")
+
     return s_enc, s_mac, initial_mac_chaining_value
 
 
@@ -225,12 +243,24 @@ class BspInstance:
         return cls(s_enc, s_mac, initial_mcv)
 
     def encrypt_and_mac_one(self, tag: int, plaintext:bytes) -> bytes:
-        """Encrypt + MAC a single plaintext TLV. Returns the protected ciphertex."""
+        """Encrypt + MAC a single plaintext TLV. Returns the protected ciphertext."""
         assert tag <= 255
         assert len(plaintext) <= self.max_payload_size
-        logger.debug("encrypt_and_mac_one(tag=0x%x, plaintext=%s)", tag, b2h(plaintext))
+
+        # DEBUG: Show what we're processing
+        logger.debug(f"BSP_DEBUG: encrypt_and_mac_one(tag=0x{tag:02x}, plaintext_len={len(plaintext)})")
+        logger.debug(f"BSP_DEBUG: plaintext[:20]: {plaintext[:20].hex()}")
+        logger.debug(f"BSP_DEBUG: s_enc[:20]: {self.c_algo.s_enc[:20].hex()}")
+        logger.debug(f"BSP_DEBUG: s_mac[:20]: {self.m_algo.s_mac[:20].hex()}")
+
+        logger.debug("encrypt_and_mac_one(tag=0x%x, plaintext=%s)", tag, b2h(plaintext)[:20])
         ciphered = self.c_algo.encrypt(plaintext)
+        logger.debug(f"BSP_DEBUG: ciphered[:20]: {ciphered[:20].hex()}")
+
         maced = self.m_algo.auth(tag, ciphered)
+        logger.debug(f"BSP_DEBUG: final_result[:20]: {maced[:20].hex()}")
+        logger.debug(f"BSP_DEBUG: final_result_len: {len(maced)}")
+
         return maced
 
     def encrypt_and_mac(self, tag: int, plaintext:bytes) -> List[bytes]:
@@ -250,11 +280,11 @@ class BspInstance:
         return result
 
     def mac_only_one(self, tag: int, plaintext: bytes) -> bytes:
-        """MAC a single plaintext TLV. Returns the protected ciphertex."""
+        """MAC a single plaintext TLV. Returns the protected ciphertext."""
         assert tag <= 255
-        assert len(plaintext) < self.max_payload_size
+        assert len(plaintext) <= self.max_payload_size
         maced = self.m_algo.auth(tag, plaintext)
-        # The data block counter for ICV caluclation is incremented also for each segment with C-MAC only.
+        # The data block counter for ICV calculation is incremented also for each segment with C-MAC only.
         self.c_algo.block_nr += 1
         return maced
 
@@ -288,7 +318,7 @@ class BspInstance:
     def demac_only_one(self, ciphertext: bytes) -> bytes:
         payload = self.m_algo.verify(ciphertext)
         _tdict, _l, val, _remain = bertlv_parse_one(payload)
-        # The data block counter for ICV caluclation is incremented also for each segment with C-MAC only.
+        # The data block counter for ICV calculation is incremented also for each segment with C-MAC only.
         self.c_algo.block_nr += 1
         return val
 

pySim/esim/es2p.py

@@ -27,7 +27,7 @@ logger.setLevel(logging.DEBUG)
 
 class param:
     class Iccid(ApiParamString):
-        """String representation of 19 or 20 digits, where the 20th digit MAY optionally be the padding
+        """String representation of 18 to 20 digits, where the 20th digit MAY optionally be the padding
         character F."""
         @classmethod
         def _encode(cls, data):
@@ -40,7 +40,7 @@ class param:
 
         @classmethod
         def verify_encoded(cls, data):
-            if len(data) not in [19, 20]:
+            if len(data) not in (18, 19, 20):
                 raise ValueError('ICCID (%s) length (%u) invalid' % (data, len(data)))
 
         @classmethod
@@ -53,7 +53,7 @@ class param:
         @classmethod
         def verify_decoded(cls, data):
             data = str(data)
-            if len(data) not in [19, 20]:
+            if len(data) not in (18, 19, 20):
                 raise ValueError('ICCID (%s) length (%u) invalid' % (data, len(data)))
             if len(data) == 19:
                 decimal_part = data
@@ -116,7 +116,7 @@ class param:
         pass
 
 class Es2PlusApiFunction(JsonHttpApiFunction):
-    """Base classs for representing an ES2+ API Function."""
+    """Base class for representing an ES2+ API Function."""
     pass
 
 # ES2+ DownloadOrder function (SGP.22 section 5.3.1)

pySim/esim/es8p.py

@@ -1,6 +1,5 @@
-# Implementation of GSMA eSIM RSP (Remote SIM Provisioning) ES8+
-# as per SGP22 v3.0 Section 5.5
-#
+"""Implementation of GSMA eSIM RSP (Remote SIM Provisioning) ES8+ as per SGP22 v3.0 Section 5.5"""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -26,6 +25,9 @@ import pySim.esim.rsp as rsp
 from pySim.esim.bsp import BspInstance
 from pySim.esim import PMO
 
+import logging
+logger = logging.getLogger(__name__)
+
 # Given that GSMA RSP uses ASN.1 in a very weird way, we actually cannot encode the full data type before
 # signing, but we have to build parts of it separately first, then sign that, so we can put the signature
 # into the same sequence as the signed data.  We use the existing pySim TLV code for this.
@@ -74,10 +76,11 @@ def gen_replace_session_keys(ppk_enc: bytes, ppk_cmac: bytes, initial_mcv: bytes
 class ProfileMetadata:
     """Representation of Profile metadata. Right now only the mandatory bits are
     supported, but in general this should follow the StoreMetadataRequest of SGP.22 5.5.3"""
-    def __init__(self, iccid_bin: bytes, spn: str, profile_name: str):
+    def __init__(self, iccid_bin: bytes, spn: str, profile_name: str, profile_class = 'operational'):
         self.iccid_bin = iccid_bin
         self.spn = spn
         self.profile_name = profile_name
+        self.profile_class = profile_class
         self.icon = None
         self.icon_type = None
         self.notifications = []
@@ -97,12 +100,20 @@ class ProfileMetadata:
         self.notifications.append((event, address))
 
     def gen_store_metadata_request(self) -> bytes:
-        """Generate encoded (but unsigned) StoreMetadataReqest DO (SGP.22 5.5.3)"""
+        """Generate encoded (but unsigned) StoreMetadataRequest DO (SGP.22 5.5.3)"""
         smr = {
             'iccid': self.iccid_bin,
             'serviceProviderName': self.spn,
             'profileName': self.profile_name,
         }
+        if self.profile_class == 'test':
+            smr['profileClass'] = 0
+        elif self.profile_class == 'provisioning':
+            smr['profileClass'] = 1
+        elif self.profile_class == 'operational':
+            smr['profileClass'] = 2
+        else:
+            raise ValueError('Unsupported Profile Class %s' % self.profile_class)
         if self.icon:
             smr['icon'] = self.icon
             smr['iconType'] = self.icon_type
@@ -197,8 +208,12 @@ class BoundProfilePackage(ProfilePackage):
         # 'initialiseSecureChannelRequest'
         bpp_seq = rsp.asn1.encode('InitialiseSecureChannelRequest', iscr)
         # firstSequenceOf87
+        logger.debug("BPP_ENCODE_DEBUG: Encrypting ConfigureISDP with BSP keys")
+        logger.debug(f"BPP_ENCODE_DEBUG: BSP S-ENC: {bsp.c_algo.s_enc.hex()}")
+        logger.debug(f"BPP_ENCODE_DEBUG: BSP S-MAC: {bsp.m_algo.s_mac.hex()}")
         bpp_seq += encode_seq(0xa0, bsp.encrypt_and_mac(0x87, conf_idsp_bin))
         # sequenceOF88
+        logger.debug("BPP_ENCODE_DEBUG: MAC-only StoreMetadata with BSP keys")
         bpp_seq += encode_seq(0xa1, bsp.mac_only(0x88, smr_bin))
 
         if self.ppp: # we have to use session keys

pySim/esim/es9p.py

@@ -1,4 +1,4 @@
-"""GSMA eSIM RSP ES9+ interface according ot SGP.22 v2.5"""
+"""GSMA eSIM RSP ES9+ interface according to SGP.22 v2.5"""
 
 # (C) 2024 by Harald Welte <laforge@osmocom.org>
 #

pySim/esim/http_json_api.py

@@ -26,15 +26,15 @@ logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
 class ApiParam(abc.ABC):
-    """A class reprsenting a single parameter in the API."""
+    """A class representing a single parameter in the API."""
     @classmethod
     def verify_decoded(cls, data):
-        """Verify the decoded reprsentation of a value. Should raise an exception if somthing is odd."""
+        """Verify the decoded representation of a value. Should raise an exception if something is odd."""
         pass
 
     @classmethod
     def verify_encoded(cls, data):
-        """Verify the encoded reprsentation of a value. Should raise an exception if somthing is odd."""
+        """Verify the encoded representation of a value. Should raise an exception if something is odd."""
         pass
 
     @classmethod
@@ -149,7 +149,8 @@ class ApiError(Exception):
             'message': None,
         }
         actual_sec = func_ex_status.get('statusCodeData', None)
-        sec.update(actual_sec)
+        if actual_sec:
+            sec.update(actual_sec)
         self.subject_code = sec['subjectCode']
         self.reason_code = sec['reasonCode']
         self.subject_id = sec['subjectIdentifier']
@@ -159,7 +160,7 @@ class ApiError(Exception):
         return f'{self.status}("{self.subject_code}","{self.reason_code}","{self.subject_id}","{self.message}")'
 
 class JsonHttpApiFunction(abc.ABC):
-    """Base classs for representing an HTTP[s] API Function."""
+    """Base class for representing an HTTP[s] API Function."""
      # the below class variables are expected to be overridden in derived classes
 
     path = None
@@ -255,5 +256,10 @@ class JsonHttpApiFunction(abc.ABC):
             raise HttpHeaderError(response)
 
         if response.content:
-            return self.decode(response.json())
+            if response.headers.get('Content-Type').startswith('application/json'):
+                return self.decode(response.json())
+            elif response.headers.get('Content-Type').startswith('text/plain;charset=UTF-8'):
+                return { 'data': response.content.decode('utf-8') }
+            raise HttpHeaderError(f'unimplemented response Content-Type: {response.headers=!r}')
+
         return None

pySim/esim/rsp.py

@@ -1,6 +1,5 @@
-# Implementation of GSMA eSIM RSP (Remote SIM Provisioning)
-# as per SGP22 v3.0
-#
+"""Implementation of GSMA eSIM RSP (Remote SIM Provisioning) as per SGP22 v3.0"""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -91,13 +90,61 @@ class RspSessionState:
             # FIXME: how to add the public key from smdp_otpk to an instance of EllipticCurvePrivateKey?
             del state['_smdp_otsk']
             del state['_smdp_ot_curve']
-        # automatically recover all the remainig state
+        # automatically recover all the remaining state
         self.__dict__.update(state)
 
 
-class RspSessionStore(shelve.DbfilenameShelf):
-    """A derived class as wrapper around the database-backed non-volatile storage 'shelve', in case we might
-    need to extend it in the future. We use it to store RspSessionState objects indexed by transactionId."""
+class RspSessionStore:
+    """A wrapper around the database-backed storage 'shelve' for storing RspSessionState objects.
+    Can be configured to use either file-based storage or in-memory storage.
+    We use it to store RspSessionState objects indexed by transactionId."""
+
+    def __init__(self, filename: Optional[str] = None, in_memory: bool = False):
+        self._in_memory = in_memory
+
+        if in_memory:
+            self._shelf = shelve.Shelf(dict())
+        else:
+            if filename is None:
+                raise ValueError("filename is required for file-based session store")
+            self._shelf = shelve.open(filename)
+
+    # dunder magic
+    def __getitem__(self, key):
+        return self._shelf[key]
+
+    def __setitem__(self, key, value):
+        self._shelf[key] = value
+
+    def __delitem__(self, key):
+        del self._shelf[key]
+
+    def __contains__(self, key):
+        return key in self._shelf
+
+    def __iter__(self):
+        return iter(self._shelf)
+
+    def __len__(self):
+        return len(self._shelf)
+
+    # everything else
+    def __getattr__(self, name):
+        """Delegate attribute access to the underlying shelf object."""
+        return getattr(self._shelf, name)
+
+    def close(self):
+        """Close the session store."""
+        if hasattr(self._shelf, 'close'):
+            self._shelf.close()
+        if self._in_memory:
+            # For in-memory store, clear the reference
+            self._shelf = None
+
+    def sync(self):
+        """Synchronize the cache with the underlying storage."""
+        if hasattr(self._shelf, 'sync'):
+            self._shelf.sync()
 
 def extract_euiccSigned1(authenticateServerResponse: bytes) -> bytes:
     """Extract the raw, DER-encoded binary euiccSigned1 field from the given AuthenticateServerResponse. This

pySim/esim/saip/__init__.py

@@ -1,5 +1,5 @@
-# Implementation of SimAlliance/TCA Interoperable Profile handling
-#
+"""Implementation of SimAlliance/TCA Interoperable Profile handling"""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -18,13 +18,19 @@
 import logging
 import abc
 import io
+import os
 from typing import Tuple, List, Optional, Dict, Union
 from collections import OrderedDict
+from difflib import SequenceMatcher, Match
+
 import asn1tools
+import zipfile
+from pySim import javacard
 from osmocom.utils import b2h, h2b, Hexstr
 from osmocom.tlv import BER_TLV_IE, bertlv_parse_tag, bertlv_parse_len
-from osmocom.construct import build_construct, parse_construct, GreedyInteger
+from osmocom.construct import build_construct, parse_construct, GreedyInteger, GreedyBytes, StripHeaderAdapter
 
+from pySim import ts_102_222
 from pySim.utils import dec_imsi
 from pySim.ts_102_221 import FileDescriptor
 from pySim.filesystem import CardADF, Path
@@ -40,8 +46,31 @@ asn1 = compile_asn1_subdir('saip')
 
 logger = logging.getLogger(__name__)
 
+class NonMatch(Match):
+    """Representing a contiguous non-matching block of data; the opposite of difflib.Match"""
+    @classmethod
+    def from_matchlist(cls, l: List[Match], size:int) -> List['NonMatch']:
+        """Build a list of non-matching blocks of data from its inverse (list of matching blocks).
+        The caller must ensure that the input list is ordered, non-overlapping and only contains
+        matches at equal offsets in a and b."""
+        res = []
+        cur = 0
+        for match in l:
+            if match.a != match.b:
+                raise ValueError('only works for equal-offset matches')
+            assert match.a >= cur
+            nm_len = match.a - cur
+            if nm_len > 0:
+                # there's no point in generating zero-lenth non-matching sections
+                res.append(cls(a=cur, b=cur, size=nm_len))
+            cur = match.a + match.size
+        if size > cur:
+            res.append(cls(a=cur, b=cur, size=size-cur))
+
+        return res
+
 class Naa:
-    """A class defining a Network Access Application (NAA)."""
+    """A class defining a Network Access Application (NAA)"""
     name = None
     # AID prefix, as used for ADF and EF.DIR
     aid = None
@@ -57,6 +86,7 @@ class Naa:
         return 'adf-' + cls.mandatory_services[0]
 
 class NaaCsim(Naa):
+    """A class representing the CSIM (CDMA) Network Access Application (NAA)"""
     name = "csim"
     aid = h2b("")
     mandatory_services = ["csim"]
@@ -64,6 +94,7 @@ class NaaCsim(Naa):
     templates = [oid.ADF_CSIM_by_default, oid.ADF_CSIMopt_not_by_default]
 
 class NaaUsim(Naa):
+    """A class representing the USIM Network Access Application (NAA)"""
     name = "usim"
     aid = h2b("a0000000871002")
     mandatory_services = ["usim"]
@@ -76,6 +107,7 @@ class NaaUsim(Naa):
     adf = ADF_USIM()
 
 class NaaIsim(Naa):
+    """A class representing the ISIM Network Access Application (NAA)"""
     name = "isim"
     aid = h2b("a0000000871004")
     mandatory_services = ["isim"]
@@ -104,7 +136,7 @@ class File:
         self.pe_name = pename
         self._name = name
         self.template = template
-        self.body: Optional[bytes] = None
+        self._body: Optional[bytes] = None
         self.node: Optional['FsNode'] = None
         self.file_type = None
         self.fid: Optional[int] = None
@@ -114,8 +146,13 @@ class File:
         self.nb_rec: Optional[int] = None
         self._file_size = 0
         self.high_update: bool = False
+        self.read_and_update_when_deact: bool = False
         self.shareable: bool = True
         self.df_name = None
+        self.fill_pattern = None
+        self.fill_pattern_repeat = False
+        self.pstdo = None # pinStatusTemplateDO, mandatory for DF/ADF
+        self.lcsi = None # optional life cycle status indicator
         # apply some defaults from profile
         if self.template:
             self.from_template(self.template)
@@ -134,6 +171,9 @@ class File:
     def file_size(self) -> Optional[int]:
         """Return the size of the file in bytes."""
         if self.file_type in ['LF', 'CY']:
+            if self._file_size and self.nb_rec is None and self.rec_len:
+                self.nb_rec = self._file_size // self.rec_len
+
             return self.nb_rec * self.rec_len
         elif self.file_type in ['TR', 'BT']:
             return self._file_size
@@ -173,9 +213,7 @@ class File:
         self.file_type = template.file_type
         self.fid = template.fid
         self.sfi = template.sfi
-        self.arr = template.arr
-        #self.default_val = template.default_val
-        #self.default_val_repeat = template.default_val_repeat
+        self.arr = template.arr.to_bytes(1, 'big')
         if hasattr(template, 'rec_len'):
             self.rec_len = template.rec_len
         else:
@@ -188,19 +226,38 @@ class File:
         # All the files defined in the templates shall have, by default, shareable/not-shareable bit in the file descriptor set to "shareable".
         self.shareable = True
         self._template_derived = True
+        if hasattr(template, 'file_size'):
+            self._file_size = template.file_size
+
+    def _recompute_size(self):
+        """recompute the file size, if needed (body larger than current size)"""
+        body_size = len(self.body)
+        if self.file_size == None or self.file_size < body_size:
+            self._file_size = body_size
+
+    @property
+    def body(self):
+        return self._body
+
+    @body.setter
+    def body(self, value: bytes):
+        self._body = value
+        # we need to potentially update the file size after changing the body [size]
+        self._recompute_size()
 
     def to_fileDescriptor(self) -> dict:
         """Convert from internal representation to 'fileDescriptor' as used by asn1tools for SAIP"""
         fileDescriptor = {}
         fdb_dec = {}
         pefi = {}
-        if self.fid:
+        spfi = 0
+        if self.fid and self.fid != self.template.fid:
             fileDescriptor['fileID'] = self.fid.to_bytes(2, 'big')
-        if self.sfi:
+        if self.sfi and self.sfi != self.template.sfi:
             fileDescriptor['shortEFID'] = bytes([self.sfi])
         if self.df_name:
             fileDescriptor['dfName'] = self.df_name
-        if self.arr:
+        if self.arr and self.arr != self.template.arr.to_bytes(1, 'big'):
             fileDescriptor['securityAttributesReferenced'] = self.arr
         if self.file_type in ['LF', 'CY']:
             fdb_dec['file_type'] = 'working_ef'
@@ -223,6 +280,8 @@ class File:
         elif self.file_type in ['MF', 'DF', 'ADF']:
             fdb_dec['file_type'] = 'df'
             fdb_dec['structure'] = 'no_info_given'
+            # pinStatusTemplateDO is mandatory for DF/ADF
+            fileDescriptor['pinStatusTemplateDO'] = self.pstdo
         # build file descriptor based on above input data
         fd_dict = {}
         if len(fdb_dec):
@@ -233,21 +292,24 @@ class File:
         if len(fd_dict):
             fileDescriptor['fileDescriptor'] = build_construct(FileDescriptor._construct, fd_dict)
         if self.high_update:
-            pefi['specialFileInformation'] = b'\x80' # TS 102 222 Table 5
-        try:
-            if self.template and self.template.default_val_repeat:
-                pefi['repeatPattern'] = self.template.expand_default_value_pattern()
-            elif self.template and self.template.default_val:
-                pefi['fillPattern'] = self.template.expand_default_value_pattern()
-        except ValueError:
-            # ignore this here as without a file or record length we cannot do this
-            pass
+            spfi |= 0x80 # TS 102 222 Table 5
+        if self.read_and_update_when_deact:
+            spfi |= 0x40 # TS 102 222 Table 5
+        if spfi != 0x00:
+            pefi['specialFileInformation'] = spfi.to_bytes(1, 'big')
+        if self.fill_pattern:
+            if not self.fill_pattern_repeat:
+                pefi['fillPattern'] = self.fill_pattern
+            else:
+                pefi['repeatPattern'] = self.fill_pattern
         if len(pefi.keys()):
             # TODO: When overwriting the default "proprietaryEFInfo" for a template EF for which a
             # default fill or repeat pattern is defined; it is hence recommended to provide the
             # desired fill or repeat pattern in the "proprietaryEFInfo" element for the EF in Profiles
             # downloaded to a V2.2 or earlier eUICC.
             fileDescriptor['proprietaryEFInfo'] = pefi
+        if self.lcsi:
+            fileDescriptor['lcsi'] = self.lcsi
         logger.debug("%s: to_fileDescriptor(%s)" % (self, fileDescriptor))
         return fileDescriptor
 
@@ -263,6 +325,12 @@ class File:
         dfName = fileDescriptor.get('dfName', None)
         if dfName:
             self.df_name = dfName
+        efFileSize = fileDescriptor.get('efFileSize', None)
+        if efFileSize:
+            self._file_size = self._decode_file_size(efFileSize)
+
+        self.pstdo = fileDescriptor.get('pinStatusTemplateDO', None)
+        self.lcsi = fileDescriptor.get('lcsi', None)
         pefi = fileDescriptor.get('proprietaryEFInfo', {})
         securityAttributesReferenced = fileDescriptor.get('securityAttributesReferenced', None)
         if securityAttributesReferenced:
@@ -272,13 +340,14 @@ class File:
             fdb_dec = fd_dec['file_descriptor_byte']
             self.shareable = fdb_dec['shareable']
             if fdb_dec['file_type'] == 'working_ef':
-                efFileSize = fileDescriptor.get('efFileSize', None)
-                if efFileSize:
-                    self._file_size = self._decode_file_size(efFileSize)
                 if fd_dec['num_of_rec']:
                     self.nb_rec = fd_dec['num_of_rec']
                 if fd_dec['record_len']:
                     self.rec_len = fd_dec['record_len']
+                if efFileSize:
+                    if self.rec_len and self.nb_rec == None:
+                        # compute the number of records from file size and record length
+                        self.nb_rec = self._file_size // self.rec_len
                 if fdb_dec['structure'] == 'linear_fixed':
                     self.file_type = 'LF'
                 elif fdb_dec['structure'] == 'cyclic':
@@ -291,14 +360,19 @@ class File:
                         self._file_size = self._decode_file_size(pefi['maximumFileSize'])
                 specialFileInformation = pefi.get('specialFileInformation', None)
                 if specialFileInformation:
+                    # TS 102 222 Table 5
                     if specialFileInformation[0] & 0x80:
-                        self.hihgi_update = True
+                        self.high_update = True
+                    if specialFileInformation[0] & 0x40:
+                        self.read_and_update_when_deact = True
                 if 'repeatPattern' in pefi:
-                    self.repeat_pattern = pefi['repeatPattern']
-                if 'defaultPattern' in pefi:
-                    self.repeat_pattern = pefi['defaultPattern']
+                    self.fill_pattern = pefi['repeatPattern']
+                    self.fill_pattern_repeat = True
+                if 'fillPattern' in pefi:
+                    self.fill_pattern = pefi['fillPattern']
+                    self.fill_pattern_repeat = False
             elif fdb_dec['file_type'] == 'df':
-                # only set it, if an earlier call to from_template() didn't alrady set it, as
+                # only set it, if an earlier call to from_template() didn't already set it, as
                 # the template can differentiate between MF, DF and ADF (unlike FDB)
                 if not self.file_type:
                     self.file_type = 'DF'
@@ -319,7 +393,7 @@ class File:
         if fd:
             self.from_fileDescriptor(dict(fd))
         # BODY
-        self.body = self.file_content_from_tuples(l)
+        self._body = self.file_content_from_tuples(l)
 
     @staticmethod
     def path_from_gfm(bin_path: bytes):
@@ -341,30 +415,69 @@ class File:
         ret += self.file_content_to_tuples()
         return ret
 
-    @staticmethod
-    def file_content_from_tuples(l: List[Tuple]) -> Optional[bytes]:
+    def expand_fill_pattern(self) -> bytes:
+        """Expand the fill/repeat pattern as per TS 102 222 Section 6.3.2.2.2"""
+        return ts_102_222.expand_pattern(self.fill_pattern, self.fill_pattern_repeat, self.file_size)
+
+    def file_content_from_tuples(self, l: List[Tuple]) -> Optional[bytes]:
         """linearize a list of fillFileContent / fillFileOffset tuples into a stream of bytes."""
         stream = io.BytesIO()
+        # Providing file content within "fillFileContent" / "fillFileOffset" shall have the same effect as
+        # creating a file with a fill/repeat pattern and thereafter updating the content via Update.
+        # Step 1: Fill with pattern from Fcp or Template
+        if self.fill_pattern:
+            stream.write(self.expand_fill_pattern())
+        elif self.template and self.template.default_val:
+            stream.write(self.template.expand_default_value_pattern(self.file_size))
+        stream.seek(0)
+        # then process the fillFileContent/fillFileOffset
         for k, v in l:
             if k == 'doNotCreate':
                 return None
             if k == 'fileDescriptor':
                 pass
             elif k == 'fillFileOffset':
-                # FIXME: respect the fillPattern!
-                stream.write(b'\xff' * v)
+                stream.seek(v, os.SEEK_CUR)
             elif k == 'fillFileContent':
                 stream.write(v)
             else:
                 return ValueError("Unknown key '%s' in tuple list" % k)
         return stream.getvalue()
 
-    def file_content_to_tuples(self) -> List[Tuple]:
-        # FIXME: simplistic approach. needs optimization. We should first check if the content
-        # matches the expanded default value from the template. If it does, return empty list.
-        # Next, we should compute the diff between the default value and self.body, and encode
-        # that as a sequence of fillFileOffset and fillFileContent tuples.
-        return [('fillFileContent', self.body)]
+    def file_content_to_tuples(self, optimize:bool = False) -> List[Tuple]:
+        """Encode the file contents into a list of fillFileContent / fillFileOffset tuples that can be fed
+        into the asn.1 encoder.  If optimize is True, it will try to encode only the differences from the
+        fillFileContent of the profile template.  Otherwise, the entire file contents will be encoded
+        as-is."""
+        if not self.file_type in ['TR', 'LF', 'CY', 'BT']:
+            return []
+        if not optimize:
+            # simplistic approach: encode the full file, ignoring the template/default
+            return [('fillFileContent', self.body)]
+        # Try to 'compress' the file body, based on the default file contents.
+        if self.template:
+            default = self.template.expand_default_value_pattern(length=len(self.body))
+            if not default:
+                sm = SequenceMatcher(a=b'\xff'*len(self.body), b=self.body)
+            else:
+                if default == self.body:
+                    # 100% match: return an empty tuple list to make eUICC use the default
+                    return []
+                sm = SequenceMatcher(a=default, b=self.body)
+        else:
+            # no template at all: we can only remove padding
+            sm = SequenceMatcher(a=b'\xff'*len(self.body), b=self.body)
+        matching_blocks = sm.get_matching_blocks()
+        # we can only make use of matches that have the same offset in 'a' and 'b'
+        matching_blocks = [x for x in matching_blocks if x.size > 0 and x.a == x.b]
+        non_matching_blocks = NonMatch.from_matchlist(matching_blocks, self.file_size)
+        ret = []
+        cur = 0
+        for block in non_matching_blocks:
+            ret.append(('fillFileOffset', block.a - cur))
+            ret.append(('fillFileContent', self.body[block.a:block.a+block.size]))
+            cur += block.size
+        return ret
 
     def __str__(self) -> str:
         return "File(%s)" % self.pe_name
@@ -380,7 +493,7 @@ class File:
 
 class ProfileElement:
     """Generic Class representing a Profile Element (PE) within a SAIP Profile. This may be used directly,
-    but ist more likely sub-classed with a specific class for the specific profile element type, like e.g
+    but it's more likely sub-classed with a specific class for the specific profile element type, like e.g
     ProfileElementHeader, ProfileElementMF, ...
     """
     FILE_BEARING = ['mf', 'cd', 'telecom', 'usim', 'opt-usim', 'isim', 'opt-isim', 'phonebook', 'gsm-access',
@@ -393,7 +506,7 @@ class ProfileElement:
         'genericFileManagement':    'gfm-header',
         'akaParameter':             'aka-header',
         'cdmaParameter':            'cdma-header',
-        # note how they couldn't even consistently captialize the 'header' suffix :(
+        # note how they couldn't even consistently capitalize the 'header' suffix :(
         'application':              'app-Header',
         'pukCodes':                 'puk-Header',
         'pinCodes':                 'pin-Header',
@@ -426,7 +539,7 @@ class ProfileElement:
     @property
     def header_name(self) -> str:
         """Return the name of the header field within the profile element."""
-        # unneccessarry compliaction by inconsistent naming :(
+        # unnecessary complication by inconsistent naming :(
         if self.type.startswith('opt-'):
             return self.type.replace('-','') + '-header'
         if self.type in self.header_name_translation_dict:
@@ -464,7 +577,7 @@ class ProfileElement:
             # TODO: cdmaParameter
             'securityDomain': ProfileElementSD,
             'rfm': ProfileElementRFM,
-            # TODO: application
+            'application': ProfileElementApplication,
             # TODO: nonStandard
             'end': ProfileElementEnd,
             'mf': ProfileElementMF,
@@ -581,13 +694,20 @@ class FsProfileElement(ProfileElement):
                     # this is a template that belongs into the [A]DF of another template
                     # 1) find the PE for the referenced template
                     parent_pe = self.pe_sequence.get_closest_prev_pe_for_templateID(self, template.parent.oid)
-                    # 2) resolve te [A]DF that forms the base of that parent PE
+                    # 2) resolve the [A]DF that forms the base of that parent PE
                     pe_df = parent_pe.files[template.parent.base_df().pe_name].node
                     self.pe_sequence.cur_df = pe_df
                 self.pe_sequence.cur_df = self.pe_sequence.cur_df.add_file(file)
 
+    def file2pe(self, file: File):
+        """Update the "decoded" member for the given file with the contents from the given File instance.
+        We expect that the File instance is part of self.files"""
+        if self.files[file.pe_name] != file:
+            raise ValueError("The file you passed is not part of this ProfileElement")
+        self.decoded[file.pe_name] = file.to_tuples()
+
     def files2pe(self):
-        """Update the "decoded" member with the contents of the "files" member."""
+        """Update the "decoded" member for each file with the contents of the "files" member."""
         for k, f in self.files.items():
             self.decoded[k] = f.to_tuples()
 
@@ -601,6 +721,14 @@ class FsProfileElement(ProfileElement):
                 file = File(k, v, template.files_by_pename.get(k, None))
                 self.add_file(file)
 
+    def create_file(self, pename: str) -> File:
+        """Programmatically create a file by its PE-Name."""
+        template = templates.ProfileTemplateRegistry.get_by_oid(self.templateID)
+        file = File(pename, None, template.files_by_pename.get(pename, None))
+        self.add_file(file)
+        self.decoded[pename] = []
+        return file
+
     def _post_decode(self):
         # not entirely sure about doing this this automatism
         self.pe2files()
@@ -698,6 +826,7 @@ class ProfileElementGFM(ProfileElement):
 
 
 class ProfileElementMF(FsProfileElement):
+    """Class representing the ProfileElement for the MF (Master File)"""
     type = 'mf'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -711,6 +840,7 @@ class ProfileElementMF(FsProfileElement):
         # TODO: resize EF.DIR?
 
 class ProfileElementPuk(ProfileElement):
+    """Class representing the ProfileElement for a PUK (PIN Unblocking Code)"""
     type = 'pukCodes'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -741,6 +871,7 @@ class ProfileElementPuk(ProfileElement):
 
 
 class ProfileElementPin(ProfileElement):
+    """Class representing the ProfileElement for a PIN (Personal Identification Number)"""
     type = 'pinCodes'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -777,6 +908,7 @@ class ProfileElementPin(ProfileElement):
 
 
 class ProfileElementTelecom(FsProfileElement):
+    """Class representing the ProfileElement for DF.TELECOM"""
     type = 'telecom'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -789,6 +921,7 @@ class ProfileElementTelecom(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementCD(FsProfileElement):
+    """Class representing the ProfileElement for DF.CD"""
     type = 'cd'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -801,6 +934,7 @@ class ProfileElementCD(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementPhonebook(FsProfileElement):
+    """Class representing the ProfileElement for DF.PHONEBOOK"""
     type = 'phonebook'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -813,6 +947,7 @@ class ProfileElementPhonebook(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementGsmAccess(FsProfileElement):
+    """Class representing the ProfileElement for ADF.USIM/DF.GSM-ACCESS"""
     type = 'gsm-access'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -825,6 +960,7 @@ class ProfileElementGsmAccess(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementDf5GS(FsProfileElement):
+    """Class representing the ProfileElement for ADF.USIM/DF.5GS"""
     type = 'df-5gs'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -837,6 +973,7 @@ class ProfileElementDf5GS(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementEAP(FsProfileElement):
+    """Class representing the ProfileElement for DF.EAP"""
     type = 'eap'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -849,6 +986,7 @@ class ProfileElementEAP(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementDfSAIP(FsProfileElement):
+    """Class representing the ProfileElement for DF.SAIP"""
     type = 'df-saip'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -861,6 +999,7 @@ class ProfileElementDfSAIP(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementDfSNPN(FsProfileElement):
+    """Class representing the ProfileElement for DF.SNPN"""
     type = 'df-snpn'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -873,6 +1012,7 @@ class ProfileElementDfSNPN(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementDf5GProSe(FsProfileElement):
+    """Class representing the ProfileElement for DF.5GProSe"""
     type = 'df-5gprose'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -909,7 +1049,7 @@ class SecurityDomainKeyComponent:
                 'macLength': self.mac_length}
 
 class SecurityDomainKey:
-    """Represenation of a key used for SCP access to a security domain."""
+    """Representation of a key used for SCP access to a security domain."""
     def __init__(self, key_version_number: int, key_id: int, key_usage_qualifier: dict,
                  key_components: List[SecurityDomainKeyComponent]):
         self.key_usage_qualifier = key_usage_qualifier
@@ -918,9 +1058,9 @@ class SecurityDomainKey:
         self.key_components = key_components
 
     def __repr__(self) -> str:
-        return 'SdKey(KVN=0x%02x, ID=0x%02x, Usage=%s, Comp=%s)' % (self.key_version_number,
+        return 'SdKey(KVN=0x%02x, ID=0x%02x, Usage=0x%x, Comp=%s)' % (self.key_version_number,
                                                                     self.key_identifier,
-                                                                    self.key_usage_qualifier,
+                                                                    build_construct(KeyUsageQualifier, self.key_usage_qualifier)[0],
                                                                     repr(self.key_components))
 
     @classmethod
@@ -939,6 +1079,13 @@ class SecurityDomainKey:
                 'keyVersionNumber': bytes([self.key_version_number]),
                 'keyComponents': [k.to_saip_dict() for k in self.key_components]}
 
+    def get_key_component(self, key_type):
+        for kc in self.key_components:
+            if kc.key_type == key_type:
+                return kc.key_data
+        return None
+
+
 class ProfileElementSD(ProfileElement):
     """Class representing a securityDomain ProfileElement."""
     type = 'securityDomain'
@@ -953,6 +1100,7 @@ class ProfileElementSD(ProfileElement):
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
         super().__init__(decoded, **kwargs)
         if decoded:
+            self._post_decode()
             return
         # provide some reasonable defaults for a MNO-SD
         self.decoded['instance'] = {
@@ -1037,7 +1185,136 @@ class ProfileElementSSD(ProfileElementSD):
                 'uiccToolkitApplicationSpecificParametersField': h2b('01000001000000020112036C756500'),
             }
 
+class ProfileElementApplication(ProfileElement):
+    """Class representing an application ProfileElement."""
+    type = 'application'
+
+    def __init__(self, decoded: Optional[dict] = None, **kwargs):
+        super().__init__(decoded, **kwargs)
+
+    @classmethod
+    def from_file(cls,
+                  filename:str,
+                  aid:Hexstr,
+                  sd_aid:Hexstr = None,
+                  non_volatile_code_limit:int = None,
+                  volatile_data_limit:int = None,
+                  non_volatile_data_limit:int = None,
+                  hash_value:Hexstr = None) -> 'ProfileElementApplication':
+        """Fill contents of application ProfileElement from a .cap file."""
+
+        inst = cls()
+        Construct_data_limit = StripHeaderAdapter(GreedyBytes, 4, steps = [2,4])
+
+        if filename.lower().endswith('.cap'):
+            cap = javacard.CapFile(filename)
+            load_block_object = cap.get_loadfile()
+        elif filename.lower().endswith('.ijc'):
+            fd = open(filename, 'rb')
+            load_block_object = fd.read()
+        else:
+            raise ValueError('Invalid file type, file must either .cap or .ijc')
+
+        # Mandatory
+        inst.decoded['loadBlock'] = {
+            'loadPackageAID': h2b(aid),
+            'loadBlockObject': load_block_object
+        }
+
+        # Optional
+        if sd_aid:
+            inst.decoded['loadBlock']['securityDomainAID'] = h2b(sd_aid)
+        if non_volatile_code_limit:
+            inst.decoded['loadBlock']['nonVolatileCodeLimitC6'] = Construct_data_limit.build(non_volatile_code_limit)
+        if volatile_data_limit:
+            inst.decoded['loadBlock']['volatileDataLimitC7'] = Construct_data_limit.build(volatile_data_limit)
+        if non_volatile_data_limit:
+            inst.decoded['loadBlock']['nonVolatileDataLimitC8'] = Construct_data_limit.build(non_volatile_data_limit)
+        if hash_value:
+            inst.decoded['loadBlock']['hashValue'] = h2b(hash_value)
+
+        return inst
+
+    def to_file(self, filename:str):
+        """Write loadBlockObject contents of application ProfileElement to a .cap or .ijc file."""
+
+        load_package_aid = b2h(self.decoded['loadBlock']['loadPackageAID'])
+        load_block_object = self.decoded['loadBlock']['loadBlockObject']
+
+        if filename.lower().endswith('.cap'):
+            with io.BytesIO(load_block_object) as f, zipfile.ZipFile(filename, 'w') as z:
+                javacard.ijc_to_cap(f, z, load_package_aid)
+        elif filename.lower().endswith('.ijc'):
+            with open(filename, 'wb') as f:
+                f.write(load_block_object)
+        else:
+            raise ValueError('Invalid file type, file must either .cap or .ijc')
+
+    def add_instance(self,
+                     aid:Hexstr,
+                     class_aid:Hexstr,
+                     inst_aid:Hexstr,
+                     app_privileges:Hexstr,
+                     app_spec_pars:Hexstr,
+                     uicc_toolkit_app_spec_pars:Hexstr = None,
+                     uicc_access_app_spec_pars:Hexstr = None,
+                     uicc_adm_access_app_spec_pars:Hexstr = None,
+                     volatile_memory_quota:Hexstr = None,
+                     non_volatile_memory_quota:Hexstr = None,
+                     process_data:list[Hexstr] = None):
+        """Create a new instance and add it to the instanceList"""
+
+        # Mandatory
+        inst = {'applicationLoadPackageAID': h2b(aid),
+                'classAID': h2b(class_aid),
+                'instanceAID': h2b(inst_aid),
+                'applicationPrivileges': h2b(app_privileges),
+                'applicationSpecificParametersC9': h2b(app_spec_pars)}
+
+        # Optional
+        if uicc_toolkit_app_spec_pars or uicc_access_app_spec_pars or uicc_adm_access_app_spec_pars:
+            inst['applicationParameters'] = {}
+            if uicc_toolkit_app_spec_pars:
+                inst['applicationParameters']['uiccToolkitApplicationSpecificParametersField'] = \
+                    h2b(uicc_toolkit_app_spec_pars)
+            if uicc_access_app_spec_pars:
+                inst['applicationParameters']['uiccAccessApplicationSpecificParametersField'] = \
+                    h2b(uicc_access_app_spec_pars)
+            if uicc_adm_access_app_spec_pars:
+                inst['applicationParameters']['uiccAdministrativeAccessApplicationSpecificParametersField'] = \
+                    h2b(uicc_adm_access_app_spec_pars)
+        if volatile_memory_quota is not None or non_volatile_memory_quota is not None:
+            inst['systemSpecificParameters'] = {}
+            Construct_data_limit = StripHeaderAdapter(GreedyBytes, 4, steps = [2,4])
+            if volatile_memory_quota is not None:
+                inst['systemSpecificParameters']['volatileMemoryQuotaC7'] = \
+                    Construct_data_limit.build(volatile_memory_quota)
+            if non_volatile_memory_quota is not None:
+                inst['systemSpecificParameters']['nonVolatileMemoryQuotaC8'] = \
+                    Construct_data_limit.build(non_volatile_memory_quota)
+        if len(process_data) > 0:
+            inst['processData'] = []
+        for proc in process_data:
+            inst['processData'].append(h2b(proc))
+
+        # Append created instance to instance list
+        if 'instanceList' not in self.decoded.keys():
+            self.decoded['instanceList'] = []
+        self.decoded['instanceList'].append(inst)
+
+    def remove_instance(self, inst_aid:Hexstr):
+        """Remove an instance from the instanceList"""
+        inst_list = self.decoded.get('instanceList', [])
+        for inst in enumerate(inst_list):
+            if b2h(inst[1].get('instanceAID', None)) == inst_aid:
+                inst_list.pop(inst[0])
+                return
+        raise ValueError("instance AID: '%s' not present in instanceList, cannot remove instance" % inst[1])
+
+
+
 class ProfileElementRFM(ProfileElement):
+    """Class representing the ProfileElement for RFM (Remote File Management)."""
     type = 'rfm'
 
     def __init__(self, decoded: Optional[dict] = None,
@@ -1063,6 +1340,7 @@ class ProfileElementRFM(ProfileElement):
                 }
 
 class ProfileElementUSIM(FsProfileElement):
+    """Class representing the ProfileElement for ADF.USIM Mandatory Files"""
     type = 'usim'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -1080,10 +1358,12 @@ class ProfileElementUSIM(FsProfileElement):
 
     @property
     def imsi(self) -> Optional[str]:
-        f = File('ef-imsi', self.decoded['ef-imsi'])
+        template = templates.ProfileTemplateRegistry.get_by_oid(self.templateID)
+        f = File('ef-imsi', self.decoded['ef-imsi'], template.files_by_pename.get('ef-imsi', None))
         return dec_imsi(b2h(f.body))
 
 class ProfileElementOptUSIM(FsProfileElement):
+    """Class representing the ProfileElement for ADF.USIM Optional Files"""
     type = 'opt-usim'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -1094,6 +1374,7 @@ class ProfileElementOptUSIM(FsProfileElement):
         self.decoded['templateID'] = str(oid.ADF_USIMopt_not_by_default_v2)
 
 class ProfileElementISIM(FsProfileElement):
+    """Class representing the ProfileElement for ADF.ISIM Mandatory Files"""
     type = 'isim'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -1110,6 +1391,7 @@ class ProfileElementISIM(FsProfileElement):
         return b2h(self.decoded['adf-isim'][0][1]['dfName'])
 
 class ProfileElementOptISIM(FsProfileElement):
+    """Class representing the ProfileElement for ADF.ISIM Optional Files"""
     type = 'opt-isim'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -1121,6 +1403,7 @@ class ProfileElementOptISIM(FsProfileElement):
 
 
 class ProfileElementAKA(ProfileElement):
+    """Class representing the ProfileElement for Authentication and Key Agreement (AKA)."""
     type = 'akaParameter'
     # TODO: RES size for USIM test algorithm can be set to 32, 64 or 128 bits. This value was
     # previously limited to 128 bits.  Recommendation: Avoid using RES size 32 or 64 in Profiles
@@ -1200,13 +1483,14 @@ class ProfileElementAKA(ProfileElement):
         })
 
 class ProfileElementHeader(ProfileElement):
+    """Class representing the ProfileElement for the Header of the PE-Sequence."""
     type = 'header'
     def __init__(self, decoded: Optional[dict] = None,
                  ver_major: Optional[int] = 2, ver_minor: Optional[int] = 3,
                  iccid: Optional[Hexstr] = '0'*20, profile_type: Optional[str] = None,
                  **kwargs):
         """You would usually initialize an instance either with a "decoded" argument (as read from
-        a DER-encoded SAIP file via asn1tools), or [some of] the othe arguments in case you're
+        a DER-encoded SAIP file via asn1tools), or [some of] the other arguments in case you're
         constructing a Profile Header from scratch.
 
         Args:
@@ -1230,7 +1514,17 @@ class ProfileElementHeader(ProfileElement):
         if profile_type:
             self.decoded['profileType'] = profile_type
 
+    def mandatory_service_add(self, service_name):
+        self.decoded['eUICC-Mandatory-services'][service_name] = None
+
+    def mandatory_service_remove(self, service_name):
+        if service_name in self.decoded['eUICC-Mandatory-services'].keys():
+            del self.decoded['eUICC-Mandatory-services'][service_name]
+        else:
+            raise ValueError("service not in eUICC-Mandatory-services list, cannot remove")
+
 class ProfileElementEnd(ProfileElement):
+    """Class representing the ProfileElement for the End of the PE-Sequence."""
     type = 'end'
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
         super().__init__(decoded, **kwargs)
@@ -1252,7 +1546,7 @@ class ProfileElementSequence:
     sequence."""
     def __init__(self):
         """After calling the constructor, you have to further initialize the instance by either
-        calling the parse_der() method, or by manually adding individual PEs, including the hedaer and
+        calling the parse_der() method, or by manually adding individual PEs, including the header and
         end PEs."""
         self.pe_list: List[ProfileElement] = []
         self.pe_by_type: Dict = {}
@@ -1274,7 +1568,7 @@ class ProfileElementSequence:
     def add_hdr_and_end(self):
         """Initialize the PE Sequence with a header and end PE."""
         if len(self.pe_list):
-            raise ValueError("Cannot add header + end PE to a non-enmpty PE-Sequence")
+            raise ValueError("Cannot add header + end PE to a non-empty PE-Sequence")
         # start with a minimal/empty sequence of header + end
         self.append(ProfileElementHeader())
         self.append(ProfileElementEnd())
@@ -1291,7 +1585,7 @@ class ProfileElementSequence:
 
     def get_pe_for_type(self, tname: str) -> Optional[ProfileElement]:
         """Return a single profile element for given profile element type. Works only for
-        types of which there is only a signle instance in the PE Sequence!"""
+        types of which there is only a single instance in the PE Sequence!"""
         l = self.get_pes_for_type(tname)
         if len(l) == 0:
             return None
@@ -1299,7 +1593,7 @@ class ProfileElementSequence:
         return l[0]
 
     def get_pes_for_templateID(self, tid: oid.OID) -> List[ProfileElement]:
-        """Return list of profile elements present for given profile eleemnt type."""
+        """Return list of profile elements present for given profile element type."""
         res = []
         for pe in self.pe_list:
             if not pe.templateID:
@@ -1349,7 +1643,7 @@ class ProfileElementSequence:
 
     def _rebuild_pes_by_naa(self) -> None:
         """rebuild the self.pes_by_naa dict {naa: [ [pe, pe, pe], [pe, pe] ]} form,
-        which basically means for every NAA there's a lsit of instances, and each consists
+        which basically means for every NAA there's a list of instances, and each consists
         of a list of a list of PEs."""
         self.pres_by_naa = {}
         petype_not_naa_related = ['securityDomain', 'rfm', 'application', 'end']
@@ -1476,8 +1770,29 @@ class ProfileElementSequence:
             pe.header['identification'] = i
             i += 1
 
+    def get_index_by_pe(self, pe: ProfileElement) -> int:
+        """Return a list with the indices of all instances of PEs of petype."""
+        ret = []
+        i = 0
+        for cur in self.pe_list:
+            if cur == pe:
+                return i
+            i += 1
+        raise ValueError('PE %s is not part of PE Sequence' % (pe))
+
+    def insert_at_index(self, idx: int, pe: ProfileElement) -> None:
+        """Insert a given [new] ProfileElement at given index into the PE Sequence."""
+        self.pe_list.insert(idx, pe)
+        self._process_pelist()
+        self.renumber_identification()
+
+    def insert_after_pe(self, pe_before: ProfileElement, pe_new: ProfileElement) -> None:
+        """Insert a given [new] ProfileElement after a given [existing] PE in the PE Sequence."""
+        idx = self.get_index_by_pe(pe_before)
+        self.insert_at_index(idx+1, pe_new)
+
     def get_index_by_type(self, petype: str) -> List[int]:
-        """Return a list with the indicies of all instances of PEs of petype."""
+        """Return a list with the indices of all instances of PEs of petype."""
         ret = []
         i = 0
         for pe in self.pe_list:
@@ -1491,9 +1806,7 @@ class ProfileElementSequence:
         # find MNO-SD index
         idx = self.get_index_by_type('securityDomain')[0]
         # insert _after_ MNO-SD
-        self.pe_list.insert(idx+1, ssd)
-        self._process_pelist()
-        self.renumber_identification()
+        self.insert_at_index(idx+1, ssd)
 
     def remove_naas_of_type(self, naa: Naa) -> None:
         """Remove all instances of NAAs of given type. This can be used, for example,
@@ -1504,10 +1817,9 @@ class ProfileElementSequence:
         for service in naa.mandatory_services:
             if service in hdr.decoded['eUICC-Mandatory-services']:
                 del hdr.decoded['eUICC-Mandatory-services'][service]
-        # remove any associaed mandatory filesystem templates
+        # remove any associated mandatory filesystem templates
         for template in naa.templates:
-            if template in hdr.decoded['eUICC-Mandatory-GFSTEList']:
-                hdr.decoded['eUICC-Mandatory-GFSTEList'] = [x for x in hdr.decoded['eUICC-Mandatory-GFSTEList'] if not template.prefix_match(x)]
+            hdr.decoded['eUICC-Mandatory-GFSTEList'] = [x for x in hdr.decoded['eUICC-Mandatory-GFSTEList'] if not template.prefix_match(x)]
         # determine the ADF names (AIDs) of all NAA ADFs
         naa_adf_names = []
         if naa.pe_types[0] in self.pe_by_type:
@@ -1550,7 +1862,7 @@ class ProfileElementSequence:
             return None
 
     @staticmethod
-    def peclass_for_path(path: Path) -> Optional[ProfileElement]:
+    def peclass_for_path(path: Path) -> Tuple[Optional[ProfileElement], Optional[templates.FileTemplate]]:
         """Return the ProfileElement class that can contain a file with given path."""
         naa = ProfileElementSequence.naa_for_path(path)
         if naa:
@@ -1583,7 +1895,7 @@ class ProfileElementSequence:
                 return ProfileElementTelecom, ft
         return ProfileElementGFM, None
 
-    def pe_for_path(self, path: Path) -> Optional[ProfileElement]:
+    def pe_for_path(self, path: Path) -> Tuple[Optional[ProfileElement], Optional[templates.FileTemplate]]:
         """Return the ProfileElement instance that can contain a file with matching path. This will
         either be an existing PE within the sequence, or it will be a newly-allocated PE that is
         inserted into the sequence."""
@@ -1649,7 +1961,10 @@ class ProfileElementSequence:
 
 
 class FsNode:
-    """A node in the filesystem hierarchy."""
+    """A node in the filesystem hierarchy. Each node can have a parent node and any number of children.
+    Each node is identified uniquely within the parent by its numeric FID and its optional human-readable
+    name.  Each node usually is associated with an instance of the File class for the actual content of
+    the file.  FsNode is the base class used by more  specific nodes, such as FsNode{EF,DF,ADF,MF}."""
     def __init__(self, fid: int, parent: Optional['FsNode'], file: Optional[File] = None,
                  name: Optional[str] = None):
         self.fid = fid
@@ -1704,7 +2019,7 @@ class FsNode:
         return x
 
     def walk(self, fn, **kwargs):
-        """call 'fn(self, **kwargs) for the File."""
+        """call 'fn(self, ``**kwargs``) for the File."""
         return [fn(self, **kwargs)]
 
 class FsNodeEF(FsNode):
@@ -1794,7 +2109,7 @@ class FsNodeDF(FsNode):
         return cur
 
     def walk(self, fn, **kwargs):
-        """call 'fn(self, **kwargs) for the DF and recursively for all children."""
+        """call 'fn(self, ``**kwargs``) for the DF and recursively for all children."""
         ret = super().walk(fn, **kwargs)
         for c in self.children.values():
             ret += c.walk(fn, **kwargs)
@@ -1808,7 +2123,8 @@ class FsNodeADF(FsNodeDF):
         super().__init__(fid, parent, file, name)
 
     def __str__(self):
-        return '%s(%s)' % (self.__class__.__name__, b2h(self.df_name))
+        # self.df_name is usually None for an ADF like ADF.USIM or ADF.ISIM so we need to guard against it
+        return '%s(%s)' % (self.__class__.__name__, b2h(self.df_name) if self.df_name else None)
 
 class FsNodeMF(FsNodeDF):
     """The MF (Master File) in the filesystem hierarchy."""

pySim/esim/saip/batch.py

@@ -0,0 +1,360 @@
+"""Implementation of Personalization of eSIM profiles in SimAlliance/TCA Interoperable Profile:
+   Run a batch of N personalizations"""
+
+# (C) 2025-2026 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: nhofmeyr@sysmocom.de
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import copy
+import pprint
+from typing import List, Generator
+from pySim.esim.saip.personalization import ConfigurableParameter
+from pySim.esim.saip import param_source
+from pySim.esim.saip import ProfileElementSequence, ProfileElementSD
+from pySim.global_platform import KeyUsageQualifier
+from osmocom.utils import b2h
+
+class BatchPersonalization:
+    """Produce a series of eSIM profiles from predefined parameters.
+    Personalization parameters are derived from pysim.esim.saip.param_source.ParamSource.
+
+    Usage example:
+
+    der_input = some_file.open('rb').read()
+    pes = ProfileElementSequence.from_der(der_input)
+    p = pers.BatchPersonalization(
+        n=10,
+        src_pes=pes,
+        csv_rows=get_csv_reader())
+
+    p.add_param_and_src(
+        personalization.Iccid(),
+        param_source.IncDigitSource(
+            num_digits=18,
+            first_value=123456789012340001,
+            last_value=123456789012340010))
+
+    # add more parameters here, using ConfigurableParameter and ParamSource subclass instances to define the profile
+    # ...
+
+    # generate all 10 profiles (from n=10 above)
+    for result_pes in p.generate_profiles():
+        upp = result_pes.to_der()
+        store_upp(upp)
+    """
+
+    class ParamAndSrc:
+        """tie a ConfigurableParameter to a source of actual values"""
+        def __init__(self, param: ConfigurableParameter, src: param_source.ParamSource):
+            if isinstance(param, type):
+                self.param_cls = param
+            else:
+                self.param_cls = param.__class__
+            self.src = src
+
+    def __init__(self,
+                 n: int,
+                 src_pes: ProfileElementSequence,
+                 params: list[ParamAndSrc]=[],
+                 csv_rows: Generator=None,
+                ):
+        """
+        n: number of eSIM profiles to generate.
+        src_pes: a decoded eSIM profile as ProfileElementSequence, to serve as template. This is not modified, only
+                 copied.
+        params: list of ParamAndSrc instances, defining a ConfigurableParameter and corresponding ParamSource to fill in
+                profile values.
+        csv_rows: A generator (e.g. iter(list_of_rows)) producing all CSV rows one at a time, starting with a row
+                  containing the column headers. This is compatible with the python csv.reader. Each row gets passed to
+                  ParamSource.get_next(), such that ParamSource implementations can access the row items. See
+                  param_source.CsvSource.
+        """
+        self.n = n
+        self.params = params or []
+        self.src_pes = src_pes
+        self.csv_rows = csv_rows
+
+    def add_param_and_src(self, param:ConfigurableParameter, src:param_source.ParamSource):
+        self.params.append(BatchPersonalization.ParamAndSrc(param, src))
+
+    def generate_profiles(self):
+        # get first row of CSV: column names
+        csv_columns = None
+        if self.csv_rows:
+            try:
+                csv_columns = next(self.csv_rows)
+            except StopIteration as e:
+                raise ValueError('the input CSV file appears to be empty') from e
+
+        for i in range(self.n):
+            csv_row = None
+            if self.csv_rows and csv_columns:
+                try:
+                    csv_row_list = next(self.csv_rows)
+                except StopIteration as e:
+                    raise ValueError(f'not enough rows in the input CSV for eSIM nr {i+1} of {self.n}') from e
+
+                csv_row = dict(zip(csv_columns, csv_row_list))
+
+            pes = copy.deepcopy(self.src_pes)
+
+            for p in self.params:
+                try:
+                    input_value = p.src.get_next(csv_row=csv_row)
+                    assert input_value is not None
+                    value = p.param_cls.validate_val(input_value)
+                    p.param_cls.apply_val(pes, value)
+                except Exception as e:
+                    raise ValueError(f'{p.param_cls.get_name()} fed by {p.src.name}: {e}') from e
+
+            yield pes
+
+
+class UppAudit(dict):
+    """
+    Key-value pairs collected from a single UPP DER or PES.
+
+    UppAudit itself is a dict, callers may use the standard python dict API to access key-value pairs read from the UPP.
+    """
+
+    @classmethod
+    def from_der(cls, der: bytes, params: List, der_size=False, additional_sd_keys=False):
+        """return a dict of parameter name and set of selected parameter values found in a DER encoded profile. Note:
+        some ConfigurableParameter implementations return more than one key-value pair, for example, Imsi returns
+        both 'IMSI' and 'IMSI-ACC' parameters.
+
+        e.g.
+            UppAudit.from_der(my_der, [Imsi, ])
+            --> {'IMSI': '001010000000023', 'IMSI-ACC': '5'}
+
+        (where 'IMSI' == Imsi.name)
+
+        Read all parameters listed in params. params is a list of either ConfigurableParameter classes or
+        ConfigurableParameter class instances. This calls only classmethods, so each entry in params can either be the
+        class itself, or a class-instance of, a (non-abstract) ConfigurableParameter subclass.
+        For example, params = [Imsi, ] is equivalent to params = [Imsi(), ].
+
+        For der_size=True, also include a {'der_size':12345} entry.
+
+        For additional_sd_keys=True, output also all Security Domain KVN that there are *no* ConfigurableParameter
+        subclasses for. For example, SCP80 has reserved kvn 0x01..0x0f, but we offer only Scp80Kvn01, Scp80Kvn02,
+        Scp80Kvn03. So we would not show kvn 0x04..0x0f in an audit. additional_sd_keys=True includes audits of all SD
+        key KVN there may be in the UPP. This helps to spot SD keys that may already be present in a UPP template, with
+        unexpected / unusual kvn.
+        """
+
+        # make an instance of this class
+        upp_audit = cls()
+
+        if der_size:
+            upp_audit['der_size'] = set((len(der), ))
+
+        pes = ProfileElementSequence.from_der(der)
+        for param in params:
+            try:
+                for valdict in param.get_values_from_pes(pes):
+                    upp_audit.add_values(valdict)
+            except Exception as e:
+                raise ValueError(f'Error during audit for parameter {param}: {e}') from e
+
+        if not additional_sd_keys:
+            return upp_audit
+
+        # additional_sd_keys
+        for pe in pes.pe_list:
+            if pe.type != 'securityDomain':
+                continue
+            assert isinstance(pe, ProfileElementSD)
+
+            for key in pe.keys:
+                audit_key = f'SdKey_KVN{key.key_version_number:02x}_ID{key.key_identifier:02x}'
+                kuq_bin = KeyUsageQualifier.build(key.key_usage_qualifier).hex()
+                audit_val = f'{key.key_components=!r} key_usage_qualifier=0x{kuq_bin}={key.key_usage_qualifier!r}'
+                upp_audit[audit_key] = set((audit_val, ))
+
+        return upp_audit
+
+    def get_single_val(self, key, validate=True, allow_absent=False, absent_val=None):
+        """
+        Return the audit's value for the given audit key (like 'IMSI' or 'IMSI-ACC').
+        Any kind of value may occur multiple times in a profile. When all of these agree to the same unambiguous value,
+        return that value. When they do not agree, raise a ValueError.
+        """
+        # key should be a string, but if someone passes a ConfigurableParameter, just use its default name
+        if ConfigurableParameter.is_super_of(key):
+            key = key.get_name()
+
+        assert isinstance(key, str)
+        v = self.get(key)
+        if v is None and allow_absent:
+            return absent_val
+        if not isinstance(v, set):
+            raise ValueError(f'audit value should be a set(), got {v!r}')
+        if len(v) != 1:
+            raise ValueError(f'expected a single value for {key}, got {v!r}')
+        v = tuple(v)[0]
+        return v
+
+    @staticmethod
+    def audit_val_to_str(v):
+        """
+        Usually, we want to see a single value in an audit. Still, to be able to collect multiple ambiguous values,
+        audit values are always python sets. Turn it into a nice string representation: only the value when it is
+        unambiguous, otherwise a list of the ambiguous values.
+        A value may also be completely absent, then return 'not present'.
+        """
+        def try_single_val(w):
+            'change single-entry sets to just the single value'
+            if isinstance(w, set):
+                if len(w) == 1:
+                    return tuple(w)[0]
+                if len(w) == 0:
+                    return None
+            return w
+
+        v = try_single_val(v)
+        if isinstance(v, bytes):
+            v = bytes_to_hexstr(v)
+        if v is None:
+            return 'not present'
+        return str(v)
+
+    def get_val_str(self, key):
+        """Return a string of the value stored for the given key"""
+        return UppAudit.audit_val_to_str(self.get(key))
+
+    def add_values(self, src:dict):
+        """self and src are both a dict of sets.
+        For example from
+            self == { 'a': set((123,)) }
+        and
+            src == { 'a': set((456,)), 'b': set((789,)) }
+        then after this function call:
+            self == { 'a': set((123, 456,)), 'b': set((789,)) }
+        """
+        assert isinstance(src, dict)
+        for key, srcvalset in src.items():
+            dstvalset = self.get(key)
+            if dstvalset is None:
+                dstvalset = set()
+                self[key] = dstvalset
+            dstvalset.add(srcvalset)
+
+    def __str__(self):
+        return '\n'.join(f'{key}: {self.get_val_str(key)}' for key in sorted(self.keys()))
+
+class BatchAudit(list):
+    """
+    Collect UppAudit instances for a batch of UPP, for example from a personalization.BatchPersonalization.
+    Produce an output CSV.
+
+    Usage example:
+
+        ba = BatchAudit(params=(personalization.Iccid, ))
+        for upp_der in upps:
+            ba.add_audit(upp_der)
+        print(ba.summarize())
+
+        with open('output.csv', 'wb') as csv_data:
+            csv_str = io.TextIOWrapper(csv_data, 'utf-8', newline='')
+            csv.writer(csv_str).writerows( ba.to_csv_rows() )
+            csv_str.flush()
+
+    BatchAudit itself is a list, callers may use the standard python list API to access the UppAudit instances.
+    """
+
+    def __init__(self, params:List):
+        assert params
+        self.params = params
+
+    def add_audit(self, upp_der:bytes):
+        audit = UppAudit.from_der(upp_der, self.params)
+        self.append(audit)
+        return audit
+
+    def summarize(self):
+        batch_audit = UppAudit()
+
+        audits = self
+
+        if len(audits) > 2:
+            val_sep = ', ..., '
+        else:
+            val_sep = ', '
+
+        first_audit = None
+        last_audit = None
+        if len(audits) >= 1:
+            first_audit = audits[0]
+        if len(audits) >= 2:
+            last_audit = audits[-1]
+
+        if first_audit:
+            if last_audit:
+                for key in first_audit.keys():
+                    first_val = first_audit.get_val_str(key)
+                    last_val = last_audit.get_val_str(key)
+
+                    if first_val == last_val:
+                        val = first_val
+                    else:
+                        val_sep_with_newline = f"{val_sep.rstrip()}\n{' ' * (len(key) + 2)}"
+                        val = val_sep_with_newline.join((first_val, last_val))
+                    batch_audit[key] = val
+            else:
+                batch_audit.update(first_audit)
+
+        return batch_audit
+
+    def to_csv_rows(self, headers=True, sort_key=None):
+        """generator that yields all audits' values as rows, useful feed to a csv.writer."""
+        columns = set()
+        for audit in self:
+            columns.update(audit.keys())
+
+        columns = tuple(sorted(columns, key=sort_key))
+
+        if headers:
+            yield columns
+
+        for audit in self:
+            yield (audit.get_single_val(col, allow_absent=True, absent_val="") for col in columns)
+
+def bytes_to_hexstr(b:bytes, sep=''):
+    return sep.join(f'{x:02x}' for x in b)
+
+def esim_profile_introspect(upp):
+    pes = ProfileElementSequence.from_der(upp.read())
+    d = {}
+    d['upp'] = repr(pes)
+
+    def show_bytes_as_hexdump(item):
+        if isinstance(item, bytes):
+            return bytes_to_hexstr(item)
+        if isinstance(item, list):
+            return list(show_bytes_as_hexdump(i) for i in item)
+        if isinstance(item, tuple):
+            return tuple(show_bytes_as_hexdump(i) for i in item)
+        if isinstance(item, dict):
+            d = {}
+            for k, v in item.items():
+                d[k] = show_bytes_as_hexdump(v)
+            return d
+        return item
+
+    l = list((pe.type, show_bytes_as_hexdump(pe.decoded)) for pe in pes)
+    d['pp'] = pprint.pformat(l, width=120)
+    return d

pySim/esim/saip/oid.py

@@ -1,5 +1,5 @@
-# Implementation of SimAlliance/TCA Interoperable Profile OIDs
-#
+"""Implementation of SimAlliance/TCA Interoperable Profile OIDs"""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify

pySim/esim/saip/param_source.py

@@ -0,0 +1,229 @@
+# Implementation of SimAlliance/TCA Interoperable Profile handling: parameter sources for batch personalization.
+#
+# (C) 2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
+#
+# Author: nhofmeyr@sysmocom.de
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import secrets
+import re
+from osmocom.utils import b2h
+
+class ParamSourceExn(Exception):
+    pass
+
+class ParamSourceExhaustedExn(ParamSourceExn):
+    pass
+
+class ParamSourceUndefinedExn(ParamSourceExn):
+    pass
+
+class ParamSource:
+    """abstract parameter source. For usage, see personalization.BatchPersonalization."""
+
+    # This name should be short but descriptive, useful for a user interface, like 'random decimal digits'.
+    name = "none"
+    numeric_base = None # or 10 or 16
+
+    def __init__(self, input_str:str):
+        """Subclasses should call super().__init__(input_str) before evaluating self.input_str. Each subclass __init__()
+        may in turn manipulate self.input_str to apply expansions or decodings."""
+        self.input_str = input_str
+
+    def get_next(self, csv_row:dict=None):
+        """Subclasses implement this: return the next value from the parameter source.
+           When there are no more values from the source, raise a ParamSourceExhaustedExn.
+           This default implementation is an empty source."""
+        raise ParamSourceExhaustedExn()
+
+    @classmethod
+    def from_str(cls, input_str:str):
+        """compatibility with earlier version of ParamSource. Just use the constructor."""
+        return cls(input_str)
+
+class ConstantSource(ParamSource):
+    """one value for all"""
+    name = "constant"
+
+    def get_next(self, csv_row:dict=None):
+        return self.input_str
+
+class InputExpandingParamSource(ParamSource):
+
+    def __init__(self, input_str:str):
+        super().__init__(input_str)
+        self.input_str = self.expand_input_str(self.input_str)
+
+    @classmethod
+    def expand_input_str(cls, input_str:str):
+        # user convenience syntax '0*32' becomes '00000000000000000000000000000000'
+        if "*" not in input_str:
+            return input_str
+        # re:                "XX            *        123" with optional spaces
+        tokens = re.split(r"([^ \t]+)[ \t]*\*[ \t]*([0-9]+)", input_str)
+        if len(tokens) < 3:
+            return input_str
+        parts = []
+        for unchanged, snippet, repeat_str in zip(tokens[0::3], tokens[1::3], tokens[2::3]):
+            parts.append(unchanged)
+            repeat = int(repeat_str)
+            parts.append(snippet * repeat)
+
+        return "".join(parts)
+
+class DecimalRangeSource(InputExpandingParamSource):
+    """abstract: decimal numbers with a value range"""
+
+    numeric_base = 10
+
+    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
+        """Constructor to set up values from a (user entered) string: DecimalRangeSource(input_str).
+        Constructor to set up values directly: DecimalRangeSource(num_digits=3, first_value=123, last_value=456)
+
+        num_digits produces leading zeros when first_value..last_value are shorter.
+        """
+        assert ((input_str is not None and (num_digits, first_value, last_value) == (None, None, None))
+                or (input_str is None and None not in (num_digits, first_value, last_value)))
+
+        if input_str is not None:
+            super().__init__(input_str)
+
+            input_str = self.input_str
+
+            if ".." in input_str:
+                first_str, last_str = input_str.split('..')
+                first_str = first_str.strip()
+                last_str = last_str.strip()
+            else:
+                first_str = input_str.strip()
+                last_str = None
+
+            num_digits = len(first_str)
+            first_value = int(first_str)
+            last_value = int(last_str if last_str is not None else "9" * num_digits)
+
+        assert num_digits > 0
+        assert first_value <= last_value
+        self.num_digits = num_digits
+        self.first_value = first_value
+        self.last_value = last_value
+
+    def val_to_digit(self, val:int):
+        return "%0*d" % (self.num_digits, val)  # pylint: disable=consider-using-f-string
+
+class RandomSourceMixin:
+    random_impl = secrets.SystemRandom()
+
+class RandomDigitSource(DecimalRangeSource, RandomSourceMixin):
+    """return a different sequence of random decimal digits each"""
+    name = "random decimal digits"
+    used_keys = set()
+
+    def get_next(self, csv_row:dict=None):
+        # try to generate random digits that are always different from previously produced random bytes
+        attempts = 10
+        while True:
+            val = self.random_impl.randint(self.first_value, self.last_value)
+            if val in RandomDigitSource.used_keys:
+                attempts -= 1
+                if attempts:
+                    continue
+            RandomDigitSource.used_keys.add(val)
+            break
+        return self.val_to_digit(val)
+
+class RandomHexDigitSource(InputExpandingParamSource, RandomSourceMixin):
+    """return a different sequence of random hexadecimal digits each"""
+    name = "random hexadecimal digits"
+    numeric_base = 16
+    used_keys = set()
+
+    def __init__(self, input_str:str):
+        super().__init__(input_str)
+        input_str = self.input_str
+
+        num_digits = len(input_str.strip())
+        if num_digits < 1:
+            raise ValueError("zero number of digits")
+        # hex digits always come in two
+        if (num_digits & 1) != 0:
+            raise ValueError(f"hexadecimal value should have even number of digits, not {num_digits}")
+        self.num_digits = num_digits
+
+    def get_next(self, csv_row:dict=None):
+        # try to generate random bytes that are always different from previously produced random bytes
+        attempts = 10
+        while True:
+            val = self.random_impl.randbytes(self.num_digits // 2)
+            if val in RandomHexDigitSource.used_keys:
+                attempts -= 1
+                if attempts:
+                    continue
+            RandomHexDigitSource.used_keys.add(val)
+            break
+
+        return b2h(val)
+
+class IncDigitSource(DecimalRangeSource):
+    """incrementing sequence of digits"""
+    name = "incrementing decimal digits"
+
+    def __init__(self, input_str:str=None, num_digits:int=None, first_value:int=None, last_value:int=None):
+        """input_str: the first value to return, a string of an integer number with optional leading zero digits. The
+        leading zero digits are preserved."""
+        super().__init__(input_str, num_digits, first_value, last_value)
+        self.next_val = None
+        self.reset()
+
+    def reset(self):
+        """Restart from the first value of the defined range passed to __init__()."""
+        self.next_val = self.first_value
+
+    def get_next(self, csv_row:dict=None):
+        val = self.next_val
+        if val is None:
+            raise ParamSourceExhaustedExn()
+
+        returnval = self.val_to_digit(val)
+
+        val += 1
+        if val > self.last_value:
+            self.next_val = None
+        else:
+            self.next_val = val
+
+        return returnval
+
+class CsvSource(ParamSource):
+    """apply a column from a CSV row, as passed in to ParamSource.get_next(csv_row)"""
+    name = "from CSV"
+
+    def __init__(self, input_str:str):
+        """self.csv_column = input_str:
+        column name indicating the column to use for this parameter.
+        This name is used in get_next(): the caller passes the current CSV row to get_next(), from which
+        CsvSource picks the column with the name matching csv_column.
+        """
+        """Parse input_str into self.num_digits, self.first_value, self.last_value."""
+        super().__init__(input_str)
+        self.csv_column = self.input_str
+
+    def get_next(self, csv_row:dict=None):
+        val = None
+        if csv_row:
+            val = csv_row.get(self.csv_column)
+        if not val:
+            raise ParamSourceUndefinedExn(f"no value for CSV column {self.csv_column!r}")
+        return val

pySim/esim/saip/templates.py

@@ -1,5 +1,5 @@
-# Implementation of SimAlliance/TCA Interoperable Profile Template handling
-#
+"""Implementation of SimAlliance/TCA Interoperable Profile Templates."""
+
 # (C) 2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -224,8 +224,8 @@ class ProfileTemplateRegistry:
 # below are transcribed template definitions from "ANNEX A (Normative): File Structure Templates Definition"
 # of "Profile interoperability specification V3.3.1 Final" (unless other version explicitly specified).
 
-# Section 9.2
 class FilesAtMF(ProfileTemplate):
+    """Files at MF as per Section 9.2"""
     created_by_default = True
     oid = OID.MF
     files = [
@@ -238,8 +238,8 @@ class FilesAtMF(ProfileTemplate):
     ]
 
 
-# Section 9.3
 class FilesCD(ProfileTemplate):
+    """Files at DF.CD as per Section 9.3"""
     created_by_default = False
     oid = OID.DF_CD
     files = [
@@ -287,8 +287,8 @@ for i in range(0x90, 0x98):
 for i in range(0x98, 0xa0):
     df_pb_files.append(FileTemplate(0x4f00+i, 'EF.CCP1', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 
-# Section 9.4 v2.3.1
 class FilesTelecom(ProfileTemplate):
+    """Files at DF.TELECOM as per Section 9.4 v2.3.1"""
     created_by_default = False
     oid = OID.DF_TELECOM
     base_path = Path('MF')
@@ -328,8 +328,8 @@ class FilesTelecom(ProfileTemplate):
     ]
 
 
-# Section 9.4
 class FilesTelecomV2(ProfileTemplate):
+    """Files at DF.TELECOM as per Section 9.4"""
     created_by_default = False
     oid = OID.DF_TELECOM_v2
     base_path = Path('MF')
@@ -379,8 +379,8 @@ class FilesTelecomV2(ProfileTemplate):
     ]
 
 
-# Section 9.5.1 v2.3.1
 class FilesUsimMandatory(ProfileTemplate):
+    """Mandatory Files at ADF.USIM as per Section 9.5.1 v2.3.1"""
     created_by_default = True
     oid = OID.ADF_USIM_by_default
     files = [
@@ -410,8 +410,8 @@ class FilesUsimMandatory(ProfileTemplate):
         FileTemplate(0x6fe4, 'EF.EPSNSC',    'LF',    1,   80,   5, 0x18, 'FF...FF', False, ass_serv=[85], high_update=True),
     ]
 
-# Section 9.5.1
 class FilesUsimMandatoryV2(ProfileTemplate):
+    """Mandatory Files at ADF.USIM as per Section 9.5.1"""
     created_by_default = True
     oid = OID.ADF_USIM_by_default_v2
     files = [
@@ -442,8 +442,8 @@ class FilesUsimMandatoryV2(ProfileTemplate):
     ]
 
 
-# Section 9.5.2 v2.3.1
 class FilesUsimOptional(ProfileTemplate):
+    """Optional Files at ADF.USIM as per Section 9.5.2 v2.3.1"""
     created_by_default = False
     optional = True
     oid = OID.ADF_USIMopt_not_by_default
@@ -529,6 +529,7 @@ class FilesUsimOptional(ProfileTemplate):
 
 # Section 9.5.2
 class FilesUsimOptionalV2(ProfileTemplate):
+    """Optional Files at ADF.USIM as per Section 9.5.2"""
     created_by_default = False
     optional = True
     oid = OID.ADF_USIMopt_not_by_default_v2
@@ -622,8 +623,8 @@ class FilesUsimOptionalV2(ProfileTemplate):
         FileTemplate(0x6ffd, 'EF.MudMidCfgdata','BT', None, None,2, None, None, True, ['size'], ass_serv=[134]),
     ]
 
-# Section 9.5.2.3 v3.3.1
 class FilesUsimOptionalV3(ProfileTemplate):
+    """Optional Files at ADF.USIM as per Section 9.5.2.3 v3.3.1"""
     created_by_default = False
     optional = True
     oid = OID.ADF_USIMopt_not_by_default_v3
@@ -633,16 +634,16 @@ class FilesUsimOptionalV3(ProfileTemplate):
         FileTemplate(0x6f01, 'EF.eAKA', 'TR', None, 1, 3, None, None, True, ['size'], ass_serv=[134]),
     ]
 
-# Section 9.5.3
 class FilesUsimDfPhonebook(ProfileTemplate):
+    """DF.PHONEBOOK Files at ADF.USIM as per Section 9.5.3"""
     created_by_default = False
     oid = OID.DF_PHONEBOOK_ADF_USIM
     base_path = Path('ADF.USIM')
     files = df_pb_files
 
 
-# Section 9.5.4
 class FilesUsimDfGsmAccess(ProfileTemplate):
+    """DF.GSM-ACCESS Files at ADF.USIM as per Section 9.5.4"""
     created_by_default = False
     oid = OID.DF_GSM_ACCESS_ADF_USIM
     base_path = Path('ADF.USIM')
@@ -656,8 +657,8 @@ class FilesUsimDfGsmAccess(ProfileTemplate):
     ]
 
 
-# Section 9.5.11 v2.3.1
 class FilesUsimDf5GS(ProfileTemplate):
+    """DF.5GS Files at ADF.USIM as per Section 9.5.11 v2.3.1"""
     created_by_default = False
     oid = OID.DF_5GS
     base_path = Path('ADF.USIM')
@@ -672,13 +673,13 @@ class FilesUsimDf5GS(ProfileTemplate):
         FileTemplate(0x4f06, 'EF.UAC_AIC',           'TR', None,    4,   2, 0x06, None, True, ass_serv=[126]),
         FileTemplate(0x4f07, 'EF.SUCI_Calc_Info',    'TR', None, None,   2, 0x07, 'FF...FF', False, ass_serv=[124]),
         FileTemplate(0x4f08, 'EF.OPL5G',             'LF', None,   10,  10, 0x08, 'FF...FF', False, ['nb_rec'], ass_serv=[129]),
-        FileTemplate(0x4f09, 'EF.SUPI_NAI',          'TR', None, None,   2, 0x09, None, True, ['size'], ass_serv=[130]),
+        FileTemplate(0x4f09, 'EF.SUPI_NAI',          'TR', None, None,   2, 0x09, None, True, ['size'], ass_serv=[130], pe_name='ef-supinai'),
         FileTemplate(0x4f0a, 'EF.Routing_Indicator', 'TR', None,    4,   2, 0x0a, 'F0FFFFFF', False, ass_serv=[124]),
     ]
 
 
-# Section 9.5.11.2
 class FilesUsimDf5GSv2(ProfileTemplate):
+    """DF.5GS Files at ADF.USIM as per Section 9.5.11.2"""
     created_by_default = False
     oid = OID.DF_5GS_v2
     base_path = Path('ADF.USIM')
@@ -700,8 +701,8 @@ class FilesUsimDf5GSv2(ProfileTemplate):
     ]
 
 
-# Section 9.5.11.3
 class FilesUsimDf5GSv3(ProfileTemplate):
+    """DF.5GS Files at ADF.USIM as per Section 9.5.11.3"""
     created_by_default = False
     oid = OID.DF_5GS_v3
     base_path = Path('ADF.USIM')
@@ -724,8 +725,8 @@ class FilesUsimDf5GSv3(ProfileTemplate):
         FileTemplate(0x4f0c, 'EF.TN3GPPSNN',         'TR', None,    1,   2, 0x0c, '00', False, ass_serv=[135]),
     ]
 
-# Section 9.5.11.4
 class FilesUsimDf5GSv4(ProfileTemplate):
+    """DF.5GS Files at ADF.USIM as per Section 9.5.11.4"""
     created_by_default = False
     oid = OID.DF_5GS_v4
     base_path = Path('ADF.USIM')
@@ -756,8 +757,8 @@ class FilesUsimDf5GSv4(ProfileTemplate):
     ]
 
 
-# Section 9.5.12
 class FilesUsimDfSaip(ProfileTemplate):
+    """DF.SAIP Files at ADF.USIM as per Section 9.5.12"""
     created_by_default = False
     oid = OID.DF_SAIP
     base_path = Path('ADF.USIM')
@@ -767,8 +768,8 @@ class FilesUsimDfSaip(ProfileTemplate):
         FileTemplate(0x4f01, 'EF.SUCICalcInfo','TR', None, None, 3, None, 'FF...FF', False, ['size'], ass_serv=[125], pe_name='ef-suci-calc-info-usim'),
     ]
 
-# Section 9.5.13
 class FilesDfSnpn(ProfileTemplate):
+    """DF.SNPN Files at ADF.USIM as per Section 9.5.13"""
     created_by_default = False
     oid = OID.DF_SNPN
     base_path = Path('ADF.USIM')
@@ -778,8 +779,8 @@ class FilesDfSnpn(ProfileTemplate):
         FileTemplate(0x4f01, 'EF.PWS_SNPN',     'TR', None,    1,  10, None, None, True, ass_serv=[143]),
     ]
 
-# Section 9.5.14
 class FilesDf5GProSe(ProfileTemplate):
+    """DF.ProSe Files at ADF.USIM as per Section 9.5.14"""
     created_by_default = False
     oid = OID.DF_5GProSe
     base_path = Path('ADF.USIM')
@@ -794,8 +795,8 @@ class FilesDf5GProSe(ProfileTemplate):
         FileTemplate(0x4f06, 'EF.5G_PROSE_UIR',   'TR', None,   32,   2, 0x06, None,  True, ass_serv=[139,1005]),
     ]
 
-# Section 9.6.1
 class FilesIsimMandatory(ProfileTemplate):
+    """Mandatory Files at ADF.ISIM as per Section 9.6.1"""
     created_by_default = True
     oid = OID.ADF_ISIM_by_default
     files = [
@@ -809,15 +810,15 @@ class FilesIsimMandatory(ProfileTemplate):
     ]
 
 
-# Section 9.6.2 v2.3.1
 class FilesIsimOptional(ProfileTemplate):
+    """Optional Files at ADF.ISIM as per Section 9.6.2 of v2.3.1"""
     created_by_default = False
     optional = True
     oid = OID.ADF_ISIMopt_not_by_default
     base_path = Path('ADF.ISIM')
     extends = FilesIsimMandatory
     files = [
-        FileTemplate(0x6f09, 'EF.P-CSCF',      'LF',    1, None,   2, None, None, True, ['size'], ass_serv=[1,5]),
+        FileTemplate(0x6f09, 'EF.P-CSCF',      'LF',    1, None,   2, None, None, True, ['size'], ass_serv=[1,5], pe_name='ef-pcscf'),
         FileTemplate(0x6f3c, 'EF.SMS',         'LF',   10,  176,   5, None, '00FF...FF', False, ass_serv=[6,8]),
         FileTemplate(0x6f42, 'EF.SMSP',        'LF',    1,   38,   5, None, 'FF...FF', False, ass_serv=[8]),
         FileTemplate(0x6f43, 'EF.SMSS',        'TR', None,    2,   5, None, 'FFFF', False, ass_serv=[6,8]),
@@ -829,8 +830,8 @@ class FilesIsimOptional(ProfileTemplate):
     ]
 
 
-# Section 9.6.2
 class FilesIsimOptionalv2(ProfileTemplate):
+    """Optional Files at ADF.ISIM as per Section 9.6.2"""
     created_by_default = False
     optional = True
     oid = OID.ADF_ISIMopt_not_by_default_v2
@@ -857,8 +858,8 @@ class FilesIsimOptionalv2(ProfileTemplate):
 # TODO: CSIM
 
 
-# Section 9.8
 class FilesEap(ProfileTemplate):
+    """Files at DF.EAP as per Section 9.8"""
     created_by_default = False
     oid = OID.DF_EAP
     files = [

pySim/esim/saip/validation.py

@@ -1,5 +1,5 @@
-# Implementation of SimAlliance/TCA Interoperable Profile handling
-#
+"""Implementation of SimAlliance/TCA Interoperable Profile validation."""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -19,16 +19,21 @@
 from pySim.esim.saip import *
 
 class ProfileError(Exception):
+    """Raised when a ProfileConstraintChecker finds an error in a file [structure]."""
     pass
 
 class ProfileConstraintChecker:
+    """Base class of a constraint checker for a ProfileElementSequence."""
     def check(self, pes: ProfileElementSequence):
+        """Execute all the check_* methods of the ProfileConstraintChecker against the given
+        ProfileElementSequence"""
         for name in dir(self):
             if name.startswith('check_'):
                 method = getattr(self, name)
                 method(pes)
 
 class CheckBasicStructure(ProfileConstraintChecker):
+    """ProfileConstraintChecker for the basic profile structure constraints."""
     def _is_after_if_exists(self, pes: ProfileElementSequence, opt:str, after:str):
         opt_pe = pes.get_pe_for_type(opt)
         if opt_pe:
@@ -38,6 +43,7 @@ class CheckBasicStructure(ProfileConstraintChecker):
             # FIXME: check order
 
     def check_start_and_end(self, pes: ProfileElementSequence):
+        """Check for mandatory header and end ProfileElements at the right position."""
         if pes.pe_list[0].type != 'header':
             raise ProfileError('first element is not header')
         if pes.pe_list[1].type != 'mf':
@@ -47,6 +53,7 @@ class CheckBasicStructure(ProfileConstraintChecker):
             raise ProfileError('last element is not end')
 
     def check_number_of_occurrence(self, pes: ProfileElementSequence):
+        """Check The number of occurrence of various ProfileElements."""
         # check for invalid number of occurrences
         if len(pes.get_pes_for_type('header')) != 1:
             raise ProfileError('multiple ProfileHeader')
@@ -60,7 +67,8 @@ class CheckBasicStructure(ProfileConstraintChecker):
                 raise ProfileError('multiple PE-%s' % tn.upper())
 
     def check_optional_ordering(self, pes: ProfileElementSequence):
-        # ordering and required depenencies
+        """Check the ordering of optional PEs following the respective mandatory ones."""
+        # ordering and required dependencies
         self._is_after_if_exists(pes,'opt-usim', 'usim')
         self._is_after_if_exists(pes,'opt-isim', 'isim')
         self._is_after_if_exists(pes,'gsm-access', 'usim')
@@ -95,6 +103,26 @@ class CheckBasicStructure(ProfileConstraintChecker):
         if 'profile-a-p256' in m_svcs and not ('usim' in m_svcs or 'isim' in m_svcs):
             raise ProfileError('profile-a-p256 mandatory, but no usim or isim')
 
+    def check_mandatory_services_aka(self, pes: ProfileElementSequence):
+        """Ensure that no unnecessary authentication related services are marked as mandatory but not
+        actually used within the profile"""
+        m_svcs = pes.get_pe_for_type('header').decoded['eUICC-Mandatory-services']
+        # list of tuples (algo_id, key_len_in_octets) for all the akaParameters in the PE Sequence
+        algo_id_klen = [(x.decoded['algoConfiguration'][1]['algorithmID'],
+                         len(x.decoded['algoConfiguration'][1]['key'])) for x in pes.get_pes_for_type('akaParameter')]
+        # just a plain list of algorithm IDs in akaParameters
+        algorithm_ids = [x[0] for x in algo_id_klen]
+        if 'milenage' in m_svcs and not 1 in algorithm_ids:
+            raise ProfileError('milenage mandatory, but no related algorithm_id in akaParameter')
+        if 'tuak128' in m_svcs and not (2, 128/8) in algo_id_klen:
+            raise ProfileError('tuak128 mandatory, but no related algorithm_id in akaParameter')
+        if 'cave' in m_svcs and not pes.get_pe_for_type('cdmaParameter'):
+            raise ProfileError('cave mandatory, but no related cdmaParameter')
+        if 'tuak256' in m_svcs and (2, 256/8) in algo_id_klen:
+            raise ProfileError('tuak256 mandatory, but no related algorithm_id in akaParameter')
+        if 'usim-test-algorithm' in m_svcs and not 3 in algorithm_ids:
+            raise ProfileError('usim-test-algorithm mandatory, but no related algorithm_id in akaParameter')
+
     def check_identification_unique(self, pes: ProfileElementSequence):
         """Ensure that each PE has a unique identification value."""
         id_list = [pe.header['identification'] for pe in pes.pe_list if pe.header]
@@ -104,17 +132,21 @@ class CheckBasicStructure(ProfileConstraintChecker):
 FileChoiceList = List[Tuple]
 
 class FileError(ProfileError):
+    """Raised when a FileConstraintChecker finds an error in a file [structure]."""
     pass
 
 class FileConstraintChecker:
     def check(self, l: FileChoiceList):
+        """Execute all the check_* methods of the FileConstraintChecker against the given FileChoiceList"""
         for name in dir(self):
             if name.startswith('check_'):
                 method = getattr(self, name)
                 method(l)
 
 class FileCheckBasicStructure(FileConstraintChecker):
+    """Validator for the basic structure of a decoded file."""
     def check_seqence(self, l: FileChoiceList):
+        """Check the sequence/ordering."""
         by_type = {}
         for k, v in l:
             if k in by_type:

pySim/esim/x509_cert.py

@@ -1,6 +1,5 @@
-# Implementation of X.509 certificate handling in GSMA eSIM
-# as per SGP22 v3.0
-#
+"""Implementation of X.509 certificate handling in GSMA eSIM as per SGP22 v3.0"""
+
 # (C) 2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -31,7 +30,7 @@ def check_signed(signed: x509.Certificate, signer: x509.Certificate) -> bool:
     """Verify if 'signed' certificate was signed using 'signer'."""
     # this code only works for ECDSA, but this is all we need for GSMA eSIM
     pkey = signer.public_key()
-    # this 'signed.signature_algorithm_parameters' below requires cryptopgraphy 41.0.0 :(
+    # this 'signed.signature_algorithm_parameters' below requires cryptography 41.0.0 :(
     pkey.verify(signed.signature, signed.tbs_certificate_bytes, signed.signature_algorithm_parameters)
 
 def cert_get_subject_key_id(cert: x509.Certificate) -> bytes:
@@ -150,7 +149,7 @@ class CertificateSet:
                 check_signed(c, self.root_cert)
                 return
             parent_cert = self.intermediate_certs.get(aki, None)
-            if not aki:
+            if not parent_cert:
                 raise VerifyError('Could not find intermediate certificate for AuthKeyId %s' % b2h(aki))
             check_signed(c, parent_cert)
             # if we reach here, we passed (no exception raised)
@@ -189,7 +188,7 @@ class CertAndPrivkey:
     def ecdsa_sign(self, plaintext: bytes) -> bytes:
         """Sign some input-data using an ECDSA signature compliant with SGP.22,
         which internally refers to Global Platform 2.2 Annex E, which in turn points
-        to BSI TS-03111 which states "concatengated raw R + S values". """
+        to BSI TS-03111 which states "concatenated raw R + S values". """
         sig = self.priv_key.sign(plaintext, ec.ECDSA(hashes.SHA256()))
         # convert from DER format to BSI TR-03111; first get long integers; then convert those to bytes
         return ecdsa_dss_to_tr03111(sig)

pySim/euicc.py

@@ -34,6 +34,7 @@ from osmocom.construct import *
 from pySim.exceptions import SwMatchError
 from pySim.utils import Hexstr, SwHexstr, SwMatchstr
 from pySim.commands import SimCardCommands
+from pySim.ts_102_221 import CardProfileUICC
 import pySim.global_platform
 
 # SGP.02 Section 2.2.2
@@ -119,7 +120,7 @@ class SetDefaultDpAddress(BER_TLV_IE, tag=0xbf3f, nested=[DefaultDpAddress, SetD
 
 # SGP.22 Section 5.7.7: GetEUICCChallenge
 class EuiccChallenge(BER_TLV_IE, tag=0x80):
-    _construct = HexAdapter(Bytes(16))
+    _construct = Bytes(16)
 class GetEuiccChallenge(BER_TLV_IE, tag=0xbf2e, nested=[EuiccChallenge]):
     pass
 
@@ -127,7 +128,7 @@ class GetEuiccChallenge(BER_TLV_IE, tag=0xbf2e, nested=[EuiccChallenge]):
 class SVN(BER_TLV_IE, tag=0x82):
     _construct = VersionType
 class SubjectKeyIdentifier(BER_TLV_IE, tag=0x04):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class EuiccCiPkiListForVerification(BER_TLV_IE, tag=0xa9, nested=[SubjectKeyIdentifier]):
     pass
 class EuiccCiPkiListForSigning(BER_TLV_IE, tag=0xaa, nested=[SubjectKeyIdentifier]):
@@ -139,15 +140,15 @@ class ProfileVersion(BER_TLV_IE, tag=0x81):
 class EuiccFirmwareVer(BER_TLV_IE, tag=0x83):
     _construct = VersionType
 class ExtCardResource(BER_TLV_IE, tag=0x84):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class UiccCapability(BER_TLV_IE, tag=0x85):
-    _construct = HexAdapter(GreedyBytes) # FIXME
+    _construct = GreedyBytes # FIXME
 class TS102241Version(BER_TLV_IE, tag=0x86):
     _construct = VersionType
 class GlobalPlatformVersion(BER_TLV_IE, tag=0x87):
     _construct = VersionType
 class RspCapability(BER_TLV_IE, tag=0x88):
-    _construct = HexAdapter(GreedyBytes) # FIXME
+    _construct = GreedyBytes # FIXME
 class EuiccCategory(BER_TLV_IE, tag=0x8b):
     _construct = Enum(Int8ub, other=0, basicEuicc=1, mediumEuicc=2, contactlessEuicc=3)
 class PpVersion(BER_TLV_IE, tag=0x04):
@@ -180,7 +181,7 @@ class SeqNumber(BER_TLV_IE, tag=0x80):
 class NotificationAddress(BER_TLV_IE, tag=0x0c):
     _construct = Utf8Adapter(GreedyBytes)
 class Iccid(BER_TLV_IE, tag=0x5a):
-    _construct = BcdAdapter(GreedyBytes)
+    _construct = PaddedBcdAdapter(GreedyBytes)
 class NotificationMetadata(BER_TLV_IE, tag=0xbf2f, nested=[SeqNumber, ProfileMgmtOperation,
                                                            NotificationAddress, Iccid]):
     pass
@@ -210,7 +211,7 @@ class TagList(BER_TLV_IE, tag=0x5c):
 class ProfileInfoListReq(BER_TLV_IE, tag=0xbf2d, nested=[TagList]): # FIXME: SearchCriteria
     pass
 class IsdpAid(BER_TLV_IE, tag=0x4f):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class ProfileState(BER_TLV_IE, tag=0x9f70):
     _construct = Enum(Int8ub, disabled=0, enabled=1)
 class ProfileNickname(BER_TLV_IE, tag=0x90):
@@ -267,9 +268,20 @@ class DeleteProfileReq(BER_TLV_IE, tag=0xbf33, nested=[IsdpAid, Iccid]):
 class DeleteProfileResp(BER_TLV_IE, tag=0xbf33, nested=[DeleteResult]):
     pass
 
+# SGP.22 Section 5.7.19: EuiccMemoryReset
+class ResetOptions(BER_TLV_IE, tag=0x82):
+    _construct = FlagsEnum(Byte, deleteOperationalProfiles=0x80, deleteFieldLoadedTestProfiles=0x40,
+                           resetDefaultSmdpAddress=0x20)
+class ResetResult(BER_TLV_IE, tag=0x80):
+    _construct = Enum(Int8ub, ok=0, nothingToDelete=1, undefinedError=127)
+class EuiccMemoryResetReq(BER_TLV_IE, tag=0xbf34, nested=[ResetOptions]):
+    pass
+class EuiccMemoryResetResp(BER_TLV_IE, tag=0xbf34, nested=[ResetResult]):
+    pass
+
 # SGP.22 Section 5.7.20 GetEID
 class EidValue(BER_TLV_IE, tag=0x5a):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class GetEuiccData(BER_TLV_IE, tag=0xbf3e, nested=[TagList, EidValue]):
     pass
 
@@ -361,7 +373,7 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
         ged_cmd = GetEuiccData(children=[TagList(decoded=[0x5A])])
         ged = CardApplicationISDR.store_data_tlv(scc, ged_cmd, GetEuiccData)
         d = ged.to_dict()
-        return flatten_dict_lists(d['get_euicc_data'])['eid_value']
+        return b2h(flatten_dict_lists(d['get_euicc_data'])['eid_value'])
 
     def decode_select_response(self, data_hex: Hexstr) -> object:
         t = FciTemplate()
@@ -503,6 +515,30 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             d = dp.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['delete_profile_resp']))
 
+        mem_res_parser = argparse.ArgumentParser()
+        mem_res_parser.add_argument('--delete-operational', action='store_true',
+                                    help='Delete all operational profiles')
+        mem_res_parser.add_argument('--delete-test-field-installed', action='store_true',
+                                    help='Delete all test profiles, except pre-installed ones')
+        mem_res_parser.add_argument('--reset-smdp-address', action='store_true',
+                                    help='Reset the SM-DP+ address')
+
+        @cmd2.with_argparser(mem_res_parser)
+        def do_euicc_memory_reset(self, opts):
+            """Perform an ES10c eUICCMemoryReset function. This will permanently delete the selected subset of
+            profiles from the eUICC."""
+            flags = {}
+            if opts.delete_operational:
+                flags['deleteOperationalProfiles'] = True
+            if opts.delete_test_field_installed:
+                flags['deleteFieldLoadedTestProfiles'] = True
+            if opts.reset_smdp_address:
+                flags['resetDefaultSmdpAddress'] = True
+
+            mr_cmd = EuiccMemoryResetReq(children=[ResetOptions(decoded=flags)])
+            mr = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, mr_cmd, EuiccMemoryResetResp)
+            d = mr.to_dict()
+            self._cmd.poutput_json(flatten_dict_lists(d['euicc_memory_reset_resp']))
 
         def do_get_eid(self, _opts):
             """Perform an ES10c GetEID function."""
@@ -556,3 +592,43 @@ class CardApplicationECASD(pySim.global_platform.CardApplicationSD):
     @with_default_category('Application-Specific Commands')
     class AddlShellCommands(CommandSet):
         pass
+
+class CardProfileEuiccSGP32(CardProfileUICC):
+    ORDER = 5
+
+    def __init__(self):
+        super().__init__(name='IoT eUICC (SGP.32)')
+
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        # try a command only supported by SGP.32
+        scc.cla_byte = "00"
+        scc.select_adf(AID_ISD_R)
+        CardApplicationISDR.store_data_tlv(scc, GetCertsReq(), GetCertsResp)
+
+class CardProfileEuiccSGP22(CardProfileUICC):
+    ORDER = 6
+
+    def __init__(self):
+        super().__init__(name='Consumer eUICC (SGP.22)')
+
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        # try to read EID from ISD-R
+        scc.cla_byte = "00"
+        scc.select_adf(AID_ISD_R)
+        eid = CardApplicationISDR.get_eid(scc)
+        # TODO: Store EID identity?
+
+class CardProfileEuiccSGP02(CardProfileUICC):
+    ORDER = 7
+
+    def __init__(self):
+        super().__init__(name='M2M eUICC (SGP.02)')
+
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        scc.cla_byte = "00"
+        scc.select_adf(AID_ECASD)
+        scc.get_data(0x5a)
+        # TODO: Store EID identity?

pySim/filesystem.py

@@ -39,7 +39,7 @@ from osmocom.utils import h2b, b2h, is_hex, auto_int, auto_uint8, auto_uint16, i
 from osmocom.tlv import bertlv_parse_one
 from osmocom.construct import filter_dict, parse_construct, build_construct
 
-from pySim.utils import sw_match
+from pySim.utils import sw_match, decomposeATR
 from pySim.jsonpath import js_path_modify
 from pySim.commands import SimCardCommands
 from pySim.exceptions import SwMatchError
@@ -86,7 +86,7 @@ class CardFile:
         self.service = service
         self.shell_commands = []  # type: List[CommandSet]
 
-        # Note: the basic properties (fid, name, ect.) are verified when
+        # Note: the basic properties (fid, name, etc.) are verified when
         # the file is attached to a parent file. See method add_file() in
         # class Card DF
 
@@ -266,7 +266,7 @@ class CardFile:
     def get_profile(self):
         """Get the profile associated with this file. If this file does not have any
         profile assigned, try to find a file above (usually the MF) in the filesystem
-        hirarchy that has a profile assigned
+        hierarchy that has a profile assigned
         """
 
         # If we have a profile set, return it
@@ -301,7 +301,7 @@ class CardFile:
 
     @staticmethod
     def export(as_json: bool, lchan):
-        """
+        r"""
         Export file contents in the form of commandline script. This method is meant to be overloaded by a subclass in
         case any exportable contents are present. The generated script may contain multiple command lines separated by
         line breaks ("\n"), where the last commandline shall have no line break at the end
@@ -661,7 +661,7 @@ class TransparentEF(CardEF):
                 filename = '%s/file' % dirname
                 # write existing data as JSON to file
                 with open(filename, 'w') as text_file:
-                    json.dump(orig_json, text_file, indent=4)
+                    json.dump(orig_json, text_file, indent=4, cls=JsonEncoder)
                 # run a text editor
                 self._cmd.run_editor(filename)
                 with open(filename, 'r') as text_file:
@@ -679,7 +679,7 @@ class TransparentEF(CardEF):
         Args:
             fid : File Identifier (4 hex digits)
             sfid : Short File Identifier (2 hex digits, optional)
-            name : Brief name of the file, lik EF_ICCID
+            name : Brief name of the file, like EF_ICCID
             desc : Description of the file
             parent : Parent CardFile object within filesystem hierarchy
             size : tuple of (minimum_size, recommended_size)
@@ -963,7 +963,7 @@ class LinFixedEF(CardEF):
                 filename = '%s/file' % dirname
                 # write existing data as JSON to file
                 with open(filename, 'w') as text_file:
-                    json.dump(orig_json, text_file, indent=4)
+                    json.dump(orig_json, text_file, indent=4, cls=JsonEncoder)
                 # run a text editor
                 self._cmd.run_editor(filename)
                 with open(filename, 'r') as text_file:
@@ -982,11 +982,11 @@ class LinFixedEF(CardEF):
         Args:
             fid : File Identifier (4 hex digits)
             sfid : Short File Identifier (2 hex digits, optional)
-            name : Brief name of the file, lik EF_ICCID
+            name : Brief name of the file, like EF_ICCID
             desc : Description of the file
             parent : Parent CardFile object within filesystem hierarchy
             rec_len : Tuple of (minimum_length, recommended_length)
-            leftpad: On write, data must be padded from the left to fit pysical record length
+            leftpad: On write, data must be padded from the left to fit physical record length
         """
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent, **kwargs)
         self.rec_len = rec_len
@@ -1224,6 +1224,13 @@ class TransRecEF(TransparentEF):
         Returns:
             abstract_data; dict representing the decoded data
         """
+
+        # The record data length should always be equal or at least greater than the record length defined for the
+        # TransRecEF. Short records may be occur when the length of the underlying TransparentEF is not a multiple
+        # of the TransRecEF record length.
+        if len(raw_hex_data) // 2 < self.__get_rec_len():
+            return {'raw': raw_hex_data}
+
         method = getattr(self, '_decode_record_hex', None)
         if callable(method):
             return method(raw_hex_data)
@@ -1251,6 +1258,11 @@ class TransRecEF(TransparentEF):
         Returns:
             abstract_data; dict representing the decoded data
         """
+
+        # See comment in decode_record_hex (above)
+        if len(raw_bin_data) < self.__get_rec_len():
+            return {'raw': b2h(raw_bin_data)}
+
         method = getattr(self, '_decode_record_bin', None)
         if callable(method):
             return method(raw_bin_data)
@@ -1410,7 +1422,7 @@ class BerTlvEF(CardEF):
         Args:
             fid : File Identifier (4 hex digits)
             sfid : Short File Identifier (2 hex digits, optional)
-            name : Brief name of the file, lik EF_ICCID
+            name : Brief name of the file, like EF_ICCID
             desc : Description of the file
             parent : Parent CardFile object within filesystem hierarchy
             size : tuple of (minimum_size, recommended_size)
@@ -1443,7 +1455,7 @@ class BerTlvEF(CardEF):
             export_str += "delete_all\n"
             for t in tags:
                 result = lchan.retrieve_data(t)
-                (tag, l, val, remainer) = bertlv_parse_one(h2b(result[0]))
+                (tag, l, val, remainder) = bertlv_parse_one(h2b(result[0]))
                 export_str += ("set_data 0x%02x %s\n" % (t, b2h(val)))
         return export_str.strip()
 
@@ -1483,7 +1495,7 @@ class CardApplication:
         self.name = name
         self.adf = adf
         self.sw = sw or {}
-        # back-reference from ADF to Applicaiton
+        # back-reference from ADF to Application
         if self.adf:
             self.aid = aid or self.adf.aid
             self.adf.application = self
@@ -1530,8 +1542,14 @@ class CardModel(abc.ABC):
         """Test if given card matches this model."""
         card_atr = scc.get_atr()
         for atr in cls._atrs:
-            atr_bin = toBytes(atr)
-            if atr_bin == card_atr:
+            if atr == card_atr:
+                print("Detected CardModel:", cls.__name__)
+                return True
+        # if nothing found try to just compare the Historical Bytes of the ATR
+        card_atr_hb = decomposeATR(card_atr)['hb']
+        for atr in cls._atrs:
+            atr_hb = decomposeATR(atr)['hb']
+            if atr_hb == card_atr_hb:
                 print("Detected CardModel:", cls.__name__)
                 return True
         return False
@@ -1554,7 +1572,7 @@ class Path:
             p = p.split('/')
         elif len(p) and isinstance(p[0], int):
             p = ['%04x' % x for x in p]
-        # make sure internal representation alwas is uppercase only
+        # make sure internal representation always is uppercase only
         self.list = [x.upper() for x in p]
 
     def __str__(self) -> str:

pySim/global_platform/__init__.py

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+import io
 from copy import deepcopy
 from typing import Optional, List, Dict, Tuple
 from construct import Optional as COptional
@@ -29,9 +30,11 @@ from osmocom.construct import *
 from pySim.utils import ResTuple
 from pySim.card_key_provider import card_key_provider_get_field
 from pySim.global_platform.scp import SCP02, SCP03
+from pySim.global_platform.install_param import gen_install_parameters
 from pySim.filesystem import *
 from pySim.profile import CardProfile
 from pySim.ota import SimFileAccessAndToolkitAppSpecParams
+from pySim.javacard import CapFile
 
 # GPCS Table 11-48 Load Parameter Tags
 class NonVolatileCodeMinMemoryReq(BER_TLV_IE, tag=0xC6):
@@ -315,7 +318,7 @@ class CurrentSecurityLevel(BER_TLV_IE, tag=0xd3):
 
 # GlobalPlatform v2.3.1 Section 11.3.3.1.3
 class ApplicationAID(BER_TLV_IE, tag=0x4f):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class ApplicationTemplate(BER_TLV_IE, tag=0x61, ntested=[ApplicationAID]):
     pass
 class ListOfApplications(BER_TLV_IE, tag=0x2f00, nested=[ApplicationTemplate]):
@@ -422,10 +425,10 @@ class FciTemplate(BER_TLV_IE, tag=0x6f, nested=FciTemplateNestedList):
     pass
 
 class IssuerIdentificationNumber(BER_TLV_IE, tag=0x42):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 class CardImageNumber(BER_TLV_IE, tag=0x45):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 class SequenceCounterOfDefaultKvn(BER_TLV_IE, tag=0xc1):
     _construct = GreedyInteger()
@@ -464,8 +467,7 @@ class LifeCycleState(BER_TLV_IE, tag=0x9f70):
 
 # Section 11.4.3.1 Table 11-36 + Section 11.1.2
 class Privileges(BER_TLV_IE, tag=0xc5):
-    # we only support 3-byte encoding. Can't use StripTrailerAdapter as length==2 is not permitted. sigh.
-    _construct = FlagsEnum(Int24ub,
+    _construct = FlagsEnum(StripTrailerAdapter(GreedyBytes, 3, steps = [1, 3]),
                            security_domain=0x800000, dap_verification=0x400000,
                            delegated_management=0x200000, card_lock=0x100000, card_terminate=0x080000,
                            card_reset=0x040000, cvm_management=0x020000,
@@ -485,7 +487,7 @@ class ImplicitSelectionParameter(BER_TLV_IE, tag=0xcf):
 
 # Section 11.4.3.1 Table 11-36
 class ExecutableLoadFileAID(BER_TLV_IE, tag=0xc4):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # Section 11.4.3.1 Table 11-36
 class ExecutableLoadFileVersionNumber(BER_TLV_IE, tag=0xce):
@@ -493,15 +495,15 @@ class ExecutableLoadFileVersionNumber(BER_TLV_IE, tag=0xce):
     # specification. It shall consist of the version information contained in the original Load File: on a
     # Java Card based card, this version number represents the major and minor version attributes of the
     # original Load File Data Block.
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # Section 11.4.3.1 Table 11-36
 class ExecutableModuleAID(BER_TLV_IE, tag=0x84):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # Section 11.4.3.1 Table 11-36
 class AssociatedSecurityDomainAID(BER_TLV_IE, tag=0xcc):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # Section 11.4.3.1 Table 11-36
 class GpRegistryRelatedData(BER_TLV_IE, tag=0xe3, nested=[ApplicationAID, LifeCycleState, Privileges,
@@ -625,7 +627,7 @@ class ADF_SD(CardADF):
                     kcv_bin = compute_kcv(opts.key_type[i], h2b(opts.key_data[i])) or b''
                     kcv = b2h(kcv_bin)
                 if self._cmd.lchan.scc.scp:
-                    # encrypte key data with DEK of current SCP
+                    # encrypted key data with DEK of current SCP
                     kcb = b2h(self._cmd.lchan.scc.scp.encrypt_key(h2b(opts.key_data[i])))
                 else:
                     # (for example) during personalization, DEK might not be required)
@@ -638,8 +640,8 @@ class ADF_SD(CardADF):
 
         # Table 11-68: Key Data Field - Format 1 (Basic Format)
         KeyDataBasic = GreedyRange(Struct('key_type'/KeyType,
-                                          'kcb'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
-                                          'kcv'/HexAdapter(Prefixed(Int8ub, GreedyBytes))))
+                                          'kcb'/Prefixed(Int8ub, GreedyBytes),
+                                          'kcv'/Prefixed(Int8ub, GreedyBytes)))
 
         def put_key(self, old_kvn:int, kvn: int, kid: int, key_dict: dict) -> bytes:
             """Perform the GlobalPlatform PUT KEY command in order to store a new key on the card.
@@ -702,7 +704,7 @@ class ADF_SD(CardADF):
         def set_status(self, scope:str, status:str, aid:Hexstr = ''):
             SetStatus = Struct(Const(0x80, Byte), Const(0xF0, Byte),
                                'scope'/SetStatusScope, 'status'/CLifeCycleState,
-                               'aid'/HexAdapter(Prefixed(Int8ub, COptional(GreedyBytes))))
+                               'aid'/Prefixed(Int8ub, COptional(GreedyBytes)))
             apdu = build_construct(SetStatus, {'scope':scope, 'status':status, 'aid':aid})
             _data, _sw = self._cmd.lchan.scc.send_apdu_checksw(b2h(apdu))
 
@@ -724,7 +726,7 @@ class ADF_SD(CardADF):
         inst_inst_parser.add_argument('--application-aid', type=is_hexstr, required=True,
                                       help='Application AID')
         inst_inst_parser.add_argument('--install-parameters', type=is_hexstr, default='',
-                                      help='Install Parameters')
+                                      help='Install Parameters (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
         inst_inst_parser.add_argument('--privilege', action='append', dest='privileges', default=[],
                                       choices=list(Privileges._construct.flags.keys()),
                                       help='Privilege granted to newly installed Application')
@@ -736,12 +738,12 @@ class ADF_SD(CardADF):
         @cmd2.with_argparser(inst_inst_parser)
         def do_install_for_install(self, opts):
             """Perform GlobalPlatform INSTALL [for install] command in order to install an application."""
-            InstallForInstallCD = Struct('load_file_aid'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
-                                         'module_aid'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
-                                         'application_aid'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
+            InstallForInstallCD = Struct('load_file_aid'/Prefixed(Int8ub, GreedyBytes),
+                                         'module_aid'/Prefixed(Int8ub, GreedyBytes),
+                                         'application_aid'/Prefixed(Int8ub, GreedyBytes),
                                          'privileges'/Prefixed(Int8ub, Privileges._construct),
-                                         'install_parameters'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
-                                         'install_token'/HexAdapter(Prefixed(Int8ub, GreedyBytes)))
+                                         'install_parameters'/Prefixed(Int8ub, GreedyBytes),
+                                         'install_token'/Prefixed(Int8ub, GreedyBytes))
             p1 = 0x04
             if opts.make_selectable:
                 p1 |= 0x08
@@ -751,6 +753,31 @@ class ADF_SD(CardADF):
             ifi_bytes = build_construct(InstallForInstallCD, decoded)
             self.install(p1, 0x00, b2h(ifi_bytes))
 
+        inst_load_parser = argparse.ArgumentParser()
+        inst_load_parser.add_argument('--load-file-aid', type=is_hexstr, required=True,
+                                      help='AID of the loaded file')
+        inst_load_parser.add_argument('--security-domain-aid', type=is_hexstr, default='',
+                                      help='AID of the Security Domain into which the file shalle be added')
+        inst_load_parser.add_argument('--load-file-hash', type=is_hexstr, default='',
+                                      help='Load File Data Block Hash (GPC_SPE_034, section C.2)')
+        inst_load_parser.add_argument('--load-parameters', type=is_hexstr, default='',
+                                      help='Load Parameters (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        inst_load_parser.add_argument('--load-token', type=is_hexstr, default='',
+                                      help='Load Token (GPC_SPE_034, section C.4.1)')
+
+        @cmd2.with_argparser(inst_load_parser)
+        def do_install_for_load(self, opts):
+            """Perform GlobalPlatform INSTALL [for load] command in order to prepare to load an application."""
+            if opts.load_token != '' and opts.load_file_hash == '':
+                raise ValueError('Load File Data Block Hash is mandatory if a Load Token is present')
+            InstallForLoadCD = Struct('load_file_aid'/Prefixed(Int8ub, GreedyBytes),
+                                      'security_domain_aid'/Prefixed(Int8ub, GreedyBytes),
+                                      'load_file_hash'/Prefixed(Int8ub, GreedyBytes),
+                                      'load_parameters'/Prefixed(Int8ub, GreedyBytes),
+                                      'load_token'/Prefixed(Int8ub, GreedyBytes))
+            ifl_bytes = build_construct(InstallForLoadCD, vars(opts))
+            self.install(0x02, 0x00, b2h(ifl_bytes))
+
         def install(self, p1:int, p2:int, data:Hexstr) -> ResTuple:
             cmd_hex = "80E6%02x%02x%02x%s00" % (p1, p2, len(data)//2, data)
             return self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)
@@ -791,11 +818,101 @@ class ADF_SD(CardADF):
             self.delete(0x00, p2, cmd)
 
         def delete(self, p1:int, p2:int, data:Hexstr) -> ResTuple:
-            cmd_hex = "80E4%02x%02x%02x%s" % (p1, p2, len(data)//2, data)
+            cmd_hex = "80E4%02x%02x%02x%s00" % (p1, p2, len(data)//2, data)
             return self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)
 
+        load_parser = argparse.ArgumentParser()
+        load_parser_from_grp = load_parser.add_mutually_exclusive_group(required=True)
+        load_parser_from_grp.add_argument('--from-hex', type=is_hexstr, help='load from hex string')
+        load_parser_from_grp.add_argument('--from-file', type=argparse.FileType('rb', 0), help='load from binary file')
+        load_parser_from_grp.add_argument('--from-cap-file', type=argparse.FileType('rb', 0), help='load from JAVA-card CAP file')
+
+        @cmd2.with_argparser(load_parser)
+        def do_load(self, opts):
+            """Perform a GlobalPlatform LOAD command. (We currently only support loading without DAP and
+            without ciphering.)"""
+            if opts.from_hex is not None:
+                self.load(h2b(opts.from_hex))
+            elif opts.from_file is not None:
+                self.load(opts.from_file.read())
+            elif opts.from_cap_file is not None:
+                cap = CapFile(opts.from_cap_file)
+                self.load(cap.get_loadfile())
+            else:
+                raise ValueError('load source not specified!')
+
+        def load(self, contents:bytes, chunk_len:int = 240):
+            # TODO:tune chunk_len based on the overhead of the used SCP?
+            # build TLV according to GPC_SPE_034 section 11.6.2.3 / Table 11-58 for unencrypted case
+            remainder = b'\xC4' + bertlv_encode_len(len(contents)) + contents
+            # transfer this in various chunks to the card
+            total_size = len(remainder)
+            block_nr = 0
+            while len(remainder):
+                block = remainder[:chunk_len]
+                remainder = remainder[chunk_len:]
+                # build LOAD command APDU according to GPC_SPE_034 section 11.6.2 / Table 11-56
+                p1 = 0x00 if len(remainder) else 0x80
+                p2 = block_nr % 256
+                block_nr += 1
+                cmd_hex = "80E8%02x%02x%02x%s00" % (p1, p2, len(block), b2h(block))
+                _rsp_hex, _sw = self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)
+            self._cmd.poutput("Loaded a total of %u bytes in %u blocks. Don't forget install_for_install (and make selectable) now!" % (total_size, block_nr))
+
+        install_cap_parser = argparse.ArgumentParser()
+        install_cap_parser.add_argument('cap_file', type=str, metavar='FILE',
+                                        help='JAVA-CARD CAP file to install')
+        install_cap_parser_inst_prm_g = install_cap_parser.add_mutually_exclusive_group()
+        install_cap_parser_inst_prm_g.add_argument('--install-parameters', type=is_hexstr, default=None,
+                                                   help='install Parameters (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        install_cap_parser_inst_prm_g_grp = install_cap_parser_inst_prm_g.add_argument_group()
+        install_cap_parser_inst_prm_g_grp.add_argument('--install-parameters-volatile-memory-quota',
+                                                       type=int, default=None,
+                                                       help='volatile memory quota (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        install_cap_parser_inst_prm_g_grp.add_argument('--install-parameters-non-volatile-memory-quota',
+                                                       type=int, default=None,
+                                                       help='non volatile memory quota (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        install_cap_parser_inst_prm_g_grp.add_argument('--install-parameters-stk',
+                                                       type=is_hexstr, default=None,
+                                                       help='Load Parameters (ETSI TS 102 226, section 8.2.1.3.2.1)')
+
+        @cmd2.with_argparser(install_cap_parser)
+        def do_install_cap(self, opts):
+            """Perform a .cap file installation using GlobalPlatform LOAD and INSTALL commands."""
+
+            self._cmd.poutput("loading cap file: %s ..." % opts.cap_file)
+            cap = CapFile(opts.cap_file)
+
+            security_domain_aid = self._cmd.lchan.selected_file.aid
+            load_file = cap.get_loadfile()
+            load_file_aid = cap.get_loadfile_aid()
+            module_aid = cap.get_applet_aid()
+            application_aid = module_aid
+            if opts.install_parameters:
+                install_parameters = opts.install_parameters;
+            else:
+                install_parameters = gen_install_parameters(opts.install_parameters_non_volatile_memory_quota,
+                                                            opts.install_parameters_volatile_memory_quota,
+                                                            opts.install_parameters_stk)
+            self._cmd.poutput("parameters:")
+            self._cmd.poutput(" security-domain-aid: %s" % security_domain_aid)
+            self._cmd.poutput(" load-file: %u bytes" % len(load_file))
+            self._cmd.poutput(" load-file-aid: %s" % load_file_aid)
+            self._cmd.poutput(" module-aid: %s" % module_aid)
+            self._cmd.poutput(" application-aid: %s" % application_aid)
+            self._cmd.poutput(" install-parameters: %s" % install_parameters)
+
+            self._cmd.poutput("step #1: install for load...")
+            self.do_install_for_load("--load-file-aid %s --security-domain-aid %s" % (load_file_aid, security_domain_aid))
+            self._cmd.poutput("step #2: load...")
+            self.load(load_file)
+            self._cmd.poutput("step #3: install_for_install (and make selectable)...")
+            self.do_install_for_install("--load-file-aid %s --module-aid %s --application-aid %s --install-parameters %s --make-selectable" %
+                                        (load_file_aid, module_aid, application_aid, install_parameters))
+            self._cmd.poutput("done.")
+
         est_scp02_parser = argparse.ArgumentParser()
-        est_scp02_parser.add_argument('--key-ver', type=auto_uint8, required=True, help='Key Version Number (KVN)')
+        est_scp02_parser.add_argument('--key-ver', type=auto_uint8, default=0, help='Key Version Number (KVN)')
         est_scp02_parser.add_argument('--host-challenge', type=is_hexstr,
                                       help='Hard-code the host challenge; default: random')
         est_scp02_parser.add_argument('--security-level', type=auto_uint8, default=0x01,
@@ -897,17 +1014,12 @@ class CardApplicationISD(CardApplicationSD):
         super().__init__(aid=aid, name='ADF.ISD', desc='Issuer Security Domain')
         self.adf.scp_key_identity = 'ICCID'
 
-#class CardProfileGlobalPlatform(CardProfile):
-#    ORDER = 23
-#
-#    def __init__(self, name='GlobalPlatform'):
-#        super().__init__(name, desc='GlobalPlatfomr 2.1.1', cla=['00','80','84'], sw=sw_table)
-
-
 class GpCardKeyset:
     """A single set of GlobalPlatform card keys and the associated KVN."""
     def __init__(self, kvn: int, enc: bytes, mac: bytes, dek: bytes):
-        assert 0 < kvn < 256
+        # The Key Version Number is an 8 bit integer number, where 0 refers to the first available key,
+        # see also: GPC_SPE_034, section E.5.1.3
+        assert 0 <= kvn < 256
         assert len(enc) == len(mac) == len(dek)
         self.kvn = kvn
         self.enc = enc

pySim/global_platform/http.py

@@ -17,9 +17,10 @@ Also known as SCP81 for SIM/USIM/UICC/eUICC/eSIM OTA.
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from construct import Struct, Int8ub, Int16ub, Bytes, GreedyBytes, GreedyString, BytesInteger
+from construct import Struct, Int8ub, Int16ub, GreedyString, BytesInteger
 from construct import this, len_, Rebuild, Const
 from construct import Optional as COptional
+from osmocom.construct import Bytes, GreedyBytes
 from osmocom.tlv import BER_TLV_IE
 
 from pySim import cat

pySim/global_platform/install_param.py

@@ -0,0 +1,72 @@
+# GlobalPlatform install parameter generator
+#
+# (C) 2024 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from osmocom.construct import *
+from osmocom.utils import *
+from osmocom.tlv import *
+
+class AppSpecificParams(BER_TLV_IE, tag=0xC9):
+    # GPD_SPE_013, table 11-49
+    _construct = GreedyBytes
+
+class VolatileMemoryQuota(BER_TLV_IE, tag=0xC7):
+    # GPD_SPE_013, table 11-49
+    _construct = StripHeaderAdapter(GreedyBytes, 4, steps = [2,4])
+
+class NonVolatileMemoryQuota(BER_TLV_IE, tag=0xC8):
+    # GPD_SPE_013, table 11-49
+    _construct = StripHeaderAdapter(GreedyBytes, 4, steps = [2,4])
+
+class StkParameter(BER_TLV_IE, tag=0xCA):
+    # GPD_SPE_013, table 11-49
+    # ETSI TS 102 226, section 8.2.1.3.2.1
+    _construct = GreedyBytes
+
+class SystemSpecificParams(BER_TLV_IE, tag=0xEF, nested=[VolatileMemoryQuota, NonVolatileMemoryQuota, StkParameter]):
+    # GPD_SPE_013 v1.1 Table 6-5
+    pass
+
+class InstallParams(TLV_IE_Collection, nested=[AppSpecificParams, SystemSpecificParams]):
+    # GPD_SPE_013, table 11-49
+    pass
+
+def gen_install_parameters(non_volatile_memory_quota:int, volatile_memory_quota:int, stk_parameter:str):
+
+    # GPD_SPE_013, table 11-49
+
+    #Mandatory
+    install_params = InstallParams()
+    install_params_dict = [{'app_specific_params': None}]
+
+    #Conditional
+    if non_volatile_memory_quota and volatile_memory_quota and stk_parameter:
+        system_specific_params = []
+        #Optional
+        if non_volatile_memory_quota:
+            system_specific_params += [{'non_volatile_memory_quota': non_volatile_memory_quota}]
+        #Optional
+        if volatile_memory_quota:
+            system_specific_params += [{'volatile_memory_quota': volatile_memory_quota}]
+        #Optional
+        if stk_parameter:
+            system_specific_params += [{'stk_parameter': stk_parameter}]
+        install_params_dict += [{'system_specific_params': system_specific_params}]
+
+    install_params.from_dict(install_params_dict)
+    return b2h(install_params.to_bytes())

pySim/global_platform/scp.py

@@ -20,8 +20,9 @@ import logging
 from typing import Optional
 from Cryptodome.Cipher import DES3, DES
 from Cryptodome.Util.strxor import strxor
-from construct import Struct, Bytes, Int8ub, Int16ub, Const
+from construct import Struct, Int8ub, Int16ub, Const
 from construct import Optional as COptional
+from osmocom.construct import Bytes
 from osmocom.utils import b2h
 from osmocom.tlv import bertlv_parse_len, bertlv_encode_len
 from pySim.utils import parse_command_apdu
@@ -113,7 +114,39 @@ CLA_SM = 0x04
 class SCP(SecureChannel, abc.ABC):
     """Abstract base class containing some common interface + functionality for SCP protocols."""
     def __init__(self, card_keys: 'GpCardKeyset', lchan_nr: int = 0):
-        if hasattr(self, 'kvn_range'):
+
+        # Spec references that explain KVN ranges:
+        # TS 102 225 Annex A.1 states KVN 0x01..0x0F shall be used for SCP80
+        # GPC_GUI_003 states
+        #   * For the Issuer Security Domain, this is initially Key Version Number 'FF' which has been deliberately
+        #     chosen to be outside of the allowable range ('01' to '7F') for a Key Version Number.
+        #   * It is logical that the initial keys in the Issuer Security Domain be replaced by an initial issuer Key
+        #     Version Number in the range '01' to '6F'.
+        #   * Key Version Numbers '70' to '72' and '74' to '7F' are reserved for future use.
+        #   * On an implementation supporting Supplementary Security Domains, the RSA public key with a Key Version
+        #     Number '73' and a Key Identifier of '01' has the following functionality in a Supplementary Security
+        #     Domain with the DAP Verification privilege [...]
+        # GPC_GUI_010 V1.0.1 Section 6 states
+        #   * Key Version number range ('20' to '2F') is reserved for SCP02
+        #   * Key Version 'FF' is reserved for use by an Issuer Security Domain supporting SCP02, and cannot be used
+        #     for SCP80. This initial key set shall be replaced by a key set with a Key Version Number in the
+        #     ('20' to '2F') range.
+        #   * Key Version number range ('01' to '0F') is reserved for SCP80
+        #   * Key Version number '70' with Key Identifier '01' is reserved for the Token Key, which is either a RSA
+        #     public key or a DES key
+        #   * Key Version number '71' with Key Identifier '01' is reserved for the Receipt Key, which is a DES key
+        #   * Key Version Number '11' is reserved for DAP as specified in ETSI TS 102 226 [2]
+        #   * Key Version Number '73' with Key Identifier '01' is reserved for the DAP verification key as specified
+        #     in sections 3.3.3 and 4 of [4], which is either an RSA public key or DES key
+        #   * Key Version Number '74' is reserved for the CASD Keys (cf. section 9.2)
+        #   * Key Version Number '75' with Key Identifier '01' is reserved for the key used to decipher the Ciphered
+        #     Load File Data Block described in section 4.8 of [5].
+
+        if card_keys.kvn == 0:
+            # Key Version Number 0x00 refers to the first available key, so we won't carry out
+            # a range check in this case. See also: GPC_SPE_034, section E.5.1.3
+            pass
+        elif hasattr(self, 'kvn_range'):
             if not card_keys.kvn in range(self.kvn_range[0], self.kvn_range[1]+1):
                 raise ValueError('%s cannot be used with KVN outside range 0x%02x..0x%02x' %
                                  (self.__class__.__name__, self.kvn_range[0], self.kvn_range[1]))
@@ -224,19 +257,22 @@ class SCP02(SCP):
 
     constr_iur = Struct('key_div_data'/Bytes(10), 'key_ver'/Int8ub, Const(b'\x02'),
                         'seq_counter'/Int16ub, 'card_challenge'/Bytes(6), 'card_cryptogram'/Bytes(8))
-    # The 0x70 is a non-spec special-case of sysmoISIM-SJA2/SJA5 and possibly more sysmocom products
-    kvn_ranges = [[0x20, 0x2f], [0x70, 0x70]]
+    # Key Version Number 0x70 is a non-spec special-case of sysmoISIM-SJA2/SJA5 and possibly more sysmocom products
+    # Key Version Number 0x01 is a non-spec special-case of sysmoUSIM-SJS1
+    kvn_ranges = [[0x01, 0x01], [0x20, 0x2f], [0x70, 0x70]]
 
     def __init__(self, *args, **kwargs):
         self.overhead = 8
         super().__init__(*args, **kwargs)
 
     def dek_encrypt(self, plaintext:bytes) -> bytes:
-        cipher = DES.new(self.card_keys.dek[:8], DES.MODE_ECB)
+        # See also GPC section B.1.1.2, E.4.7, and E.4.1
+        cipher = DES3.new(self.sk.data_enc, DES.MODE_ECB)
         return cipher.encrypt(plaintext)
 
     def dek_decrypt(self, ciphertext:bytes) -> bytes:
-        cipher = DES.new(self.card_keys.dek[:8], DES.MODE_ECB)
+        # See also GPC section B.1.1.2, E.4.7, and E.4.1
+        cipher = DES3.new(self.sk.data_enc, DES.MODE_ECB)
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self, card_challenge: bytes, host_challenge: bytes):

pySim/global_platform/uicc.py

@@ -91,6 +91,7 @@ class UiccSdInstallParams(TLV_IE_Collection, nested=[UiccScp, AcceptExtradAppsAn
 
 # Key Usage:
 # KVN 0x01 .. 0x0F reserved for SCP80
+# KVN 0x81 .. 0x8f reserved for SCP81
 # KVN 0x11 reserved for DAP specified in ETSI TS 102 226
 # KVN 0x20 .. 0x2F reserved for SCP02
 #   KID 0x01 = ENC; 0x02 = MAC; 0x03 = DEK
@@ -104,4 +105,4 @@ class UiccSdInstallParams(TLV_IE_Collection, nested=[UiccScp, AcceptExtradAppsAn
 #   KID 0x02: SK.CASD.AUT (PK) and KS.CASD.AUT (Non-PK)
 #   KID 0x03: SK.CASD.CT (P) and KS.CASD.CT (Non-PK)
 # KVN 0x75 KID 0x01: 16-byte DES key for  Ciphered Load File Data Block
-# KVN 0xFF reserved for ISD with SCP02 without SCP80 s upport
+# KVN 0xFF reserved for ISD with SCP02 without SCP80 s support

pySim/gsm_r.py

@@ -26,7 +26,7 @@ order to describe the files specified in UIC Reference P38 T 9001 5.0 "FFFIS for
 
 from pySim.utils import *
 from struct import pack, unpack
-from construct import Struct, Bytes, Int8ub, Int16ub, Int24ub, Int32ub, FlagsEnum
+from construct import Struct, Int8ub, Int16ub, Int24ub, Int32ub, FlagsEnum
 from construct import Optional as COptional
 from osmocom.construct import *
 
@@ -184,13 +184,13 @@ class EF_CallconfI(LinFixedEF):
 class EF_Shunting(TransparentEF):
     """Section 7.6"""
     _test_de_encode = [
-        ( "03f8ffffff000000", { "common_gid": 3, "shunting_gid": "f8ffffff000000" } ),
+        ( "03f8ffffff000000", { "common_gid": 3, "shunting_gid": h2b("f8ffffff000000") } ),
     ]
     def __init__(self):
         super().__init__(fid='6ff4', sfid=None,
                          name='EF.Shunting', desc='Shunting', size=(8, 8))
         self._construct = Struct('common_gid'/Int8ub,
-                                 'shunting_gid'/HexAdapter(Bytes(7)))
+                                 'shunting_gid'/Bytes(7))
 
 
 class EF_GsmrPLMN(LinFixedEF):
@@ -199,13 +199,13 @@ class EF_GsmrPLMN(LinFixedEF):
         ( "22f860f86f8d6f8e01", { "plmn": "228-06", "class_of_network": {
                                     "supported": { "vbs": True, "vgcs": True, "emlpp": True,
                                     "fn": True, "eirene": True }, "preference": 0 },
-                                  "ic_incoming_ref_tbl": "6f8d", "outgoing_ref_tbl": "6f8e",
-                                  "ic_table_ref": "01" } ),
+                                  "ic_incoming_ref_tbl": h2b("6f8d"), "outgoing_ref_tbl": h2b("6f8e"),
+                                  "ic_table_ref": h2b("01") } ),
         ( "22f810416f8d6f8e02", { "plmn": "228-01", "class_of_network": {
                                     "supported": { "vbs": False, "vgcs": False, "emlpp": False,
                                     "fn": True, "eirene": False }, "preference": 1 },
-                                  "ic_incoming_ref_tbl": "6f8d", "outgoing_ref_tbl": "6f8e",
-                                  "ic_table_ref": "02" } ),
+                                  "ic_incoming_ref_tbl": h2b("6f8d"), "outgoing_ref_tbl": h2b("6f8e"),
+                                  "ic_table_ref": h2b("02") } ),
     ]
     def __init__(self):
         super().__init__(fid='6ff5', sfid=None, name='EF.GsmrPLMN',
@@ -213,24 +213,24 @@ class EF_GsmrPLMN(LinFixedEF):
         self._construct = Struct('plmn'/PlmnAdapter(Bytes(3)),
                                  'class_of_network'/BitStruct('supported'/FlagsEnum(BitsInteger(5), vbs=1, vgcs=2, emlpp=4, fn=8, eirene=16),
                                                               'preference'/BitsInteger(3)),
-                                 'ic_incoming_ref_tbl'/HexAdapter(Bytes(2)),
-                                 'outgoing_ref_tbl'/HexAdapter(Bytes(2)),
-                                 'ic_table_ref'/HexAdapter(Bytes(1)))
+                                 'ic_incoming_ref_tbl'/Bytes(2),
+                                 'outgoing_ref_tbl'/Bytes(2),
+                                 'ic_table_ref'/Bytes(1))
 
 
 class EF_IC(LinFixedEF):
     """Section 7.8"""
     _test_de_encode = [
-        ( "f06f8e40f10001", { "next_table_type": "decision", "id_of_next_table": "6f8e",
+        ( "f06f8e40f10001", { "next_table_type": "decision", "id_of_next_table": h2b("6f8e"),
                               "ic_decision_value": "041f", "network_string_table_index": 1 } ),
-        ( "ffffffffffffff", { "next_table_type": "empty", "id_of_next_table": "ffff",
+        ( "ffffffffffffff", { "next_table_type": "empty", "id_of_next_table": h2b("ffff"),
                               "ic_decision_value": "ffff", "network_string_table_index": 65535 } ),
     ]
     def __init__(self):
         super().__init__(fid='6f8d', sfid=None, name='EF.IC',
                          desc='International Code', rec_len=(7, 7))
         self._construct = Struct('next_table_type'/NextTableType,
-                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'id_of_next_table'/Bytes(2),
                                  'ic_decision_value'/BcdAdapter(Bytes(2)),
                                  'network_string_table_index'/Int16ub)
 
@@ -252,18 +252,18 @@ class EF_NW(LinFixedEF):
 class EF_Switching(LinFixedEF):
     """Section 8.4"""
     _test_de_encode = [
-        ( "f26f87f0ff00", { "next_table_type": "num_dial_digits", "id_of_next_table": "6f87",
+        ( "f26f87f0ff00", { "next_table_type": "num_dial_digits", "id_of_next_table": h2b("6f87"),
                             "decision_value": "0fff", "string_table_index": 0 } ),
-        ( "f06f8ff1ff01", { "next_table_type": "decision", "id_of_next_table": "6f8f",
+        ( "f06f8ff1ff01", { "next_table_type": "decision", "id_of_next_table": h2b("6f8f"),
                             "decision_value": "1fff", "string_table_index": 1 } ),
-        ( "f16f89f5ff05", { "next_table_type": "predefined", "id_of_next_table": "6f89",
+        ( "f16f89f5ff05", { "next_table_type": "predefined", "id_of_next_table": h2b("6f89"),
                             "decision_value": "5fff", "string_table_index": 5 } ),
     ]
     def __init__(self, fid='1234', name='Switching', desc=None):
         super().__init__(fid=fid, sfid=None,
                          name=name, desc=desc, rec_len=(6, 6))
         self._construct = Struct('next_table_type'/NextTableType,
-                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'id_of_next_table'/Bytes(2),
                                  'decision_value'/BcdAdapter(Bytes(2)),
                                  'string_table_index'/Int8ub)
 
@@ -271,12 +271,12 @@ class EF_Switching(LinFixedEF):
 class EF_Predefined(LinFixedEF):
     """Section 8.5"""
     _test_de_encode = [
-        ( "f26f85", 1, { "next_table_type": "num_dial_digits", "id_of_next_table": "6f85" } ),
+        ( "f26f85", 1, { "next_table_type": "num_dial_digits", "id_of_next_table": h2b("6f85") } ),
         ( "f0ffc8", 2, { "predefined_value1": "0fff", "string_table_index1": 200 } ),
     ]
     # header and other records have different structure. WTF !?!
     construct_first = Struct('next_table_type'/NextTableType,
-                             'id_of_next_table'/HexAdapter(Bytes(2)))
+                             'id_of_next_table'/Bytes(2))
     construct_others = Struct('predefined_value1'/BcdAdapter(Bytes(2)),
                               'string_table_index1'/Int8ub)
 
@@ -301,13 +301,13 @@ class EF_Predefined(LinFixedEF):
 class EF_DialledVals(TransparentEF):
     """Section 8.6"""
     _test_de_encode = [
-        ( "ffffff22", { "next_table_type": "empty", "id_of_next_table": "ffff", "dialed_digits": "22" } ),
-        ( "f16f8885", { "next_table_type": "predefined", "id_of_next_table": "6f88", "dialed_digits": "58" }),
+        ( "ffffff22", { "next_table_type": "empty", "id_of_next_table": h2b("ffff"), "dialed_digits": "22" } ),
+        ( "f16f8885", { "next_table_type": "predefined", "id_of_next_table": h2b("6f88"), "dialed_digits": "58" }),
     ]
     def __init__(self, fid='1234', name='DialledVals', desc=None):
         super().__init__(fid=fid, sfid=None, name=name, desc=desc, size=(4, 4))
         self._construct = Struct('next_table_type'/NextTableType,
-                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'id_of_next_table'/Bytes(2),
                                  'dialed_digits'/BcdAdapter(Bytes(1)))
 
 

pySim/iso7816_4.py

@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
-from construct import GreedyBytes, GreedyString
+from construct import GreedyString
 from osmocom.tlv import *
 from osmocom.construct import *
 

pySim/javacard.py

@@ -1,19 +1,142 @@
 # JavaCard related utilities
-
+#
+# (C) 2024 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
 import zipfile
 import struct
 import sys
 import io
+from osmocom.utils import b2h, Hexstr
+from construct import Struct, Array, this, Int32ub, Int16ub, Int8ub
+from osmocom.construct import *
+from osmocom.tlv import *
+from construct import Optional as COptional
 
 def ijc_to_cap(in_file: io.IOBase, out_zip: zipfile.ZipFile, p : str = "foo"):
-    """Convert an ICJ (Interoperable Java Card) file [back] to a CAP file."""
-    TAGS = ["Header", "Directory", "Applet", "Import", "ConstantPool", "Class", "Method", "StaticField", "RefLocation", "Export", "Descriptor", "Debug"]
+    """Convert an ICJ (Interoperable Java Card) file [back] to a CAP file.
+    example usage:
+        with io.open(sys.argv[1],"rb") as f, zipfile.ZipFile(sys.argv[2], "wb") as z:
+        ijc_to_cap(f, z)
+    """
+    TAGS = ["Header", "Directory", "Applet", "Import", "ConstantPool", "Class", "Method", "StaticField", "RefLocation",
+            "Export", "Descriptor", "Debug"]
     b = in_file.read()
     while len(b):
         tag, size = struct.unpack('!BH', b[0:3])
         out_zip.writestr(p+"/javacard/"+TAGS[tag-1]+".cap", b[0:3+size])
         b = b[3+size:]
 
-# example usage:
-# with io.open(sys.argv[1],"rb") as f, zipfile.ZipFile(sys.argv[2], "wb") as z:
-#     ijc_to_cap(f, z)
+class CapFile():
+
+    # Java Card Platform Virtual Machine Specification, v3.2, section 6.4
+    __header_component_compact = Struct('tag'/Int8ub,
+                                        'size'/Int16ub,
+                                        'magic'/Int32ub,
+                                        'minor_version'/Int8ub,
+                                        'major_version'/Int8ub,
+                                        'flags'/Int8ub,
+                                        'package'/Struct('minor_version'/Int8ub,
+                                                         'major_version'/Int8ub,
+                                                         'AID'/LV),
+                                        'package_name'/COptional(LV)) #since CAP format 2.2
+
+    # Java Card Platform Virtual Machine Specification, v3.2, section 6.6
+    __applet_component_compact = Struct('tag'/Int8ub,
+                                        'size'/Int16ub,
+                                        'count'/Int8ub,
+                                        'applets'/Array(this.count, Struct('AID'/LV,
+                                                                           'install_method_offset'/Int16ub)),
+                                       )
+
+    def __init__(self, filename:str):
+
+        # In this dictionary we will keep all nested .cap file components by their file names (without .cap suffix)
+        # See also: Java Card Platform Virtual Machine Specification, v3.2, section 6.2.1
+        self.__component = {}
+
+        # Extract the nested .cap components from the .cap file
+        # See also: Java Card Platform Virtual Machine Specification, v3.2, section 6.2.1
+        cap = zipfile.ZipFile(filename)
+        cap_namelist = cap.namelist()
+        for i, filename in enumerate(cap_namelist):
+            if filename.lower().endswith('.capx') and not filename.lower().endswith('.capx'):
+                #TODO: At the moment we only support the compact .cap format, add support for the extended .cap format.
+                raise ValueError("incompatible .cap file, extended .cap format not supported!")
+
+            if filename.lower().endswith('.cap'):
+                key = filename.split('/')[-1].removesuffix('.cap')
+                self.__component[key] = cap.read(filename)
+
+        # Make sure that all mandatory components are present
+        # See also: Java Card Platform Virtual Machine Specification, v3.2, section 6.2
+        required_components = {'Header' : 'COMPONENT_Header',
+                               'Directory' : 'COMPONENT_Directory',
+                               'Import' : 'COMPONENT_Import',
+                               'ConstantPool' : 'COMPONENT_ConstantPool',
+                               'Class' : 'COMPONENT_Class',
+                               'Method' : 'COMPONENT_Method',
+                               'StaticField' : 'COMPONENT_StaticField',
+                               'RefLocation' : 'COMPONENT_ReferenceLocation',
+                               'Descriptor' : 'COMPONENT_Descriptor'}
+        for component in required_components:
+            if component not in self.__component.keys():
+                raise ValueError("invalid cap file, %s missing!" % required_components[component])
+
+    def get_loadfile(self) -> bytes:
+        """Get the executable loadfile as hexstring"""
+        # Concatenate all cap file components in the specified order
+        # see also: Java Card Platform Virtual Machine Specification, v3.2, section 6.3
+        loadfile = self.__component['Header']
+        loadfile += self.__component['Directory']
+        loadfile += self.__component['Import']
+        if 'Applet' in self.__component.keys():
+            loadfile += self.__component['Applet']
+        loadfile += self.__component['Class']
+        loadfile += self.__component['Method']
+        loadfile += self.__component['StaticField']
+        if 'Export' in self.__component.keys():
+            loadfile += self.__component['Export']
+        loadfile += self.__component['ConstantPool']
+        loadfile += self.__component['RefLocation']
+        if 'Descriptor' in self.__component.keys():
+            loadfile += self.__component['Descriptor']
+        return loadfile
+
+    def get_loadfile_aid(self) -> Hexstr:
+        """Get the loadfile AID as hexstring"""
+        header = self.__header_component_compact.parse(self.__component['Header'])
+        magic = header['magic'] or 0
+        if magic != 0xDECAFFED:
+            raise ValueError("invalid cap file, COMPONENT_Header lacks magic number (0x%08X!=0xDECAFFED)!" % magic)
+        #TODO: check cap version and make sure we are compatible with it
+        return header['package']['AID']
+
+    def get_applet_aid(self, index:int = 0) -> Hexstr:
+        """Get the applet AID as hexstring"""
+        #To get the module AID, we must look into COMPONENT_Applet. Unfortunately, even though this component should
+        #be present in any .cap file, it is defined as an optional component.
+        if 'Applet' not in self.__component.keys():
+            raise ValueError("can't get the AID, this cap file lacks the optional COMPONENT_Applet component!")
+
+        applet = self.__applet_component_compact.parse(self.__component['Applet'])
+
+        if index > applet['count']:
+            raise ValueError("can't get the AID for applet with index=%u, this .cap file only has %u applets!" %
+                             (index, applet['count']))
+
+        return applet['applets'][index]['AID']
+

pySim/legacy/cards.py

@@ -3,7 +3,6 @@
 ################################################################################
 
 import abc
-from smartcard.util import toBytes
 from pytlv.TLV import *
 
 from pySim.cards import SimCardBase, UiccCardBase
@@ -16,7 +15,7 @@ from pySim.legacy.ts_51_011 import EF, DF
 from pySim.legacy.ts_31_102 import EF_USIM_ADF_map
 from pySim.legacy.ts_31_103 import EF_ISIM_ADF_map
 
-from pySim.profile.ts_51_011 import EF_AD, EF_SPN
+from pySim.ts_51_011 import EF_AD, EF_SPN
 
 def format_addr(addr: str, addr_type: str) -> str:
     """
@@ -496,7 +495,7 @@ class IsimCard(UiccCardBase):
 
 class MagicSimBase(abc.ABC, SimCard):
     """
-    Theses cards uses several record based EFs to store the provider infos,
+    These cards uses several record based EFs to store the provider infos,
     each possible provider uses a specific record number in each EF. The
     indexes used are ( where N is the number of providers supported ) :
      - [2 .. N+1] for the operator name
@@ -645,7 +644,7 @@ class MagicSim(MagicSimBase):
 
 class FakeMagicSim(SimCard):
     """
-    Theses cards have a record based EF 3f00/000c that contains the provider
+    These cards have a record based EF 3f00/000c that contains the provider
     information. See the program method for its format. The records go from
     1 to N.
     """
@@ -781,7 +780,7 @@ class SysmoSIMgr1(GrcardSim):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 99 18 00 11 88 22 33 44 55 66 77 60"):
+            if scc.get_atr() == "3b991800118822334455667760":
                 return kls(scc)
         except:
             return None
@@ -826,7 +825,7 @@ class SysmoSIMgr2(SimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 7D 94 00 00 55 55 53 0A 74 86 93 0B 24 7C 4D 54 68"):
+            if scc.get_atr() == "3b7d9400005555530a7486930b247c4d5468":
                 return kls(scc)
         except:
             return None
@@ -904,7 +903,7 @@ class SysmoUSIMSJS1(UsimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 9F 96 80 1F C7 80 31 A0 73 BE 21 13 67 43 20 07 18 00 00 01 A5"):
+            if scc.get_atr() == "3b9f96801fc78031a073be21136743200718000001a5":
                 return kls(scc)
         except:
             return None
@@ -1032,7 +1031,7 @@ class FairwavesSIM(UsimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 9F 96 80 1F C7 80 31 A0 73 BE 21 13 67 44 22 06 10 00 00 01 A9"):
+            if scc.get_atr() == "3b9f96801fc78031a073be21136744220610000001a9":
                 return kls(scc)
         except:
             return None
@@ -1166,7 +1165,7 @@ class OpenCellsSim(SimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 9F 95 80 1F C3 80 31 E0 73 FE 21 13 57 86 81 02 86 98 44 18 A8"):
+            if scc.get_atr() == "3b9f95801fc38031e073fe21135786810286984418a8":
                 return kls(scc)
         except:
             return None
@@ -1215,7 +1214,7 @@ class WavemobileSim(UsimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 9F 95 80 1F C7 80 31 E0 73 F6 21 13 67 4D 45 16 00 43 01 00 8F"):
+            if scc.get_atr() == "3b9f95801fc78031e073f62113674d4516004301008f":
                 return kls(scc)
         except:
             return None
@@ -1305,18 +1304,18 @@ class SysmoISIMSJA2(UsimCard, IsimCard):
     def autodetect(kls, scc):
         try:
             # Try card model #1
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 30 34 05 4B A9"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a4c753034054ba9"
+            if scc.get_atr() == atr:
                 return kls(scc)
 
             # Try card model #2
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 31 33 02 51 B2"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a4c7531330251b2"
+            if scc.get_atr() == atr:
                 return kls(scc)
 
             # Try card model #3
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 52 75 31 04 51 D5"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a4c5275310451d5"
+            if scc.get_atr() == atr:
                 return kls(scc)
         except:
             return None
@@ -1554,16 +1553,16 @@ class SysmoISIMSJA5(SysmoISIMSJA2):
     def autodetect(kls, scc):
         try:
             # Try card model #1 (9FJ)
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 51 CC"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a357530350251cc"
+            if scc.get_atr() == atr:
                 return kls(scc)
             # Try card model #2 (SLM17)
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 65 F8"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a357530350265f8"
+            if scc.get_atr() == atr:
                 return kls(scc)
             # Try card model #3 (9FV)
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 59 C4"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a357530350259c4"
+            if scc.get_atr() == atr:
                 return kls(scc)
         except:
             return None
@@ -1592,7 +1591,7 @@ class GialerSim(UsimCard):
     def autodetect(cls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes('3B 9F 95 80 1F C7 80 31 A0 73 B6 A1 00 67 CF 32 15 CA 9C D7 09 20'):
+            if scc.get_atr() == '3b9f95801fc78031a073b6a10067cf3215ca9cd70920':
                 return cls(scc)
         except:
             return None

pySim/legacy/utils.py

@@ -148,7 +148,7 @@ def dec_st(st, table="sim") -> str:
         from pySim.ts_31_102 import EF_UST_map
         lookup_map = EF_UST_map
     else:
-        from pySim.profile.ts_51_011 import EF_SST_map
+        from pySim.ts_51_011 import EF_SST_map
         lookup_map = EF_SST_map
 
     st_bytes = [st[i:i+2] for i in range(0, len(st), 2)]
@@ -296,7 +296,7 @@ def dec_addr_tlv(hexstr):
 
         elif addr_type == 0x01:  # IPv4
             # Skip address tye byte i.e. first byte in value list
-            # Skip the unused byte in Octect 4 after address type byte as per 3GPP TS 31.102
+            # Skip the unused byte in Octet 4 after address type byte as per 3GPP TS 31.102
             ipv4 = tlv[2][2:]
             content = '.'.join(str(x) for x in ipv4)
             return (content, '01')

pySim/log.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+
+""" pySim: Logging
+"""
+
+#
+# (C) 2025 by sysmocom - s.f.m.c. GmbH
+# All Rights Reserved
+#
+# Author: Philipp Maier <pmaier@sysmocom.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging
+from cmd2 import style
+
+class _PySimLogHandler(logging.Handler):
+    def __init__(self, log_callback):
+        super().__init__()
+        self.log_callback = log_callback
+
+    def emit(self, record):
+        formatted_message = self.format(record)
+        self.log_callback(formatted_message, record)
+
+class PySimLogger:
+    """
+    Static class to centralize the log output of PySim applications. This class can be used to print log messages from
+    any pySim module. Configuration of the log behaviour (see setup and set_ methods) is entirely optional. In case no
+    print callback is set (see setup method), the logger will pass the log messages directly to print() without applying
+    any formatting to the original log message.
+    """
+
+    LOG_FMTSTR = "%(levelname)s: %(message)s"
+    LOG_FMTSTR_VERBOSE = "%(module)s.%(lineno)d -- " + LOG_FMTSTR
+    __formatter = logging.Formatter(LOG_FMTSTR)
+    __formatter_verbose = logging.Formatter(LOG_FMTSTR_VERBOSE)
+
+    # No print callback by default, means that log messages are passed directly to print()
+    print_callback = None
+
+    # No specific color scheme by default
+    colors = {}
+
+    # The logging default is non-verbose logging on logging level DEBUG. This is a safe default that works for
+    # applications that ignore the presence of the PySimLogger class.
+    verbose = False
+    logging.root.setLevel(logging.DEBUG)
+
+    def __init__(self):
+        raise RuntimeError('static class, do not instantiate')
+
+    @staticmethod
+    def setup(print_callback = None, colors:dict = {}):
+        """
+        Set a print callback function and color scheme. This function call is optional. In case this method is not
+        called, default settings apply.
+        Args:
+            print_callback : A callback function that accepts the resulting log string as input. The callback should
+                             have the following format: print_callback(message:str)
+            colors : An optional dict through which certain log levels can be assigned a color.
+                     (e.g. {logging.WARN: YELLOW})
+        """
+        PySimLogger.print_callback = print_callback
+        PySimLogger.colors = colors
+
+    @staticmethod
+    def set_verbose(verbose:bool = False):
+        """
+        Enable/disable verbose logging. (has no effect in case no print callback is set, see method setup)
+        Args:
+            verbose: verbosity (True = verbose logging, False = normal logging)
+        """
+        PySimLogger.verbose = verbose;
+
+    @staticmethod
+    def set_level(level:int = logging.DEBUG):
+        """
+        Set the logging level.
+        Args:
+            level: Logging level, valis log leves are: DEBUG, INFO, WARNING, ERROR and CRITICAL
+        """
+        logging.root.setLevel(level)
+
+    @staticmethod
+    def _log_callback(message, record):
+        if not PySimLogger.print_callback:
+            # In case no print callback has been set display the message as if it were printed trough a normal
+            # python print statement.
+            print(record.message)
+        else:
+            # When a print callback is set, use it to display the log line. Apply color if the API user chose one
+            if PySimLogger.verbose:
+                formatted_message = logging.Formatter.format(PySimLogger.__formatter_verbose, record)
+            else:
+                formatted_message = logging.Formatter.format(PySimLogger.__formatter, record)
+            color = PySimLogger.colors.get(record.levelno)
+            if color:
+                if isinstance(color, str):
+                    PySimLogger.print_callback(color + formatted_message + "\033[0m")
+                else:
+                    PySimLogger.print_callback(style(formatted_message, fg = color))
+            else:
+                PySimLogger.print_callback(formatted_message)
+
+    @staticmethod
+    def get(log_facility: str):
+        """
+        Set up and return a new python logger object
+        Args:
+            log_facility : Name of log facility (e.g. "MAIN", "RUNTIME"...)
+        """
+        logger = logging.getLogger(log_facility)
+        handler = _PySimLogHandler(log_callback=PySimLogger._log_callback)
+        logger.addHandler(handler)
+        return logger

pySim/ota.py

@@ -19,7 +19,7 @@ import zlib
 import abc
 import struct
 from typing import Optional, Tuple
-from construct import Enum, Int8ub, Int16ub, Struct, Bytes, GreedyBytes, BitsInteger, BitStruct
+from construct import Enum, Int8ub, Int16ub, Struct, BitsInteger, BitStruct
 from construct import Flag, Padding, Switch, this, PrefixedArray, GreedyRange
 from osmocom.construct import *
 from osmocom.utils import b2h
@@ -57,12 +57,13 @@ CompactRemoteResp = Struct('number_of_commands'/Int8ub,
                            'last_response_data'/HexAdapter(GreedyBytes))
 
 RC_CC_DS = Enum(BitsInteger(2), no_rc_cc_ds=0, rc=1, cc=2, ds=3)
+CNTR_REQ = Enum(BitsInteger(2), no_counter=0, counter_no_replay_or_seq=1, counter_must_be_higher=2, counter_must_be_lower=3)
+POR_REQ = Enum(BitsInteger(2), no_por=0, por_required=1, por_only_when_error=2)
 
 # TS 102 225 Section 5.1.1 + TS 31.115 Section 4.2
 SPI = BitStruct(  # first octet
     Padding(3),
-    'counter'/Enum(BitsInteger(2), no_counter=0, counter_no_replay_or_seq=1,
-                   counter_must_be_higher=2, counter_must_be_lower=3),
+    'counter'/CNTR_REQ,
     'ciphering'/Flag,
     'rc_cc_ds'/RC_CC_DS,
     # second octet
@@ -70,8 +71,7 @@ SPI = BitStruct(  # first octet
     'por_in_submit'/Flag,
     'por_shall_be_ciphered'/Flag,
     'por_rc_cc_ds'/RC_CC_DS,
-    'por'/Enum(BitsInteger(2), no_por=0,
-               por_required=1, por_only_when_error=2)
+    'por'/POR_REQ
 )
 
 # TS 102 225 Section 5.1.2
@@ -410,6 +410,7 @@ class OtaDialectSms(OtaDialect):
             ciph = encoded[2+8:]
             envelope_data = otak.crypt.decrypt(ciph)
         else:
+            cpl = None # FIXME this line was just added to silence pylint possibly-used-before-assignment
             part_head = encoded[:8]
             envelope_data = encoded[8:]
 

pySim/profile.py

@@ -4,7 +4,7 @@
 """
 
 #
-# (C) 2021 by Sysmocom s.f.m.c. GmbH
+# (C) 2021 by sysmocom - s.f.m.c. GmbH
 # All Rights Reserved
 #
 # This program is free software: you can redistribute it and/or modify
@@ -110,7 +110,7 @@ class CardProfile:
     @abc.abstractmethod
     def _try_match_card(cls, scc: SimCardCommands) -> None:
         """Try to see if the specific profile matches the card. This method is a
-        placeholder that is overloaded by specific dirived classes. The method
+        placeholder that is overloaded by specific derived classes. The method
         actively probes the card to make sure the profile class matches the
         physical card. This usually also means that the card is reset during
         the process, so this method must not be called at random times. It may

pySim/profile/euicc.py

@@ -1,58 +0,0 @@
-# Copyright (C) 2023 Harald Welte <laforge@osmocom.org>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from pySim.profile.ts_102_221 import CardProfileUICC
-from pySim.commands import SimCardCommands
-from pySim.euicc import CardApplicationISDR, AID_ISD_R, AID_ECASD, GetCertsReq, GetCertsResp
-
-class CardProfileEuiccSGP32(CardProfileUICC):
-    ORDER = 5
-
-    def __init__(self):
-        super().__init__(name='IoT eUICC (SGP.32)')
-
-    @classmethod
-    def _try_match_card(cls, scc: SimCardCommands) -> None:
-        # try a command only supported by SGP.32
-        scc.cla_byte = "00"
-        scc.select_adf(AID_ISD_R)
-        CardApplicationISDR.store_data_tlv(scc, GetCertsReq(), GetCertsResp)
-
-class CardProfileEuiccSGP22(CardProfileUICC):
-    ORDER = 6
-
-    def __init__(self):
-        super().__init__(name='Consumer eUICC (SGP.22)')
-
-    @classmethod
-    def _try_match_card(cls, scc: SimCardCommands) -> None:
-        # try to read EID from ISD-R
-        scc.cla_byte = "00"
-        scc.select_adf(AID_ISD_R)
-        eid = CardApplicationISDR.get_eid(scc)
-        # TODO: Store EID identity?
-
-class CardProfileEuiccSGP02(CardProfileUICC):
-    ORDER = 7
-
-    def __init__(self):
-        super().__init__(name='M2M eUICC (SGP.02)')
-
-    @classmethod
-    def _try_match_card(cls, scc: SimCardCommands) -> None:
-        scc.cla_byte = "00"
-        scc.select_adf(AID_ECASD)
-        scc.get_data(0x5a)
-        # TODO: Store EID identity?

pySim/profile/ts_102_221.py

@@ -1,394 +0,0 @@
-# coding=utf-8
-"""Card Profile of ETSI TS 102 221, the core UICC spec.
-
-(C) 2021-2024 by Harald Welte <laforge@osmocom.org>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 2 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-"""
-
-from construct import Struct, FlagsEnum, GreedyString
-
-from osmocom.construct import *
-from osmocom.utils import *
-from osmocom.tlv import BER_TLV_IE
-
-from pySim.utils import *
-from pySim.filesystem import *
-from pySim.profile import CardProfile
-from pySim import iso7816_4
-from pySim.ts_102_221 import decode_select_response, ts_102_22x_cmdset
-from pySim.ts_102_221 import AM_DO_EF, SC_DO, AdditionalInterfacesSupport, AdditionalTermCapEuicc
-from pySim.ts_102_221 import TerminalPowerSupply, ExtendedLchanTerminalSupport, TerminalCapability
-
-# A UICC will usually also support 2G functionality. If this is the case, we
-# need to add DF_GSM and DF_TELECOM along with the UICC related files
-from pySim.profile.ts_51_011 import AddonSIM, EF_ICCID, EF_PL
-from pySim.profile.gsm_r import AddonGSMR
-from pySim.profile.cdma_ruim import AddonRUIM
-
-
-# TS 102 221 Section 13.1
-class EF_DIR(LinFixedEF):
-    _test_de_encode = [
-        ( '61294f10a0000000871002ffffffff890709000050055553696d31730ea00c80011781025f608203454150',
-          { "application_template": [ { "application_id": h2b("a0000000871002ffffffff8907090000") },
-                                      { "application_label": "USim1" },
-                                      { "discretionary_template": h2b("a00c80011781025f608203454150") } ] }
-        ),
-        ( '61194f10a0000000871004ffffffff890709000050054953696d31',
-          { "application_template": [ { "application_id": h2b("a0000000871004ffffffff8907090000") },
-                                      { "application_label": "ISim1" } ] }
-        ),
-    ]
-    class ApplicationLabel(BER_TLV_IE, tag=0x50):
-        # TODO: UCS-2 coding option as per Annex A of TS 102 221
-        _construct = GreedyString('ascii')
-
-    # see https://github.com/PyCQA/pylint/issues/5794
-    #pylint: disable=undefined-variable
-    class ApplicationTemplate(BER_TLV_IE, tag=0x61,
-                              nested=[iso7816_4.ApplicationId, ApplicationLabel, iso7816_4.FileReference,
-                                      iso7816_4.CommandApdu, iso7816_4.DiscretionaryData,
-                                      iso7816_4.DiscretionaryTemplate, iso7816_4.URL,
-                                      iso7816_4.ApplicationRelatedDOSet]):
-        pass
-
-    def __init__(self, fid='2f00', sfid=0x1e, name='EF.DIR', desc='Application Directory'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(5, 54))
-        self._tlv = EF_DIR.ApplicationTemplate
-
-
-# TS 102 221 Section 13.4
-class EF_ARR(LinFixedEF):
-    _test_de_encode = [
-        ( '800101a40683010a950108800106900080016097008401d4a40683010a950108',
-         [ [ { "access_mode": [ "read_search_compare" ] },
-             { "control_reference_template": "ADM1" } ],
-           [ { "access_mode": [ "write_append", "update_erase" ] },
-             { "always": None } ],
-           [ { "access_mode": [ "delete_file", "terminate_ef" ] },
-             { "never": None } ],
-           [ { "command_header": { "INS": 212 } },
-             { "control_reference_template": "ADM1" } ]
-         ] ),
-        ( '80010190008001029700800118a40683010a9501088401d4a40683010a950108',
-         [ [ { "access_mode": [ "read_search_compare" ] },
-             { "always": None } ],
-           [ { "access_mode": [ "update_erase" ] },
-             { "never": None } ],
-           [ { "access_mode": [ "activate_file_or_record", "deactivate_file_or_record" ] },
-             { "control_reference_template": "ADM1" } ],
-           [ { "command_header": { "INS": 212 } },
-             { "control_reference_template": "ADM1" } ]
-         ] ),
-    ]
-    def __init__(self, fid='2f06', sfid=0x06, name='EF.ARR', desc='Access Rule Reference'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc)
-        # add those commands to the general commands of a TransparentEF
-        self.shell_commands += [self.AddlShellCommands()]
-
-    @staticmethod
-    def flatten(inp: list):
-        """Flatten the somewhat deep/complex/nested data returned from decoder."""
-        def sc_abbreviate(sc):
-            if 'always' in sc:
-                return 'always'
-            elif 'never' in sc:
-                return 'never'
-            elif 'control_reference_template' in sc:
-                return sc['control_reference_template']
-            else:
-                return sc
-
-        by_mode = {}
-        for t in inp:
-            am = t[0]
-            sc = t[1]
-            sc_abbr = sc_abbreviate(sc)
-            if 'access_mode' in am:
-                for m in am['access_mode']:
-                    by_mode[m] = sc_abbr
-            elif 'command_header' in am:
-                ins = am['command_header']['INS']
-                if 'CLA' in am['command_header']:
-                    cla = am['command_header']['CLA']
-                else:
-                    cla = None
-                cmd = ts_102_22x_cmdset.lookup(ins, cla)
-                if cmd:
-                    name = cmd.name.lower().replace(' ', '_')
-                    by_mode[name] = sc_abbr
-                else:
-                    raise ValueError
-            else:
-                raise ValueError
-        return by_mode
-
-    def _decode_record_bin(self, raw_bin_data, **kwargs):
-        # we can only guess if we should decode for EF or DF here :(
-        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
-        dec = arr_seq.decode_multi(raw_bin_data)
-        # we cannot pass the result through flatten() here, as we don't have a related
-        # 'un-flattening' decoder, and hence would be unable to encode :(
-        return dec[0]
-
-    def _encode_record_bin(self, in_json, **kwargs):
-        # we can only guess if we should decode for EF or DF here :(
-        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
-        return arr_seq.encode_multi(in_json)
-
-    @with_default_category('File-Specific Commands')
-    class AddlShellCommands(CommandSet):
-        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
-        def do_read_arr_record(self, opts):
-            """Read one EF.ARR record in flattened, human-friendly form."""
-            (data, _sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
-            data = self._cmd.lchan.selected_file.flatten(data)
-            self._cmd.poutput_json(data, opts.oneline)
-
-        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
-        def do_read_arr_records(self, opts):
-            """Read + decode all EF.ARR records in flattened, human-friendly form."""
-            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
-            # collect all results in list so they are rendered as JSON list when printing
-            data_list = []
-            for recnr in range(1, 1 + num_of_rec):
-                (data, _sw) = self._cmd.lchan.read_record_dec(recnr)
-                data = self._cmd.lchan.selected_file.flatten(data)
-                data_list.append(data)
-            self._cmd.poutput_json(data_list, opts.oneline)
-
-
-# TS 102 221 Section 13.6
-class EF_UMPC(TransparentEF):
-    _test_de_encode = [
-        ( '3cff02', { "max_current_mA": 60, "t_op_s": 255,
-                      "addl_info": { "req_inc_idle_current": False, "support_uicc_suspend": True } } ),
-        ( '320500', { "max_current_mA": 50, "t_op_s": 5, "addl_info": {"req_inc_idle_current": False,
-                                                                       "support_uicc_suspend": False } } ),
-    ]
-    def __init__(self, fid='2f08', sfid=0x08, name='EF.UMPC', desc='UICC Maximum Power Consumption'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(5, 5))
-        addl_info = FlagsEnum(Byte, req_inc_idle_current=1,
-                              support_uicc_suspend=2)
-        self._construct = Struct(
-            'max_current_mA'/Int8ub, 't_op_s'/Int8ub, 'addl_info'/addl_info)
-
-
-class CardProfileUICC(CardProfile):
-
-    ORDER = 10
-
-    def __init__(self, name='UICC'):
-        files = [
-            EF_DIR(),
-            EF_ICCID(),
-            EF_PL(),
-            EF_ARR(),
-            # FIXME: DF.CD
-            EF_UMPC(),
-        ]
-        addons = [
-            AddonSIM,
-            AddonGSMR,
-            AddonRUIM,
-        ]
-        sw = {
-            'Normal': {
-                '9000': 'Normal ending of the command',
-                '91xx': 'Normal ending of the command, with extra information from the proactive UICC containing a command for the terminal',
-                '92xx': 'Normal ending of the command, with extra information concerning an ongoing data transfer session',
-            },
-            'Postponed processing': {
-                '9300': 'SIM Application Toolkit is busy. Command cannot be executed at present, further normal commands are allowed',
-            },
-            'Warnings': {
-                '6200': 'No information given, state of non-volatile memory unchanged',
-                '6281': 'Part of returned data may be corrupted',
-                '6282': 'End of file/record reached before reading Le bytes or unsuccessful search',
-                '6283': 'Selected file invalidated/disabled; needs to be activated before use',
-                '6284': 'Selected file in termination state',
-                '62f1': 'More data available',
-                '62f2': 'More data available and proactive command pending',
-                '62f3': 'Response data available',
-                '63f1': 'More data expected',
-                '63f2': 'More data expected and proactive command pending',
-                '63cx': 'Command successful but after using an internal update retry routine X times',
-            },
-            'Execution errors': {
-                '6400': 'No information given, state of non-volatile memory unchanged',
-                '6500': 'No information given, state of non-volatile memory changed',
-                '6581': 'Memory problem',
-            },
-            'Checking errors': {
-                '6700': 'Wrong length',
-                '67xx': 'The interpretation of this status word is command dependent',
-                '6b00': 'Wrong parameter(s) P1-P2',
-                '6d00': 'Instruction code not supported or invalid',
-                '6e00': 'Class not supported',
-                '6f00': 'Technical problem, no precise diagnosis',
-                '6fxx': 'The interpretation of this status word is command dependent',
-            },
-            'Functions in CLA not supported': {
-                '6800': 'No information given',
-                '6881': 'Logical channel not supported',
-                '6882': 'Secure messaging not supported',
-            },
-            'Command not allowed': {
-                '6900': 'No information given',
-                '6981': 'Command incompatible with file structure',
-                '6982': 'Security status not satisfied',
-                '6983': 'Authentication/PIN method blocked',
-                '6984': 'Referenced data invalidated',
-                '6985': 'Conditions of use not satisfied',
-                '6986': 'Command not allowed (no EF selected)',
-                '6989': 'Command not allowed - secure channel - security not satisfied',
-            },
-            'Wrong parameters': {
-                '6a80': 'Incorrect parameters in the data field',
-                '6a81': 'Function not supported',
-                '6a82': 'File not found',
-                '6a83': 'Record not found',
-                '6a84': 'Not enough memory space',
-                '6a86': 'Incorrect parameters P1 to P2',
-                '6a87': 'Lc inconsistent with P1 to P2',
-                '6a88': 'Referenced data not found',
-            },
-            'Application errors': {
-                '9850': 'INCREASE cannot be performed, max value reached',
-                '9862': 'Authentication error, application specific',
-                '9863': 'Security session or association expired',
-                '9864': 'Minimum UICC suspension time is too long',
-            },
-        }
-
-        super().__init__(name, desc='ETSI TS 102 221', cla="00",
-                         sel_ctrl="0004", files_in_mf=files, sw=sw,
-                         shell_cmdsets = [self.AddlShellCommands()], addons = addons)
-
-    @staticmethod
-    def decode_select_response(data_hex: str) -> object:
-        """ETSI TS 102 221 Section 11.1.1.3"""
-        return decode_select_response(data_hex)
-
-    @classmethod
-    def _try_match_card(cls, scc: SimCardCommands) -> None:
-        """ Try to access MF via UICC APDUs (3GPP TS 102.221), if this works, the
-        card is considered a UICC card."""
-        cls._mf_select_test(scc, "00", "0004", ["3f00"])
-
-    @with_default_category('TS 102 221 Specific Commands')
-    class AddlShellCommands(CommandSet):
-        suspend_uicc_parser = argparse.ArgumentParser()
-        suspend_uicc_parser.add_argument('--min-duration-secs', type=int, default=60,
-                                         help='Proposed minimum duration of suspension')
-        suspend_uicc_parser.add_argument('--max-duration-secs', type=int, default=24*60*60,
-                                         help='Proposed maximum duration of suspension')
-
-        # not ISO7816-4 but TS 102 221
-        @cmd2.with_argparser(suspend_uicc_parser)
-        def do_suspend_uicc(self, opts):
-            """Perform the SUSPEND UICC command. Only supported on some UICC (check EF.UMPC)."""
-            (duration, token, sw) = self._cmd.card._scc.suspend_uicc(min_len_secs=opts.min_duration_secs,
-                                                                     max_len_secs=opts.max_duration_secs)
-            self._cmd.poutput(
-                'Negotiated Duration: %u secs, Token: %s, SW: %s' % (duration, token, sw))
-
-        resume_uicc_parser = argparse.ArgumentParser()
-        resume_uicc_parser.add_argument('TOKEN', type=str, help='Token provided during SUSPEND')
-
-        @cmd2.with_argparser(resume_uicc_parser)
-        def do_resume_uicc(self, opts):
-            """Perform the REUSME UICC operation. Only supported on some UICC. Also: A power-cycle
-            of the card is required between SUSPEND and RESUME, and only very few non-RESUME
-            commands are permitted between SUSPEND and RESUME.  See TS 102 221 Section 11.1.22."""
-            self._cmd.card._scc.resume_uicc(opts.TOKEN)
-
-        term_cap_parser = argparse.ArgumentParser()
-        # power group
-        tc_power_grp = term_cap_parser.add_argument_group('Terminal Power Supply')
-        tc_power_grp.add_argument('--used-supply-voltage-class', type=str, choices=['a','b','c','d','e'],
-                                  help='Actual used Supply voltage class')
-        tc_power_grp.add_argument('--maximum-available-power-supply', type=auto_uint8,
-                                  help='Maximum available power supply of the terminal')
-        tc_power_grp.add_argument('--actual-used-freq-100k', type=auto_uint8,
-                                  help='Actual used clock frequency (in units of 100kHz)')
-        # no separate groups for those two
-        tc_elc_grp = term_cap_parser.add_argument_group('Extended logical channels terminal support')
-        tc_elc_grp.add_argument('--extended-logical-channel', action='store_true',
-                                help='Extended Logical Channel supported')
-        tc_aif_grp = term_cap_parser.add_argument_group('Additional interfaces support')
-        tc_aif_grp.add_argument('--uicc-clf', action='store_true',
-                                help='Local User Interface in the Device (LUId) supported')
-        # eUICC group
-        tc_euicc_grp = term_cap_parser.add_argument_group('Additional Terminal capability indications related to eUICC')
-        tc_euicc_grp.add_argument('--lui-d', action='store_true',
-                                  help='Local User Interface in the Device (LUId) supported')
-        tc_euicc_grp.add_argument('--lpd-d', action='store_true',
-                                  help='Local Profile Download in the Device (LPDd) supported')
-        tc_euicc_grp.add_argument('--lds-d', action='store_true',
-                                  help='Local Discovery Service in the Device (LPDd) supported')
-        tc_euicc_grp.add_argument('--lui-e-scws', action='store_true',
-                                  help='LUIe based on SCWS supported')
-        tc_euicc_grp.add_argument('--metadata-update-alerting', action='store_true',
-                                  help='Metadata update alerting supported')
-        tc_euicc_grp.add_argument('--enterprise-capable-device', action='store_true',
-                                  help='Enterprise Capable Device')
-        tc_euicc_grp.add_argument('--lui-e-e4e', action='store_true',
-                                  help='LUIe using E4E (ENVELOPE tag E4) supported')
-        tc_euicc_grp.add_argument('--lpr', action='store_true',
-                                  help='LPR (LPA Proxy) supported')
-
-        @cmd2.with_argparser(term_cap_parser)
-        def do_terminal_capability(self, opts):
-            """Perform the TERMINAL CAPABILITY function. Used to inform the UICC about terminal capability."""
-            ps_flags = {}
-            addl_if_flags = {}
-            euicc_flags = {}
-
-            opts_dict = vars(opts)
-
-            power_items = ['used_supply_voltage_class', 'maximum_available_power_supply', 'actual_used_freq_100k']
-            if any(opts_dict[x] for x in power_items):
-                if not all(opts_dict[x] for x in power_items):
-                    raise argparse.ArgumentTypeError('If any of the Terminal Power Supply group options are used, all must be specified')
-
-            for k, v in opts_dict.items():
-                if k in AdditionalInterfacesSupport._construct.flags.keys():
-                    addl_if_flags[k] = v
-                elif k in AdditionalTermCapEuicc._construct.flags.keys():
-                    euicc_flags[k] = v
-                elif k in [f.name for f in TerminalPowerSupply._construct.subcons]:
-                    if k == 'used_supply_voltage_class' and v:
-                        v = {v: True}
-                    ps_flags[k] = v
-
-            child_list = []
-            if any(x for x in ps_flags.values()):
-                child_list.append(TerminalPowerSupply(decoded=ps_flags))
-
-            if opts.extended_logical_channel:
-                child_list.append(ExtendedLchanTerminalSupport())
-            if any(x for x in addl_if_flags.values()):
-                child_list.append(AdditionalInterfacesSupport(decoded=addl_if_flags))
-            if any(x for x in euicc_flags.values()):
-                child_list.append(AdditionalTermCapEuicc(decoded=euicc_flags))
-
-            print(child_list)
-            tc = TerminalCapability(children=child_list)
-            self.terminal_capability(b2h(tc.to_tlv()))
-
-        def terminal_capability(self, data:Hexstr):
-            cmd_hex = "80AA0000%02x%s" % (len(data)//2, data)
-            _rsp_hex, _sw = self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)

pySim/runtime.py

@@ -1,4 +1,5 @@
 # coding=utf-8
+
 """Representation of the runtime state of an application like pySim-shell.
 """
 
@@ -23,6 +24,9 @@ from osmocom.tlv import bertlv_parse_one
 
 from pySim.exceptions import *
 from pySim.filesystem import *
+from pySim.log import PySimLogger
+
+log = PySimLogger.get(__name__)
 
 def lchan_nr_from_cla(cla: int) -> int:
     """Resolve the logical channel number from the CLA byte."""
@@ -44,6 +48,7 @@ class RuntimeState:
             card : pysim.cards.Card instance
             profile : CardProfile instance
         """
+
         self.mf = CardMF(profile=profile)
         self.card = card
         self.profile = profile
@@ -53,16 +58,20 @@ class RuntimeState:
         # this is a dict of card identities which different parts of the code might populate,
         # typically with something like ICCID, EID, ATR, ...
         self.identity = {}
+        self.adm_verified = False
 
         # make sure the class and selection control bytes, which are specified
         # by the card profile are used
         self.card.set_apdu_parameter(
             cla=self.profile.cla, sel_ctrl=self.profile.sel_ctrl)
 
+        # make sure MF is selected before probing for Addons
+        self.lchan[0].select('MF')
+
         for addon_cls in self.profile.addons:
             addon = addon_cls()
             if addon.probe(self.card):
-                print("Detected %s Add-on \"%s\"" % (self.profile, addon))
+                log.info("Detected %s Add-on \"%s\"" % (self.profile, addon))
                 for f in addon.files_in_mf:
                     self.mf.add_file(f)
 
@@ -96,18 +105,18 @@ class RuntimeState:
         apps_taken = []
         if aids_card:
             aids_taken = []
-            print("AIDs on card:")
+            log.info("AIDs on card:")
             for a in aids_card:
                 for f in apps_profile:
                     if f.aid in a:
-                        print(" %s: %s (EF.DIR)" % (f.name, a))
+                        log.info(" %s: %s (EF.DIR)" % (f.name, a))
                         aids_taken.append(a)
                         apps_taken.append(f)
             aids_unknown = set(aids_card) - set(aids_taken)
             for a in aids_unknown:
-                print(" unknown: %s (EF.DIR)" % a)
+                log.info(" unknown: %s (EF.DIR)" % a)
         else:
-            print("warning: EF.DIR seems to be empty!")
+            log.warning("EF.DIR seems to be empty!")
 
         # Some card applications may not be registered in EF.DIR, we will actively
         # probe for those applications
@@ -122,7 +131,7 @@ class RuntimeState:
                 _data, sw = self.card.select_adf_by_aid(f.aid)
                 self.selected_adf = f
                 if sw == "9000":
-                    print(" %s: %s" % (f.name, f.aid))
+                    log.info(" %s: %s" % (f.name, f.aid))
                     apps_taken.append(f)
             except (SwMatchError, ProtocolError):
                 pass
@@ -139,13 +148,14 @@ class RuntimeState:
             if lchan_nr == 0:
                 continue
             del self.lchan[lchan_nr]
-        atr = i2h(self.card.reset())
+        self.adm_verified = False
+        atr = self.card.reset()
         if cmd_app:
             cmd_app.lchan = self.lchan[0]
         # select MF to reset internal state and to verify card really works
         self.lchan[0].select('MF', cmd_app)
         self.lchan[0].selected_adf = None
-        # store ATR as part of our card identies dict
+        # store ATR as part of our card identities dict
         self.identity['ATR'] = atr
         return atr
 
@@ -319,7 +329,7 @@ class RuntimeLchan:
             # If we succeed, we know that the file exists on the card and we may
             # proceed with creating a new CardEF object in the local file model at
             # run time. In case the file does not exist on the card, we just abort.
-            # The state on the card (selected file/application) wont't be changed,
+            # The state on the card (selected file/application) won't be changed,
             # so we do not have to update any state in that case.
             (data, _sw) = self.scc.select_file(fid)
         except SwMatchError as swm:
@@ -468,11 +478,15 @@ class RuntimeLchan:
 
     def get_file_for_filename(self, name: str):
         """Get the related CardFile object for a specified filename."""
+        if is_hex(name):
+            name = name.lower()
         sels = self.selected_file.get_selectables()
         return sels[name]
 
     def activate_file(self, name: str):
         """Request ACTIVATE FILE of specified file."""
+        if is_hex(name):
+            name = name.lower()
         sels = self.selected_file.get_selectables()
         f = sels[name]
         data, sw = self.scc.activate_file(f.fid)
@@ -505,6 +519,47 @@ class RuntimeLchan:
         dec_data = self.selected_file.decode_hex(data)
         return (dec_data, sw)
 
+    def __get_writeable_size(self):
+        """ Determine the writable size (file or record) using the cached FCP parameters of the currently selected
+            file. Return None in case the writeable size cannot be determined (no FCP available, FCP lacks size
+            information).
+        """
+        fcp = self.selected_file_fcp
+        if not fcp:
+            return None
+
+        structure = fcp.get('file_descriptor', {}).get('file_descriptor_byte', {}).get('structure')
+        if not structure:
+            return None
+
+        if structure == 'transparent':
+            return fcp.get('file_size')
+        elif structure == 'linear_fixed':
+            return fcp.get('file_descriptor', {}).get('record_len')
+        else:
+            return None
+
+    def __check_writeable_size(self, data_len):
+        """ Guard against unsuccessful writes caused by attempts to write data that exceeds the file limits. """
+
+        writeable_size = self.__get_writeable_size()
+        if not writeable_size:
+            return
+
+        if isinstance(self.selected_file, TransparentEF):
+            writeable_name = "file"
+        elif isinstance(self.selected_file, LinFixedEF):
+            writeable_name = "record"
+        else:
+            writeable_name = "object"
+
+        if data_len > writeable_size:
+            raise TypeError("Data length (%u) exceeds %s size (%u) by %u bytes" %
+                            (data_len, writeable_name, writeable_size, data_len - writeable_size))
+        elif data_len < writeable_size:
+            log.warning("Data length (%u) less than %s size (%u), leaving %u unwritten bytes at the end of the %s" %
+                        (data_len, writeable_name, writeable_size, writeable_size - data_len, writeable_name))
+
     def update_binary(self, data_hex: str, offset: int = 0):
         """Update transparent EF binary data.
 
@@ -515,6 +570,7 @@ class RuntimeLchan:
         if not isinstance(self.selected_file, TransparentEF):
             raise TypeError("Only works with TransparentEF, but %s is %s" % (self.selected_file,
                                                                              self.selected_file.__class__.__mro__))
+        self.__check_writeable_size(len(data_hex) // 2 + offset)
         return self.scc.update_binary(self.selected_file.fid, data_hex, offset, conserve=self.rs.conserve_write)
 
     def update_binary_dec(self, data: dict):
@@ -562,6 +618,7 @@ class RuntimeLchan:
         if not isinstance(self.selected_file, LinFixedEF):
             raise TypeError("Only works with Linear Fixed EF, but %s is %s" % (self.selected_file,
                                                                                self.selected_file.__class__.__mro__))
+        self.__check_writeable_size(len(data_hex) // 2)
         return self.scc.update_record(self.selected_file.fid, rec_nr, data_hex,
 					       conserve=self.rs.conserve_write,
 					       leftpad=self.selected_file.leftpad)

pySim/secure_channel.py

@@ -32,7 +32,7 @@ class SecureChannel(abc.ABC):
         pass
 
     def send_apdu_wrapper(self, send_fn: callable, pdu: Hexstr, *args, **kwargs) -> ResTuple:
-        """Wrapper function to wrap command APDU and unwrap repsonse APDU around send_apdu callable."""
+        """Wrapper function to wrap command APDU and unwrap response APDU around send_apdu callable."""
         pdu_wrapped = b2h(self.wrap_cmd_apdu(h2b(pdu)))
         res, sw = send_fn(pdu_wrapped, *args, **kwargs)
         res_unwrapped = b2h(self.unwrap_rsp_apdu(h2b(sw), h2b(res)))

pySim/sms.py

@@ -20,10 +20,10 @@
 import typing
 import abc
 from bidict import bidict
-from construct import Int8ub, Byte, Bytes, Bit, Flag, BitsInteger
+from construct import Int8ub, Byte, Bit, Flag, BitsInteger
 from construct import Struct, Enum, Tell, BitStruct, this, Padding
-from construct import Prefixed, GreedyRange, GreedyBytes
-from osmocom.construct import HexAdapter, BcdAdapter, TonNpi
+from construct import Prefixed, GreedyRange
+from osmocom.construct import BcdAdapter, TonNpi, Bytes, GreedyBytes
 from osmocom.utils import Hexstr, h2b, b2h
 
 from smpp.pdu import pdu_types, operations
@@ -253,6 +253,49 @@ class SMS_DELIVER(SMS_TPDU):
             }
         return cls(**d)
 
+    @classmethod
+    def from_submit(cls, submit: 'SMS_SUBMIT') -> 'SMS_DELIVER':
+        """Construct a SMS_DELIVER instance from a SMS_SUBMIT instance."""
+        d = {
+            # common fields (SMS_TPDU base class) which exist in submit, so we can copy them
+            'tp_mti': submit.tp_mti,
+            'tp_rp': submit.tp_rp,
+            'tp_udhi': submit.tp_udhi,
+            'tp_pid': submit.tp_pid,
+            'tp_dcs': submit.tp_dcs,
+            'tp_udl': submit.tp_udl,
+            'tp_ud': submit.tp_ud,
+            # SMS_DELIVER specific fields
+            'tp_lp': False,
+            'tp_mms': False,
+            'tp_oa': None,
+            'tp_scts': h2b('22705200000000'), # FIXME
+            'tp_sri': False,
+            }
+        return cls(**d)
+
+    def to_smpp(self) -> pdu_types.PDU:
+        """Translate a SMS_DELIVER instance to a smpp.pdu.operations.DeliverSM instance."""
+        # we only deal with binary SMS here:
+        if self.tp_dcs != 0xF6:
+            raise ValueError('Unsupported DCS: We only support DCS=0xF6 for now')
+        dcs = pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED)
+        esm_class = pdu_types.EsmClass(pdu_types.EsmClassMode.DEFAULT, pdu_types.EsmClassType.DEFAULT,
+                                       gsmFeatures=[pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET])
+        if self.tp_oa:
+            oa_digits, oa_ton, oa_npi = self.tp_oa.to_smpp()
+        else:
+            oa_digits, oa_ton, oa_npi = None, None, None
+        return operations.DeliverSM(source_addr=oa_digits,
+                                    source_addr_ton=oa_ton,
+                                    source_addr_npi=oa_npi,
+                                    #destination_addr=ESME_MSISDN,
+                                    esm_class=esm_class,
+                                    protocol_id=self.tp_pid,
+                                    data_coding=dcs,
+                                    short_message=self.tp_ud)
+
 
 
 class SMS_SUBMIT(SMS_TPDU):

pySim/sysmocom_sja2.py

@@ -18,14 +18,13 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
 from struct import unpack
-from construct import FlagsEnum, Byte, Struct, Int8ub, Bytes, Mapping, Enum, Padding, BitsInteger
+from construct import FlagsEnum, Byte, Struct, Int8ub, Mapping, Enum, Padding, BitsInteger
 from construct import Bit, this, Int32ub, Int16ub, Nibble, BytesInteger, GreedyRange, Const
 from construct import Optional as COptional
 from osmocom.utils import *
 from osmocom.construct import *
 
 from pySim.filesystem import *
-from pySim.profile.ts_102_221 import CardProfileUICC
 from pySim.runtime import RuntimeState
 import pySim
 
@@ -52,13 +51,13 @@ class EF_PIN(TransparentEF):
         ( 'f1030331323334ffffffff0a0a3132333435363738',
           { 'state': { 'valid': True, 'change_able': True, 'unblock_able': True, 'disable_able': True,
                        'not_initialized': False, 'disabled': True },
-           'attempts_remaining': 3, 'maximum_attempts': 3, 'pin': '31323334',
-           'puk': { 'attempts_remaining': 10, 'maximum_attempts': 10, 'puk': '3132333435363738' }
+           'attempts_remaining': 3, 'maximum_attempts': 3, 'pin': b'1234',
+           'puk': { 'attempts_remaining': 10, 'maximum_attempts': 10, 'puk': b'12345678' }
           } ),
         ( 'f003039999999999999999',
           { 'state': { 'valid': True, 'change_able': True, 'unblock_able': True, 'disable_able': True,
                        'not_initialized': False, 'disabled': False },
-           'attempts_remaining': 3, 'maximum_attempts': 3, 'pin': '9999999999999999',
+           'attempts_remaining': 3, 'maximum_attempts': 3, 'pin': h2b('9999999999999999'),
            'puk': None } ),
     ]
     def __init__(self, fid='6f01', name='EF.CHV1'):
@@ -67,29 +66,32 @@ class EF_PIN(TransparentEF):
                                     change_able=0x40, valid=0x80)
         PukStruct = Struct('attempts_remaining'/Int8ub,
                            'maximum_attempts'/Int8ub,
-                           'puk'/HexAdapter(Rpad(Bytes(8))))
+                           'puk'/Rpad(Bytes(8)))
         self._construct = Struct('state'/StateByte,
                                  'attempts_remaining'/Int8ub,
                                  'maximum_attempts'/Int8ub,
-                                 'pin'/HexAdapter(Rpad(Bytes(8))),
+                                 'pin'/Rpad(Bytes(8)),
                                  'puk'/COptional(PukStruct))
 
 
 class EF_MILENAGE_CFG(TransparentEF):
     _test_de_encode = [
         ( '40002040600000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000020000000000000000000000000000000400000000000000000000000000000008',
-          {"r1": 64, "r2": 0, "r3": 32, "r4": 64, "r5": 96, "c1": "00000000000000000000000000000000", "c2":
-           "00000000000000000000000000000001", "c3": "00000000000000000000000000000002", "c4":
-           "00000000000000000000000000000004", "c5": "00000000000000000000000000000008"} ),
+          {"r1": 64, "r2": 0, "r3": 32, "r4": 64, "r5": 96,
+           "c1": h2b("00000000000000000000000000000000"),
+           "c2": h2b("00000000000000000000000000000001"),
+           "c3": h2b("00000000000000000000000000000002"),
+           "c4": h2b("00000000000000000000000000000004"),
+           "c5": h2b("00000000000000000000000000000008")} ),
       ]
     def __init__(self, fid='6f21', name='EF.MILENAGE_CFG', desc='Milenage connfiguration'):
         super().__init__(fid, name=name, desc=desc)
         self._construct = Struct('r1'/Int8ub, 'r2'/Int8ub, 'r3'/Int8ub, 'r4'/Int8ub, 'r5'/Int8ub,
-                                 'c1'/HexAdapter(Bytes(16)),
-                                 'c2'/HexAdapter(Bytes(16)),
-                                 'c3'/HexAdapter(Bytes(16)),
-                                 'c4'/HexAdapter(Bytes(16)),
-                                 'c5'/HexAdapter(Bytes(16)))
+                                 'c1'/Bytes(16),
+                                 'c2'/Bytes(16),
+                                 'c3'/Bytes(16),
+                                 'c4'/Bytes(16),
+                                 'c5'/Bytes(16))
 
 
 class EF_0348_KEY(LinFixedEF):
@@ -103,18 +105,18 @@ class EF_0348_KEY(LinFixedEF):
         self._construct = Struct('security_domain'/Int8ub,
                                  'key_set_version'/Int8ub,
                                  'key_len_and_type'/KeyLenAndType,
-                                 'key'/HexAdapter(Bytes(this.key_len_and_type.key_length)))
+                                 'key'/Bytes(this.key_len_and_type.key_length))
 
 
 class EF_0348_COUNT(LinFixedEF):
     _test_de_encode = [
-        ( 'fe010000000000', {"sec_domain": 254, "key_set_version": 1, "counter": "0000000000"} ),
+        ( 'fe010000000000', {"sec_domain": 254, "key_set_version": 1, "counter": h2b("0000000000")} ),
       ]
     def __init__(self, fid='6f23', name='EF.0348_COUNT', desc='TS 03.48 OTA Counters'):
         super().__init__(fid, name=name, desc=desc, rec_len=(7, 7))
         self._construct = Struct('sec_domain'/Int8ub,
                                  'key_set_version'/Int8ub,
-                                 'counter'/HexAdapter(Bytes(5)))
+                                 'counter'/Bytes(5))
 
 
 class EF_SIM_AUTH_COUNTER(TransparentEF):
@@ -146,8 +148,9 @@ class EF_GP_DIV_DATA(LinFixedEF):
 class EF_SIM_AUTH_KEY(TransparentEF):
     _test_de_encode = [
         ( '14000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f',
-          {"cfg": {"sres_deriv_func": 1, "use_opc_instead_of_op": True, "algorithm": "milenage"}, "key":
-           "000102030405060708090a0b0c0d0e0f", "op_opc": "101112131415161718191a1b1c1d1e1f"} ),
+          {"cfg": {"sres_deriv_func": 1, "use_opc_instead_of_op": True, "algorithm": "milenage"},
+           "key": h2b("000102030405060708090a0b0c0d0e0f"),
+           "op_opc": h2b("101112131415161718191a1b1c1d1e1f")} ),
       ]
     def __init__(self, fid='6f20', name='EF.SIM_AUTH_KEY'):
         super().__init__(fid, name=name, desc='USIM authentication key')
@@ -156,8 +159,8 @@ class EF_SIM_AUTH_KEY(TransparentEF):
                             'use_opc_instead_of_op'/Flag,
                             'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/HexAdapter(Bytes(16)),
-                                 'op_opc' /HexAdapter(Bytes(16)))
+                                 'key'/Bytes(16),
+                                 'op_opc' /Bytes(16))
 
 
 class DF_SYSTEM(CardDF):
@@ -181,7 +184,7 @@ class DF_SYSTEM(CardDF):
         self.add_files(files)
 
     def decode_select_response(self, resp_hex):
-        return CardProfileUICC.decode_select_response(resp_hex)
+        return pySim.ts_102_221.CardProfileUICC.decode_select_response(resp_hex)
 
 
 class EF_USIM_SQN(TransparentEF):
@@ -210,13 +213,13 @@ class EF_USIM_AUTH_KEY(TransparentEF):
     _test_de_encode = [
         ( '141898d827f70120d33b3e7462ee5fd6fe6ca53d7a0a804561646816d7b0c702fb',
           { "cfg": { "only_4bytes_res_in_3g": False, "sres_deriv_func_in_2g": 1, "use_opc_instead_of_op": True, "algorithm": "milenage" },
-            "key": "1898d827f70120d33b3e7462ee5fd6fe", "op_opc": "6ca53d7a0a804561646816d7b0c702fb" } ),
+            "key": h2b("1898d827f70120d33b3e7462ee5fd6fe"), "op_opc": h2b("6ca53d7a0a804561646816d7b0c702fb") } ),
         ( '160a04101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f000102030405060708090a0b0c0d0e0f',
          { "cfg" : { "algorithm" : "tuak", "key_length" : 128, "sres_deriv_func_in_2g" : 1, "use_opc_instead_of_op" : True },
            "tuak_cfg" : { "ck_and_ik_size" : 128, "mac_size" : 128, "res_size" : 128 },
            "num_of_keccak_iterations" : 4,
-           "k" : "000102030405060708090a0b0c0d0e0f",
-           "op_opc" : "101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f"
+           "k" : h2b("000102030405060708090a0b0c0d0e0f"),
+           "op_opc" : h2b("101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f")
          } ),
       ]
     def __init__(self, fid='af20', name='EF.USIM_AUTH_KEY'):
@@ -227,8 +230,8 @@ class EF_USIM_AUTH_KEY(TransparentEF):
                             'use_opc_instead_of_op'/Mapping(Bit, {False:0, True:1}),
                             'algorithm'/Algorithm)
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/HexAdapter(Bytes(16)),
-                                 'op_opc' /HexAdapter(Bytes(16)))
+                                 'key'/Bytes(16),
+                                 'op_opc'/Bytes(16))
         # TUAK has a rather different layout for the data, so we define a different
         # construct below and use explicit _{decode,encode}_bin() methods for separating
         # the TUAK and non-TUAK situation
@@ -244,8 +247,8 @@ class EF_USIM_AUTH_KEY(TransparentEF):
         self._constr_tuak = Struct('cfg'/CfgByteTuak,
                                    'tuak_cfg'/TuakCfgByte,
                                    'num_of_keccak_iterations'/Int8ub,
-                                   'op_opc'/HexAdapter(Bytes(32)),
-                                   'k'/HexAdapter(Bytes(this.cfg.key_length//8)))
+                                   'op_opc'/Bytes(32),
+                                   'k'/Bytes(this.cfg.key_length//8))
 
     def _decode_bin(self, raw_bin_data: bytearray) -> dict:
         if raw_bin_data[0] & 0x0F == 0x06:
@@ -264,8 +267,9 @@ class EF_USIM_AUTH_KEY_2G(TransparentEF):
     _test_de_encode = [
         ( '14000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f',
           {"cfg": {"only_4bytes_res_in_3g": False, "sres_deriv_func_in_2g": 1, "use_opc_instead_of_op": True,
-                   "algorithm": "milenage"}, "key": "000102030405060708090a0b0c0d0e0f", "op_opc":
-           "101112131415161718191a1b1c1d1e1f"} ),
+                   "algorithm": "milenage"},
+           "key": h2b("000102030405060708090a0b0c0d0e0f"),
+           "op_opc": h2b("101112131415161718191a1b1c1d1e1f")} ),
       ]
     def __init__(self, fid='af22', name='EF.USIM_AUTH_KEY_2G'):
         super().__init__(fid, name=name, desc='USIM authentication key in 2G context')
@@ -274,8 +278,8 @@ class EF_USIM_AUTH_KEY_2G(TransparentEF):
                             'use_opc_instead_of_op'/Flag,
                             'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3, xor=14))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/HexAdapter(Bytes(16)),
-                                 'op_opc' /HexAdapter(Bytes(16)))
+                                 'key'/Bytes(16),
+                                 'op_opc'/Bytes(16))
 
 
 class EF_GBA_SK(TransparentEF):
@@ -299,9 +303,9 @@ class EF_GBA_INT_KEY(LinFixedEF):
 
 
 class SysmocomSJA2(CardModel):
-    _atrs = ["3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 30 34 05 4B A9",
-             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 31 33 02 51 B2",
-             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 52 75 31 04 51 D5"]
+    _atrs = ["3b9f96801f878031e073fe211b674a4c753034054ba9",
+             "3b9f96801f878031e073fe211b674a4c7531330251b2",
+             "3b9f96801f878031e073fe211b674a4c5275310451d5"]
 
     @classmethod
     def add_files(cls, rs: RuntimeState):
@@ -330,9 +334,9 @@ class SysmocomSJA2(CardModel):
             isim_adf.add_files(files_adf_isim)
 
 class SysmocomSJA5(CardModel):
-    _atrs = ["3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 51 CC",
-             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 65 F8",
-             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 59 C4"]
+    _atrs = ["3b9f96801f878031e073fe211b674a357530350251cc",
+             "3b9f96801f878031e073fe211b674a357530350265f8",
+             "3b9f96801f878031e073fe211b674a357530350259c4"]
 
     @classmethod
     def add_files(cls, rs: RuntimeState):

pySim/transport/__init__.py

@@ -3,18 +3,6 @@
 """ pySim: PCSC reader transport link base
 """
 
-import os
-import abc
-import argparse
-from typing import Optional, Tuple
-from construct import Construct
-from osmocom.utils import b2h, h2b, i2h, Hexstr
-
-from pySim.exceptions import *
-from pySim.utils import SwHexstr, SwMatchstr, ResTuple, sw_match, parse_command_apdu
-from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
-
-#
 # Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
 # Copyright (C) 2021-2023 Harald Welte <laforge@osmocom.org>
 #
@@ -30,8 +18,20 @@ from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
 
+import os
+import abc
+import argparse
+from typing import Optional, Tuple
+from construct import Construct
+from osmocom.utils import b2h, h2b, i2h, Hexstr
+
+from pySim.exceptions import *
+from pySim.utils import SwHexstr, SwMatchstr, ResTuple, sw_match, parse_command_apdu
+from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
+from pySim.log import PySimLogger
+
+log = PySimLogger.get(__name__)
 
 class ApduTracer:
     def trace_command(self, cmd):
@@ -46,11 +46,11 @@ class ApduTracer:
 class StdoutApduTracer(ApduTracer):
     """Minimalistic APDU tracer, printing commands to stdout."""
     def trace_response(self, cmd, sw, resp):
-        print("-> %s %s" % (cmd[:10], cmd[10:]))
-        print("<- %s: %s" % (sw, resp))
+        log.info("-> %s %s", cmd[:10], cmd[10:])
+        log.info("<- %s: %s", sw, resp)
 
     def trace_reset(self):
-        print("-- RESET")
+        log.info("-- RESET")
 
 class ProactiveHandler(abc.ABC):
     """Abstract base class representing the interface of some code that handles
@@ -119,6 +119,11 @@ class LinkBase(abc.ABC):
         """Connect to a card immediately
         """
 
+    @abc.abstractmethod
+    def get_atr(self) -> Hexstr:
+        """Retrieve card ATR
+        """
+
     @abc.abstractmethod
     def disconnect(self):
         """Disconnect from card
@@ -159,8 +164,8 @@ class LinkBase(abc.ABC):
         if self.apdu_tracer:
             self.apdu_tracer.trace_response(apdu, sw, data)
 
-        # The APDU case (See aso ISO/IEC 7816-3, table 12) dictates if we should receive a response or not. If we
-        # receive a response in an APDU case that does not allow the reception of a respnse we print a warning to
+        # The APDU case (See also ISO/IEC 7816-3, table 12) dictates if we should receive a response or not. If we
+        # receive a response in an APDU case that does not allow the reception of a response we print a warning to
         # make the user/caller aware of the problem. Since the transaction is over at this point and data was received
         # we count it as a successful transaction anyway, even though the spec was violated. The problem is most likely
         # caused by a missing Le field in the APDU. This is an error that the caller/user should correct to avoid
@@ -172,7 +177,7 @@ class LinkBase(abc.ABC):
             if self.apdu_strict:
                 raise ValueError(exeption_str)
             else:
-                print('Warning: %s' % exeption_str)
+                log.warning(exeption_str)
 
         return (data, sw)
 
@@ -195,7 +200,7 @@ class LinkBase(abc.ABC):
             # It *was* successful after all -- the extra pieces FETCH handled
             # need not concern the caller.
             rv = (rv[0], '9000')
-            # proactive sim as per TS 102 221 Setion 7.4.2
+            # proactive sim as per TS 102 221 Section 7.4.2
             # TODO: Check SW manually to avoid recursing on the stack (provided this piece of code stays in this place)
             fetch_rv = self.send_apdu_checksw('80120000' + last_sw[2:], sw)
             # Setting this in case we later decide not to send a terminal
@@ -206,7 +211,7 @@ class LinkBase(abc.ABC):
             # parse the proactive command
             pcmd = ProactiveCommand()
             parsed = pcmd.from_tlv(h2b(fetch_rv[0]))
-            print("FETCH: %s (%s)" % (fetch_rv[0], type(parsed).__name__))
+            log.info("FETCH: %s (%s)", fetch_rv[0], type(parsed).__name__)
             if self.proactive_handler:
                 # Extension point: If this does return a list of TLV objects,
                 # they could be appended after the Result; if the first is a
@@ -223,7 +228,7 @@ class LinkBase(abc.ABC):
             # Structure as per TS 102 223 V4.4.0 Section 6.8
 
             # Testing hint: The value of tail does not influence the behavior
-            # of an SJA2 that sent ans SMS, so this is implemented only
+            # of an SJA2 that sent an SMS, so this is implemented only
             # following TS 102 223, and not fully tested.
             ti_list_bin = [x.to_tlv() for x in ti_list]
             tail = b''.join(ti_list_bin)
@@ -328,13 +333,11 @@ def argparse_add_reader_args(arg_parser: argparse.ArgumentParser):
     from pySim.transport.pcsc import PcscSimLink
     from pySim.transport.modem_atcmd import ModemATCommandLink
     from pySim.transport.calypso import CalypsoSimLink
-    from pySim.transport.wsrc import WsrcSimLink
 
     SerialSimLink.argparse_add_reader_args(arg_parser)
     PcscSimLink.argparse_add_reader_args(arg_parser)
     ModemATCommandLink.argparse_add_reader_args(arg_parser)
     CalypsoSimLink.argparse_add_reader_args(arg_parser)
-    WsrcSimLink.argparse_add_reader_args(arg_parser)
     arg_parser.add_argument('--apdu-trace', action='store_true',
                             help='Trace the command/response APDUs exchanged with the card')
 
@@ -357,17 +360,14 @@ def init_reader(opts, **kwargs) -> LinkBase:
     elif opts.modem_dev is not None:
         from pySim.transport.modem_atcmd import ModemATCommandLink
         sl = ModemATCommandLink(opts, **kwargs)
-    elif opts.wsrc_server_url is not None:
-        from pySim.transport.wsrc import WsrcSimLink
-        sl = WsrcSimLink(opts, **kwargs)
     else:  # Serial reader is default
-        print("No reader/driver specified; falling back to default (Serial reader)")
+        log.warning("No reader/driver specified; falling back to default (Serial reader)")
         from pySim.transport.serial import SerialSimLink
         sl = SerialSimLink(opts, **kwargs)
 
     if os.environ.get('PYSIM_INTEGRATION_TEST') == "1":
-        print("Using %s reader interface" % (sl.name))
+        log.info("Using %s reader interface" % (sl.name))
     else:
-        print("Using reader %s" % sl)
+        log.info("Using reader %s" % sl)
 
     return sl

pySim/transport/calypso.py

@@ -123,6 +123,9 @@ class CalypsoSimLink(LinkBaseTpdu):
     def connect(self):
         self.reset_card()
 
+    def get_atr(self) -> Hexstr:
+        return "3b00" # Dummy ATR
+
     def disconnect(self):
         pass  # Nothing to do really ...
 

pySim/transport/modem_atcmd.py

@@ -139,6 +139,9 @@ class ModemATCommandLink(LinkBaseTpdu):
     def connect(self):
         pass  # Nothing to do really ...
 
+    def get_atr(self) -> Hexstr:
+        return "3b00" # Dummy ATR
+
     def disconnect(self):
         pass  # Nothing to do really ...
 
@@ -163,7 +166,7 @@ class ModemATCommandLink(LinkBaseTpdu):
 
         # Make sure that the response has format: b'+CSIM: %d,\"%s\"'
         try:
-            result = re.match(b'\+CSIM: (\d+),\"([0-9A-F]+)\"', rsp)
+            result = re.match(rb'\+CSIM: (\d+),\"([0-9A-F]+)\"', rsp)
             (_rsp_tpdu_len, rsp_tpdu) = result.groups()
         except Exception as exc:
             raise ReaderError('Failed to parse response from modem: %s' % rsp) from exc

pySim/transport/pcsc.py

@@ -103,7 +103,7 @@ class PcscSimLink(LinkBaseTpdu):
             raise NoCardError() from exc
 
     def get_atr(self) -> Hexstr:
-        return self._con.getATR()
+        return i2h(self._con.getATR())
 
     def disconnect(self):
         self._con.disconnect()

pySim/transport/serial.py

@@ -21,7 +21,7 @@ import os
 import argparse
 from typing import Optional
 import serial
-from osmocom.utils import h2b, b2h, Hexstr
+from osmocom.utils import h2b, b2h, i2h, Hexstr
 
 from pySim.exceptions import NoCardError, ProtocolError
 from pySim.transport import LinkBaseTpdu
@@ -96,7 +96,7 @@ class SerialSimLink(LinkBaseTpdu):
         self.reset_card()
 
     def get_atr(self) -> Hexstr:
-        return self._atr
+        return i2h(self._atr)
 
     def disconnect(self):
         pass  # Nothing to do really ...

pySim/transport/wsrc.py

@@ -1,99 +0,0 @@
-# Copyright (C) 2024  Harald Welte <laforge@gnumonks.org>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import argparse
-from typing import Optional
-
-from osmocom.utils import h2i, i2h, Hexstr, is_hexstr
-
-from pySim.exceptions import NoCardError, ProtocolError, ReaderError
-from pySim.transport import LinkBase
-from pySim.utils import ResTuple
-from pySim.wsrc import WSRC_DEFAULT_PORT_USER
-from pySim.wsrc.client_blocking import WsClientBlocking
-
-class UserWsClientBlocking(WsClientBlocking):
-    def __init__(self, ws_uri: str, **kwargs):
-        super().__init__('user', ws_uri, **kwargs)
-
-    def select_card(self, id_type:str, id_str:str):
-        rx = self.transceive_json('select_card', {'id_type': id_type, 'id_str': id_str},
-                                  'select_card_ack')
-        return rx
-
-    def reset_card(self):
-        self.transceive_json('reset_req', {}, 'reset_resp')
-
-    def xceive_apdu_raw(self, cmd: Hexstr) -> ResTuple:
-        rx = self.transceive_json('c_apdu', {'command': cmd}, 'r_apdu')
-        return rx['response'], rx['sw']
-
-
-class WsrcSimLink(LinkBase):
-    """ pySim: WSRC (WebSocket Remote Card) reader transport link."""
-    name = 'WSRC'
-
-    def __init__(self, opts: argparse.Namespace, **kwargs):
-        super().__init__(**kwargs)
-        self.identities = {}
-        self.server_url = opts.wsrc_server_url
-        if opts.wsrc_eid:
-            self.id_type = 'eid'
-            self.id_str = opts.wsrc_eid
-        elif opts.wsrc_iccid:
-            self.id_type = 'iccid'
-            self.id_str = opts.wsrc_iccid
-        self.client = UserWsClientBlocking(self.server_url)
-        self.client.connect()
-
-    def __del__(self):
-       # FIXME: disconnect from server
-       pass
-
-    def wait_for_card(self, timeout: Optional[int] = None, newcardonly: bool = False):
-        self.connect()
-
-    def connect(self):
-        rx = self.client.select_card(self.id_type, self.id_str)
-        self.identities = rx['identities']
-
-    def get_atr(self) -> Hexstr:
-        return h2i(self.identities['ATR'])
-
-    def disconnect(self):
-        self.__delete__()
-
-    def _reset_card(self):
-        self.client.reset_card()
-        return 1
-
-    def _send_apdu_raw(self, pdu: Hexstr) -> ResTuple:
-        return self.client.xceive_apdu_raw(pdu)
-
-    def __str__(self) -> str:
-        return "WSRC[%s=%s]" % (self.id_type, self.id_str)
-
-    @staticmethod
-    def argparse_add_reader_args(arg_parser: argparse.ArgumentParser):
-        wsrc_group = arg_parser.add_argument_group('WebSocket Remote Card',
-            """WebSocket Remote Card (WSRC) is a protoocl by which remot cards / card readers
-            can be accessed via a network.""")
-        wsrc_group.add_argument('--wsrc-server-url', default='ws://localhost:%u' % WSRC_DEFAULT_PORT_USER,
-                                help='URI of the WSRC server to connect to')
-        wsrc_group.add_argument('--wsrc-iccid', type=is_hexstr,
-                                help='ICCID of the card to open via WSRC')
-        wsrc_group.add_argument('--wsrc-eid', type=is_hexstr,
-                                help='EID of the card to open via WSRC')

pySim/ts_102_221.py

@@ -18,16 +18,22 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 from bidict import bidict
 
-from construct import Select, Const, Bit, Struct, Int16ub, FlagsEnum, ValidationError
+from construct import Select, Const, Bit, Struct, Int16ub, FlagsEnum, GreedyString, ValidationError
 from construct import Optional as COptional, Computed
 
 from osmocom.construct import *
 from osmocom.utils import *
-from osmocom.tlv import BER_TLV_IE, flatten_dict_lists
+from osmocom.tlv import *
 from pySim.utils import *
+from pySim.filesystem import *
+from pySim.profile import CardProfile
+from pySim import iso7816_4
 
-#from pySim.filesystem import *
-#from pySim.profile import CardProfile
+# A UICC will usually also support 2G functionality. If this is the case, we
+# need to add DF_GSM and DF_TELECOM along with the UICC related files
+from pySim.ts_51_011 import AddonSIM, EF_ICCID, EF_PL
+from pySim.gsm_r import AddonGSMR
+from pySim.cdma_ruim import AddonRUIM
 
 ts_102_22x_cmdset = CardCommandSet('TS 102 22x', [
     # TS 102 221 Section 10.1.2 Table 10.5 "Coding of Instruction Byte"
@@ -113,11 +119,11 @@ class FileDescriptor(BER_TLV_IE, tag=0x82):
 
 # ETSI TS 102 221 11.1.1.4.4
 class FileIdentifier(BER_TLV_IE, tag=0x83):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # ETSI TS 102 221 11.1.1.4.5
 class DfName(BER_TLV_IE, tag=0x84):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # ETSI TS 102 221 11.1.1.4.6.1
 class UiccCharacteristics(BER_TLV_IE, tag=0x80):
@@ -211,7 +217,7 @@ class SecurityAttribExpanded(BER_TLV_IE, tag=0xab):
 # ETSI TS 102 221 11.1.1.4.7.3
 class SecurityAttribReferenced(BER_TLV_IE, tag=0x8b):
     # TODO: longer format with SEID
-    _construct = Struct('ef_arr_file_id'/HexAdapter(Bytes(2)), 'ef_arr_record_nr'/Int8ub)
+    _construct = Struct('ef_arr_file_id'/Bytes(2), 'ef_arr_record_nr'/Int8ub)
 
 # ETSI TS 102 221 11.1.1.4.8
 class ShortFileIdentifier(BER_TLV_IE, tag=0x88):
@@ -282,12 +288,6 @@ class FcpTemplate(BER_TLV_IE, tag=0x62, nested=[FileSize, TotalFileSize, FileDes
                                                 PinStatusTemplate_DO]):
     pass
 
-def decode_select_response(data_hex: str) -> object:
-    """ETSI TS 102 221 Section 11.1.1.3"""
-    t = FcpTemplate()
-    t.from_tlv(h2b(data_hex))
-    d = t.to_dict()
-    return flatten_dict_lists(d['fcp_template'])
 
 def tlv_key_replace(inmap, indata):
     def newkey(inmap, key):
@@ -634,3 +634,361 @@ NOT_DO = Nested_DO('not', 0xaf, NOT_Template)
 SC_DO = DataObjectChoice('security_condition', 'Security Condition',
                          members=[Always_DO, Never_DO, SecCondByte_DO(), SecCondByte_DO(0x9e), CRT_DO(),
                                   OR_DO, AND_DO, NOT_DO])
+
+# TS 102 221 Section 13.1
+class EF_DIR(LinFixedEF):
+    _test_de_encode = [
+        ( '61294f10a0000000871002ffffffff890709000050055553696d31730ea00c80011781025f608203454150',
+          { "application_template": [ { "application_id": h2b("a0000000871002ffffffff8907090000") },
+                                      { "application_label": "USim1" },
+                                      { "discretionary_template": h2b("a00c80011781025f608203454150") } ] }
+        ),
+        ( '61194f10a0000000871004ffffffff890709000050054953696d31',
+          { "application_template": [ { "application_id": h2b("a0000000871004ffffffff8907090000") },
+                                      { "application_label": "ISim1" } ] }
+        ),
+    ]
+    class ApplicationLabel(BER_TLV_IE, tag=0x50):
+        # TODO: UCS-2 coding option as per Annex A of TS 102 221
+        _construct = GreedyString('ascii')
+
+    # see https://github.com/PyCQA/pylint/issues/5794
+    #pylint: disable=undefined-variable
+    class ApplicationTemplate(BER_TLV_IE, tag=0x61,
+                              nested=[iso7816_4.ApplicationId, ApplicationLabel, iso7816_4.FileReference,
+                                      iso7816_4.CommandApdu, iso7816_4.DiscretionaryData,
+                                      iso7816_4.DiscretionaryTemplate, iso7816_4.URL,
+                                      iso7816_4.ApplicationRelatedDOSet]):
+        pass
+
+    def __init__(self, fid='2f00', sfid=0x1e, name='EF.DIR', desc='Application Directory'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(5, 54))
+        self._tlv = EF_DIR.ApplicationTemplate
+
+
+# TS 102 221 Section 13.4
+class EF_ARR(LinFixedEF):
+    _test_de_encode = [
+        ( '800101a40683010a950108800106900080016097008401d4a40683010a950108',
+         [ [ { "access_mode": [ "read_search_compare" ] },
+             { "control_reference_template": "ADM1" } ],
+           [ { "access_mode": [ "write_append", "update_erase" ] },
+             { "always": None } ],
+           [ { "access_mode": [ "delete_file", "terminate_ef" ] },
+             { "never": None } ],
+           [ { "command_header": { "INS": 212 } },
+             { "control_reference_template": "ADM1" } ]
+         ] ),
+        ( '80010190008001029700800118a40683010a9501088401d4a40683010a950108',
+         [ [ { "access_mode": [ "read_search_compare" ] },
+             { "always": None } ],
+           [ { "access_mode": [ "update_erase" ] },
+             { "never": None } ],
+           [ { "access_mode": [ "activate_file_or_record", "deactivate_file_or_record" ] },
+             { "control_reference_template": "ADM1" } ],
+           [ { "command_header": { "INS": 212 } },
+             { "control_reference_template": "ADM1" } ]
+         ] ),
+    ]
+    def __init__(self, fid='2f06', sfid=0x06, name='EF.ARR', desc='Access Rule Reference'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc)
+        # add those commands to the general commands of a TransparentEF
+        self.shell_commands += [self.AddlShellCommands()]
+
+    @staticmethod
+    def flatten(inp: list):
+        """Flatten the somewhat deep/complex/nested data returned from decoder."""
+        def sc_abbreviate(sc):
+            if 'always' in sc:
+                return 'always'
+            elif 'never' in sc:
+                return 'never'
+            elif 'control_reference_template' in sc:
+                return sc['control_reference_template']
+            else:
+                return sc
+
+        by_mode = {}
+        for t in inp:
+            am = t[0]
+            sc = t[1]
+            sc_abbr = sc_abbreviate(sc)
+            if 'access_mode' in am:
+                for m in am['access_mode']:
+                    by_mode[m] = sc_abbr
+            elif 'command_header' in am:
+                ins = am['command_header']['INS']
+                if 'CLA' in am['command_header']:
+                    cla = am['command_header']['CLA']
+                else:
+                    cla = None
+                cmd = ts_102_22x_cmdset.lookup(ins, cla)
+                if cmd:
+                    name = cmd.name.lower().replace(' ', '_')
+                    by_mode[name] = sc_abbr
+                else:
+                    raise ValueError
+            else:
+                raise ValueError
+        return by_mode
+
+    def _decode_record_bin(self, raw_bin_data, **kwargs):
+        # we can only guess if we should decode for EF or DF here :(
+        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        dec = arr_seq.decode_multi(raw_bin_data)
+        # we cannot pass the result through flatten() here, as we don't have a related
+        # 'un-flattening' decoder, and hence would be unable to encode :(
+        return dec[0]
+
+    def _encode_record_bin(self, in_json, **kwargs):
+        # we can only guess if we should decode for EF or DF here :(
+        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        return arr_seq.encode_multi(in_json)
+
+    @with_default_category('File-Specific Commands')
+    class AddlShellCommands(CommandSet):
+        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
+        def do_read_arr_record(self, opts):
+            """Read one EF.ARR record in flattened, human-friendly form."""
+            (data, _sw) = self._cmd.lchan.read_record_dec(opts.RECORD_NR)
+            data = self._cmd.lchan.selected_file.flatten(data)
+            self._cmd.poutput_json(data, opts.oneline)
+
+        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
+        def do_read_arr_records(self, opts):
+            """Read + decode all EF.ARR records in flattened, human-friendly form."""
+            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
+            # collect all results in list so they are rendered as JSON list when printing
+            data_list = []
+            for recnr in range(1, 1 + num_of_rec):
+                (data, _sw) = self._cmd.lchan.read_record_dec(recnr)
+                data = self._cmd.lchan.selected_file.flatten(data)
+                data_list.append(data)
+            self._cmd.poutput_json(data_list, opts.oneline)
+
+
+# TS 102 221 Section 13.6
+class EF_UMPC(TransparentEF):
+    _test_de_encode = [
+        ( '3cff02', { "max_current_mA": 60, "t_op_s": 255,
+                      "addl_info": { "req_inc_idle_current": False, "support_uicc_suspend": True } } ),
+        ( '320500', { "max_current_mA": 50, "t_op_s": 5, "addl_info": {"req_inc_idle_current": False,
+                                                                       "support_uicc_suspend": False } } ),
+    ]
+    def __init__(self, fid='2f08', sfid=0x08, name='EF.UMPC', desc='UICC Maximum Power Consumption'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(5, 5))
+        addl_info = FlagsEnum(Byte, req_inc_idle_current=1,
+                              support_uicc_suspend=2)
+        self._construct = Struct(
+            'max_current_mA'/Int8ub, 't_op_s'/Int8ub, 'addl_info'/addl_info)
+
+
+class CardProfileUICC(CardProfile):
+
+    ORDER = 10
+
+    def __init__(self, name='UICC'):
+        files = [
+            EF_DIR(),
+            EF_ICCID(),
+            EF_PL(),
+            EF_ARR(),
+            # FIXME: DF.CD
+            EF_UMPC(),
+        ]
+        addons = [
+            AddonSIM,
+            AddonGSMR,
+            AddonRUIM,
+        ]
+        sw = {
+            'Normal': {
+                '9000': 'Normal ending of the command',
+                '91xx': 'Normal ending of the command, with extra information from the proactive UICC containing a command for the terminal',
+                '92xx': 'Normal ending of the command, with extra information concerning an ongoing data transfer session',
+            },
+            'Postponed processing': {
+                '9300': 'SIM Application Toolkit is busy. Command cannot be executed at present, further normal commands are allowed',
+            },
+            'Warnings': {
+                '6200': 'No information given, state of non-volatile memory unchanged',
+                '6281': 'Part of returned data may be corrupted',
+                '6282': 'End of file/record reached before reading Le bytes or unsuccessful search',
+                '6283': 'Selected file invalidated/disabled; needs to be activated before use',
+                '6284': 'Selected file in termination state',
+                '62f1': 'More data available',
+                '62f2': 'More data available and proactive command pending',
+                '62f3': 'Response data available',
+                '63f1': 'More data expected',
+                '63f2': 'More data expected and proactive command pending',
+                '63cx': 'Command successful but after using an internal update retry routine X times',
+            },
+            'Execution errors': {
+                '6400': 'No information given, state of non-volatile memory unchanged',
+                '6500': 'No information given, state of non-volatile memory changed',
+                '6581': 'Memory problem',
+            },
+            'Checking errors': {
+                '6700': 'Wrong length',
+                '67xx': 'The interpretation of this status word is command dependent',
+                '6b00': 'Wrong parameter(s) P1-P2',
+                '6d00': 'Instruction code not supported or invalid',
+                '6e00': 'Class not supported',
+                '6f00': 'Technical problem, no precise diagnosis',
+                '6fxx': 'The interpretation of this status word is command dependent',
+            },
+            'Functions in CLA not supported': {
+                '6800': 'No information given',
+                '6881': 'Logical channel not supported',
+                '6882': 'Secure messaging not supported',
+            },
+            'Command not allowed': {
+                '6900': 'No information given',
+                '6981': 'Command incompatible with file structure',
+                '6982': 'Security status not satisfied',
+                '6983': 'Authentication/PIN method blocked',
+                '6984': 'Referenced data invalidated',
+                '6985': 'Conditions of use not satisfied',
+                '6986': 'Command not allowed (no EF selected)',
+                '6989': 'Command not allowed - secure channel - security not satisfied',
+            },
+            'Wrong parameters': {
+                '6a80': 'Incorrect parameters in the data field',
+                '6a81': 'Function not supported',
+                '6a82': 'File not found',
+                '6a83': 'Record not found',
+                '6a84': 'Not enough memory space',
+                '6a86': 'Incorrect parameters P1 to P2',
+                '6a87': 'Lc inconsistent with P1 to P2',
+                '6a88': 'Referenced data not found',
+            },
+            'Application errors': {
+                '9850': 'INCREASE cannot be performed, max value reached',
+                '9862': 'Authentication error, application specific',
+                '9863': 'Security session or association expired',
+                '9864': 'Minimum UICC suspension time is too long',
+            },
+        }
+
+        super().__init__(name, desc='ETSI TS 102 221', cla="00",
+                         sel_ctrl="0004", files_in_mf=files, sw=sw,
+                         shell_cmdsets = [self.AddlShellCommands()], addons = addons)
+
+    @staticmethod
+    def decode_select_response(data_hex: str) -> object:
+        """ETSI TS 102 221 Section 11.1.1.3"""
+        t = FcpTemplate()
+        t.from_tlv(h2b(data_hex))
+        d = t.to_dict()
+        return flatten_dict_lists(d['fcp_template'])
+
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        """ Try to access MF via UICC APDUs (3GPP TS 102.221), if this works, the
+        card is considered a UICC card."""
+        cls._mf_select_test(scc, "00", "0004", ["3f00"])
+
+    @with_default_category('TS 102 221 Specific Commands')
+    class AddlShellCommands(CommandSet):
+        suspend_uicc_parser = argparse.ArgumentParser()
+        suspend_uicc_parser.add_argument('--min-duration-secs', type=int, default=60,
+                                         help='Proposed minimum duration of suspension')
+        suspend_uicc_parser.add_argument('--max-duration-secs', type=int, default=24*60*60,
+                                         help='Proposed maximum duration of suspension')
+
+        # not ISO7816-4 but TS 102 221
+        @cmd2.with_argparser(suspend_uicc_parser)
+        def do_suspend_uicc(self, opts):
+            """Perform the SUSPEND UICC command. Only supported on some UICC (check EF.UMPC)."""
+            (duration, token, sw) = self._cmd.card._scc.suspend_uicc(min_len_secs=opts.min_duration_secs,
+                                                                     max_len_secs=opts.max_duration_secs)
+            self._cmd.poutput(
+                'Negotiated Duration: %u secs, Token: %s, SW: %s' % (duration, token, sw))
+
+        resume_uicc_parser = argparse.ArgumentParser()
+        resume_uicc_parser.add_argument('TOKEN', type=str, help='Token provided during SUSPEND')
+
+        @cmd2.with_argparser(resume_uicc_parser)
+        def do_resume_uicc(self, opts):
+            """Perform the REUSME UICC operation. Only supported on some UICC. Also: A power-cycle
+            of the card is required between SUSPEND and RESUME, and only very few non-RESUME
+            commands are permitted between SUSPEND and RESUME.  See TS 102 221 Section 11.1.22."""
+            self._cmd.card._scc.resume_uicc(opts.TOKEN)
+
+        term_cap_parser = argparse.ArgumentParser()
+        # power group
+        tc_power_grp = term_cap_parser.add_argument_group('Terminal Power Supply')
+        tc_power_grp.add_argument('--used-supply-voltage-class', type=str, choices=['a','b','c','d','e'],
+                                  help='Actual used Supply voltage class')
+        tc_power_grp.add_argument('--maximum-available-power-supply', type=auto_uint8,
+                                  help='Maximum available power supply of the terminal')
+        tc_power_grp.add_argument('--actual-used-freq-100k', type=auto_uint8,
+                                  help='Actual used clock frequency (in units of 100kHz)')
+        # no separate groups for those two
+        tc_elc_grp = term_cap_parser.add_argument_group('Extended logical channels terminal support')
+        tc_elc_grp.add_argument('--extended-logical-channel', action='store_true',
+                                help='Extended Logical Channel supported')
+        tc_aif_grp = term_cap_parser.add_argument_group('Additional interfaces support')
+        tc_aif_grp.add_argument('--uicc-clf', action='store_true',
+                                help='Local User Interface in the Device (LUId) supported')
+        # eUICC group
+        tc_euicc_grp = term_cap_parser.add_argument_group('Additional Terminal capability indications related to eUICC')
+        tc_euicc_grp.add_argument('--lui-d', action='store_true',
+                                  help='Local User Interface in the Device (LUId) supported')
+        tc_euicc_grp.add_argument('--lpd-d', action='store_true',
+                                  help='Local Profile Download in the Device (LPDd) supported')
+        tc_euicc_grp.add_argument('--lds-d', action='store_true',
+                                  help='Local Discovery Service in the Device (LPDd) supported')
+        tc_euicc_grp.add_argument('--lui-e-scws', action='store_true',
+                                  help='LUIe based on SCWS supported')
+        tc_euicc_grp.add_argument('--metadata-update-alerting', action='store_true',
+                                  help='Metadata update alerting supported')
+        tc_euicc_grp.add_argument('--enterprise-capable-device', action='store_true',
+                                  help='Enterprise Capable Device')
+        tc_euicc_grp.add_argument('--lui-e-e4e', action='store_true',
+                                  help='LUIe using E4E (ENVELOPE tag E4) supported')
+        tc_euicc_grp.add_argument('--lpr', action='store_true',
+                                  help='LPR (LPA Proxy) supported')
+
+        @cmd2.with_argparser(term_cap_parser)
+        def do_terminal_capability(self, opts):
+            """Perform the TERMINAL CAPABILITY function. Used to inform the UICC about terminal capability."""
+            ps_flags = {}
+            addl_if_flags = {}
+            euicc_flags = {}
+
+            opts_dict = vars(opts)
+
+            power_items = ['used_supply_voltage_class', 'maximum_available_power_supply', 'actual_used_freq_100k']
+            if any(opts_dict[x] for x in power_items):
+                if not all(opts_dict[x] for x in power_items):
+                    raise argparse.ArgumentTypeError('If any of the Terminal Power Supply group options are used, all must be specified')
+
+            for k, v in opts_dict.items():
+                if k in AdditionalInterfacesSupport._construct.flags.keys():
+                    addl_if_flags[k] = v
+                elif k in AdditionalTermCapEuicc._construct.flags.keys():
+                    euicc_flags[k] = v
+                elif k in [f.name for f in TerminalPowerSupply._construct.subcons]:
+                    if k == 'used_supply_voltage_class' and v:
+                        v = {v: True}
+                    ps_flags[k] = v
+
+            child_list = []
+            if any(x for x in ps_flags.values()):
+                child_list.append(TerminalPowerSupply(decoded=ps_flags))
+
+            if opts.extended_logical_channel:
+                child_list.append(ExtendedLchanTerminalSupport())
+            if any(x for x in addl_if_flags.values()):
+                child_list.append(AdditionalInterfacesSupport(decoded=addl_if_flags))
+            if any(x for x in euicc_flags.values()):
+                child_list.append(AdditionalTermCapEuicc(decoded=euicc_flags))
+
+            print(child_list)
+            tc = TerminalCapability(children=child_list)
+            self.terminal_capability(b2h(tc.to_tlv()))
+
+        def terminal_capability(self, data:Hexstr):
+            cmd_hex = "80AA0000%02x%s" % (len(data)//2, data)
+            _rsp_hex, _sw = self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)

pySim/ts_102_222.py

@@ -25,6 +25,16 @@ from osmocom.utils import b2h, auto_uint8, auto_uint16, is_hexstr
 
 from pySim.ts_102_221 import *
 
+def expand_pattern(pattern: bytes, repeat: bool, size: int) -> bytes:
+    """Expand the fill/repeat pattern as per TS 102 222 Section 6.3.2.2.2 Tags C1/C2."""
+    if not repeat:
+        pad_len = size - len(pattern)
+        return pattern + pattern[-1:] * pad_len
+    else:
+        count = size // len(pattern)
+        part_len = size - count * len(pattern)
+        return pattern * count + pattern[:part_len]
+
 @with_default_category('TS 102 222 Administrative Commands')
 class Ts102222Commands(CommandSet):
     """Administrative commands for telecommunication applications."""

pySim/ts_102_310.py

@@ -27,9 +27,9 @@ from pySim.filesystem import CardDF, TransparentEF
 # TS102 310 Section 7.1
 class EF_EAPKEYS(TransparentEF):
     class Msk(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
     class Emsk(BER_TLV_IE, tag=0x81):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
     class MskCollection(TLV_IE_Collection, nested=[EF_EAPKEYS.Msk, EF_EAPKEYS.Emsk]):
         pass