smdpp: proper checks, proper check order, test mode

Change-Id: Ie4db65594b2eaf6c95ffc5e73ff9ae61c4d9d3a3

.gitignore

@@ -7,3 +7,10 @@
 /.local
 /build
 /pySim.egg-info
+/smdpp-data/sm-dp-sessions*
+dist
+tags
+smdpp-data/certs/DPtls/CERT_S_SM_DP_TLS_NIST.pem
+smdpp-data/certs/DPtls/CERT_S_SM_DP_TLS_BRP.pem
+smdpp-data/generated
+smdpp-data/certs/dhparam2048.pem

README.md

@@ -1,16 +1,29 @@
-pySim - Read, Write and Browse Programmable SIM/USIM/ISIM/HPSIM Cards
-=====================================================================
+pySim - Tools for reading, decoding, browsing SIM/USIM/ISIM/HPSIM/eUICC Cards
+=============================================================================
 
-This repository contains a number of Python programs that can be used
-to read, program (write) and browse all fields/parameters/files on
-SIM/USIM/ISIM/HPSIM cards used in 3GPP cellular networks from 2G to 5G.
+This repository contains a number of Python programs related to working with
+subscriber identity modules of cellular networks, including but not limited
+to SIM, UICC, USIM, ISIM, HPSIMs and eUICCs.
+
+* `pySim-shell.py` can be used to interactively explore, read and decode contents
+  of any of the supported card models / card applications.  Furthermore, if
+  you have the credentials to your card (ADM PIN), you can also write to the card,
+  i.e. edit its contents.
+* `pySim-read.py` and `pySim-prog.py` are _legacy_ tools for batch programming
+  some very common parameters to an entire batch of programmable cards
+* `pySim-trace.py` is a tool to do an in-depth decode of SIM card protocol traces
+  such as those obtained by [Osmocom SIMtrace2](https://osmocom.org/projects/simtrace2/wiki)
+  or [osmo-qcdiag](https://osmocom.org/projects/osmo-qcdiag/wiki).
+* `osmo-smdpp.py` is a proof-of-concept GSMA SGP.22 Consumer eSIM SM-DP+ for lab/research
+* there are more related tools, particularly in the `contrib` directory.
 
 Note that the access control configuration of normal production cards
 issue by operators will restrict significantly which files a normal
 user can read, and particularly write to.
 
 The full functionality of pySim hence can only be used with on so-called
-programmable SIM/USIM/ISIM/HPSIM cards.
+programmable SIM/USIM/ISIM/HPSIM cards, such as the various
+[sysmocom programmable card products](https://shop.sysmocom.de/SIM/).
 
 Such SIM/USIM/ISIM/HPSIM cards are special cards, which - unlike those
 issued by regular commercial operators - come with the kind of keys that
@@ -49,9 +62,9 @@ pySim-shell vs. legacy tools
 ----------------------------
 
 While you will find a lot of online resources still describing the use of
-pySim-prog.py and pySim-read.py, those tools are considered legacy by
+`pySim-prog.py` and `pySim-read.py`, those tools are considered legacy by
 now and have by far been superseded by the much more capable
-pySim-shell.  We strongly encourage users to adopt pySim-shell, unless
+`pySim-shell.py`.  We strongly encourage users to adopt pySim-shell, unless
 they have very specific requirements like batch programming of large
 quantities of cards, which is about the only remaining use case for the
 legacy tools.

contrib/es2p_client.py

@@ -17,7 +17,7 @@
 
 import copy
 import argparse
-from pySim.esim import es2p
+from pySim.esim import es2p, ActivationCode
 
 EID_HELP='EID of the eUICC for which eSIM shall be made available'
 ICCID_HELP='The ICCID of the eSIM that shall be made available'
@@ -73,6 +73,11 @@ if __name__ == '__main__':
         res = peer.call_downloadOrder(data)
     elif opts.command == 'confirm-order':
         res = peer.call_confirmOrder(data)
+        matchingId = res.get('matchingId', None)
+        smdpAddress = res.get('smdpAddress', None)
+        if matchingId:
+            ac = ActivationCode(smdpAddress, matchingId, cc_required=bool(opts.confirmationCode))
+            print("Activation Code: '%s'" % ac.to_string())
     elif opts.command == 'cancel-order':
         res = peer.call_cancelOrder(data)
     elif opts.command == 'release-profile':

contrib/esim-qrcode-gen.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+# Small command line utility program to encode eSIM QR-Codes
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import argparse
+
+from pySim.esim import ActivationCode
+
+
+option_parser = argparse.ArgumentParser(description="""
+eSIM QR code generator. Will encode the given hostname + activation code
+into the eSIM RSP String format as specified in SGP.22 Section 4.1.  If
+a PNG output file is specified, it will also generate a QR code.""")
+option_parser.add_argument('hostname', help='FQDN of SM-DP+')
+option_parser.add_argument('token', help='MatchingID / Token')
+option_parser.add_argument('--oid', help='SM-DP+ OID in CERT.DPauth.ECDSA')
+option_parser.add_argument('--confirmation-code-required', action='store_true',
+                           help='Whether a Confirmation Code is required')
+option_parser.add_argument('--png', help='Output PNG file name (no PNG is written if omitted)')
+
+
+
+if __name__ == '__main__':
+    opts = option_parser.parse_args()
+
+    ac = ActivationCode(opts.hostname, opts.token, opts.oid, opts.confirmation_code_required)
+    print(ac.to_string())
+    if opts.png:
+        with open(opts.png, 'wb') as f:
+            img = ac.to_qrcode()
+            img.save(f)
+            print("# generated QR code stored to '%s'" % (opts.png))

contrib/generate_smdpp_certs.py

@@ -0,0 +1,661 @@
+#!/usr/bin/env python3
+
+"""
+Faithfully reproduces the smdpp certs contained in SGP.26_v1.5_Certificates_18_07_2024.zip
+available at https://www.gsma.com/solutions-and-impact/technologies/esim/gsma_resources/sgp-26-test-certificate-definition-v1-5/
+Only usable for testing, it obviously uses a different CI key.
+"""
+
+import os
+import binascii
+from datetime import datetime
+from cryptography import x509
+from cryptography.x509.oid import NameOID, ExtensionOID
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.backends import default_backend
+
+# Custom OIDs used in certificates
+OID_CERTIFICATE_POLICIES_CI = "2.23.146.1.2.1.0"  # CI cert policy
+OID_CERTIFICATE_POLICIES_TLS = "2.23.146.1.2.1.3"  # DPtls cert policy
+OID_CERTIFICATE_POLICIES_AUTH = "2.23.146.1.2.1.4"  # DPauth cert policy
+OID_CERTIFICATE_POLICIES_PB = "2.23.146.1.2.1.5"  # DPpb cert policy
+
+# Subject Alternative Name OIDs
+OID_CI_RID = "2.999.1"  # CI Registered ID
+OID_DP_RID = "2.999.10"  # DP+ Registered ID
+OID_DP2_RID = "2.999.12"  # DP+2 Registered ID
+OID_DP4_RID = "2.999.14"  # DP+4 Registered ID
+OID_DP8_RID = "2.999.18"  # DP+8 Registered ID
+
+
+class SimplifiedCertificateGenerator:
+    def __init__(self):
+        self.backend = default_backend()
+        # Store generated CI keys to sign other certs
+        self.ci_certs = {}  # {"BRP": cert, "NIST": cert}
+        self.ci_keys = {}   # {"BRP": key, "NIST": key}
+
+    def get_curve(self, curve_type):
+        """Get the appropriate curve object."""
+        if curve_type == "BRP":
+            return ec.BrainpoolP256R1()
+        else:
+            return ec.SECP256R1()
+
+    def generate_key_pair(self, curve):
+        """Generate a new EC key pair."""
+        private_key = ec.generate_private_key(curve, self.backend)
+        return private_key
+
+    def load_private_key_from_hex(self, hex_key, curve):
+        """Load EC private key from hex string."""
+        key_bytes = binascii.unhexlify(hex_key.replace(":", "").replace(" ", "").replace("\n", ""))
+        key_int = int.from_bytes(key_bytes, 'big')
+        return ec.derive_private_key(key_int, curve, self.backend)
+
+    def generate_ci_cert(self, curve_type):
+        """Generate CI certificate for either BRP or NIST curve."""
+        curve = self.get_curve(curve_type)
+        private_key = self.generate_key_pair(curve)
+
+        # Build subject and issuer (self-signed) - same for both
+        subject = issuer = x509.Name([
+            x509.NameAttribute(NameOID.COMMON_NAME, "Test CI"),
+            x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, "TESTCERT"),
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "RSPTEST"),
+            x509.NameAttribute(NameOID.COUNTRY_NAME, "IT"),
+        ])
+
+        # Build certificate - all parameters same for both
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(issuer)
+        builder = builder.not_valid_before(datetime(2020, 4, 1, 8, 27, 51))
+        builder = builder.not_valid_after(datetime(2055, 4, 1, 8, 27, 51))
+        builder = builder.serial_number(0xb874f3abfa6c44d3)
+        builder = builder.public_key(private_key.public_key())
+
+        # Add extensions - all same for both
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.BasicConstraints(ca=True, path_length=None),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier(OID_CERTIFICATE_POLICIES_CI),
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=False,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=True,
+                crl_sign=True,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectAlternativeName([
+                x509.RegisteredID(x509.ObjectIdentifier(OID_CI_RID))
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.CRLDistributionPoints([
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-A.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                ),
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-B.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                )
+            ]),
+            critical=False
+        )
+
+        certificate = builder.sign(private_key, hashes.SHA256(), self.backend)
+
+        self.ci_keys[curve_type] = private_key
+        self.ci_certs[curve_type] = certificate
+
+        return certificate, private_key
+
+    def generate_dp_cert(self, curve_type, subject_cn, serial, key_hex,
+                        cert_policy_oid, rid_oid, validity_start, validity_end):
+        """Generate a DP certificate signed by CI - works for both BRP and NIST."""
+        curve = self.get_curve(curve_type)
+        private_key = self.load_private_key_from_hex(key_hex, curve)
+
+        ci_cert = self.ci_certs[curve_type]
+        ci_key = self.ci_keys[curve_type]
+
+        subject = x509.Name([
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "ACME"),
+            x509.NameAttribute(NameOID.COMMON_NAME, subject_cn),
+        ])
+
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(ci_cert.subject)
+        builder = builder.not_valid_before(validity_start)
+        builder = builder.not_valid_after(validity_end)
+        builder = builder.serial_number(serial)
+        builder = builder.public_key(private_key.public_key())
+
+        builder = builder.add_extension(
+            x509.AuthorityKeyIdentifier.from_issuer_public_key(ci_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectAlternativeName([
+                x509.RegisteredID(x509.ObjectIdentifier(rid_oid))
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=True,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=False,
+                crl_sign=False,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier(cert_policy_oid),
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CRLDistributionPoints([
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-A.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                ),
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-B.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                )
+            ]),
+            critical=False
+        )
+
+        certificate = builder.sign(ci_key, hashes.SHA256(), self.backend)
+
+        return certificate, private_key
+
+    def generate_tls_cert(self, curve_type, subject_cn, dns_name, serial, key_hex,
+                         rid_oid, validity_start, validity_end):
+        """Generate a TLS certificate signed by CI."""
+        curve = self.get_curve(curve_type)
+        private_key = self.load_private_key_from_hex(key_hex, curve)
+
+        ci_cert = self.ci_certs[curve_type]
+        ci_key = self.ci_keys[curve_type]
+
+        subject = x509.Name([
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "ACME"),
+            x509.NameAttribute(NameOID.COMMON_NAME, subject_cn),
+        ])
+
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(ci_cert.subject)
+        builder = builder.not_valid_before(validity_start)
+        builder = builder.not_valid_after(validity_end)
+        builder = builder.serial_number(serial)
+        builder = builder.public_key(private_key.public_key())
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=True,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=False,
+                crl_sign=False,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.ExtendedKeyUsage([
+                x509.oid.ExtendedKeyUsageOID.SERVER_AUTH,
+                x509.oid.ExtendedKeyUsageOID.CLIENT_AUTH
+            ]),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier(OID_CERTIFICATE_POLICIES_TLS),
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.AuthorityKeyIdentifier.from_issuer_public_key(ci_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectAlternativeName([
+                x509.DNSName(dns_name),
+                x509.RegisteredID(x509.ObjectIdentifier(rid_oid))
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.CRLDistributionPoints([
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-A.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                ),
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-B.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                )
+            ]),
+            critical=False
+        )
+
+        certificate = builder.sign(ci_key, hashes.SHA256(), self.backend)
+
+        return certificate, private_key
+
+    def generate_eum_cert(self, curve_type, key_hex):
+        """Generate EUM certificate signed by CI."""
+        curve = self.get_curve(curve_type)
+        private_key = self.load_private_key_from_hex(key_hex, curve)
+
+        ci_cert = self.ci_certs[curve_type]
+        ci_key = self.ci_keys[curve_type]
+
+        subject = x509.Name([
+            x509.NameAttribute(NameOID.COUNTRY_NAME, "ES"),
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "RSP Test EUM"),
+            x509.NameAttribute(NameOID.COMMON_NAME, "EUM Test"),
+        ])
+
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(ci_cert.subject)
+        builder = builder.not_valid_before(datetime(2020, 4, 1, 9, 28, 37))
+        builder = builder.not_valid_after(datetime(2054, 3, 24, 9, 28, 37))
+        builder = builder.serial_number(0x12345678)
+        builder = builder.public_key(private_key.public_key())
+
+        builder = builder.add_extension(
+            x509.AuthorityKeyIdentifier.from_issuer_public_key(ci_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=False,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=True,
+                crl_sign=False,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier("2.23.146.1.2.1.2"),  # EUM policy
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectAlternativeName([
+                x509.RegisteredID(x509.ObjectIdentifier("2.999.5"))
+            ]),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.BasicConstraints(ca=True, path_length=0),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CRLDistributionPoints([
+                x509.DistributionPoint(
+                    full_name=[x509.UniformResourceIdentifier("http://ci.test.example.com/CRL-B.crl")],
+                    relative_name=None,
+                    reasons=None,
+                    crl_issuer=None
+                )
+            ]),
+            critical=False
+        )
+
+        # Name Constraints
+        constrained_name = x509.Name([
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "RSP Test EUM"),
+            x509.NameAttribute(NameOID.SERIAL_NUMBER, "89049032"),
+        ])
+
+        name_constraints = x509.NameConstraints(
+            permitted_subtrees=[
+                x509.DirectoryName(constrained_name)
+            ],
+            excluded_subtrees=None
+        )
+
+        builder = builder.add_extension(
+            name_constraints,
+            critical=True
+        )
+
+        certificate = builder.sign(ci_key, hashes.SHA256(), self.backend)
+
+        return certificate, private_key
+
+    def generate_euicc_cert(self, curve_type, eum_cert, eum_key, key_hex):
+        """Generate eUICC certificate signed by EUM."""
+        curve = self.get_curve(curve_type)
+        private_key = self.load_private_key_from_hex(key_hex, curve)
+
+        subject = x509.Name([
+            x509.NameAttribute(NameOID.COUNTRY_NAME, "ES"),
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, "RSP Test EUM"),
+            x509.NameAttribute(NameOID.SERIAL_NUMBER, "89049032123451234512345678901235"),
+            x509.NameAttribute(NameOID.COMMON_NAME, "Test eUICC"),
+        ])
+
+        builder = x509.CertificateBuilder()
+        builder = builder.subject_name(subject)
+        builder = builder.issuer_name(eum_cert.subject)
+        builder = builder.not_valid_before(datetime(2020, 4, 1, 9, 48, 58))
+        builder = builder.not_valid_after(datetime(7496, 1, 24, 9, 48, 58))
+        builder = builder.serial_number(0x0200000000000001)
+        builder = builder.public_key(private_key.public_key())
+
+        builder = builder.add_extension(
+            x509.AuthorityKeyIdentifier.from_issuer_public_key(eum_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.SubjectKeyIdentifier.from_public_key(private_key.public_key()),
+            critical=False
+        )
+
+        builder = builder.add_extension(
+            x509.KeyUsage(
+                digital_signature=True,
+                content_commitment=False,
+                key_encipherment=False,
+                data_encipherment=False,
+                key_agreement=False,
+                key_cert_sign=False,
+                crl_sign=False,
+                encipher_only=False,
+                decipher_only=False
+            ),
+            critical=True
+        )
+
+        builder = builder.add_extension(
+            x509.CertificatePolicies([
+                x509.PolicyInformation(
+                    x509.ObjectIdentifier("2.23.146.1.2.1.1"),  # eUICC policy
+                    policy_qualifiers=None
+                )
+            ]),
+            critical=True
+        )
+
+        certificate = builder.sign(eum_key, hashes.SHA256(), self.backend)
+
+        return certificate, private_key
+
+    def save_cert_and_key(self, cert, key, cert_path_der, cert_path_pem, key_path_sk, key_path_pk):
+        """Save certificate and key in various formats."""
+        # Create directories if needed
+        os.makedirs(os.path.dirname(cert_path_der), exist_ok=True)
+
+        with open(cert_path_der, "wb") as f:
+            f.write(cert.public_bytes(serialization.Encoding.DER))
+
+        if cert_path_pem:
+            with open(cert_path_pem, "wb") as f:
+                f.write(cert.public_bytes(serialization.Encoding.PEM))
+
+        if key and key_path_sk:
+            with open(key_path_sk, "wb") as f:
+                f.write(key.private_bytes(
+                    encoding=serialization.Encoding.PEM,
+                    format=serialization.PrivateFormat.TraditionalOpenSSL,
+                    encryption_algorithm=serialization.NoEncryption()
+                ))
+
+        if key and key_path_pk:
+            with open(key_path_pk, "wb") as f:
+                f.write(key.public_key().public_bytes(
+                    encoding=serialization.Encoding.PEM,
+                    format=serialization.PublicFormat.SubjectPublicKeyInfo
+                ))
+
+
+def main():
+    gen = SimplifiedCertificateGenerator()
+
+    output_dir = "smdpp-data/generated"
+    os.makedirs(output_dir, exist_ok=True)
+
+    print("=== Generating CI Certificates ===")
+
+    for curve_type in ["BRP", "NIST"]:
+        ci_cert, ci_key = gen.generate_ci_cert(curve_type)
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            ci_cert, ci_key,
+            f"{output_dir}/CertificateIssuer/CERT_CI{suffix}.der",
+            f"{output_dir}/CertificateIssuer/CERT_CI{suffix}.pem",
+            None, None
+        )
+        print(f"Generated CI {curve_type} certificate")
+
+    print("\n=== Generating DPauth Certificates ===")
+
+    dpauth_configs = [
+        ("BRP", "TEST SM-DP+", 256, "93:fb:33:d0:58:4f:34:9b:07:f8:b5:d2:af:93:d7:c3:e3:54:b3:49:a3:b9:13:50:2e:6a:bc:07:0e:4d:49:29", OID_DP_RID, "DPauth"),
+        ("NIST", "TEST SM-DP+", 256, "0a:7c:c1:c2:44:e6:0c:52:cd:5b:78:07:ab:8c:36:0c:26:52:46:01:50:7d:ca:bc:5d:d5:98:b5:a6:16:d5:d5", OID_DP_RID, "DPauth"),
+        ("BRP", "TEST SM-DP+2", 512, "0c:17:35:5c:01:1d:0f:e8:d7:da:dd:63:f1:97:85:cf:6c:51:cb:cd:46:6a:e8:8b:e8:f8:1b:c1:05:88:46:f6", OID_DP2_RID, "DP2auth"),
+        ("NIST", "TEST SM-DP+2", 512, "9c:32:a0:95:d4:88:42:d9:ff:a4:04:f7:12:51:2a:a2:c5:42:5a:1a:26:38:6a:b6:a1:45:d5:81:1e:03:91:41", OID_DP2_RID, "DP2auth"),
+    ]
+
+    for curve_type, cn, serial, key_hex, rid_oid, name_prefix in dpauth_configs:
+        cert, key = gen.generate_dp_cert(
+            curve_type, cn, serial, key_hex,
+            OID_CERTIFICATE_POLICIES_AUTH, rid_oid,
+            datetime(2020, 4, 1, 8, 31, 30),
+            datetime(2030, 3, 30, 8, 31, 30)
+        )
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/DPauth/CERT_S_SM_{name_prefix}{suffix}.der",
+            None,
+            f"{output_dir}/DPauth/SK_S_SM_{name_prefix}{suffix}.pem",
+            f"{output_dir}/DPauth/PK_S_SM_{name_prefix}{suffix}.pem"
+        )
+        print(f"Generated {name_prefix} {curve_type} certificate")
+
+    print("\n=== Generating DPpb Certificates ===")
+
+    dppb_configs = [
+        ("BRP", "TEST SM-DP+", 257, "75:ff:32:2f:41:66:16:da:e1:a4:84:ef:71:d4:87:4f:b0:df:32:95:fd:35:c2:cb:a4:89:fb:b2:bb:9c:7b:f6", OID_DP_RID, "DPpb"),
+        ("NIST", "TEST SM-DP+", 257, "dc:d6:94:b7:78:95:7e:8e:9a:dd:bd:d9:44:33:e9:ef:8f:73:d1:1e:49:1c:48:d4:25:a3:8a:94:91:bd:3b:ed", OID_DP_RID, "DPpb"),
+        ("BRP", "TEST SM-DP+2", 513, "9c:ae:2e:1a:56:07:a9:d5:78:38:2e:ee:93:2e:25:1f:52:30:4f:86:ee:b1:f1:70:8c:db:d3:c0:7b:e2:cd:3d", OID_DP2_RID, "DP2pb"),
+        ("NIST", "TEST SM-DP+2", 513, "66:93:11:49:63:9d:ba:ac:1d:c3:d3:06:c5:8b:d2:df:d2:2f:73:bf:63:ac:86:31:98:32:90:b5:7f:90:93:45", OID_DP2_RID, "DP2pb"),
+    ]
+
+    for curve_type, cn, serial, key_hex, rid_oid, name_prefix in dppb_configs:
+        cert, key = gen.generate_dp_cert(
+            curve_type, cn, serial, key_hex,
+            OID_CERTIFICATE_POLICIES_PB, rid_oid,
+            datetime(2020, 4, 1, 8, 34, 46),
+            datetime(2030, 3, 30, 8, 34, 46)
+        )
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/DPpb/CERT_S_SM_{name_prefix}{suffix}.der",
+            None,
+            f"{output_dir}/DPpb/SK_S_SM_{name_prefix}{suffix}.pem",
+            f"{output_dir}/DPpb/PK_S_SM_{name_prefix}{suffix}.pem"
+        )
+        print(f"Generated {name_prefix} {curve_type} certificate")
+
+    print("\n=== Generating DPtls Certificates ===")
+
+    dptls_configs = [
+        ("BRP", "testsmdpplus1.example.com", "testsmdpplus1.example.com", 9, "3f:67:15:28:02:b3:f4:c7:fa:e6:79:58:55:f6:82:54:1e:45:e3:5e:ff:f4:e8:a0:55:65:a0:f1:91:2a:78:2e", OID_DP_RID, "DP_TLS_BRP"),
+        ("NIST", "testsmdpplus1.example.com", "testsmdpplus1.example.com", 9, "a0:3e:7c:e4:55:04:74:be:a4:b7:a8:73:99:ce:5a:8c:9f:66:1b:68:0f:94:01:39:ff:f8:4e:9d:ec:6a:4d:8c", OID_DP_RID, "DP_TLS_NIST"),
+        ("NIST", "testsmdpplus2.example.com", "testsmdpplus2.example.com", 12, "4e:65:61:c6:40:88:f6:69:90:7a:db:e3:94:b1:1a:84:24:2e:03:3a:82:a8:84:02:31:63:6d:c9:1b:4e:e3:f5", OID_DP2_RID, "DP2_TLS"),
+        ("NIST", "testsmdpplus4.example.com", "testsmdpplus4.example.com", 14, "f2:65:9d:2f:52:8f:4b:11:37:40:d5:8a:0d:2a:f3:eb:2b:48:e1:22:c2:b6:0a:6a:f6:fc:96:ad:86:be:6f:a4", OID_DP4_RID, "DP4_TLS"),
+        ("NIST", "testsmdpplus8.example.com", "testsmdpplus8.example.com", 18, "ff:6e:4a:50:9b:ad:db:38:10:88:31:c2:3c:cc:2d:44:30:7a:f2:81:e9:25:96:7f:8c:df:1d:95:54:a0:28:8d", OID_DP8_RID, "DP8_TLS"),
+    ]
+
+    for curve_type, cn, dns, serial, key_hex, rid_oid, name_prefix in dptls_configs:
+        cert, key = gen.generate_tls_cert(
+            curve_type, cn, dns, serial, key_hex, rid_oid,
+            datetime(2024, 7, 9, 15, 29, 36),
+            datetime(2025, 8, 11, 15, 29, 36)
+        )
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/DPtls/CERT_S_SM_{name_prefix}.der",
+            None,
+            f"{output_dir}/DPtls/SK_S_SM_{name_prefix.replace('_BRP', '_BRP').replace('_NIST', '_NIST')}.pem",
+            f"{output_dir}/DPtls/PK_S_SM_{name_prefix.replace('_BRP', '_BRP').replace('_NIST', '_NIST')}.pem"
+        )
+        print(f"Generated {name_prefix} certificate")
+
+    print("\n=== Generating EUM Certificates ===")
+
+    eum_configs = [
+        ("BRP", "12:9b:0a:b1:3f:17:e1:4a:40:b6:fa:4e:d8:23:e0:cf:46:5b:7b:3d:73:24:05:e6:29:5d:3b:23:b0:45:c9:9a"),
+        ("NIST", "25:e6:75:77:28:e1:e9:51:13:51:9c:dc:34:55:5c:29:ba:ed:23:77:3a:c5:af:dd:dc:da:d9:84:89:8a:52:f0"),
+    ]
+
+    eum_certs = {}
+    eum_keys = {}
+
+    for curve_type, key_hex in eum_configs:
+        cert, key = gen.generate_eum_cert(curve_type, key_hex)
+        eum_certs[curve_type] = cert
+        eum_keys[curve_type] = key
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/EUM/CERT_EUM{suffix}.der",
+            None,
+            f"{output_dir}/EUM/SK_EUM{suffix}.pem",
+            f"{output_dir}/EUM/PK_EUM{suffix}.pem"
+        )
+        print(f"Generated EUM {curve_type} certificate")
+
+    print("\n=== Generating eUICC Certificates ===")
+
+    euicc_configs = [
+        ("BRP", "8d:c3:47:a7:6d:b7:bd:d6:22:2d:d7:5e:a1:a1:68:8a:ca:81:1e:4c:bc:6a:7f:6a:ef:a4:b2:64:19:62:0b:90"),
+        ("NIST", "11:e1:54:67:dc:19:4f:33:71:83:e4:60:c9:f6:32:60:09:1e:12:e8:10:26:cd:65:61:e1:7c:6d:85:39:cc:9c"),
+    ]
+
+    for curve_type, key_hex in euicc_configs:
+        cert, key = gen.generate_euicc_cert(curve_type, eum_certs[curve_type], eum_keys[curve_type], key_hex)
+        suffix = "_ECDSA_BRP" if curve_type == "BRP" else "_ECDSA_NIST"
+        gen.save_cert_and_key(
+            cert, key,
+            f"{output_dir}/eUICC/CERT_EUICC{suffix}.der",
+            None,
+            f"{output_dir}/eUICC/SK_EUICC{suffix}.pem",
+            f"{output_dir}/eUICC/PK_EUICC{suffix}.pem"
+        )
+        print(f"Generated eUICC {curve_type} certificate")
+
+    print("\n=== Certificate generation complete! ===")
+    print(f"All certificates saved to: {output_dir}/")
+
+if __name__ == "__main__":
+    main()

contrib/jenkins.sh

@@ -82,6 +82,10 @@ case "$JOB_TYPE" in
 
 	pip install -r requirements.txt
 
+	# XXX: workaround for https://github.com/python-cmd2/cmd2/issues/1414
+	# 2.4.3 was the last stable release not affected by this bug (OS#6776)
+	pip install cmd2==2.4.3
+
 	rm -rf docs/_build
 	make -C "docs" html latexpdf
 

contrib/saip-tool.py

@@ -19,14 +19,13 @@ import os
 import sys
 import argparse
 import logging
-import zipfile
 from pathlib import Path as PlPath
 from typing import List
 from osmocom.utils import h2b, b2h, swap_nibbles
+from osmocom.construct import GreedyBytes, StripHeaderAdapter
 
 from pySim.esim.saip import *
 from pySim.esim.saip.validation import CheckBasicStructure
-from pySim import javacard
 from pySim.pprint import HexBytesPrettyPrinter
 
 pp = HexBytesPrettyPrinter(indent=4,width=500)
@@ -48,9 +47,14 @@ parser_dump.add_argument('--dump-decoded', action='store_true', help='Dump decod
 
 parser_check = subparsers.add_parser('check', help='Run constraint checkers on PE-Sequence')
 
+parser_rpe = subparsers.add_parser('extract-pe', help='Extract specified PE to (DER encoded) file')
+parser_rpe.add_argument('--pe-file', required=True, help='PE file name')
+parser_rpe.add_argument('--identification', type=int, help='Extract PE matching specified identification')
+
 parser_rpe = subparsers.add_parser('remove-pe', help='Remove specified PEs from PE-Sequence')
 parser_rpe.add_argument('--output-file', required=True, help='Output file name')
-parser_rpe.add_argument('--identification', type=int, action='append', help='Remove PEs matching specified identification')
+parser_rpe.add_argument('--identification', default=[], type=int, action='append', help='Remove PEs matching specified identification')
+parser_rpe.add_argument('--type', default=[], action='append', help='Remove PEs matching specified type')
 
 parser_rn = subparsers.add_parser('remove-naa', help='Remove speciifed NAAs from PE-Sequence')
 parser_rn.add_argument('--output-file', required=True, help='Output file name')
@@ -58,13 +62,59 @@ parser_rn.add_argument('--naa-type', required=True, choices=NAAs.keys(), help='N
 # TODO: add an --naa-index or the like, so only one given instance can be removed
 
 parser_info = subparsers.add_parser('info', help='Display information about the profile')
+parser_info.add_argument('--apps', action='store_true', help='List applications and their related instances')
 
 parser_eapp = subparsers.add_parser('extract-apps', help='Extract applications as loadblock file')
 parser_eapp.add_argument('--output-dir', default='.', help='Output directory (where to store files)')
 parser_eapp.add_argument('--format', default='cap', choices=['ijc', 'cap'], help='Data format of output files')
 
+parser_aapp = subparsers.add_parser('add-app', help='Add application to PE-Sequence')
+parser_aapp.add_argument('--output-file', required=True, help='Output file name')
+parser_aapp.add_argument('--applet-file', required=True, help='Applet file name')
+parser_aapp.add_argument('--aid', required=True, help='Load package AID')
+parser_aapp.add_argument('--sd-aid', default=None, help='Security Domain AID')
+parser_aapp.add_argument('--non-volatile-code-limit', default=None, type=int, help='Non volatile code limit (C6)')
+parser_aapp.add_argument('--volatile-data-limit', default=None, type=int, help='Volatile data limit (C7)')
+parser_aapp.add_argument('--non-volatile-data-limit', default=None, type=int, help='Non volatile data limit (C8)')
+parser_aapp.add_argument('--hash-value', default=None, help='Hash value')
+
+parser_rapp = subparsers.add_parser('remove-app', help='Remove application from PE-Sequence')
+parser_rapp.add_argument('--output-file', required=True, help='Output file name')
+parser_rapp.add_argument('--aid', required=True, help='Load package AID')
+
+parser_aappi = subparsers.add_parser('add-app-inst', help='Add application instance to Application PE')
+parser_aappi.add_argument('--output-file', required=True, help='Output file name')
+parser_aappi.add_argument('--aid', required=True, help='Load package AID')
+parser_aappi.add_argument('--class-aid', required=True, help='Class AID')
+parser_aappi.add_argument('--inst-aid', required=True, help='Instance AID (must match Load package AID)')
+parser_aappi.add_argument('--app-privileges', default='000000', help='Application privileges')
+parser_aappi.add_argument('--volatile-memory-quota', default=None, type=int, help='Volatile memory quota (C7)')
+parser_aappi.add_argument('--non-volatile-memory-quota', default=None, type=int, help='Non volatile memory quota (C8)')
+parser_aappi.add_argument('--app-spec-pars', default='00', help='Application specific parameters (C9)')
+parser_aappi.add_argument('--uicc-toolkit-app-spec-pars', help='UICC toolkit application specific parameters field')
+parser_aappi.add_argument('--uicc-access-app-spec-pars', help='UICC Access application specific parameters field')
+parser_aappi.add_argument('--uicc-adm-access-app-spec-pars', help='UICC Administrative access application specific parameters field')
+parser_aappi.add_argument('--process-data', default=[], action='append', help='Process personalization APDUs')
+
+parser_rappi = subparsers.add_parser('remove-app-inst', help='Remove application instance from Application PE')
+parser_rappi.add_argument('--output-file', required=True, help='Output file name')
+parser_rappi.add_argument('--aid', required=True, help='Load package AID')
+parser_rappi.add_argument('--inst-aid', required=True, help='Instance AID')
+
+esrv_flag_choices = [t.name for t in asn1.types['ServicesList'].type.root_members]
+parser_esrv = subparsers.add_parser('edit-mand-srv-list', help='Add/Remove service flag from/to mandatory services list')
+parser_esrv.add_argument('--output-file', required=True, help='Output file name')
+parser_esrv.add_argument('--add-flag', default=[], choices=esrv_flag_choices, action='append', help='Add flag to mandatory services list')
+parser_esrv.add_argument('--remove-flag', default=[], choices=esrv_flag_choices, action='append', help='Remove flag from mandatory services list')
+
 parser_info = subparsers.add_parser('tree', help='Display the filesystem tree')
 
+def write_pes(pes: ProfileElementSequence, output_file:str):
+    """write the PE sequence to a file"""
+    print("Writing %u PEs to file '%s'..." % (len(pes.pe_list), output_file))
+    with open(output_file, 'wb') as f:
+        f.write(pes.to_der())
+
 def do_split(pes: ProfileElementSequence, opts):
     i = 0
     for pe in pes.pe_list:
@@ -120,6 +170,14 @@ def do_check(pes: ProfileElementSequence, opts):
     checker.check(pes)
     print("All good!")
 
+def do_extract_pe(pes: ProfileElementSequence, opts):
+    new_pe_list = []
+    for pe in pes.pe_list:
+        if pe.identification == opts.identification:
+            print("Extracting PE %s (id=%u) to file %s..." % (pe, pe.identification, opts.pe_file))
+            with open(opts.pe_file, 'wb') as f:
+                f.write(pe.to_der())
+
 def do_remove_pe(pes: ProfileElementSequence, opts):
     new_pe_list = []
     for pe in pes.pe_list:
@@ -128,13 +186,14 @@ def do_remove_pe(pes: ProfileElementSequence, opts):
             if identification in opts.identification:
                 print("Removing PE %s (id=%u) from Sequence..." % (pe, identification))
                 continue
+        if pe.type in opts.type:
+            print("Removing PE %s (type=%s) from Sequence..." % (pe, pe.type))
+            continue
         new_pe_list.append(pe)
 
     pes.pe_list = new_pe_list
     pes._process_pelist()
-    print("Writing %u PEs to file '%s'..." % (len(pes.pe_list), opts.output_file))
-    with open(opts.output_file, 'wb') as f:
-        f.write(pes.to_der())
+    write_pes(pes, opts.output_file)
 
 def do_remove_naa(pes: ProfileElementSequence, opts):
     if not opts.naa_type in NAAs:
@@ -142,9 +201,83 @@ def do_remove_naa(pes: ProfileElementSequence, opts):
     naa = NAAs[opts.naa_type]
     print("Removing NAAs of type '%s' from Sequence..." % opts.naa_type)
     pes.remove_naas_of_type(naa)
-    print("Writing %u PEs to file '%s'..." % (len(pes.pe_list), opts.output_file))
-    with open(opts.output_file, 'wb') as f:
-        f.write(pes.to_der())
+    write_pes(pes, opts.output_file)
+
+def info_apps(pes:ProfileElementSequence):
+    def show_member(dictionary:Optional[dict], member:str, indent:str="\t", mandatory:bool = False, limit:bool = False):
+        if dictionary is None:
+            return
+        value = dictionary.get(member, None)
+        if value is None and mandatory == True:
+            print("%s%s: (missing!)" % (indent, member))
+            return
+        elif value is None:
+            return
+
+        if limit and len(value) > 40:
+            print("%s%s: '%s...%s' (%u bytes)" % (indent, member, b2h(value[:20]), b2h(value[-20:]), len(value)))
+        else:
+            print("%s%s: '%s' (%u bytes)" % (indent, member, b2h(value), len(value)))
+
+    apps = pes.pe_by_type.get('application', [])
+    if len(apps) == 0:
+        print("No Application PE present!")
+        return;
+
+    for app_pe in enumerate(apps):
+        print("Application #%u:" % app_pe[0])
+        print("\tloadBlock:")
+        load_block = app_pe[1].decoded['loadBlock']
+        show_member(load_block, 'loadPackageAID', "\t\t", True)
+        show_member(load_block, 'securityDomainAID', "\t\t")
+        show_member(load_block, 'nonVolatileCodeLimitC6', "\t\t")
+        show_member(load_block, 'volatileDataLimitC7', "\t\t")
+        show_member(load_block, 'nonVolatileDataLimitC8', "\t\t")
+        show_member(load_block, 'hashValue', "\t\t")
+        show_member(load_block, 'loadBlockObject', "\t\t", True, True)
+        for inst in enumerate(app_pe[1].decoded.get('instanceList', [])):
+            print("\tinstanceList[%u]:" % inst[0])
+            show_member(inst[1], 'applicationLoadPackageAID', "\t\t", True)
+            if inst[1].get('applicationLoadPackageAID', None) != load_block.get('loadPackageAID', None):
+                print("\t\t(applicationLoadPackageAID should be the same as loadPackageAID!)")
+            show_member(inst[1], 'classAID', "\t\t", True)
+            show_member(inst[1], 'instanceAID', "\t\t", True)
+            show_member(inst[1], 'extraditeSecurityDomainAID', "\t\t")
+            show_member(inst[1], 'applicationPrivileges', "\t\t", True)
+            show_member(inst[1], 'lifeCycleState', "\t\t", True)
+            show_member(inst[1], 'applicationSpecificParametersC9', "\t\t", True)
+            sys_specific_pars = inst[1].get('systemSpecificParameters', None)
+            if sys_specific_pars:
+                print("\t\tsystemSpecificParameters:")
+                show_member(sys_specific_pars, 'volatileMemoryQuotaC7', "\t\t\t")
+                show_member(sys_specific_pars, 'nonVolatileMemoryQuotaC8', "\t\t\t")
+                show_member(sys_specific_pars, 'globalServiceParameters', "\t\t\t")
+                show_member(sys_specific_pars, 'implicitSelectionParameter', "\t\t\t")
+                show_member(sys_specific_pars, 'volatileReservedMemory', "\t\t\t")
+                show_member(sys_specific_pars, 'nonVolatileReservedMemory', "\t\t\t")
+                show_member(sys_specific_pars, 'ts102226SIMFileAccessToolkitParameter', "\t\t\t")
+                additional_cl_pars = inst.get('ts102226AdditionalContactlessParameters', None)
+                if additional_cl_pars:
+                    print("\t\t\tts102226AdditionalContactlessParameters:")
+                    show_member(additional_cl_pars, 'protocolParameterData', "\t\t\t\t")
+                show_member(sys_specific_pars, 'userInteractionContactlessParameters', "\t\t\t")
+                show_member(sys_specific_pars, 'cumulativeGrantedVolatileMemory', "\t\t\t")
+                show_member(sys_specific_pars, 'cumulativeGrantedNonVolatileMemory', "\t\t\t")
+            app_pars = inst[1].get('applicationParameters', None)
+            if app_pars:
+                print("\t\tapplicationParameters:")
+                show_member(app_pars, 'uiccToolkitApplicationSpecificParametersField', "\t\t\t")
+                show_member(app_pars, 'uiccAccessApplicationSpecificParametersField', "\t\t\t")
+                show_member(app_pars, 'uiccAdministrativeAccessApplicationSpecificParametersField', "\t\t\t")
+            ctrl_ref_tp = inst[1].get('controlReferenceTemplate', None)
+            if ctrl_ref_tp:
+                print("\t\tcontrolReferenceTemplate:")
+                show_member(ctrl_ref_tp, 'applicationProviderIdentifier', "\t\t\t", True)
+            process_data = inst[1].get('processData', None)
+            if process_data:
+                print("\t\tprocessData:")
+                for proc in process_data:
+                    print("\t\t\t" + b2h(proc))
 
 def do_info(pes: ProfileElementSequence, opts):
     def get_naa_count(pes: ProfileElementSequence) -> dict:
@@ -154,6 +287,10 @@ def do_info(pes: ProfileElementSequence, opts):
             ret[naa_type] = len(pes.pes_by_naa[naa_type])
         return ret
 
+    if opts.apps:
+        info_apps(pes)
+        return;
+
     pe_hdr_dec = pes.pe_by_type['header'][0].decoded
     print()
     print("SAIP Profile Version: %u.%u" % (pe_hdr_dec['major-version'], pe_hdr_dec['minor-version']))
@@ -213,16 +350,98 @@ def do_extract_apps(pes:ProfileElementSequence, opts):
     apps = pes.pe_by_type.get('application', [])
     for app_pe in apps:
         package_aid = b2h(app_pe.decoded['loadBlock']['loadPackageAID'])
-
         fname = os.path.join(opts.output_dir, '%s-%s.%s' % (pes.iccid, package_aid, opts.format))
-        load_block_obj = app_pe.decoded['loadBlock']['loadBlockObject']
         print("Writing Load Package AID: %s to file %s" % (package_aid, fname))
-        if opts.format == 'ijc':
-            with open(fname, 'wb') as f:
-                f.write(load_block_obj)
+        app_pe.to_file(fname)
+
+def do_add_app(pes:ProfileElementSequence, opts):
+    print("Applying applet file: '%s'..." % opts.applet_file)
+    app_pe = ProfileElementApplication.from_file(opts.applet_file,
+                                                 opts.aid,
+                                                 opts.sd_aid,
+                                                 opts.non_volatile_code_limit,
+                                                 opts.volatile_data_limit,
+                                                 opts.non_volatile_data_limit,
+                                                 opts.hash_value)
+
+    security_domain = pes.pe_by_type.get('securityDomain', [])
+    if len(security_domain) == 0:
+        print("profile package does not contain a securityDomain, please add a securityDomain PE first!")
+    elif len(security_domain) > 1:
+        print("adding an application PE to profiles with multiple securityDomain is not supported yet!")
+    else:
+        pes.insert_after_pe(security_domain[0], app_pe)
+        print("application PE inserted into PE Sequence after securityDomain PE AID: %s" %
+              b2h(security_domain[0].decoded['instance']['instanceAID']))
+        write_pes(pes, opts.output_file)
+
+def do_remove_app(pes:ProfileElementSequence, opts):
+    apps = pes.pe_by_type.get('application', [])
+    for app_pe in apps:
+        package_aid = b2h(app_pe.decoded['loadBlock']['loadPackageAID'])
+        if opts.aid == package_aid:
+            identification = app_pe.identification
+            opts_remove_pe = argparse.Namespace()
+            opts_remove_pe.identification = [app_pe.identification]
+            opts_remove_pe.type = []
+            opts_remove_pe.output_file = opts.output_file
+            print("Found Load Package AID: %s, removing related PE (id=%u) from Sequence..." %
+                  (package_aid, identification))
+            do_remove_pe(pes, opts_remove_pe)
+            return
+    print("Load Package AID: %s not found in PE Sequence" % opts.aid)
+
+def do_add_app_inst(pes:ProfileElementSequence, opts):
+    apps = pes.pe_by_type.get('application', [])
+    for app_pe in apps:
+        package_aid = b2h(app_pe.decoded['loadBlock']['loadPackageAID'])
+        if opts.aid == package_aid:
+            print("Found Load Package AID: %s, adding new instance AID: %s to Application PE..." %
+                  (opts.aid, opts.inst_aid))
+            app_pe.add_instance(opts.aid,
+                                opts.class_aid,
+                                opts.inst_aid,
+                                opts.app_privileges,
+                                opts.app_spec_pars,
+                                opts.uicc_toolkit_app_spec_pars,
+                                opts.uicc_access_app_spec_pars,
+                                opts.uicc_adm_access_app_spec_pars,
+                                opts.volatile_memory_quota,
+                                opts.non_volatile_memory_quota,
+                                opts.process_data)
+            write_pes(pes, opts.output_file)
+            return
+    print("Load Package AID: %s not found in PE Sequence" % opts.aid)
+
+def do_remove_app_inst(pes:ProfileElementSequence, opts):
+    apps = pes.pe_by_type.get('application', [])
+    for app_pe in apps:
+        if opts.aid == b2h(app_pe.decoded['loadBlock']['loadPackageAID']):
+            print("Found Load Package AID: %s, removing instance AID: %s from Application PE..." %
+                  (opts.aid, opts.inst_aid))
+            app_pe.remove_instance(opts.inst_aid)
+            write_pes(pes, opts.output_file)
+            return
+    print("Load Package AID: %s not found in PE Sequence" % opts.aid)
+
+def do_edit_mand_srv_list(pes: ProfileElementSequence, opts):
+    header = pes.pe_by_type.get('header', [])[0]
+
+    for s in opts.add_flag:
+        print("Adding service '%s' to mandatory services list..." % s)
+        header.mandatory_service_add(s)
+    for s in opts.remove_flag:
+        if s in header.decoded['eUICC-Mandatory-services'].keys():
+            print("Removing service '%s' from mandatory services list..." % s)
+            header.mandatory_service_remove(s)
         else:
-            with io.BytesIO(load_block_obj) as f, zipfile.ZipFile(fname, 'w') as z:
-                javacard.ijc_to_cap(f, z, package_aid)
+            print("Service '%s' not present in mandatory services list, cannot remove!" % s)
+
+    print("The following services are now set mandatory:")
+    for s in header.decoded['eUICC-Mandatory-services'].keys():
+        print("\t%s" % s)
+
+    write_pes(pes, opts.output_file)
 
 def do_tree(pes:ProfileElementSequence, opts):
     pes.mf.print_tree()
@@ -246,6 +465,8 @@ if __name__ == '__main__':
         do_dump(pes, opts)
     elif opts.command == 'check':
         do_check(pes, opts)
+    elif opts.command == 'extract-pe':
+        do_extract_pe(pes, opts)
     elif opts.command == 'remove-pe':
         do_remove_pe(pes, opts)
     elif opts.command == 'remove-naa':
@@ -254,5 +475,15 @@ if __name__ == '__main__':
         do_info(pes, opts)
     elif opts.command == 'extract-apps':
         do_extract_apps(pes, opts)
+    elif opts.command == 'add-app':
+        do_add_app(pes, opts)
+    elif opts.command == 'remove-app':
+        do_remove_app(pes, opts)
+    elif opts.command == 'add-app-inst':
+        do_add_app_inst(pes, opts)
+    elif opts.command == 'remove-app-inst':
+        do_remove_app_inst(pes, opts)
+    elif opts.command == 'edit-mand-srv-list':
+        do_edit_mand_srv_list(pes, opts)
     elif opts.command == 'tree':
         do_tree(pes, opts)

contrib/saip-tool_example_add-app.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# This is an example script to illustrate how to add JAVA card applets to an existing eUICC profile package.
+
+PYSIMPATH=../
+INPATH=../smdpp-data/upp/TS48V1-A-UNIQUE.der
+OUTPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello.der
+APPPATH=./HelloSTK_09122024.cap
+
+# Download example applet (see also https://gitea.osmocom.org/sim-card/hello-stk):
+if ! [ -f $APPPATH ]; then
+    wget https://osmocom.org/attachments/download/8931/HelloSTK_09122024.cap
+fi
+
+# Step #1: Create the application PE and load the ijc contents from the .cap file:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $INPATH add-app \
+		      --output-file $OUTPATH --applet-file $APPPATH --aid 'D07002CA44'
+
+# Step #2: Create the application instance inside the application PE created in step #1:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $OUTPATH add-app-inst --output-file $OUTPATH \
+		      --aid 'D07002CA44' \
+		      --class-aid 'D07002CA44900101' \
+		      --inst-aid 'D07002CA44900101' \
+		      --app-privileges '00' \
+		      --app-spec-pars '00' \
+		      --uicc-toolkit-app-spec-pars '01001505000000000000000000000000'
+
+# Display the contents of the resulting application PE:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $OUTPATH info --apps
+
+# For an explaination of --uicc-toolkit-app-spec-pars, see:
+# ETSI TS 102 226, section 8.2.1.3.2.2.1

contrib/saip-tool_example_extract-apps.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# This is an example script to illustrate how to extract JAVA card applets from an existing eUICC profile package.
+
+PYSIMPATH=../
+INPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello.der
+OUTPATH=./
+
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $INPATH extract-apps --output-dir ./ --format ijc

contrib/saip-tool_example_remove-app-inst.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# This is an example script to illustrate how to remove a JAVA card applet instance from an application PE inside an
+# existing eUICC profile package.
+
+PYSIMPATH=../
+INPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello.der
+OUTPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello-no-inst.der
+
+# Remove application PE entirely
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $INPATH remove-app-inst \
+		      --output-file $OUTPATH --aid 'd07002ca44' --inst-aid 'd07002ca44900101'
+
+# Display the contents of the resulting application PE:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $OUTPATH info --apps

contrib/saip-tool_example_remove-app.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# This is an example script to illustrate how to remove a JAVA card applet from an existing eUICC profile package.
+
+PYSIMPATH=../
+INPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-hello.der
+OUTPATH=../smdpp-data/upp/TS48V1-A-UNIQUE-no-hello.der
+
+# Remove application PE entirely
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $INPATH remove-app \
+		      --output-file $OUTPATH --aid 'D07002CA44'
+
+# Display the contents of the resulting application PE:
+PYTHONPATH=$PYSIMPATH python3 $PYSIMPATH/contrib/saip-tool.py $OUTPATH info --apps

contrib/suci-keytool.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+# small utility program to deal with 5G SUCI key material, at least for the ECIES Protection Scheme
+# Profile A (curve25519) and B (secp256r1)
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+# SPDX-License-Identifier: GPL-2.0+
+
+import argparse
+
+from osmocom.utils import b2h
+from Cryptodome.PublicKey import ECC
+# if used with pycryptodome < v3.21.0 you will get the following error when using curve25519:
+# "Cryptodome.PublicKey.ECC.UnsupportedEccFeature: Unsupported ECC purpose (OID: 1.3.101.110)"
+
+def gen_key(opts):
+    # FIXME: avoid overwriting key files
+    mykey = ECC.generate(curve=opts.curve)
+    data = mykey.export_key(format='PEM')
+    with open(opts.key_file, "wt") as f:
+        f.write(data)
+
+def dump_pkey(opts):
+
+    #with open("curve25519-1.key", "r") as f:
+
+    with open(opts.key_file, "r") as f:
+        data = f.read()
+        mykey = ECC.import_key(data)
+
+        der = mykey.public_key().export_key(format='raw', compress=opts.compressed)
+        print(b2h(der))
+
+arg_parser = argparse.ArgumentParser(description="""Generate or export SUCI keys for 5G SA networks""")
+arg_parser.add_argument('--key-file', help='The key file to use', required=True)
+
+subparsers = arg_parser.add_subparsers(dest='command', help="The command to perform", required=True)
+
+parser_genkey = subparsers.add_parser('generate-key', help='Generate a new key pair')
+parser_genkey.add_argument('--curve', help='The ECC curve to use', choices=['secp256r1','curve25519'], required=True)
+
+parser_dump_pkey = subparsers.add_parser('dump-pub-key', help='Dump the public key')
+parser_dump_pkey.add_argument('--compressed', help='Use point compression', action='store_true')
+
+if __name__ == '__main__':
+
+    opts = arg_parser.parse_args()
+
+    if opts.command == 'generate-key':
+        gen_key(opts)
+    elif opts.command == 'dump-pub-key':
+        dump_pkey(opts)

docs/cap-tutorial.rst

@@ -0,0 +1,103 @@
+
+Guide: Installing JAVA-card applets
+===================================
+
+Almost all modern-day UICC cards have some form of JAVA-card / Sim-Toolkit support, which allows the installation
+of customer specific JAVA-card applets. The installation of JAVA-card applets is usually done via the standardized
+GlobalPlatform (GPC_SPE_034) ISD (Issuer Security Domain) application interface during the card provisioning process.
+(it is also possible to load JAVA-card applets in field via OTA-SMS, but that is beyond the scope of this guide). In
+this guide we will go through the individual steps that are required to load JAVA-card applet onto an UICC card.
+
+
+Preparation
+~~~~~~~~~~~
+
+In this example we will install the CAP file HelloSTK_09122024.cap [1] on an sysmoISIM-SJA2 card. Since the interface
+is standardized, the exact card model does not matter.
+
+The example applet makes use of the STK (Sim-Toolkit), so we must supply STK installation parameters. Those
+parameters are supplied in the form of a hexstring and should be provided by the applet manufacturer. The available
+parameters and their exact encoding is specified in ETSI TS 102 226, section 8.2.1.3.2.1. The installation of
+HelloSTK_09122024.cap [1], will require the following STK installation parameters: "010001001505000000000000000000000000"
+
+During the installation, we also have to set a memory quota for the volatile and for the non volatile card memory.
+Those values also should be provided by the applet manufacturer. In this example, we will allow 255 bytes of volatile
+memory and 255 bytes of non volatile memory to be consumed by the applet.
+
+To install JAVA-card applets, one must be in the possession of the key material belonging to the card. The keys are
+usually provided by the card manufacturer. The following example will use the following keyset:
+
++---------+----------------------------------+
+| Keyname | Keyvalue                         |
++=========+==================================+
+| DEK/KIK | 5524F4BECFE96FB63FC29D6BAAC6058B |
++---------+----------------------------------+
+| ENC/KIC | 542C37A6043679F2F9F71116418B1CD5 |
++---------+----------------------------------+
+| MAC/KID | 34F11BAC8E5390B57F4E601372339E3C |
++---------+----------------------------------+
+
+[1] https://osmocom.org/projects/cellular-infrastructure/wiki/HelloSTK
+
+
+Applet Installation
+~~~~~~~~~~~~~~~~~~~
+
+To prepare the installation, a secure channel to the ISD must be established first:
+
+::
+
+    pySIM-shell (00:MF)> select ADF.ISD
+    {
+        "application_id": "a000000003000000",
+	"proprietary_data": {
+	    "maximum_length_of_data_field_in_command_message": 255
+        }
+    }
+    pySIM-shell (00:MF/ADF.ISD)> establish_scp02 --key-dek 5524F4BECFE96FB63FC29D6BAAC6058B --key-enc 542C37A6043679F2F9F71116418B1CD5 --key-mac 34F11BAC8E5390B57F4E601372339E3C --security-level 1
+    Successfully established a SCP02[01] secure channel
+
+.. warning:: In case you get an "EXCEPTION of type 'ValueError' occurred with message: card cryptogram doesn't match" error message, it is very likely that there is a problem with the key material. The card may lock the ISD access after a certain amount of failed tries. Carefully check the key material any try again.
+
+
+When the secure channel is established, we are ready to install the applet. The installation normally is a multi step
+procedure, where the loading of an executable load file is announced first, then loaded and then installed in a final
+step. The pySim-shell command ``install_cap`` automatically takes care of those three steps.
+
+::
+
+    pySIM-shell (SCP02[01]:00:MF/ADF.ISD)> install_cap /home/user/HelloSTK_09122024.cap --install-parameters-non-volatile-memory-quota 255 --install-parameters-volatile-memory-quota 255 --install-parameters-stk 010001001505000000000000000000000000
+    loading cap file: /home/user/HelloSTK_09122024.cap ...
+    parameters:
+     security-domain-aid: a000000003000000
+     load-file: 569 bytes
+     load-file-aid: d07002ca44
+     module-aid: d07002ca44900101
+     application-aid: d07002ca44900101
+     install-parameters: c900ef1cc80200ffc70200ffca12010001001505000000000000000000000000
+    step #1: install for load...
+    step #2: load...
+    Loaded a total of 573 bytes in 3 blocks. Don't forget install_for_install (and make selectable) now!
+    step #3: install_for_install (and make selectable)...
+    done.
+
+The applet is now installed on the card. We can now quit pySim-shell and remove the card from the reader and test the
+applet in a mobile phone. There should be a new STK application with one menu entry shown, that will greet the user
+when pressed.
+
+
+Applet Removal
+~~~~~~~~~~~~~~
+
+To remove the applet, we must establish a secure channel to the ISD (see above). Then we can delete the applet using the
+``delete_card_content`` command.
+
+::
+
+    pySIM-shell (SCP02[01]:00:MF/ADF.ISD)> delete_card_content D07002CA44 --delete-related-objects
+
+The parameter "D07002CA44" is the load-file-AID of the applet. The load-file-AID is encoded in the .cap file and also
+displayed during the installation process. It is also important to note that when the applet is installed, it cannot
+be installed (under the same AID) again until it is removed.
+
+

docs/card-key-provider.rst

@@ -1,6 +1,5 @@
-
 Retrieving card-individual keys via CardKeyProvider
-==================================================
+===================================================
 
 When working with a batch of cards, or more than one card in general, it
 is a lot of effort to manually retrieve the card-specific PIN (like

docs/index.rst

@@ -41,8 +41,13 @@ pySim consists of several parts:
    shell
    trace
    legacy
+   smpp2sim
    library
+   library-esim
    osmo-smdpp
+   sim-rest
+   suci-keytool
+   saip-tool
 
 
 Indices and tables

docs/library-esim.rst

@@ -0,0 +1,95 @@
+pySim eSIM libraries
+====================
+
+The pySim eSIM libraries implement a variety of functionality related to the GSMA eSIM universe,
+including the various interfaces of SGP.21 + SGP.22, as well as Interoperable Profile decioding,
+validation, personalization and encoding.
+
+.. automodule:: pySim.esim
+   :members:
+
+
+GSMA SGP.21/22 Remote SIM Provisioning (RSP) - High Level
+---------------------------------------------------------
+
+pySim.esim.rsp
+~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.rsp
+   :members:
+
+pySim.esim.es2p
+~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.es2p
+   :members:
+
+
+pySim.esim.es8p
+~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.es8p
+   :members:
+
+
+pySim.esim.es9p
+~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.es9p
+   :members:
+
+
+GSMA SGP.21/22 Remote SIM Provisioning (RSP) - Low Level
+--------------------------------------------------------
+
+pySim.esim.bsp
+~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.bsp
+   :members:
+
+pySim.esim.http_json_api
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.http_json_api
+   :members:
+
+pySim.esim.x509_cert
+~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.x509_cert
+   :members:
+
+SIMalliance / TCA Interoperable Profile
+---------------------------------------
+
+
+pySim.esim.saip
+~~~~~~~~~~~~~~~
+.. automodule:: pySim.esim.saip
+   :members:
+
+
+pySim.esim.saip.oid
+~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.saip.oid
+   :members:
+
+pySim.esim.saip.personalization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.saip.personalization
+   :members:
+
+pySim.esim.saip.templates
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.saip.templates
+   :members:
+
+pySim.esim.saip.validation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: pySim.esim.saip.validation
+   :members:

docs/osmo-smdpp.rst

@@ -82,7 +82,7 @@ software.
 supplementary files
 ~~~~~~~~~~~~~~~~~~~
 
-The `smdpp-data/certs`` directory contains the DPtls, DPauth and DPpb as well as CI certificates
+The `smdpp-data/certs` directory contains the DPtls, DPauth and DPpb as well as CI certificates
 used; they are copied from GSMA SGP.26 v2.  You can of course replace them with custom certificates
 if you're operating eSIM with a *private root CA*.
 
@@ -92,10 +92,20 @@ The `smdpp-data/upp` directory contains the UPP (Unprotected Profile Package) us
 commandline options
 ~~~~~~~~~~~~~~~~~~~
 
-osmo-smdpp currently doesn't have any configuration file or command line options.  You just run it,
-and it will bind its plain-HTTP ES9+ interface to local TCP port 8000.
+Typically, you just run it without any arguments, and it will bind its plain-HTTP ES9+ interface to
+`localhost` TCP port 8000.
 
+osmo-smdpp currently doesn't have any configuration file.
 
+There are command line options for binding:
+
+Bind the HTTP ES9+ to a port other than 8000::
+
+  ./osmo-smdpp.py -p 8001
+
+Bind the HTTP ES9+ to a different local interface::
+
+  ./osmo-smdpp.py -H 127.0.0.1
 
 DNS setup for your LPA
 ~~~~~~~~~~~~~~~~~~~~~~

docs/saip-tool.rst

@@ -0,0 +1,137 @@
+saip-tool
+=========
+
+eSIM profiles are stored as a sequence of profile element (PE) objects in an ASN.1 DER encoded binary file. To inspect,
+verify or make changes to those files, the `saip-tool.py` utility can be used.
+
+NOTE: The file format, eSIM SAIP (SimAlliance Interoperable Profile) is specified in `TCA eUICC Profile Package:
+Interoperable Format Technical Specification`
+
+
+Profile Package Examples
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+pySim ships with a set of TS48 profile package examples. Those examples can be found in `pysim/smdpp-data/upp`. The
+files can be used as input for `saip-tool.py`. (see also GSMA TS.48 - Generic eUICC Test Profile for Device Testing)
+
+See also: https://github.com/GSMATerminals/Generic-eUICC-Test-Profile-for-Device-Testing-Public
+
+JAVA card applets
+~~~~~~~~~~~~~~~~~
+
+The `saip-tool.py` can also be used to manage JAVA-card applets (Application PE) inside a profile package. The user has
+the option to add, remove and inspect applications and their instances. In the following we will discuss a few JAVA-card
+related use-cases of `saip-tool.py`
+
+NOTE: see also `contrib` folder for script examples (`saip-tool_example_*.sh`)
+
+Inserting applications
+----------------------
+
+An application is usually inserted in two steps. In the first step, the application PE is created and populated with
+the executable code from a provided `.cap` or `.ijc` file. The user also has to pick a suitable load block AID.
+
+The application instance, which exists inside the application PE, is created in a second step. Here the user must
+reference the load block AID and pick, among other application related parameters, a suitable class and instance AID.
+
+Example: Adding a JAVA-card applet to an existing profile package
+::
+
+  # Step #1: Create the application PE and load the ijc contents from the .cap file:
+  $ ./contrib/saip-tool.py upp.der add-app --output-file upp_with_app.der --applet-file app.cap --aid '1122334455'
+  Read 28 PEs from file 'upp.der'
+  Applying applet file: 'app.cap'...
+  application PE inserted into PE Sequence after securityDomain PE AID: a000000151000000
+  Writing 29 PEs to file 'upp_with_app.der'...
+
+  # Step #2: Create the application instance inside the application PE created in step #1:
+  $ ./contrib/saip-tool.py upp_with_app.der add-app-inst --output-file upp_with_app_and_instance.der \
+          --aid '1122334455' \
+          --class-aid '112233445501' \
+          --inst-aid '112233445501' \
+          --app-privileges '00' \
+          --app-spec-pars '00' \
+          --uicc-toolkit-app-spec-pars '01001505000000000000000000000000'
+  Read 29 PEs from file 'upp_with_app.der'
+  Found Load Package AID: 1122334455, adding new instance AID: 112233445501 to Application PE...
+  Writing 29 PEs to file 'upp_with_app_and_instance.der'...
+
+NOTE: The parameters of the sub-commands `add-app` and `add-app-inst` are application specific. It is up to the application
+developer to pick parameters that suit the application correctly. For an exact command reference see section
+`saip-tool syntax`. For parameter details see `TCA eUICC Profile Package: Interoperable Format Technical Specification`,
+section 8.7 and ETSI TS 102 226, section 8.2.1.3.2
+
+
+Inspecting applications
+-----------------------
+
+To inspect the application PE contents of an existing profile package, sub-command `info` with parameter '--apps' can
+be used. This command lists out all application and their parameters in detail. This allows an application developer
+to check if the applet insertaion was carried out as expected.
+
+Example: Listing applications and their parameters
+::
+
+  $ ./contrib/saip-tool.py upp_with_app_and_instance.der info --apps
+  Read 29 PEs from file 'upp_with_app_and_instance.der'
+  Application #0:
+          loadBlock:
+                  loadPackageAID: '1122334455' (5 bytes)
+                  loadBlockObject: '01000fdecaffed010204000105d07002ca440200...681080056810a00633b44104b431066800a10231' (569 bytes)
+          instanceList[0]:
+                  applicationLoadPackageAID: '1122334455' (5 bytes)
+                  classAID: '112233445501' (8 bytes)
+                  instanceAID: '112233445501' (8 bytes)
+                  applicationPrivileges: '00' (1 bytes)
+                  lifeCycleState: '07' (1 bytes)
+                  applicationSpecificParametersC9: '00' (1 bytes)
+                  applicationParameters:
+                          uiccToolkitApplicationSpecificParametersField: '01001505000000000000000000000000' (16 bytes)
+
+In case further analysis with external tools or transfer of applications from one profile package to another is
+necessary, the executable code in the `loadBlockObject` field can be extracted to an `.ijc` or an `.cap` file.
+
+Example: Extracting applications from a profile package
+::
+
+  $ ./contrib/saip-tool.py upp_with_app_and_instance.der extract-apps --output-dir ./apps --format ijc
+  Read 29 PEs from file 'upp_with_app_and_instance.der'
+  Writing Load Package AID: 1122334455 to file ./apps/8949449999999990023f-1122334455.ijc
+
+
+Removing applications
+---------------------
+
+An application PE can be removed using sub-command `remove-app`. The user passes the load package AID as parameter. Then
+`saip-tool.py` will search for the related application PE and delete it from the PE sequence.
+
+Example: Remove an application from a profile package
+::
+
+  $ ./contrib/saip-tool.py upp_with_app_and_instance.der remove-app --output-file upp_without_app.der --aid '1122334455'
+  Read 29 PEs from file 'upp_with_app_and_instance.der'
+  Found Load Package AID: 1122334455, removing related PE (id=23) from Sequence...
+  Removing PE application (id=23) from Sequence...
+  Writing 28 PEs to file 'upp_without_app.der'...
+
+In some cases it is useful to remove only an instance from an existing application PE. This may be the case when the
+an application developer wants to modify parameters of an application by removing and re-adding the instance. The
+operation basically rolls the state back to step 1 explained in section :ref:`Inserting applications`
+
+Example: Remove an application instance from an application PE
+::
+
+  $ ./contrib/saip-tool.py upp_with_app_and_instance.der remove-app-inst --output-file upp_without_app.der --aid '1122334455' --inst-aid '112233445501'
+  Read 29 PEs from file 'upp_with_app_and_instance.der'
+  Found Load Package AID: 1122334455, removing instance AID: 112233445501 from Application PE...
+  Removing instance from Application PE...
+  Writing 29 PEs to file 'upp_with_app.der'...
+
+
+saip-tool syntax
+~~~~~~~~~~~~~~~~
+
+.. argparse::
+   :module: contrib.saip-tool
+   :func: parser
+   :prog: contrib/saip-tool.py

docs/shell.rst

@@ -1,4 +1,4 @@
-pySim-shell
+pySim-shell
 ===========
 
 pySim-shell is an interactive command line shell for all kind of interactions with SIM cards,
@@ -67,6 +67,7 @@ Usage Examples
    :caption: Tutorials for pySIM-shell:
 
    suci-tutorial
+   cap-tutorial
 
 
 Advanced Topics
@@ -1005,6 +1006,24 @@ ARA-M applet.  Use it with caution, there is no undo.  Any rules later
 intended must be manually inserted again using :ref:`aram_store_ref_ar_do`
 
 
+aram_lock
+~~~~~~~~~
+This command allows to lock the access to the STORE DATA command. This renders
+all access rules stored within the ARA-M applet effectively read-only. The lock
+can only be removed via a secure channel to the security domain and is therefore
+suitable to prevent unauthorized changes to ARA-M rules.
+
+Removal of the lock:
+::
+
+  pySIM-shell (SCP02[01]:00:MF/ADF.ISD)> install_for_personalization A00000015141434C00
+  pySIM-shell (SCP02[01]:00:MF/ADF.ISD)> apdu --expect-sw 9000 80E2900001A2
+
+NOTE: ARA-M Locking is a proprietary feature that is specific to sysmocom's
+fork of Bertrand Martel's ARA-M implementation. ARA-M Locking is supported in
+newer (2025) applet versions from v0.1.0 onward.
+
+
 GlobalPlatform commands
 -----------------------
 
@@ -1047,6 +1066,18 @@ delete_key
    :module: pySim.global_platform
    :func: ADF_SD.AddlShellCommands.del_key_parser
 
+load
+~~~~
+.. argparse::
+   :module: pySim.global_platform
+   :func: ADF_SD.AddlShellCommands.load_parser
+
+install_cap
+~~~~~~~~~~~
+.. argparse::
+   :module: pySim.global_platform
+   :func: ADF_SD.AddlShellCommands.install_cap_parser
+
 install_for_personalization
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. argparse::
@@ -1059,6 +1090,12 @@ install_for_install
    :module: pySim.global_platform
    :func: ADF_SD.AddlShellCommands.inst_inst_parser
 
+install_for_load
+~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.global_platform
+   :func: ADF_SD.AddlShellCommands.inst_load_parser
+
 delete_card_content
 ~~~~~~~~~~~~~~~~~~~
 .. argparse::
@@ -1118,7 +1155,7 @@ es10x_store_data
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.es10x_store_data_parser
+   :func: CardApplicationISDR.AddlShellCommands.es10x_store_data_parser
 
 get_euicc_configured_addresses
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1137,7 +1174,7 @@ set_default_dp_address
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.set_def_dp_addr_parser
+   :func: CardApplicationISDR.AddlShellCommands.set_def_dp_addr_parser
 
 get_euicc_challenge
 ~~~~~~~~~~~~~~~~~~~
@@ -1280,7 +1317,7 @@ remove_notification_from_list
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.rem_notif_parser
+   :func: CardApplicationISDR.AddlShellCommands.rem_notif_parser
 
 Example::
 
@@ -1329,7 +1366,7 @@ enable_profile
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.en_prof_parser
+   :func: CardApplicationISDR.AddlShellCommands.en_prof_parser
 
 Example (successful)::
 
@@ -1351,7 +1388,7 @@ disable_profile
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.dis_prof_parser
+   :func: CardApplicationISDR.AddlShellCommands.dis_prof_parser
 
 Example (successful)::
 
@@ -1365,7 +1402,7 @@ delete_profile
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.del_prof_parser
+   :func: CardApplicationISDR.AddlShellCommands.del_prof_parser
 
 Example::
 
@@ -1374,6 +1411,13 @@ Example::
       "delete_result": "ok"
   }
 
+euicc_memory_reset
+~~~~~~~~~~~~~~~~~~
+
+.. argparse::
+   :module: pySim.euicc
+   :func: CardApplicationISDR.AddlShellCommands.mem_res_parser
+
 
 get_eid
 ~~~~~~~
@@ -1392,7 +1436,7 @@ set_nickname
 
 .. argparse::
    :module: pySim.euicc
-   :func: ADF_ISDR.AddlShellCommands.set_nickname_parser
+   :func: CardApplicationISDR.AddlShellCommands.set_nickname_parser
 
 Example::
 

docs/sim-rest.rst

@@ -0,0 +1,118 @@
+sim-rest-server
+===============
+
+Sometimes there are use cases where a [remote] application will need
+access to a USIM for authentication purposes.  This is, for example, in
+case an IMS test client needs to perform USIM based authentication
+against an IMS core.
+
+The pysim repository contains two programs: `sim-rest-server.py` and
+`sim-rest-client.py` that implement a simple approach to achieve the
+above:
+
+`sim-rest-server.py` speaks to a [usually local] USIM via the PC/SC
+API and provides a high-level REST API towards [local or remote]
+applications that wish to perform UMTS AKA using the USIM.
+
+`sim-rest-client.py` implements a small example client program to
+illustrate how the REST API provided by `sim-rest-server.py` can be
+used.
+
+REST API Calls
+--------------
+
+POST /sim-auth-api/v1/slot/SLOT_NR
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+where SLOT_NR is the integer-encoded slot number (corresponds to PC/SC
+reader number).  When using a single sysmoOCTSIM board, this is in the range of 0..7
+
+Example: `/sim-auth-api/v1/slot/0` for the first slot.
+
+Request Body
+############
+
+The request body is a JSON document, comprising of
+    1. the RAND and AUTN parameters as hex-encoded string
+    2. the application against which to authenticate (USIM, ISIM)
+
+Example:
+::
+
+        {
+            "rand": "bb685a4b2fc4d697b9d6a129dd09a091",
+            "autn": "eea7906f8210000004faf4a7df279b56"
+        }
+
+HTTP Status Codes
+#################
+
+HTTP status codes are used to represent errors within the REST server
+and the SIM reader hardware.  They are not used to communicate protocol
+level errors reported by the SIM Card.  An unsuccessful authentication
+will hence have a `200 OK` HTTP Status code and then encode the SIM
+specific error information in the Response Body.
+
+======  =========== ================================
+Status  Code        Description
+------  ----------- --------------------------------
+200     OK          Successful execution
+400     Bad Request Request body is malformed
+404     Not Found   Specified SIM Slot doesn't exist
+410     Gone        No SIM card inserted in slot
+======  =========== ================================
+
+Response Body
+#############
+
+The response body is a JSON document, either
+
+#. a successful outcome; encoding RES, CK, IK as hex-encoded string
+#. a sync failure; encoding AUTS as hex-encoded string
+#. errors
+   #. authentication error (incorrect MAC)
+   #. authentication error (security context not supported)
+   #. key freshness failure
+   #. unspecified card error
+
+Example (succcess):
+::
+
+        {
+           "successful_3g_authentication": {
+                "res": "b15379540ec93985",
+                "ck": "713fde72c28cbd282a4cd4565f3d6381",
+                "ik": "2e641727c95781f1020d319a0594f31a",
+                "kc": "771a2c995172ac42"
+            }
+        }
+
+Example (re-sync case):
+::
+
+        {
+            "synchronisation_failure": {
+                "auts": "dc2a591fe072c92d7c46ecfe97e5"
+            }
+        }
+
+Concrete example using the included sysmoISIM-SJA2
+--------------------------------------------------
+
+This was tested using SIMs ending in IMSI numbers 45890...45899
+
+The following command were executed successfully:
+
+Slot 0
+::
+
+        $ /usr/local/src/pysim/contrib/sim-rest-client.py -c 1 -n 0 -k 841EAD87BC9D974ECA1C167409357601 -o 3211CACDD64F51C3FD3013ECD9A582A0
+        -> {'rand': 'fb195c7873b20affa278887920b9dd57', 'autn': 'd420895a6aa2000089cd016f8d8ae67c'}
+        <- {'successful_3g_authentication': {'res': '131004db2ff1ce8e', 'ck': 'd42eb5aa085307903271b2422b698bad', 'ik': '485f81e6fd957fe3cad374adf12fe1ca', 'kc': '64d3f2a32f801214'}}
+
+Slot 1
+::
+
+        $ /usr/local/src/pysim/contrib/sim-rest-client.py -c 1 -n 1 -k 5C2CE9633FF9B502B519A4EACD16D9DF -o 9834D619E71A02CD76F00CC7AA34FB32
+        -> {'rand': '433dc5553db95588f1d8b93870930b66', 'autn': '126bafdcbe9e00000026a208da61075d'}
+        <- {'successful_3g_authentication': {'res': '026d7ac42d379207', 'ck': '83a90ba331f47a95c27a550b174c4a1f', 'ik': '31e1d10329ffaf0ca1684a1bf0b0a14a', 'kc': 'd15ac5b0fff73ecc'}}

docs/smpp2sim.rst

@@ -0,0 +1,57 @@
+pySim-smpp2sim
+==============
+
+This is a program to emulate the entire communication path SMSC-CN-RAN-ME
+that is usually between an OTA backend and the SIM card.  This allows
+to play with SIM OTA technology without using a mobile network or even
+a mobile phone.
+
+An external application can act as SMPP ESME and must encode (and
+encrypt/sign) the OTA SMS and submit them via SMPP to this program, just
+like it would submit it normally to a SMSC (SMS Service Centre).  The
+program then re-formats the SMPP-SUBMIT into a SMS DELIVER TPDU and
+passes it via an ENVELOPE APDU to the SIM card that is locally inserted
+into a smart card reader.
+
+The path from SIM to external OTA application works the opposite way.
+
+The default SMPP system_id is `test`.  Likewise, the default SMPP
+password is `test`
+
+Running pySim-smpp2sim
+----------------------
+
+The command accepts the same command line arguments for smart card interface device selection as pySim-shell,
+as well as a few SMPP specific arguments:
+
+.. argparse::
+   :module: pySim-smpp2sim
+   :func: option_parser
+   :prog: pySim-smpp2sim.py
+
+
+Example execution with sample output
+------------------------------------
+
+So for a simple system with a single PC/SC device, you would typically use something like
+`./pySim-smpp2sim.py -p0` to start the program.  You will see output like this at start-up
+::
+
+  Using reader PCSC[HID Global OMNIKEY 3x21 Smart Card Reader [OMNIKEY 3x21 Smart Card Reader] 00 00]
+  INFO     root: Binding Virtual SMSC to TCP Port 2775 at ::
+
+The application has hence bound to local TCP port 2775 and expects your SMS-sending applications to send their
+SMS there.  Once you do, you will see log output like below:
+::
+
+  WARNING  smpp.twisted.protocol: SMPP connection established from ::ffff:127.0.0.1 to port 2775
+  INFO     smpp.twisted.server: Added CommandId.bind_transceiver bind for 'test'. Active binds: CommandId.bind_transceiver: 1, CommandId.bind_transmitter: 0, CommandId.bind_receiver: 0. Max binds: 2
+  INFO     smpp.twisted.protocol: Bind request succeeded for test. 1 active binds
+
+And once your external program is sending SMS to the simulated SMSC, it will log something like
+::
+
+  INFO     root: SMS_DELIVER(MTI=0, MMS=False, LP=False, RP=False, UDHI=True, SRI=False, OA=AddressField(TON=international, NPI=unknown, 12), PID=7f, DCS=f6, SCTS=bytearray(b'"pR\x00\x00\x00\x00'), UDL=45, UD=b"\x02p\x00\x00(\x15\x16\x19\x12\x12\xb0\x00\x01'\xfa(\xa5\xba\xc6\x9d<^\x9d\xf2\xc7\x15]\xfd\xdeD\x9c\x82k#b\x15Ve0x{0\xe8\xbe]")
+  SMSPPDownload(DeviceIdentities({'source_dev_id': 'network', 'dest_dev_id': 'uicc'}),Address({'ton_npi': 0, 'call_number': '0123456'}),SMS_TPDU({'tpdu': '400290217ff6227052000000002d02700000281516191212b0000127fa28a5bac69d3c5e9df2c7155dfdde449c826b236215566530787b30e8be5d'}))
+  INFO     root: ENVELOPE: d147820283818604001032548b3b400290217ff6227052000000002d02700000281516191212b0000127fa28a5bac69d3c5e9df2c7155dfdde449c826b236215566530787b30e8be5d
+  INFO     root: SW 9000: 027100002412b000019a551bb7c28183652de0ace6170d0e563c5e949a3ba56747fe4c1dbbef16642c

docs/suci-keytool.rst

@@ -0,0 +1,58 @@
+suci-keytool
+============
+
+Subscriber concealment is an important feature of the 5G SA architecture: It avoids the many privacy
+issues associated with having a permanent identifier (SUPI, traditionally the IMSI) transmitted in plain text
+over the air interface.  Using SUCI solves this issue not just for the air interface; it even ensures the SUPI/IMSI
+is not known to the visited network (VPLMN) at all.
+
+In principle, the SUCI mechanism works by encrypting the SUPI by asymmetric (public key) cryptography:
+Only the HPLMN is in possession of the private key and hence can decrypt the SUCI to the SUPI, while
+each subscriber has the public key in order to encrypt their SUPI into the SUCI.  In reality, the
+details are more complex, as there are ephemeral keys and cryptographic MAC involved.
+
+In any case, in order to operate a SUCI-enabled 5G SA network, you will have to
+
+#. generate a ECC key pair of public + private key
+#. deploy the public key on your USIMs
+#. deploy the private key on your 5GC, specifically the UDM function
+
+pysim contains (int its `contrib` directory) a small utility program that can make it easy to generate
+such keys: `suci-keytool.py`
+
+Generating keys
+~~~~~~~~~~~~~~~
+
+Example: Generating a *secp256r1* ECC public key pair and storing it to `/tmp/suci.key`:
+::
+
+        $ ./contrib/suci-keytool.py --key-file /tmp/suci.key generate-key --curve secp256r1
+
+Dumping public keys
+~~~~~~~~~~~~~~~~~~~
+
+In order to store the key to SIM cards as part of `ADF.USIM/DF.5GS/EF.SUCI_Calc_Info`, you will need
+a hexadecimal representation of the public key.  You can achieve that using the `dump-pub-key` operation
+of suci-keytool:
+
+Example: Dumping the public key part from a previously generated key file:
+::
+
+        $ ./contrib/suci-keytool.py --key-file /tmp/suci.key dump-pub-key
+        0473152f32523725f5175d255da2bd909de97b1d06449a9277bc629fe42112f8643e6b69aa6dce6c86714ccbe6f2e0f4f4898d102e2b3f0c18ce26626f052539bb
+
+If you want the point-compressed representation, you can use the `--compressed` option:
+::
+
+        $ ./contrib/suci-keytool.py --key-file /tmp/suci.key dump-pub-key --compressed
+        0373152f32523725f5175d255da2bd909de97b1d06449a9277bc629fe42112f864
+
+
+
+suci-keytool syntax
+~~~~~~~~~~~~~~~~~~~
+
+.. argparse::
+   :module: contrib.suci-keytool
+   :func: arg_parser
+   :prog: contrib/suci-keytool.py

pySim-shell.py

@@ -155,6 +155,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         # When a card object and a runtime state is present, (re)equip pySim-shell with everything that is
         # needed to operate on cards.
         if self.card and self.rs:
+            self.rs.reset()
             self.lchan = self.rs.lchan[0]
             self._onchange_conserve_write(
                 'conserve_write', False, self.conserve_write)
@@ -218,18 +219,23 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
             self.cmd2.poutput("<- %s: %s" % (sw, resp))
 
     def update_prompt(self):
+        if self.rs and self.rs.adm_verified:
+            prompt_char = '#'
+        else:
+            prompt_char = '>'
+
         if self.lchan:
             path_str = self.lchan.selected_file.fully_qualified_path_str(not self.numeric_path)
             scp = self.lchan.scc.scp
             if scp:
-                self.prompt = 'pySIM-shell (%s:%02u:%s)> ' % (str(scp), self.lchan.lchan_nr, path_str)
+                self.prompt = 'pySIM-shell (%s:%02u:%s)%c ' % (str(scp), self.lchan.lchan_nr, path_str, prompt_char)
             else:
-                self.prompt = 'pySIM-shell (%02u:%s)> ' % (self.lchan.lchan_nr, path_str)
+                self.prompt = 'pySIM-shell (%02u:%s)%c ' % (self.lchan.lchan_nr, path_str, prompt_char)
         else:
             if self.card:
-                self.prompt = 'pySIM-shell (no card profile)> '
+                self.prompt = 'pySIM-shell (no card profile)%c ' % prompt_char
             else:
-                self.prompt = 'pySIM-shell (no card)> '
+                self.prompt = 'pySIM-shell (no card)%c ' % prompt_char
 
     @cmd2.with_category(CUSTOM_CATEGORY)
     def do_intro(self, _):
@@ -288,7 +294,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         if self.rs is None:
             # In case no runtime state is available we go the direct route
             self.card._scc.reset_card()
-            atr = b2h(self.card._scc.get_atr())
+            atr = self.card._scc.get_atr()
         else:
             atr = self.rs.reset(self)
         self.poutput('Card ATR: %s' % atr)
@@ -854,6 +860,8 @@ class PySimCommands(CommandSet):
             self._cmd.lchan.scc.verify_chv(adm_chv_num, h2b(pin_adm))
         else:
             raise ValueError("error: cannot authenticate, no adm-pin!")
+        self._cmd.rs.adm_verified = True
+        self._cmd.update_prompt()
 
     def do_cardinfo(self, opts):
         """Display information about the currently inserted card"""
@@ -1151,14 +1159,18 @@ if __name__ == '__main__':
     # Run optional commands
     for c in opts.execute_command:
         if not startup_errors:
-            app.onecmd_plus_hooks(c)
+            stop = app.onecmd_plus_hooks(c)
+            if stop == True:
+                sys.exit(0)
         else:
             print("Errors during startup, refusing to execute command (%s)" % c)
 
     # Run optional command
     if opts.command:
         if not startup_errors:
-            app.onecmd_plus_hooks('{} {}'.format(opts.command, ' '.join(opts.command_args)))
+            stop = app.onecmd_plus_hooks('{} {}'.format(opts.command, ' '.join(opts.command_args)))
+            if stop == True:
+                sys.exit(0)
         else:
             print("Errors during startup, refusing to execute command (%s)" % opts.command)
 
@@ -1169,7 +1181,9 @@ if __name__ == '__main__':
                 print("Error: script file (%s) not readable!" % opts.script)
                 startup_errors = True
             else:
-                app.onecmd_plus_hooks('{} {}'.format('run_script', opts.script), add_to_history = False)
+                stop = app.onecmd_plus_hooks('{} {}'.format('run_script', opts.script), add_to_history = False)
+                if stop == True:
+                    sys.exit(0)
         else:
             print("Errors during startup, refusing to execute script (%s)" % opts.script)
 

pySim-smpp2sim.py

@@ -0,0 +1,428 @@
+#!/usr/bin/env python3
+#
+# Program to emulate the entire communication path SMSC-MSC-BSC-BTS-ME
+# that is usually between an OTA backend and the SIM card.  This allows
+# to play with SIM OTA technology without using a mobile network or even
+# a mobile phone.
+#
+# An external application must encode (and encrypt/sign) the OTA SMS
+# and submit them via SMPP to this program, just like it would submit
+# it normally to a SMSC (SMS Service Centre).  The program then re-formats
+# the SMPP-SUBMIT into a SMS DELIVER TPDU and passes it via an ENVELOPE
+# APDU to the SIM card that is locally inserted into a smart card reader.
+#
+# The path from SIM to external OTA application works the opposite way.
+
+# (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import logging
+import colorlog
+
+from twisted.protocols import basic
+from twisted.internet import defer, endpoints, protocol, reactor, task
+from twisted.cred.portal import IRealm
+from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
+from twisted.cred.portal import Portal
+from zope.interface import implementer
+
+from smpp.twisted.config import SMPPServerConfig
+from smpp.twisted.server import SMPPServerFactory, SMPPBindManager
+from smpp.twisted.protocol import SMPPSessionStates, DataHandlerResponse
+
+from smpp.pdu import pdu_types, operations, pdu_encoding
+
+from pySim.sms import SMS_DELIVER, SMS_SUBMIT, AddressField
+
+from pySim.transport import LinkBase, ProactiveHandler, argparse_add_reader_args, init_reader, ApduTracer
+from pySim.commands import SimCardCommands
+from pySim.cards import UiccCardBase
+from pySim.exceptions import *
+from pySim.cat import ProactiveCommand, SendShortMessage, SMS_TPDU, SMSPPDownload, BearerDescription
+from pySim.cat import DeviceIdentities, Address, OtherAddress, UiccTransportLevel, BufferSize
+from pySim.cat import ChannelStatus, ChannelData, ChannelDataLength
+from pySim.utils import b2h, h2b
+
+logger = logging.getLogger(__name__)
+
+# MSISDNs to use when generating proactive SMS messages
+SIM_MSISDN='23'
+ESME_MSISDN='12'
+
+# HACK: we need some kind of mapping table between system_id and card-reader
+# or actually route based on MSISDNs
+hackish_global_smpp = None
+
+class MyApduTracer(ApduTracer):
+    def trace_response(self, cmd, sw, resp):
+        print("-> %s %s" % (cmd[:10], cmd[10:]))
+        print("<- %s: %s" % (sw, resp))
+
+class TcpProtocol(protocol.Protocol):
+    def dataReceived(self, data):
+        pass
+
+    def connectionLost(self, reason):
+        pass
+
+
+def tcp_connected_callback(p: protocol.Protocol):
+    """called by twisted TCP client."""
+    logger.error("%s: connected!" % p)
+
+class ProactChannel:
+    """Representation of a single proective channel."""
+    def __init__(self, channels: 'ProactChannels', chan_nr: int):
+        self.channels = channels
+        self.chan_nr = chan_nr
+        self.ep = None
+
+    def close(self):
+        """Close the channel."""
+        if self.ep:
+            self.ep.disconnect()
+        self.channels.channel_delete(self.chan_nr)
+
+class ProactChannels:
+    """Wrapper class for maintaining state of proactive channels."""
+    def __init__(self):
+        self.channels = {}
+
+    def channel_create(self) -> ProactChannel:
+        """Create a new proactive channel, allocating its integer number."""
+        for i in range(1, 9):
+            if not i in self.channels:
+                self.channels[i] = ProactChannel(self, i)
+                return self.channels[i]
+        raise ValueError('Cannot allocate another channel: All channels active')
+
+    def channel_delete(self, chan_nr: int):
+        del self.channels[chan_nr]
+
+class Proact(ProactiveHandler):
+    #def __init__(self, smpp_factory):
+    #    self.smpp_factory = smpp_factory
+    def __init__(self):
+        self.channels = ProactChannels()
+
+    @staticmethod
+    def _find_first_element_of_type(instlist, cls):
+        for i in instlist:
+            if isinstance(i, cls):
+                return i
+        return None
+
+    """Call-back which the pySim transport core calls whenever it receives a
+    proactive command from the SIM."""
+    def handle_SendShortMessage(self, pcmd: ProactiveCommand):
+        # {'smspp_download': [{'device_identities': {'source_dev_id': 'network',
+        #                                            'dest_dev_id': 'uicc'}},
+        #                     {'address': {'ton_npi': {'ext': True,
+        #                                              'type_of_number': 'international',
+        #                                              'numbering_plan_id': 'isdn_e164'},
+        #                                  'call_number': '79'}},
+        #                     {'sms_tpdu': {'tpdu': '40048111227ff6407070611535004d02700000481516011212000001fe4c0943aea42e45021c078ae06c66afc09303608874b72f58bacadb0dcf665c29349c799fbb522e61709c9baf1890015e8e8e196e36153106c8b92f95153774'}}
+        #                    ]}
+        """Card requests sending a SMS. We need to pass it on to the ESME via SMPP."""
+        logger.info("SendShortMessage")
+        logger.info(pcmd)
+        # Relevant parts in pcmd: Address, SMS_TPDU
+        addr_ie = Proact._find_first_element_of_type(pcmd.children, Address)
+        sms_tpdu_ie = Proact._find_first_element_of_type(pcmd.children, SMS_TPDU)
+        raw_tpdu = sms_tpdu_ie.decoded['tpdu']
+        submit = SMS_SUBMIT.from_bytes(raw_tpdu)
+        submit.tp_da = AddressField(addr_ie.decoded['call_number'], addr_ie.decoded['ton_npi']['type_of_number'],
+                                    addr_ie.decoded['ton_npi']['numbering_plan_id'])
+        logger.info(submit)
+        self.send_sms_via_smpp(submit)
+
+    def handle_OpenChannel(self, pcmd: ProactiveCommand):
+        """Card requests opening a new channel via a UDP/TCP socket."""
+        # {'open_channel': [{'command_details': {'command_number': 1,
+        #                                        'type_of_command': 'open_channel',
+        #                                        'command_qualifier': 3}},
+        #                   {'device_identities': {'source_dev_id': 'uicc',
+        #                                          'dest_dev_id': 'terminal'}},
+        #                   {'bearer_description': {'bearer_type': 'default',
+        #                                           'bearer_parameters': ''}},
+        #                   {'buffer_size': 1024},
+        #                   {'uicc_transport_level': {'protocol_type': 'tcp_uicc_client_remote',
+        #                                             'port_number': 32768}},
+        #                   {'other_address': {'type_of_address': 'ipv4',
+        #                                      'address': '01020304'}}
+        #                  ]}
+        logger.info("OpenChannel")
+        logger.info(pcmd)
+        transp_lvl_ie = Proact._find_first_element_of_type(pcmd.children, UiccTransportLevel)
+        other_addr_ie = Proact._find_first_element_of_type(pcmd.children, OtherAddress)
+        bearer_desc_ie = Proact._find_first_element_of_type(pcmd.children, BearerDescription)
+        buffer_size_ie = Proact._find_first_element_of_type(pcmd.children, BufferSize)
+        if transp_lvl_ie.decoded['protocol_type'] != 'tcp_uicc_client_remote':
+            raise ValueError('Unsupported protocol_type')
+        if other_addr_ie.decoded.get('type_of_address', None) != 'ipv4':
+            raise ValueError('Unsupported type_of_address')
+        ipv4_bytes = h2b(other_addr_ie.decoded['address'])
+        ipv4_str = '%u.%u.%u.%u' % (ipv4_bytes[0], ipv4_bytes[1], ipv4_bytes[2], ipv4_bytes[3])
+        port_nr = transp_lvl_ie.decoded['port_number']
+        print("%s:%u" % (ipv4_str, port_nr))
+        channel = self.channels.channel_create()
+        channel.ep = endpoints.TCP4ClientEndpoint(reactor, ipv4_str, port_nr)
+        channel.prot = TcpProtocol()
+        d = endpoints.connectProtocol(channel.ep, channel.prot)
+        # FIXME: why is this never called despite the client showing the inbound connection?
+        d.addCallback(tcp_connected_callback)
+
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'open_channel',
+        #                       'command_qualifier': 3}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_status': '8100'},
+        #  {'bearer_description': {'bearer_type': 'default', 'bearer_parameters': ''}},
+        #  {'buffer_size': 1024}
+        # ]
+        return self.prepare_response(pcmd) + [ChannelStatus(decoded='8100'), bearer_desc_ie, buffer_size_ie]
+
+    def handle_CloseChannel(self, pcmd: ProactiveCommand):
+        """Close a channel."""
+        logger.info("CloseChannel")
+        logger.info(pcmd)
+
+    def handle_ReceiveData(self, pcmd: ProactiveCommand):
+        """Receive/read data from the socket."""
+        # {'receive_data': [{'command_details': {'command_number': 1,
+        #                                        'type_of_command': 'receive_data',
+        #                                        'command_qualifier': 0}},
+        #                   {'device_identities': {'source_dev_id': 'uicc',
+        #                                          'dest_dev_id': 'channel_1'}},
+        #                   {'channel_data_length': 9}
+        #                  ]}
+        logger.info("ReceiveData")
+        logger.info(pcmd)
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'receive_data',
+        #                       'command_qualifier': 0}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_data': '16030100040e000000'},
+        #  {'channel_data_length': 0}
+        # ]
+        return self.prepare_response(pcmd) + []
+
+    def handle_SendData(self, pcmd: ProactiveCommand):
+        """Send/write data received from the SIM to the socket."""
+        # {'send_data': [{'command_details': {'command_number': 1,
+        #                                     'type_of_command': 'send_data',
+        #                                     'command_qualifier': 1}},
+        #                {'device_identities': {'source_dev_id': 'uicc',
+        #                                       'dest_dev_id': 'channel_1'}},
+        #                {'channel_data': '160301003c010000380303d0f45e12b52ce5bb522750dd037738195334c87a46a847fe2b6886cada9ea6bf00000a00ae008c008b00b0002c010000050001000101'}
+        #               ]}
+        logger.info("SendData")
+        logger.info(pcmd)
+        dev_id_ie = Proact._find_first_element_of_type(pcmd.children, DeviceIdentities)
+        chan_data_ie = Proact._find_first_element_of_type(pcmd.children, ChannelData)
+        chan_str = dev_id_ie.decoded['dest_dev_id']
+        chan_nr = 1 # FIXME
+        chan = self.channels.channels.get(chan_nr, None)
+        # FIXME chan.prot.transport.write(h2b(chan_data_ie.decoded))
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'send_data',
+        #                       'command_qualifier': 1}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}},
+        #  {'channel_data_length': 255}
+        # ]
+        return self.prepare_response(pcmd) + [ChannelDataLength(decoded=255)]
+
+    def handle_SetUpEventList(self, pcmd: ProactiveCommand):
+        # {'set_up_event_list': [{'command_details': {'command_number': 1,
+        #                                             'type_of_command': 'set_up_event_list',
+        #                                             'command_qualifier': 0}},
+        #                        {'device_identities': {'source_dev_id': 'uicc',
+        #                                               'dest_dev_id': 'terminal'}},
+        #                        {'event_list': ['data_available', 'channel_status']}
+        #                       ]}
+        logger.info("SetUpEventList")
+        logger.info(pcmd)
+        # Terminal Response example: [
+        #  {'command_details': {'command_number': 1,
+        #                       'type_of_command': 'set_up_event_list',
+        #                       'command_qualifier': 0}},
+        #  {'device_identities': {'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'}},
+        #  {'result': {'general_result': 'performed_successfully', 'additional_information': ''}}
+        # ]
+        return self.prepare_response(pcmd)
+
+    def getChannelStatus(self, pcmd: ProactiveCommand):
+        logger.info("GetChannelStatus")
+        logger.info(pcmd)
+        return self.prepare_response(pcmd) + []
+
+    def send_sms_via_smpp(self, submit: SMS_SUBMIT):
+        # while in a normal network the phone/ME would *submit* a message to the SMSC,
+        # we are actually emulating the SMSC itself, so we must *deliver* the message
+        # to the ESME
+        deliver = SMS_DELIVER.from_submit(submit)
+        deliver_smpp = deliver.to_smpp()
+
+        hackish_global_smpp.sendDataRequest(deliver_smpp)
+#       # obtain the connection/binding of system_id to be used for delivering MO-SMS to the ESME
+#       connection = smpp_server.getBoundConnections[system_id].getNextBindingForDelivery()
+#       connection.sendDataRequest(deliver_smpp)
+
+
+
+def dcs_is_8bit(dcs):
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED):
+        return True
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED_COMMON):
+        return True
+    # pySim-smpp2sim.py:150:21: E1101: Instance of 'DataCodingScheme' has no 'GSM_MESSAGE_CLASS' member (no-member)
+    # pylint: disable=no-member
+    if dcs.scheme == pdu_types.DataCodingScheme.GSM_MESSAGE_CLASS and dcs.schemeData['msgCoding'] == pdu_types.DataCodingGsmMsgCoding.DATA_8BIT:
+        return True
+    else:
+        return False
+
+
+class MyServer:
+
+    @implementer(IRealm)
+    class SmppRealm:
+        def requestAvatar(self, avatarId, mind, *interfaces):
+            return ('SMPP', avatarId, lambda: None)
+
+    def __init__(self, tcp_port:int = 2775, bind_ip = '::', system_id:str = 'test', password:str = 'test'):
+        smpp_config = SMPPServerConfig(msgHandler=self._msgHandler,
+                                       systems={system_id: {'max_bindings': 2}})
+        portal = Portal(self.SmppRealm())
+        credential_checker = InMemoryUsernamePasswordDatabaseDontUse()
+        credential_checker.addUser(system_id, password)
+        portal.registerChecker(credential_checker)
+        self.factory = SMPPServerFactory(smpp_config, auth_portal=portal)
+        logger.info('Binding Virtual SMSC to TCP Port %u at %s' % (tcp_port, bind_ip))
+        smppEndpoint = endpoints.TCP6ServerEndpoint(reactor, tcp_port, interface=bind_ip)
+        smppEndpoint.listen(self.factory)
+        self.tp = self.scc = self.card = None
+
+    def connect_to_card(self, tp: LinkBase):
+        self.tp = tp
+        self.scc = SimCardCommands(self.tp)
+        self.card = UiccCardBase(self.scc)
+        # this should be part of UiccCardBase, but FairewavesSIM breaks with that :/
+        self.scc.cla_byte = "00"
+        self.scc.sel_ctrl = "0004"
+        self.card.read_aids()
+        self.card.select_adf_by_aid(adf='usim')
+        # FIXME: create a more realistic profile than ffffff
+        self.scc.terminal_profile('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff')
+
+    def _msgHandler(self, system_id, smpp, pdu):
+        """Handler for incoming messages received via SMPP from ESME."""
+        # HACK: we need some kind of mapping table between system_id and card-reader
+        # or actually route based on MSISDNs
+        global hackish_global_smpp
+        hackish_global_smpp = smpp
+        if pdu.id == pdu_types.CommandId.submit_sm:
+            return self.handle_submit_sm(system_id, smpp, pdu)
+        else:
+            logger.warning('Rejecting non-SUBMIT commandID')
+            return pdu_types.CommandStatus.ESME_RINVCMDID
+
+    def handle_submit_sm(self, system_id, smpp, pdu):
+        """SUBMIT-SM was received via SMPP from ESME. We need to deliver it to the SIM."""
+        # check for valid data coding scheme + PID
+        if not dcs_is_8bit(pdu.params['data_coding']):
+            logger.warning('Rejecting non-8bit DCS')
+            return pdu_types.CommandStatus.ESME_RINVDCS
+        if pdu.params['protocol_id'] != 0x7f:
+            logger.warning('Rejecting non-SIM PID')
+            return pdu_types.CommandStatus.ESME_RINVDCS
+
+        # 1) build a SMS-DELIVER (!) from the SMPP-SUBMIT
+        tpdu = SMS_DELIVER.from_smpp_submit(pdu)
+        logger.info(tpdu)
+        # 2) wrap into the CAT ENVELOPE for SMS-PP-Download
+        tpdu_ie = SMS_TPDU(decoded={'tpdu': b2h(tpdu.to_bytes())})
+        addr_ie = Address(decoded={'ton_npi': {'ext':False, 'type_of_number':'unknown', 'numbering_plan_id':'unknown'}, 'call_number': '0123456'})
+        dev_ids = DeviceIdentities(decoded={'source_dev_id': 'network', 'dest_dev_id': 'uicc'})
+        sms_dl = SMSPPDownload(children=[dev_ids, addr_ie, tpdu_ie])
+        # 3) send to the card
+        envelope_hex = b2h(sms_dl.to_tlv())
+        logger.info("ENVELOPE: %s" % envelope_hex)
+        (data, sw) = self.scc.envelope(envelope_hex)
+        logger.info("SW %s: %s" % (sw, data))
+        if sw in ['9200', '9300']:
+            # TODO send back RP-ERROR message with TP-FCS == 'SIM Application Toolkit Busy'
+            return pdu_types.CommandStatus.ESME_RSUBMITFAIL
+        elif sw == '9000' or sw[0:2] in ['6f', '62', '63'] and len(data):
+            # data something like 027100000e0ab000110000000000000001612f or
+            # 027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c
+            # which is the user-data portion of the SMS starting with the UDH (027100)
+            # TODO: return the response back to the sender in an RP-ACK; PID/DCS like in CMD
+            deliver = operations.DeliverSM(service_type=pdu.params['service_type'],
+                                           source_addr_ton=pdu.params['dest_addr_ton'],
+                                           source_addr_npi=pdu.params['dest_addr_npi'],
+                                           source_addr=pdu.params['destination_addr'],
+                                           dest_addr_ton=pdu.params['source_addr_ton'],
+                                           dest_addr_npi=pdu.params['source_addr_npi'],
+                                           destination_addr=pdu.params['source_addr'],
+                                           esm_class=pdu.params['esm_class'],
+                                           protocol_id=pdu.params['protocol_id'],
+                                           priority_flag=pdu.params['priority_flag'],
+                                           data_coding=pdu.params['data_coding'],
+                                           short_message=h2b(data))
+            smpp.sendDataRequest(deliver)
+            return pdu_types.CommandStatus.ESME_ROK
+        else:
+            return pdu_types.CommandStatus.ESME_RSUBMITFAIL
+
+
+option_parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+argparse_add_reader_args(option_parser)
+smpp_group = option_parser.add_argument_group('SMPP Options')
+smpp_group.add_argument('--smpp-bind-port', type=int, default=2775,
+                        help='TCP Port to bind the SMPP socket to')
+smpp_group.add_argument('--smpp-bind-ip', default='::',
+                        help='IPv4/IPv6 address to bind the SMPP socket to')
+smpp_group.add_argument('--smpp-system-id', default='test',
+                        help='SMPP System-ID used by ESME to bind')
+smpp_group.add_argument('--smpp-password', default='test',
+                        help='SMPP Password used by ESME to bind')
+
+if __name__ == '__main__':
+    log_format='%(log_color)s%(levelname)-8s%(reset)s %(name)s: %(message)s'
+    colorlog.basicConfig(level=logging.INFO, format = log_format)
+    logger = colorlog.getLogger()
+
+    opts = option_parser.parse_args()
+
+    tp = init_reader(opts, proactive_handler = Proact())
+    if tp is None:
+        exit(1)
+    tp.connect()
+
+    global g_ms
+    g_ms = MyServer(opts.smpp_bind_port, opts.smpp_bind_ip, opts.smpp_system_id, opts.smpp_password)
+    g_ms.connect_to_card(tp)
+    reactor.run()
+

pySim/apdu/__init__.py

@@ -29,7 +29,7 @@ import abc
 import typing
 from typing import List, Dict, Optional
 from termcolor import colored
-from construct import Byte, GreedyBytes
+from construct import Byte
 from construct import Optional as COptional
 from osmocom.construct import *
 from osmocom.utils import *

pySim/apdu/ts_31_102.py

@@ -9,7 +9,7 @@ APDU commands of 3GPP TS 31.102 V16.6.0
 """
 from typing import Dict
 
-from construct import BitStruct, Enum, BitsInteger, Int8ub, Bytes, this, Struct, If, Switch, Const
+from construct import BitStruct, Enum, BitsInteger, Int8ub, this, Struct, If, Switch, Const
 from construct import Optional as COptional
 from osmocom.construct import *
 

pySim/ara_m.py

@@ -26,7 +26,7 @@ Support for the Secure Element Access Control, specifically the ARA-M inside an
 #
 
 
-from construct import GreedyBytes, GreedyString, Struct, Enum, Int8ub, Int16ub
+from construct import GreedyString, Struct, Enum, Int8ub, Int16ub
 from construct import Optional as COptional
 from osmocom.construct import *
 from osmocom.tlv import *
@@ -38,17 +38,17 @@ import pySim.global_platform
 
 
 class AidRefDO(BER_TLV_IE, tag=0x4f):
-    # SEID v1.1 Table 6-3
+    # GPD_SPE_013 v1.1 Table 6-3
     _construct = HexAdapter(GreedyBytes)
 
 
 class AidRefEmptyDO(BER_TLV_IE, tag=0xc0):
-    # SEID v1.1 Table 6-3
+    # GPD_SPE_013 v1.1 Table 6-3
     pass
 
 
 class DevAppIdRefDO(BER_TLV_IE, tag=0xc1):
-    # SEID v1.1 Table 6-4
+    # GPD_SPE_013 v1.1 Table 6-4
     _construct = HexAdapter(GreedyBytes)
 
 
@@ -58,12 +58,12 @@ class PkgRefDO(BER_TLV_IE, tag=0xca):
 
 
 class RefDO(BER_TLV_IE, tag=0xe1, nested=[AidRefDO, AidRefEmptyDO, DevAppIdRefDO, PkgRefDO]):
-    # SEID v1.1 Table 6-5
+    # GPD_SPE_013 v1.1 Table 6-5
     pass
 
 
 class ApduArDO(BER_TLV_IE, tag=0xd0):
-    # SEID v1.1 Table 6-8
+    # GPD_SPE_013 v1.1 Table 6-8
     def _from_bytes(self, do: bytes):
         if len(do) == 1:
             if do[0] == 0x00:
@@ -108,7 +108,7 @@ class ApduArDO(BER_TLV_IE, tag=0xd0):
 
 
 class NfcArDO(BER_TLV_IE, tag=0xd1):
-    # SEID v1.1 Table 6-9
+    # GPD_SPE_013 v1.1 Table 6-9
     _construct = Struct('nfc_event_access_rule' /
                         Enum(Int8ub, never=0, always=1))
 
@@ -120,122 +120,122 @@ class PermArDO(BER_TLV_IE, tag=0xdb):
 
 
 class ArDO(BER_TLV_IE, tag=0xe3, nested=[ApduArDO, NfcArDO, PermArDO]):
-    # SEID v1.1 Table 6-7
+    # GPD_SPE_013 v1.1 Table 6-7
     pass
 
 
 class RefArDO(BER_TLV_IE, tag=0xe2, nested=[RefDO, ArDO]):
-    # SEID v1.1 Table 6-6
+    # GPD_SPE_013 v1.1 Table 6-6
     pass
 
 
 class ResponseAllRefArDO(BER_TLV_IE, tag=0xff40, nested=[RefArDO]):
-    # SEID v1.1 Table 4-2
+    # GPD_SPE_013 v1.1 Table 4-2
     pass
 
 
 class ResponseArDO(BER_TLV_IE, tag=0xff50, nested=[ArDO]):
-    # SEID v1.1 Table 4-3
+    # GPD_SPE_013 v1.1 Table 4-3
     pass
 
 
 class ResponseRefreshTagDO(BER_TLV_IE, tag=0xdf20):
-    # SEID v1.1 Table 4-4
+    # GPD_SPE_013 v1.1 Table 4-4
     _construct = Struct('refresh_tag'/HexAdapter(Bytes(8)))
 
 
 class DeviceInterfaceVersionDO(BER_TLV_IE, tag=0xe6):
-    # SEID v1.1 Table 6-12
+    # GPD_SPE_013 v1.1 Table 6-12
     _construct = Struct('major'/Int8ub, 'minor'/Int8ub, 'patch'/Int8ub)
 
 
 class DeviceConfigDO(BER_TLV_IE, tag=0xe4, nested=[DeviceInterfaceVersionDO]):
-    # SEID v1.1 Table 6-10
+    # GPD_SPE_013 v1.1 Table 6-10
     pass
 
 
 class ResponseDeviceConfigDO(BER_TLV_IE, tag=0xff7f, nested=[DeviceConfigDO]):
-    # SEID v1.1 Table 5-14
+    # GPD_SPE_013 v1.1 Table 5-14
     pass
 
 
 class AramConfigDO(BER_TLV_IE, tag=0xe5, nested=[DeviceInterfaceVersionDO]):
-    # SEID v1.1 Table 6-11
+    # GPD_SPE_013 v1.1 Table 6-11
     pass
 
 
 class ResponseAramConfigDO(BER_TLV_IE, tag=0xdf21, nested=[AramConfigDO]):
-    # SEID v1.1 Table 4-5
+    # GPD_SPE_013 v1.1 Table 4-5
     pass
 
 
 class CommandStoreRefArDO(BER_TLV_IE, tag=0xf0, nested=[RefArDO]):
-    # SEID v1.1 Table 5-2
+    # GPD_SPE_013 v1.1 Table 5-2
     pass
 
 
 class CommandDelete(BER_TLV_IE, tag=0xf1, nested=[AidRefDO, AidRefEmptyDO, RefDO, RefArDO]):
-    # SEID v1.1 Table 5-4
+    # GPD_SPE_013 v1.1 Table 5-4
     pass
 
 
 class CommandUpdateRefreshTagDO(BER_TLV_IE, tag=0xf2):
-    # SEID V1.1 Table 5-6
+    # GPD_SPE_013 V1.1 Table 5-6
     pass
 
 
 class CommandRegisterClientAidsDO(BER_TLV_IE, tag=0xf7, nested=[AidRefDO, AidRefEmptyDO]):
-    # SEID v1.1 Table 5-7
+    # GPD_SPE_013 v1.1 Table 5-7
     pass
 
 
 class CommandGet(BER_TLV_IE, tag=0xf3, nested=[AidRefDO, AidRefEmptyDO]):
-    # SEID v1.1 Table 5-8
+    # GPD_SPE_013 v1.1 Table 5-8
     pass
 
 
 class CommandGetAll(BER_TLV_IE, tag=0xf4):
-    # SEID v1.1 Table 5-9
+    # GPD_SPE_013 v1.1 Table 5-9
     pass
 
 
 class CommandGetClientAidsDO(BER_TLV_IE, tag=0xf6):
-    # SEID v1.1 Table 5-10
+    # GPD_SPE_013 v1.1 Table 5-10
     pass
 
 
 class CommandGetNext(BER_TLV_IE, tag=0xf5):
-    # SEID v1.1 Table 5-11
+    # GPD_SPE_013 v1.1 Table 5-11
     pass
 
 
 class CommandGetDeviceConfigDO(BER_TLV_IE, tag=0xf8):
-    # SEID v1.1 Table 5-12
+    # GPD_SPE_013 v1.1 Table 5-12
     pass
 
 
 class ResponseAracAidDO(BER_TLV_IE, tag=0xff70, nested=[AidRefDO, AidRefEmptyDO]):
-    # SEID v1.1 Table 5-13
+    # GPD_SPE_013 v1.1 Table 5-13
     pass
 
 
 class BlockDO(BER_TLV_IE, tag=0xe7):
-    # SEID v1.1 Table 6-13
+    # GPD_SPE_013 v1.1 Table 6-13
     _construct = Struct('offset'/Int16ub, 'length'/Int8ub)
 
 
-# SEID v1.1 Table 4-1
+# GPD_SPE_013 v1.1 Table 4-1
 class GetCommandDoCollection(TLV_IE_Collection, nested=[RefDO, DeviceConfigDO]):
     pass
 
 
-# SEID v1.1 Table 4-2
+# GPD_SPE_013 v1.1 Table 4-2
 class GetResponseDoCollection(TLV_IE_Collection, nested=[ResponseAllRefArDO, ResponseArDO,
                                                          ResponseRefreshTagDO, ResponseAramConfigDO]):
     pass
 
 
-# SEID v1.1 Table 5-1
+# GPD_SPE_013 v1.1 Table 5-1
 class StoreCommandDoCollection(TLV_IE_Collection,
                                nested=[BlockDO, CommandStoreRefArDO, CommandDelete,
                                        CommandUpdateRefreshTagDO, CommandRegisterClientAidsDO,
@@ -244,7 +244,7 @@ class StoreCommandDoCollection(TLV_IE_Collection,
     pass
 
 
-# SEID v1.1 Section 5.1.2
+# GPD_SPE_013 v1.1 Section 5.1.2
 class StoreResponseDoCollection(TLV_IE_Collection,
                                 nested=[ResponseAllRefArDO, ResponseAracAidDO, ResponseDeviceConfigDO]):
     pass
@@ -320,9 +320,9 @@ class ADF_ARAM(CardADF):
             '--device-app-id', required=True, help='Identifies the specific device application that the rule appplies to. Hash of Certificate of Application Provider, or UUID. (20/32 hex bytes)')
         aid_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
         aid_grp.add_argument(
-            '--aid', help='Identifies the specific SE application for which rules are to be stored. Can be a partial AID, containing for example only the RID.  (5-16 hex bytes)')
+            '--aid', help='Identifies the specific SE application for which rules are to be stored. Can be a partial AID, containing for example only the RID. (5-16 or 0 hex bytes)')
         aid_grp.add_argument('--aid-empty', action='store_true',
-                             help='No specific SE application, applies to all applications')
+                             help='No specific SE application, applies to implicitly selected application (all channels)')
         store_ref_ar_do_parse.add_argument(
             '--pkg-ref', help='Full Android Java package name (up to 127 chars ASCII)')
         # AR-DO
@@ -389,6 +389,11 @@ class ADF_ARAM(CardADF):
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
+        def do_aram_lock(self, opts):
+            """Lock STORE DATA command to prevent unauthorized changes
+            (Proprietary feature that is specific to sysmocom's fork of Bertrand Martel’s ARA-M implementation.)"""
+            self._cmd.lchan.scc.send_apdu_checksw('80e2900001A1', '9000')
+
 
 # SEAC v1.1 Section 4.1.2.2 + 5.1.2.2
 sw_aram = {
@@ -423,10 +428,13 @@ class CardApplicationARAM(CardApplication):
         # matching key.
         if dictlist is None:
             return None
-        obj = None
         for d in dictlist:
-            obj = d.get(key, obj)
-        return obj
+            if key in d:
+                obj = d.get(key)
+                if obj is None:
+                    return ""
+                return obj
+        return None
 
     @staticmethod
     def __export_ref_ar_do_list(ref_ar_do_list):
@@ -437,6 +445,7 @@ class CardApplicationARAM(CardApplication):
         if ref_do_list and ar_do_list:
             # Get ref_do parameters
             aid_ref_do = CardApplicationARAM.__export_get_from_dictlist('aid_ref_do', ref_do_list)
+            aid_ref_empty_do = CardApplicationARAM.__export_get_from_dictlist('aid_ref_empty_do', ref_do_list)
             dev_app_id_ref_do = CardApplicationARAM.__export_get_from_dictlist('dev_app_id_ref_do', ref_do_list)
             pkg_ref_do = CardApplicationARAM.__export_get_from_dictlist('pkg_ref_do', ref_do_list)
 
@@ -447,9 +456,11 @@ class CardApplicationARAM(CardApplication):
 
             # Write command-line
             export_str += "aram_store_ref_ar_do"
-            if aid_ref_do:
+            if aid_ref_do is not None and len(aid_ref_do) > 0:
                 export_str += (" --aid %s" % aid_ref_do)
-            else:
+            elif aid_ref_do is not None:
+                export_str += " --aid \"\""
+            if aid_ref_empty_do is not None:
                 export_str += " --aid-empty"
             if dev_app_id_ref_do:
                 export_str += (" --device-app-id %s" % dev_app_id_ref_do)

pySim/cards.py

@@ -73,6 +73,16 @@ class CardBase:
         # callers having to do hasattr('read_aids') ahead of every call.
         return []
 
+    def adf_present(self, adf: str = "usim") -> bool:
+        # a non-UICC doesn't have any applications. Convenience helper to avoid
+        # callers having to do hasattr('adf_present') ahead of every call.
+        return False
+
+    def select_adf_by_aid(self, adf: str = "usim", scc: Optional[SimCardCommands] = None) -> Tuple[Optional[Hexstr], Optional[SwHexstr]]:
+        # a non-UICC doesn't have any applications. Convenience helper to avoid
+        # callers having to do hasattr('select_adf_by_aid') ahead of every call.
+        return (None, None)
+
 
 class SimCardBase(CardBase):
     """Here we only add methods for commands specified in TS 51.011, without

pySim/cat.py

@@ -20,19 +20,19 @@ as described in 3GPP TS 31.111."""
 
 from typing import List
 from bidict import bidict
-from construct import Int8ub, Int16ub, Byte, Bytes, BitsInteger
+from construct import Int8ub, Int16ub, Byte, BitsInteger
 from construct import Struct, Enum, BitStruct, this
-from construct import GreedyBytes, Switch, GreedyRange, FlagsEnum
+from construct import Switch, GreedyRange, FlagsEnum
 from osmocom.tlv import TLV_IE, COMPR_TLV_IE, BER_TLV_IE, TLV_IE_Collection
-from osmocom.construct import PlmnAdapter, BcdAdapter, HexAdapter, GsmStringAdapter, TonNpi, GsmString
-from osmocom.utils import b2h
+from osmocom.construct import PlmnAdapter, BcdAdapter, GsmStringAdapter, TonNpi, GsmString, Bytes, GreedyBytes
+from osmocom.utils import b2h, h2b
 from pySim.utils import dec_xplmn_w_act
 
 # Tag values as per TS 101 220 Table 7.23
 
 # TS 102 223 Section 8.1
 class Address(COMPR_TLV_IE, tag=0x86):
-    _construct = Struct('ton_npi'/Int8ub,
+    _construct = Struct('ton_npi'/TonNpi,
                         'call_number'/BcdAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.2
@@ -255,24 +255,24 @@ class Result(COMPR_TLV_IE, tag=0x83):
                             'launch_browser_generic_error': AddlInfoLaunchBrowser,
                             'bearer_independent_protocol_error': AddlInfoBip,
                             'frames_error': AddlInfoFrames
-                        }, default=HexAdapter(GreedyBytes)))
+                        }, default=GreedyBytes))
 
 # TS 102 223 Section 8.13  + TS 31.111 Section 8.13
 class SMS_TPDU(COMPR_TLV_IE, tag=0x8B):
-    _construct = Struct('tpdu'/HexAdapter(GreedyBytes))
+    _construct = Struct('tpdu'/GreedyBytes)
 
 # TS 31.111 Section 8.14
 class SsString(COMPR_TLV_IE, tag=0x89):
-    _construct = Struct('ton_npi'/TonNpi, 'ss_string'/HexAdapter(GreedyBytes))
+    _construct = Struct('ton_npi'/TonNpi, 'ss_string'/GreedyBytes)
 
 
 # TS 102 223 Section 8.15
 class TextString(COMPR_TLV_IE, tag=0x8D):
     _test_de_encode = [
-        ( '8d090470617373776f7264', {'dcs': 4, 'text_string': '70617373776f7264'} ),
+        ( '8d090470617373776f7264', {'dcs': 4, 'text_string': b'password'} )
     ]
     _construct = Struct('dcs'/Int8ub, # TS 03.38
-                        'text_string'/HexAdapter(GreedyBytes))
+                        'text_string'/GreedyBytes)
 
 # TS 102 223 Section 8.16
 class Tone(COMPR_TLV_IE, tag=0x8E):
@@ -308,11 +308,11 @@ class Tone(COMPR_TLV_IE, tag=0x8E):
 # TS 31 111 Section 8.17
 class USSDString(COMPR_TLV_IE, tag=0x8A):
     _construct = Struct('dcs'/Int8ub,
-                        'ussd_string'/HexAdapter(GreedyBytes))
+                        'ussd_string'/GreedyBytes)
 
 # TS 102 223 Section 8.18
 class FileList(COMPR_TLV_IE, tag=0x92):
-    FileId=HexAdapter(Bytes(2))
+    FileId=Bytes(2)
     _construct = Struct('number_of_files'/Int8ub,
                         'files'/GreedyRange(FileId))
 
@@ -335,7 +335,7 @@ class NetworkMeasurementResults(COMPR_TLV_IE, tag=0x96):
 # TS 102 223 Section 8.23
 class DefaultText(COMPR_TLV_IE, tag=0x97):
     _construct = Struct('dcs'/Int8ub,
-                        'text_string'/HexAdapter(GreedyBytes))
+                        'text_string'/GreedyBytes)
 
 # TS 102 223 Section 8.24
 class ItemsNextActionIndicator(COMPR_TLV_IE, tag=0x98):
@@ -394,7 +394,7 @@ class ItemIconIdentifierList(COMPR_TLV_IE, tag=0x9f):
 
 # TS 102 223 Section 8.35
 class CApdu(COMPR_TLV_IE, tag=0xA2):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.37
 class TimerIdentifier(COMPR_TLV_IE, tag=0xA4):
@@ -406,7 +406,7 @@ class TimerValue(COMPR_TLV_IE, tag=0xA5):
 
 # TS 102 223 Section 8.40
 class AtCommand(COMPR_TLV_IE, tag=0xA8):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.43
 class ImmediateResponse(COMPR_TLV_IE, tag=0xAB):
@@ -418,7 +418,7 @@ class DtmfString(COMPR_TLV_IE, tag=0xAC):
 
 # TS 102 223 Section 8.45
 class Language(COMPR_TLV_IE, tag=0xAD):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 31.111 Section 8.46
 class TimingAdvance(COMPR_TLV_IE, tag=0xC6):
@@ -440,7 +440,7 @@ class Bearer(COMPR_TLV_IE, tag=0xB2):
 
 # TS 102 223 Section 8.50
 class ProvisioningFileReference(COMPR_TLV_IE, tag=0xB3):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.51
 class BrowserTerminationCause(COMPR_TLV_IE, tag=0xB4):
@@ -449,7 +449,7 @@ class BrowserTerminationCause(COMPR_TLV_IE, tag=0xB4):
 # TS 102 223 Section 8.52
 class BearerDescription(COMPR_TLV_IE, tag=0xB5):
     _test_de_encode = [
-        ( 'b50103', {'bearer_parameters': '', 'bearer_type': 'default'} ),
+        ( 'b50103', {'bearer_parameters': b'', 'bearer_type': 'default'} ),
     ]
     # TS 31.111 Section 8.52.1
     BearerParsCs = Struct('data_rate'/Int8ub,
@@ -492,11 +492,11 @@ class BearerDescription(COMPR_TLV_IE, tag=0xB5):
                                                         'packet_grps_utran_eutran': BearerParsPacket,
                                                         'packet_with_extd_params': BearerParsPacketExt,
                                                         'ng_ran': BearerParsNgRan,
-                                                    }, default=HexAdapter(GreedyBytes)))
+                                                    }, default=GreedyBytes))
 
 # TS 102 223 Section 8.53
 class ChannelData(COMPR_TLV_IE, tag = 0xB6):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.54
 class ChannelDataLength(COMPR_TLV_IE, tag = 0xB7):
@@ -510,15 +510,15 @@ class BufferSize(COMPR_TLV_IE, tag = 0xB9):
 class ChannelStatus(COMPR_TLV_IE, tag = 0xB8):
     # complex decoding, depends on out-of-band context/knowledge :(
     # for default / TCP Client mode: bit 8 of first byte indicates connected, 3 LSB indicate channel nr
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.58
 class OtherAddress(COMPR_TLV_IE, tag = 0xBE):
     _test_de_encode = [
-        ( 'be052101020304', {'address': '01020304', 'type_of_address': 'ipv4'} ),
+        ( 'be052101020304', {'address': h2b('01020304'), 'type_of_address': 'ipv4'} ),
     ]
     _construct = Struct('type_of_address'/Enum(Int8ub, ipv4=0x21, ipv6=0x57),
-                        'address'/HexAdapter(GreedyBytes))
+                        'address'/GreedyBytes)
 
 # TS 102 223 Section 8.59
 class UiccTransportLevel(COMPR_TLV_IE, tag = 0xBC):
@@ -532,7 +532,7 @@ class UiccTransportLevel(COMPR_TLV_IE, tag = 0xBC):
 
 # TS 102 223 Section 8.60
 class Aid(COMPR_TLV_IE, tag=0xAF):
-    _construct = Struct('aid'/HexAdapter(GreedyBytes))
+    _construct = Struct('aid'/GreedyBytes)
 
 # TS 102 223 Section 8.61
 class AccessTechnology(COMPR_TLV_IE, tag=0xBF):
@@ -546,35 +546,35 @@ class ServiceRecord(COMPR_TLV_IE, tag=0xC1):
     BearerTechId = Enum(Int8ub, technology_independent=0, bluetooth=1, irda=2, rs232=3, usb=4)
     _construct = Struct('local_bearer_technology'/BearerTechId,
                         'service_identifier'/Int8ub,
-                        'service_record'/HexAdapter(GreedyBytes))
+                        'service_record'/GreedyBytes)
 
 # TS 102 223 Section 8.64
 class DeviceFilter(COMPR_TLV_IE, tag=0xC2):
     _construct = Struct('local_bearer_technology'/ServiceRecord.BearerTechId,
-                        'device_filter'/HexAdapter(GreedyBytes))
+                        'device_filter'/GreedyBytes)
 
 # TS 102 223 Section 8.65
 class ServiceSearchIE(COMPR_TLV_IE, tag=0xC3):
     _construct = Struct('local_bearer_technology'/ServiceRecord.BearerTechId,
-                        'service_search'/HexAdapter(GreedyBytes))
+                        'service_search'/GreedyBytes)
 
 # TS 102 223 Section 8.66
 class AttributeInformation(COMPR_TLV_IE, tag=0xC4):
     _construct = Struct('local_bearer_technology'/ServiceRecord.BearerTechId,
-                        'attribute_information'/HexAdapter(GreedyBytes))
+                        'attribute_information'/GreedyBytes)
 
 
 # TS 102 223 Section 8.68
 class RemoteEntityAddress(COMPR_TLV_IE, tag=0xC9):
     _construct = Struct('coding_type'/Enum(Int8ub, ieee802_16=0, irda=1),
-                        'address'/HexAdapter(GreedyBytes))
+                        'address'/GreedyBytes)
 
 # TS 102 223 Section 8.70
 class NetworkAccessName(COMPR_TLV_IE, tag=0xC7):
     _test_de_encode = [
-        ( 'c704036e6161', '036e6161' ),
+        ( 'c704036e6161', h2b('036e6161') ),
     ]
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.72
 class TextAttribute(COMPR_TLV_IE, tag=0xD0):
@@ -618,15 +618,15 @@ class FrameIdentifier(COMPR_TLV_IE, tag=0xE8):
 
 # TS 102 223 Section 8.82
 class MultimediaMessageReference(COMPR_TLV_IE, tag=0xEA):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.83
 class MultimediaMessageIdentifier(COMPR_TLV_IE, tag=0xEB):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.85
 class MmContentIdentifier(COMPR_TLV_IE, tag=0xEE):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.89
 class ActivateDescriptor(COMPR_TLV_IE, tag=0xFB):
@@ -649,7 +649,7 @@ class ContactlessFunctionalityState(COMPR_TLV_IE, tag=0xD4):
 # TS 31.111 Section 8.91
 class RoutingAreaIdentification(COMPR_TLV_IE, tag=0xF3):
     _construct = Struct('mcc_mnc'/PlmnAdapter(Bytes(3)),
-                        'lac'/HexAdapter(Bytes(2)),
+                        'lac'/Bytes(2),
                         'rac'/Int8ub)
 
 # TS 31.111 Section 8.92
@@ -709,15 +709,15 @@ class EcatSequenceNumber(COMPR_TLV_IE, tag=0xA1):
 
 # TS 102 223 Section 8.99
 class EncryptedTlvList(COMPR_TLV_IE, tag=0xA2):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.100
 class Mac(COMPR_TLV_IE, tag=0xE0):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.101
 class SaTemplate(COMPR_TLV_IE, tag=0xA3):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.103
 class RefreshEnforcementPolicy(COMPR_TLV_IE, tag=0xBA):
@@ -725,7 +725,7 @@ class RefreshEnforcementPolicy(COMPR_TLV_IE, tag=0xBA):
 
 # TS 102 223 Section 8.104
 class DnsServerAddress(COMPR_TLV_IE, tag=0xC0):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # TS 102 223 Section 8.105
 class SupportedRadioAccessTechnologies(COMPR_TLV_IE, tag=0xB4):

pySim/cdma_ruim.py

@@ -115,7 +115,7 @@ class EF_AD(TransparentEF):
     '''3.4.33 Administrative Data'''
 
     _test_de_encode = [
-        ( "000000", { 'ms_operation_mode' : 'normal', 'additional_info' : '0000', 'rfu' : '' } ),
+        ( "000000", { 'ms_operation_mode' : 'normal', 'additional_info' : b'\x00\x00', 'rfu' : b'' } ),
     ]
     _test_no_pad = True
 
@@ -134,9 +134,9 @@ class EF_AD(TransparentEF):
             # Byte 1: Display Condition
             'ms_operation_mode'/Enum(Byte, self.OP_MODE),
             # Bytes 2-3: Additional information
-            'additional_info'/HexAdapter(Bytes(2)),
+            'additional_info'/Bytes(2),
             # Bytes 4..: RFU
-            'rfu'/HexAdapter(GreedyBytesRFU),
+            'rfu'/GreedyBytesRFU,
         )
 
 

pySim/commands.py

@@ -735,7 +735,7 @@ class SimCardCommands:
         Args:
                 payload : payload as hex string
         """
-        return self.send_apdu_checksw('80c20000%02x%s' % (len(payload)//2, payload), apply_lchan = False)
+        return self.send_apdu_checksw('80c20000%02x%s' % (len(payload)//2, payload) + "00", apply_lchan = False)
 
     def terminal_profile(self, payload: Hexstr) -> ResTuple:
         """Send TERMINAL PROFILE to card

pySim/esim/__init__.py

@@ -61,8 +61,8 @@ def compile_asn1_subdir(subdir_name:str, codec='der'):
     return asn1tools.compile_string(asn_txt, codec=codec)
 
 
-# SGP.22 section 4.1 Activation Code
 class ActivationCode:
+    """SGP.22 section 4.1 Activation Code"""
     def __init__(self, hostname:str, token:str, oid: Optional[str] = None, cc_required: Optional[bool] = False):
         if '$' in hostname:
             raise ValueError('$ sign not permitted in hostname')
@@ -78,6 +78,7 @@ class ActivationCode:
 
     @staticmethod
     def decode_str(ac: str) -> dict:
+        """decode an activation code from its string representation."""
         if ac[0] != '1':
             raise ValueError("Unsupported AC_Format '%s'!" % ac[0])
         ac_elements = ac.split('$')

pySim/esim/bsp.py

@@ -1,11 +1,9 @@
-# Early proof-of-concept implementation of
-# GSMA eSIM RSP (Remote SIM Provisioning BSP (BPP Protection Protocol),
-# where BPP is the Bound  Profile Package.  So the full expansion is the
-# "GSMA eSIM Remote SIM Provisioning Bound Profile Packate Protection Protocol"
-#
-# Originally (SGP.22 v2.x) this was called SCP03t, but it has since been
-# renamed to BSP.
-#
+"""Implementation of GSMA eSIM RSP (Remote SIM Provisioning BSP (BPP Protection Protocol),
+where BPP is the Bound  Profile Package.  So the full expansion is the
+"GSMA eSIM Remote SIM Provisioning Bound Profile Packate Protection Protocol"
+
+Originally (SGP.22 v2.x) this was called SCP03t, but it has since been renamed to BSP."""
+
 # (C) 2023 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -45,6 +43,7 @@ logger.addHandler(logging.NullHandler())
 MAX_SEGMENT_SIZE = 1020
 
 class BspAlgo(abc.ABC):
+    """Base class representing a cryptographic algorithm within the BSP (BPP Security Protocol)."""
     blocksize: int
 
     def _get_padding(self, in_len: int, multiple: int, padding: int = 0) -> bytes:
@@ -62,6 +61,7 @@ class BspAlgo(abc.ABC):
         return self.__class__.__name__
 
 class BspAlgoCrypt(BspAlgo, abc.ABC):
+    """Base class representing an encryption/decryption algorithm within the BSP (BPP Security Protocol)."""
 
     def __init__(self, s_enc: bytes):
         self.s_enc = s_enc
@@ -73,7 +73,7 @@ class BspAlgoCrypt(BspAlgo, abc.ABC):
         block_nr = self.block_nr
         ciphertext = self._encrypt(padded_data)
         logger.debug("encrypt(block_nr=%u, s_enc=%s, plaintext=%s, padded=%s) -> %s",
-                     block_nr, b2h(self.s_enc), b2h(data), b2h(padded_data), b2h(ciphertext))
+                     block_nr, b2h(self.s_enc)[:20], b2h(data)[:20], b2h(padded_data)[:20], b2h(ciphertext)[:20])
         return ciphertext
 
     def decrypt(self, data:bytes) -> bytes:
@@ -93,6 +93,7 @@ class BspAlgoCrypt(BspAlgo, abc.ABC):
         """Actual implementation, to be implemented by derived class."""
 
 class BspAlgoCryptAES128(BspAlgoCrypt):
+    """AES-CBC-128 implementation of the BPP Security Protocol for GSMA SGP.22 eSIM."""
     name = 'AES-CBC-128'
     blocksize = 16
 
@@ -133,6 +134,7 @@ class BspAlgoCryptAES128(BspAlgoCrypt):
 
 
 class BspAlgoMac(BspAlgo, abc.ABC):
+    """Base class representing a message authentication code algorithm within the BSP (BPP Security Protocol)."""
     l_mac = 0 # must be overridden by derived class
 
     def __init__(self, s_mac: bytes, initial_mac_chaining_value: bytes):
@@ -147,10 +149,20 @@ class BspAlgoMac(BspAlgo, abc.ABC):
         temp_data = self.mac_chain + tag_and_length + data
         old_mcv = self.mac_chain
         c_mac = self._auth(temp_data)
+
+        # DEBUG: Show MAC computation details
+        logger.debug(f"MAC_DEBUG: tag=0x{tag:02x}, lcc={lcc}")
+        logger.debug(f"MAC_DEBUG: tag_and_length: {tag_and_length.hex()}")
+        logger.debug(f"MAC_DEBUG: mac_chain[:20]: {old_mcv[:20].hex()}")
+        logger.debug(f"MAC_DEBUG: temp_data[:20]: {temp_data[:20].hex()}")
+        logger.debug(f"MAC_DEBUG: c_mac: {c_mac.hex()}")
+
         # The output data is computed by concatenating the following data: the tag, the final length, the result of step 2 and the C-MAC value.
         ret = tag_and_length + data + c_mac
+        logger.debug(f"MAC_DEBUG: final_output[:20]: {ret[:20].hex()}")
+
         logger.debug("auth(tag=0x%x, mcv=%s, s_mac=%s, plaintext=%s, temp=%s) -> %s",
-                     tag, b2h(old_mcv), b2h(self.s_mac), b2h(data), b2h(temp_data), b2h(ret))
+                     tag, b2h(old_mcv)[:20], b2h(self.s_mac)[:20], b2h(data)[:20], b2h(temp_data)[:20], b2h(ret)[:20])
         return ret
 
     def verify(self, ciphertext: bytes) -> bool:
@@ -167,6 +179,7 @@ class BspAlgoMac(BspAlgo, abc.ABC):
         """To be implemented by algorithm specific derived class."""
 
 class BspAlgoMacAES128(BspAlgoMac):
+    """AES-CMAC-128 implementation of the BPP Security Protocol for GSMA SGP.22 eSIM."""
     name = 'AES-CMAC-128'
     l_mac = 8
 
@@ -201,6 +214,11 @@ def bsp_key_derivation(shared_secret: bytes, key_type: int, key_length: int, hos
     s_enc = out[l:2*l]
     s_mac = out[l*2:3*l]
 
+    logger.debug(f"BSP_KDF_DEBUG: kdf_out = {b2h(out)}")
+    logger.debug(f"BSP_KDF_DEBUG: initial_mcv = {b2h(initial_mac_chaining_value)}")
+    logger.debug(f"BSP_KDF_DEBUG: s_enc = {b2h(s_enc)}")
+    logger.debug(f"BSP_KDF_DEBUG: s_mac = {b2h(s_mac)}")
+
     return s_enc, s_mac, initial_mac_chaining_value
 
 
@@ -225,12 +243,24 @@ class BspInstance:
         return cls(s_enc, s_mac, initial_mcv)
 
     def encrypt_and_mac_one(self, tag: int, plaintext:bytes) -> bytes:
-        """Encrypt + MAC a single plaintext TLV. Returns the protected ciphertex."""
+        """Encrypt + MAC a single plaintext TLV. Returns the protected ciphertext."""
         assert tag <= 255
         assert len(plaintext) <= self.max_payload_size
-        logger.debug("encrypt_and_mac_one(tag=0x%x, plaintext=%s)", tag, b2h(plaintext))
+
+        # DEBUG: Show what we're processing
+        logger.debug(f"BSP_DEBUG: encrypt_and_mac_one(tag=0x{tag:02x}, plaintext_len={len(plaintext)})")
+        logger.debug(f"BSP_DEBUG: plaintext[:20]: {plaintext[:20].hex()}")
+        logger.debug(f"BSP_DEBUG: s_enc[:20]: {self.c_algo.s_enc[:20].hex()}")
+        logger.debug(f"BSP_DEBUG: s_mac[:20]: {self.m_algo.s_mac[:20].hex()}")
+
+        logger.debug("encrypt_and_mac_one(tag=0x%x, plaintext=%s)", tag, b2h(plaintext)[:20])
         ciphered = self.c_algo.encrypt(plaintext)
+        logger.debug(f"BSP_DEBUG: ciphered[:20]: {ciphered[:20].hex()}")
+
         maced = self.m_algo.auth(tag, ciphered)
+        logger.debug(f"BSP_DEBUG: final_result[:20]: {maced[:20].hex()}")
+        logger.debug(f"BSP_DEBUG: final_result_len: {len(maced)}")
+
         return maced
 
     def encrypt_and_mac(self, tag: int, plaintext:bytes) -> List[bytes]:
@@ -250,11 +280,11 @@ class BspInstance:
         return result
 
     def mac_only_one(self, tag: int, plaintext: bytes) -> bytes:
-        """MAC a single plaintext TLV. Returns the protected ciphertex."""
+        """MAC a single plaintext TLV. Returns the protected ciphertext."""
         assert tag <= 255
         assert len(plaintext) < self.max_payload_size
         maced = self.m_algo.auth(tag, plaintext)
-        # The data block counter for ICV caluclation is incremented also for each segment with C-MAC only.
+        # The data block counter for ICV calculation is incremented also for each segment with C-MAC only.
         self.c_algo.block_nr += 1
         return maced
 
@@ -288,7 +318,7 @@ class BspInstance:
     def demac_only_one(self, ciphertext: bytes) -> bytes:
         payload = self.m_algo.verify(ciphertext)
         _tdict, _l, val, _remain = bertlv_parse_one(payload)
-        # The data block counter for ICV caluclation is incremented also for each segment with C-MAC only.
+        # The data block counter for ICV calculation is incremented also for each segment with C-MAC only.
         self.c_algo.block_nr += 1
         return val
 

pySim/esim/es8p.py

@@ -1,6 +1,5 @@
-# Implementation of GSMA eSIM RSP (Remote SIM Provisioning) ES8+
-# as per SGP22 v3.0 Section 5.5
-#
+"""Implementation of GSMA eSIM RSP (Remote SIM Provisioning) ES8+ as per SGP22 v3.0 Section 5.5"""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -26,6 +25,9 @@ import pySim.esim.rsp as rsp
 from pySim.esim.bsp import BspInstance
 from pySim.esim import PMO
 
+import logging
+logger = logging.getLogger(__name__)
+
 # Given that GSMA RSP uses ASN.1 in a very weird way, we actually cannot encode the full data type before
 # signing, but we have to build parts of it separately first, then sign that, so we can put the signature
 # into the same sequence as the signed data.  We use the existing pySim TLV code for this.
@@ -97,7 +99,7 @@ class ProfileMetadata:
         self.notifications.append((event, address))
 
     def gen_store_metadata_request(self) -> bytes:
-        """Generate encoded (but unsigned) StoreMetadataReqest DO (SGP.22 5.5.3)"""
+        """Generate encoded (but unsigned) StoreMetadataRequest DO (SGP.22 5.5.3)"""
         smr = {
             'iccid': self.iccid_bin,
             'serviceProviderName': self.spn,
@@ -197,8 +199,12 @@ class BoundProfilePackage(ProfilePackage):
         # 'initialiseSecureChannelRequest'
         bpp_seq = rsp.asn1.encode('InitialiseSecureChannelRequest', iscr)
         # firstSequenceOf87
+        logger.debug(f"BPP_ENCODE_DEBUG: Encrypting ConfigureISDP with BSP keys")
+        logger.debug(f"BPP_ENCODE_DEBUG: BSP S-ENC: {bsp.c_algo.s_enc.hex()}")
+        logger.debug(f"BPP_ENCODE_DEBUG: BSP S-MAC: {bsp.m_algo.s_mac.hex()}")
         bpp_seq += encode_seq(0xa0, bsp.encrypt_and_mac(0x87, conf_idsp_bin))
         # sequenceOF88
+        logger.debug(f"BPP_ENCODE_DEBUG: MAC-only StoreMetadata with BSP keys")
         bpp_seq += encode_seq(0xa1, bsp.mac_only(0x88, smr_bin))
 
         if self.ppp: # we have to use session keys

pySim/esim/http_json_api.py

@@ -26,15 +26,15 @@ logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
 class ApiParam(abc.ABC):
-    """A class reprsenting a single parameter in the API."""
+    """A class representing a single parameter in the API."""
     @classmethod
     def verify_decoded(cls, data):
-        """Verify the decoded reprsentation of a value. Should raise an exception if somthing is odd."""
+        """Verify the decoded representation of a value. Should raise an exception if something is odd."""
         pass
 
     @classmethod
     def verify_encoded(cls, data):
-        """Verify the encoded reprsentation of a value. Should raise an exception if somthing is odd."""
+        """Verify the encoded representation of a value. Should raise an exception if something is odd."""
         pass
 
     @classmethod

pySim/esim/rsp.py

@@ -1,6 +1,5 @@
-# Implementation of GSMA eSIM RSP (Remote SIM Provisioning)
-# as per SGP22 v3.0
-#
+"""Implementation of GSMA eSIM RSP (Remote SIM Provisioning) as per SGP22 v3.0"""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -95,9 +94,57 @@ class RspSessionState:
         self.__dict__.update(state)
 
 
-class RspSessionStore(shelve.DbfilenameShelf):
-    """A derived class as wrapper around the database-backed non-volatile storage 'shelve', in case we might
-    need to extend it in the future. We use it to store RspSessionState objects indexed by transactionId."""
+class RspSessionStore:
+    """A wrapper around the database-backed storage 'shelve' for storing RspSessionState objects.
+    Can be configured to use either file-based storage or in-memory storage.
+    We use it to store RspSessionState objects indexed by transactionId."""
+
+    def __init__(self, filename: Optional[str] = None, in_memory: bool = False):
+        self._in_memory = in_memory
+
+        if in_memory:
+            self._shelf = shelve.Shelf(dict())
+        else:
+            if filename is None:
+                raise ValueError("filename is required for file-based session store")
+            self._shelf = shelve.open(filename)
+
+    # dunder magic
+    def __getitem__(self, key):
+        return self._shelf[key]
+
+    def __setitem__(self, key, value):
+        self._shelf[key] = value
+
+    def __delitem__(self, key):
+        del self._shelf[key]
+
+    def __contains__(self, key):
+        return key in self._shelf
+
+    def __iter__(self):
+        return iter(self._shelf)
+
+    def __len__(self):
+        return len(self._shelf)
+
+    # everything else
+    def __getattr__(self, name):
+        """Delegate attribute access to the underlying shelf object."""
+        return getattr(self._shelf, name)
+
+    def close(self):
+        """Close the session store."""
+        if hasattr(self._shelf, 'close'):
+            self._shelf.close()
+        if self._in_memory:
+            # For in-memory store, clear the reference
+            self._shelf = None
+
+    def sync(self):
+        """Synchronize the cache with the underlying storage."""
+        if hasattr(self._shelf, 'sync'):
+            self._shelf.sync()
 
 def extract_euiccSigned1(authenticateServerResponse: bytes) -> bytes:
     """Extract the raw, DER-encoded binary euiccSigned1 field from the given AuthenticateServerResponse. This

pySim/esim/saip/__init__.py

@@ -1,5 +1,5 @@
-# Implementation of SimAlliance/TCA Interoperable Profile handling
-#
+"""Implementation of SimAlliance/TCA Interoperable Profile handling"""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -18,13 +18,17 @@
 import logging
 import abc
 import io
+import os
 from typing import Tuple, List, Optional, Dict, Union
 from collections import OrderedDict
 import asn1tools
+import zipfile
+from pySim import javacard
 from osmocom.utils import b2h, h2b, Hexstr
 from osmocom.tlv import BER_TLV_IE, bertlv_parse_tag, bertlv_parse_len
-from osmocom.construct import build_construct, parse_construct, GreedyInteger
+from osmocom.construct import build_construct, parse_construct, GreedyInteger, GreedyBytes, StripHeaderAdapter
 
+from pySim import ts_102_222
 from pySim.utils import dec_imsi
 from pySim.ts_102_221 import FileDescriptor
 from pySim.filesystem import CardADF, Path
@@ -41,7 +45,7 @@ asn1 = compile_asn1_subdir('saip')
 logger = logging.getLogger(__name__)
 
 class Naa:
-    """A class defining a Network Access Application (NAA)."""
+    """A class defining a Network Access Application (NAA)"""
     name = None
     # AID prefix, as used for ADF and EF.DIR
     aid = None
@@ -57,6 +61,7 @@ class Naa:
         return 'adf-' + cls.mandatory_services[0]
 
 class NaaCsim(Naa):
+    """A class representing the CSIM (CDMA) Network Access Application (NAA)"""
     name = "csim"
     aid = h2b("")
     mandatory_services = ["csim"]
@@ -64,6 +69,7 @@ class NaaCsim(Naa):
     templates = [oid.ADF_CSIM_by_default, oid.ADF_CSIMopt_not_by_default]
 
 class NaaUsim(Naa):
+    """A class representing the USIM Network Access Application (NAA)"""
     name = "usim"
     aid = h2b("a0000000871002")
     mandatory_services = ["usim"]
@@ -76,6 +82,7 @@ class NaaUsim(Naa):
     adf = ADF_USIM()
 
 class NaaIsim(Naa):
+    """A class representing the ISIM Network Access Application (NAA)"""
     name = "isim"
     aid = h2b("a0000000871004")
     mandatory_services = ["isim"]
@@ -104,7 +111,7 @@ class File:
         self.pe_name = pename
         self._name = name
         self.template = template
-        self.body: Optional[bytes] = None
+        self._body: Optional[bytes] = None
         self.node: Optional['FsNode'] = None
         self.file_type = None
         self.fid: Optional[int] = None
@@ -114,8 +121,11 @@ class File:
         self.nb_rec: Optional[int] = None
         self._file_size = 0
         self.high_update: bool = False
+        self.read_and_update_when_deact: bool = False
         self.shareable: bool = True
         self.df_name = None
+        self.fill_pattern = None
+        self.fill_pattern_repeat = False
         # apply some defaults from profile
         if self.template:
             self.from_template(self.template)
@@ -173,9 +183,7 @@ class File:
         self.file_type = template.file_type
         self.fid = template.fid
         self.sfi = template.sfi
-        self.arr = template.arr
-        #self.default_val = template.default_val
-        #self.default_val_repeat = template.default_val_repeat
+        self.arr = template.arr.to_bytes(1)
         if hasattr(template, 'rec_len'):
             self.rec_len = template.rec_len
         else:
@@ -188,19 +196,38 @@ class File:
         # All the files defined in the templates shall have, by default, shareable/not-shareable bit in the file descriptor set to "shareable".
         self.shareable = True
         self._template_derived = True
+        if hasattr(template, 'file_size'):
+            self._file_size = template.file_size
+
+    def _recompute_size(self):
+        """recompute the file size, if needed (body larger than current size)"""
+        body_size = len(self.body)
+        if self.file_size == None or self.file_size < body_size:
+            self._file_size = body_size
+
+    @property
+    def body(self):
+        return self._body
+
+    @body.setter
+    def body(self, value: bytes):
+        self._body = value
+        # we need to potentially update the file size after changing the body [size]
+        self._recompute_size()
 
     def to_fileDescriptor(self) -> dict:
         """Convert from internal representation to 'fileDescriptor' as used by asn1tools for SAIP"""
         fileDescriptor = {}
         fdb_dec = {}
         pefi = {}
-        if self.fid:
+        spfi = 0
+        if self.fid and self.fid != self.template.fid:
             fileDescriptor['fileID'] = self.fid.to_bytes(2, 'big')
-        if self.sfi:
+        if self.sfi and self.sfi != self.template.sfi:
             fileDescriptor['shortEFID'] = bytes([self.sfi])
         if self.df_name:
             fileDescriptor['dfName'] = self.df_name
-        if self.arr:
+        if self.arr and self.arr != self.template.arr.to_bytes(1):
             fileDescriptor['securityAttributesReferenced'] = self.arr
         if self.file_type in ['LF', 'CY']:
             fdb_dec['file_type'] = 'working_ef'
@@ -233,15 +260,16 @@ class File:
         if len(fd_dict):
             fileDescriptor['fileDescriptor'] = build_construct(FileDescriptor._construct, fd_dict)
         if self.high_update:
-            pefi['specialFileInformation'] = b'\x80' # TS 102 222 Table 5
-        try:
-            if self.template and self.template.default_val_repeat:
-                pefi['repeatPattern'] = self.template.expand_default_value_pattern()
-            elif self.template and self.template.default_val:
-                pefi['fillPattern'] = self.template.expand_default_value_pattern()
-        except ValueError:
-            # ignore this here as without a file or record length we cannot do this
-            pass
+            spfi |= 0x80 # TS 102 222 Table 5
+        if self.read_and_update_when_deact:
+            spfi |= 0x40 # TS 102 222 Table 5
+        if spfi != 0x00:
+            pefi['specialFileInformation'] = spfi.to_bytes(1)
+        if self.fill_pattern:
+            if not self.fill_pattern_repeat:
+                pefi['fillPattern'] = self.fill_pattern
+            else:
+                pefi['repeatPattern'] = self.fill_pattern
         if len(pefi.keys()):
             # TODO: When overwriting the default "proprietaryEFInfo" for a template EF for which a
             # default fill or repeat pattern is defined; it is hence recommended to provide the
@@ -273,12 +301,15 @@ class File:
             self.shareable = fdb_dec['shareable']
             if fdb_dec['file_type'] == 'working_ef':
                 efFileSize = fileDescriptor.get('efFileSize', None)
-                if efFileSize:
-                    self._file_size = self._decode_file_size(efFileSize)
                 if fd_dec['num_of_rec']:
                     self.nb_rec = fd_dec['num_of_rec']
                 if fd_dec['record_len']:
                     self.rec_len = fd_dec['record_len']
+                if efFileSize:
+                    self._file_size = self._decode_file_size(efFileSize)
+                    if self.rec_len and self.nb_rec == None:
+                        # compute the number of records from file size and record length
+                        self.nb_rec = self._file_size // self.rec_len
                 if fdb_dec['structure'] == 'linear_fixed':
                     self.file_type = 'LF'
                 elif fdb_dec['structure'] == 'cyclic':
@@ -291,12 +322,17 @@ class File:
                         self._file_size = self._decode_file_size(pefi['maximumFileSize'])
                 specialFileInformation = pefi.get('specialFileInformation', None)
                 if specialFileInformation:
+                    # TS 102 222 Table 5
                     if specialFileInformation[0] & 0x80:
-                        self.hihgi_update = True
+                        self.high_update = True
+                    if specialFileInformation[0] & 0x40:
+                        self.read_and_update_when_deact = True
                 if 'repeatPattern' in pefi:
-                    self.repeat_pattern = pefi['repeatPattern']
-                if 'defaultPattern' in pefi:
-                    self.repeat_pattern = pefi['defaultPattern']
+                    self.fill_pattern = pefi['repeatPattern']
+                    self.fill_pattern_repeat = True
+                if 'fillPattern' in pefi:
+                    self.fill_pattern = pefi['fillPattern']
+                    self.fill_pattern_repeat = False
             elif fdb_dec['file_type'] == 'df':
                 # only set it, if an earlier call to from_template() didn't alrady set it, as
                 # the template can differentiate between MF, DF and ADF (unlike FDB)
@@ -319,7 +355,7 @@ class File:
         if fd:
             self.from_fileDescriptor(dict(fd))
         # BODY
-        self.body = self.file_content_from_tuples(l)
+        self._body = self.file_content_from_tuples(l)
 
     @staticmethod
     def path_from_gfm(bin_path: bytes):
@@ -341,18 +377,29 @@ class File:
         ret += self.file_content_to_tuples()
         return ret
 
-    @staticmethod
-    def file_content_from_tuples(l: List[Tuple]) -> Optional[bytes]:
+    def expand_fill_pattern(self) -> bytes:
+        """Expand the fill/repeat pattern as per TS 102 222 Section 6.3.2.2.2"""
+        return ts_102_222.expand_pattern(self.fill_pattern, self.fill_pattern_repeat, self.file_size)
+
+    def file_content_from_tuples(self, l: List[Tuple]) -> Optional[bytes]:
         """linearize a list of fillFileContent / fillFileOffset tuples into a stream of bytes."""
         stream = io.BytesIO()
+        # Providing file content within "fillFileContent" / "fillFileOffset" shall have the same effect as
+        # creating a file with a fill/repeat pattern and thereafter updating the content via Update.
+        # Step 1: Fill with pattern from Fcp or Template
+        if self.fill_pattern:
+            stream.write(self.expand_fill_pattern())
+        elif self.template and self.template.default_val:
+            stream.write(self.template.expand_default_value_pattern(self.file_size))
+        stream.seek(0)
+        # then process the fillFileContent/fillFileOffset
         for k, v in l:
             if k == 'doNotCreate':
                 return None
             if k == 'fileDescriptor':
                 pass
             elif k == 'fillFileOffset':
-                # FIXME: respect the fillPattern!
-                stream.write(b'\xff' * v)
+                stream.seek(v, os.SEEK_CUR)
             elif k == 'fillFileContent':
                 stream.write(v)
             else:
@@ -426,7 +473,7 @@ class ProfileElement:
     @property
     def header_name(self) -> str:
         """Return the name of the header field within the profile element."""
-        # unneccessarry compliaction by inconsistent naming :(
+        # unnecessary complication by inconsistent naming :(
         if self.type.startswith('opt-'):
             return self.type.replace('-','') + '-header'
         if self.type in self.header_name_translation_dict:
@@ -464,7 +511,7 @@ class ProfileElement:
             # TODO: cdmaParameter
             'securityDomain': ProfileElementSD,
             'rfm': ProfileElementRFM,
-            # TODO: application
+            'application': ProfileElementApplication,
             # TODO: nonStandard
             'end': ProfileElementEnd,
             'mf': ProfileElementMF,
@@ -601,6 +648,14 @@ class FsProfileElement(ProfileElement):
                 file = File(k, v, template.files_by_pename.get(k, None))
                 self.add_file(file)
 
+    def create_file(self, pename: str) -> File:
+        """Programatically create a file by its PE-Name."""
+        template = templates.ProfileTemplateRegistry.get_by_oid(self.templateID)
+        file = File(pename, None, template.files_by_pename.get(pename, None))
+        self.add_file(file)
+        self.decoded[pename] = []
+        return file
+
     def _post_decode(self):
         # not entirely sure about doing this this automatism
         self.pe2files()
@@ -698,6 +753,7 @@ class ProfileElementGFM(ProfileElement):
 
 
 class ProfileElementMF(FsProfileElement):
+    """Class representing the ProfileElement for the MF (Master File)"""
     type = 'mf'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -711,6 +767,7 @@ class ProfileElementMF(FsProfileElement):
         # TODO: resize EF.DIR?
 
 class ProfileElementPuk(ProfileElement):
+    """Class representing the ProfileElement for a PUK (PIN Unblocking Code)"""
     type = 'pukCodes'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -741,6 +798,7 @@ class ProfileElementPuk(ProfileElement):
 
 
 class ProfileElementPin(ProfileElement):
+    """Class representing the ProfileElement for a PIN (Personal Identification Number)"""
     type = 'pinCodes'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -777,6 +835,7 @@ class ProfileElementPin(ProfileElement):
 
 
 class ProfileElementTelecom(FsProfileElement):
+    """Class representing the ProfileElement for DF.TELECOM"""
     type = 'telecom'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -789,6 +848,7 @@ class ProfileElementTelecom(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementCD(FsProfileElement):
+    """Class representing the ProfileElement for DF.CD"""
     type = 'cd'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -801,6 +861,7 @@ class ProfileElementCD(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementPhonebook(FsProfileElement):
+    """Class representing the ProfileElement for DF.PHONEBOOK"""
     type = 'phonebook'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -813,6 +874,7 @@ class ProfileElementPhonebook(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementGsmAccess(FsProfileElement):
+    """Class representing the ProfileElement for ADF.USIM/DF.GSM-ACCESS"""
     type = 'gsm-access'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -825,6 +887,7 @@ class ProfileElementGsmAccess(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementDf5GS(FsProfileElement):
+    """Class representing the ProfileElement for ADF.USIM/DF.5GS"""
     type = 'df-5gs'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -837,6 +900,7 @@ class ProfileElementDf5GS(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementEAP(FsProfileElement):
+    """Class representing the ProfileElement for DF.EAP"""
     type = 'eap'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -849,6 +913,7 @@ class ProfileElementEAP(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementDfSAIP(FsProfileElement):
+    """Class representing the ProfileElement for DF.SAIP"""
     type = 'df-saip'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -861,6 +926,7 @@ class ProfileElementDfSAIP(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementDfSNPN(FsProfileElement):
+    """Class representing the ProfileElement for DF.SNPN"""
     type = 'df-snpn'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -873,6 +939,7 @@ class ProfileElementDfSNPN(FsProfileElement):
             self.decoded[fname] = []
 
 class ProfileElementDf5GProSe(FsProfileElement):
+    """Class representing the ProfileElement for DF.5GProSe"""
     type = 'df-5gprose'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -909,7 +976,7 @@ class SecurityDomainKeyComponent:
                 'macLength': self.mac_length}
 
 class SecurityDomainKey:
-    """Represenation of a key used for SCP access to a security domain."""
+    """Representation of a key used for SCP access to a security domain."""
     def __init__(self, key_version_number: int, key_id: int, key_usage_qualifier: dict,
                  key_components: List[SecurityDomainKeyComponent]):
         self.key_usage_qualifier = key_usage_qualifier
@@ -1037,7 +1104,136 @@ class ProfileElementSSD(ProfileElementSD):
                 'uiccToolkitApplicationSpecificParametersField': h2b('01000001000000020112036C756500'),
             }
 
+class ProfileElementApplication(ProfileElement):
+    """Class representing an application ProfileElement."""
+    type = 'application'
+
+    def __init__(self, decoded: Optional[dict] = None, **kwargs):
+        super().__init__(decoded, **kwargs)
+
+    @classmethod
+    def from_file(cls,
+                  filename:str,
+                  aid:Hexstr,
+                  sd_aid:Hexstr = None,
+                  non_volatile_code_limit:int = None,
+                  volatile_data_limit:int = None,
+                  non_volatile_data_limit:int = None,
+                  hash_value:Hexstr = None) -> 'ProfileElementApplication':
+        """Fill contents of application ProfileElement from a .cap file."""
+
+        inst = cls()
+        Construct_data_limit = StripHeaderAdapter(GreedyBytes, 4, steps = [2,4])
+
+        if filename.lower().endswith('.cap'):
+            cap = javacard.CapFile(filename)
+            load_block_object = cap.get_loadfile()
+        elif filename.lower().endswith('.ijc'):
+            fd = open(filename, 'rb')
+            load_block_object = fd.read()
+        else:
+            raise ValueError('Invalid file type, file must either .cap or .ijc')
+
+        # Mandatory
+        inst.decoded['loadBlock'] = {
+            'loadPackageAID': h2b(aid),
+            'loadBlockObject': load_block_object
+        }
+
+        # Optional
+        if sd_aid:
+            inst.decoded['loadBlock']['securityDomainAID'] = h2b(sd_aid)
+        if non_volatile_code_limit:
+            inst.decoded['loadBlock']['nonVolatileCodeLimitC6'] = Construct_data_limit.build(non_volatile_code_limit)
+        if volatile_data_limit:
+            inst.decoded['loadBlock']['volatileDataLimitC7'] = Construct_data_limit.build(volatile_data_limit)
+        if non_volatile_data_limit:
+            inst.decoded['loadBlock']['nonVolatileDataLimitC8'] = Construct_data_limit.build(non_volatile_data_limit)
+        if hash_value:
+            inst.decoded['loadBlock']['hashValue'] = h2b(hash_value)
+
+        return inst
+
+    def to_file(self, filename:str):
+        """Write loadBlockObject contents of application ProfileElement to a .cap or .ijc file."""
+
+        load_package_aid = b2h(self.decoded['loadBlock']['loadPackageAID'])
+        load_block_object = self.decoded['loadBlock']['loadBlockObject']
+
+        if filename.lower().endswith('.cap'):
+            with io.BytesIO(load_block_object) as f, zipfile.ZipFile(filename, 'w') as z:
+                javacard.ijc_to_cap(f, z, load_package_aid)
+        elif filename.lower().endswith('.ijc'):
+            with open(filename, 'wb') as f:
+                f.write(load_block_object)
+        else:
+            raise ValueError('Invalid file type, file must either .cap or .ijc')
+
+    def add_instance(self,
+                     aid:Hexstr,
+                     class_aid:Hexstr,
+                     inst_aid:Hexstr,
+                     app_privileges:Hexstr,
+                     app_spec_pars:Hexstr,
+                     uicc_toolkit_app_spec_pars:Hexstr = None,
+                     uicc_access_app_spec_pars:Hexstr = None,
+                     uicc_adm_access_app_spec_pars:Hexstr = None,
+                     volatile_memory_quota:Hexstr = None,
+                     non_volatile_memory_quota:Hexstr = None,
+                     process_data:list[Hexstr] = None):
+        """Create a new instance and add it to the instanceList"""
+
+        # Mandatory
+        inst = {'applicationLoadPackageAID': h2b(aid),
+                'classAID': h2b(class_aid),
+                'instanceAID': h2b(inst_aid),
+                'applicationPrivileges': h2b(app_privileges),
+                'applicationSpecificParametersC9': h2b(app_spec_pars)}
+
+        # Optional
+        if uicc_toolkit_app_spec_pars or uicc_access_app_spec_pars or uicc_adm_access_app_spec_pars:
+            inst['applicationParameters'] = {}
+            if uicc_toolkit_app_spec_pars:
+                inst['applicationParameters']['uiccToolkitApplicationSpecificParametersField'] = \
+                    h2b(uicc_toolkit_app_spec_pars)
+            if uicc_access_app_spec_pars:
+                inst['applicationParameters']['uiccAccessApplicationSpecificParametersField'] = \
+                    h2b(uicc_access_app_spec_pars)
+            if uicc_adm_access_app_spec_pars:
+                inst['applicationParameters']['uiccAdministrativeAccessApplicationSpecificParametersField'] = \
+                    h2b(uicc_adm_access_app_spec_pars)
+        if volatile_memory_quota is not None or non_volatile_memory_quota is not None:
+            inst['systemSpecificParameters'] = {}
+            Construct_data_limit = StripHeaderAdapter(GreedyBytes, 4, steps = [2,4])
+            if volatile_memory_quota is not None:
+                inst['systemSpecificParameters']['volatileMemoryQuotaC7'] = \
+                    Construct_data_limit.build(volatile_memory_quota)
+            if non_volatile_memory_quota is not None:
+                inst['systemSpecificParameters']['nonVolatileMemoryQuotaC8'] = \
+                    Construct_data_limit.build(non_volatile_memory_quota)
+        if len(process_data) > 0:
+            inst['processData'] = []
+        for proc in process_data:
+            inst['processData'].append(h2b(proc))
+
+        # Append created instance to instance list
+        if 'instanceList' not in self.decoded.keys():
+            self.decoded['instanceList'] = []
+        self.decoded['instanceList'].append(inst)
+
+    def remove_instance(self, inst_aid:Hexstr):
+        """Remove an instance from the instanceList"""
+        inst_list = self.decoded.get('instanceList', [])
+        for inst in enumerate(inst_list):
+            if b2h(inst[1].get('instanceAID', None)) == inst_aid:
+                inst_list.pop(inst[0])
+                return
+        raise ValueError("instance AID: '%s' not present in instanceList, cannot remove instance" % inst[1])
+
+
+
 class ProfileElementRFM(ProfileElement):
+    """Class representing the ProfileElement for RFM (Remote File Management)."""
     type = 'rfm'
 
     def __init__(self, decoded: Optional[dict] = None,
@@ -1063,6 +1259,7 @@ class ProfileElementRFM(ProfileElement):
                 }
 
 class ProfileElementUSIM(FsProfileElement):
+    """Class representing the ProfileElement for ADF.USIM Mandatory Files"""
     type = 'usim'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -1080,10 +1277,12 @@ class ProfileElementUSIM(FsProfileElement):
 
     @property
     def imsi(self) -> Optional[str]:
-        f = File('ef-imsi', self.decoded['ef-imsi'])
+        template = templates.ProfileTemplateRegistry.get_by_oid(self.templateID)
+        f = File('ef-imsi', self.decoded['ef-imsi'], template.files_by_pename.get('ef-imsi', None))
         return dec_imsi(b2h(f.body))
 
 class ProfileElementOptUSIM(FsProfileElement):
+    """Class representing the ProfileElement for ADF.USIM Optional Files"""
     type = 'opt-usim'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -1094,6 +1293,7 @@ class ProfileElementOptUSIM(FsProfileElement):
         self.decoded['templateID'] = str(oid.ADF_USIMopt_not_by_default_v2)
 
 class ProfileElementISIM(FsProfileElement):
+    """Class representing the ProfileElement for ADF.ISIM Mandatory Files"""
     type = 'isim'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -1110,6 +1310,7 @@ class ProfileElementISIM(FsProfileElement):
         return b2h(self.decoded['adf-isim'][0][1]['dfName'])
 
 class ProfileElementOptISIM(FsProfileElement):
+    """Class representing the ProfileElement for ADF.ISIM Optional Files"""
     type = 'opt-isim'
 
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
@@ -1121,6 +1322,7 @@ class ProfileElementOptISIM(FsProfileElement):
 
 
 class ProfileElementAKA(ProfileElement):
+    """Class representing the ProfileElement for Authentication and Key Agreement (AKA)."""
     type = 'akaParameter'
     # TODO: RES size for USIM test algorithm can be set to 32, 64 or 128 bits. This value was
     # previously limited to 128 bits.  Recommendation: Avoid using RES size 32 or 64 in Profiles
@@ -1200,6 +1402,7 @@ class ProfileElementAKA(ProfileElement):
         })
 
 class ProfileElementHeader(ProfileElement):
+    """Class representing the ProfileElement for the Header of the PE-Sequence."""
     type = 'header'
     def __init__(self, decoded: Optional[dict] = None,
                  ver_major: Optional[int] = 2, ver_minor: Optional[int] = 3,
@@ -1230,7 +1433,17 @@ class ProfileElementHeader(ProfileElement):
         if profile_type:
             self.decoded['profileType'] = profile_type
 
+    def mandatory_service_add(self, service_name):
+        self.decoded['eUICC-Mandatory-services'][service_name] = None
+
+    def mandatory_service_remove(self, service_name):
+        if service_name in self.decoded['eUICC-Mandatory-services'].keys():
+            del self.decoded['eUICC-Mandatory-services'][service_name]
+        else:
+            raise ValueError("service not in eUICC-Mandatory-services list, cannot remove")
+
 class ProfileElementEnd(ProfileElement):
+    """Class representing the ProfileElement for the End of the PE-Sequence."""
     type = 'end'
     def __init__(self, decoded: Optional[dict] = None, **kwargs):
         super().__init__(decoded, **kwargs)
@@ -1252,7 +1465,7 @@ class ProfileElementSequence:
     sequence."""
     def __init__(self):
         """After calling the constructor, you have to further initialize the instance by either
-        calling the parse_der() method, or by manually adding individual PEs, including the hedaer and
+        calling the parse_der() method, or by manually adding individual PEs, including the header and
         end PEs."""
         self.pe_list: List[ProfileElement] = []
         self.pe_by_type: Dict = {}
@@ -1274,7 +1487,7 @@ class ProfileElementSequence:
     def add_hdr_and_end(self):
         """Initialize the PE Sequence with a header and end PE."""
         if len(self.pe_list):
-            raise ValueError("Cannot add header + end PE to a non-enmpty PE-Sequence")
+            raise ValueError("Cannot add header + end PE to a non-empty PE-Sequence")
         # start with a minimal/empty sequence of header + end
         self.append(ProfileElementHeader())
         self.append(ProfileElementEnd())
@@ -1291,7 +1504,7 @@ class ProfileElementSequence:
 
     def get_pe_for_type(self, tname: str) -> Optional[ProfileElement]:
         """Return a single profile element for given profile element type. Works only for
-        types of which there is only a signle instance in the PE Sequence!"""
+        types of which there is only a single instance in the PE Sequence!"""
         l = self.get_pes_for_type(tname)
         if len(l) == 0:
             return None
@@ -1299,7 +1512,7 @@ class ProfileElementSequence:
         return l[0]
 
     def get_pes_for_templateID(self, tid: oid.OID) -> List[ProfileElement]:
-        """Return list of profile elements present for given profile eleemnt type."""
+        """Return list of profile elements present for given profile element type."""
         res = []
         for pe in self.pe_list:
             if not pe.templateID:
@@ -1476,6 +1689,27 @@ class ProfileElementSequence:
             pe.header['identification'] = i
             i += 1
 
+    def get_index_by_pe(self, pe: ProfileElement) -> int:
+        """Return a list with the indicies of all instances of PEs of petype."""
+        ret = []
+        i = 0
+        for cur in self.pe_list:
+            if cur == pe:
+                return i
+            i += 1
+        raise ValueError('PE %s is not part of PE Sequence' % (pe))
+
+    def insert_at_index(self, idx: int, pe: ProfileElement) -> None:
+        """Insert a given [new] ProfileElement at given index into the PE Sequence."""
+        self.pe_list.insert(idx, pe)
+        self._process_pelist()
+        self.renumber_identification()
+
+    def insert_after_pe(self, pe_before: ProfileElement, pe_new: ProfileElement) -> None:
+        """Insert a given [new] ProfileElement after a given [existing] PE in the PE Sequence."""
+        idx = self.get_index_by_pe(pe_before)
+        self.insert_at_index(idx+1, pe_new)
+
     def get_index_by_type(self, petype: str) -> List[int]:
         """Return a list with the indicies of all instances of PEs of petype."""
         ret = []
@@ -1491,9 +1725,7 @@ class ProfileElementSequence:
         # find MNO-SD index
         idx = self.get_index_by_type('securityDomain')[0]
         # insert _after_ MNO-SD
-        self.pe_list.insert(idx+1, ssd)
-        self._process_pelist()
-        self.renumber_identification()
+        self.insert_at_index(idx+1, ssd)
 
     def remove_naas_of_type(self, naa: Naa) -> None:
         """Remove all instances of NAAs of given type. This can be used, for example,

pySim/esim/saip/oid.py

@@ -1,5 +1,5 @@
-# Implementation of SimAlliance/TCA Interoperable Profile OIDs
-#
+"""Implementation of SimAlliance/TCA Interoperable Profile OIDs"""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify

pySim/esim/saip/personalization.py

@@ -1,5 +1,5 @@
-# Implementation of SimAlliance/TCA Interoperable Profile handling
-#
+"""Implementation of Personalization of eSIM profiles in SimAlliance/TCA Interoperable Profile."""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -67,7 +67,7 @@ class Iccid(ConfigurableParameter):
     If the string of digits is only 18 digits long, a Luhn check digit will be added."""
 
     def validate(self):
-        # convert to string as it migt be an integer
+        # convert to string as it might be an integer
         iccid_str = str(self.input_value)
         if len(iccid_str) < 18 or len(iccid_str) > 20:
             raise ValueError('ICCID must be 18, 19 or 20 digits long')
@@ -86,7 +86,7 @@ class Imsi(ConfigurableParameter):
     the last digit of the IMSI."""
 
     def validate(self):
-        # convert to string as it migt be an integer
+        # convert to string as it might be an integer
         imsi_str = str(self.input_value)
         if len(imsi_str) < 6 or len(imsi_str) > 15:
             raise ValueError('IMSI must be 6..15 digits long')
@@ -112,7 +112,7 @@ class SdKey(ConfigurableParameter, metaclass=ClassVarMeta):
     key_id = None
     kvn = None
     key_usage_qual = None
-    permitted_len = None
+    permitted_len = []
 
     def validate(self):
         if not isinstance(self.input_value, (io.BytesIO, bytes, bytearray)):
@@ -300,7 +300,7 @@ class Adm2(Pin, keyReference=0x0B):
 
 
 class AlgoConfig(ConfigurableParameter, metaclass=ClassVarMeta):
-    """Configurable Algorithm parameter.  bytes."""
+    """Configurable Algorithm parameter."""
     key = None
     def validate(self):
         if not isinstance(self.input_value, (io.BytesIO, bytes, bytearray)):

pySim/esim/saip/templates.py

@@ -1,5 +1,5 @@
-# Implementation of SimAlliance/TCA Interoperable Profile Template handling
-#
+"""Implementation of SimAlliance/TCA Interoperable Profile Templates."""
+
 # (C) 2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -224,8 +224,8 @@ class ProfileTemplateRegistry:
 # below are transcribed template definitions from "ANNEX A (Normative): File Structure Templates Definition"
 # of "Profile interoperability specification V3.3.1 Final" (unless other version explicitly specified).
 
-# Section 9.2
 class FilesAtMF(ProfileTemplate):
+    """Files at MF as per Section 9.2"""
     created_by_default = True
     oid = OID.MF
     files = [
@@ -238,8 +238,8 @@ class FilesAtMF(ProfileTemplate):
     ]
 
 
-# Section 9.3
 class FilesCD(ProfileTemplate):
+    """Files at DF.CD as per Section 9.3"""
     created_by_default = False
     oid = OID.DF_CD
     files = [
@@ -287,8 +287,8 @@ for i in range(0x90, 0x98):
 for i in range(0x98, 0xa0):
     df_pb_files.append(FileTemplate(0x4f00+i, 'EF.CCP1', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 
-# Section 9.4 v2.3.1
 class FilesTelecom(ProfileTemplate):
+    """Files at DF.TELECOM as per Section 9.4 v2.3.1"""
     created_by_default = False
     oid = OID.DF_TELECOM
     base_path = Path('MF')
@@ -328,8 +328,8 @@ class FilesTelecom(ProfileTemplate):
     ]
 
 
-# Section 9.4
 class FilesTelecomV2(ProfileTemplate):
+    """Files at DF.TELECOM as per Section 9.4"""
     created_by_default = False
     oid = OID.DF_TELECOM_v2
     base_path = Path('MF')
@@ -379,8 +379,8 @@ class FilesTelecomV2(ProfileTemplate):
     ]
 
 
-# Section 9.5.1 v2.3.1
 class FilesUsimMandatory(ProfileTemplate):
+    """Mandatory Files at ADF.USIM as per Section 9.5.1 v2.3.1"""
     created_by_default = True
     oid = OID.ADF_USIM_by_default
     files = [
@@ -410,8 +410,8 @@ class FilesUsimMandatory(ProfileTemplate):
         FileTemplate(0x6fe4, 'EF.EPSNSC',    'LF',    1,   80,   5, 0x18, 'FF...FF', False, ass_serv=[85], high_update=True),
     ]
 
-# Section 9.5.1
 class FilesUsimMandatoryV2(ProfileTemplate):
+    """Mandatory Files at ADF.USIM as per Section 9.5.1"""
     created_by_default = True
     oid = OID.ADF_USIM_by_default_v2
     files = [
@@ -442,8 +442,8 @@ class FilesUsimMandatoryV2(ProfileTemplate):
     ]
 
 
-# Section 9.5.2 v2.3.1
 class FilesUsimOptional(ProfileTemplate):
+    """Optional Files at ADF.USIM as per Section 9.5.2 v2.3.1"""
     created_by_default = False
     optional = True
     oid = OID.ADF_USIMopt_not_by_default
@@ -529,6 +529,7 @@ class FilesUsimOptional(ProfileTemplate):
 
 # Section 9.5.2
 class FilesUsimOptionalV2(ProfileTemplate):
+    """Optional Files at ADF.USIM as per Section 9.5.2"""
     created_by_default = False
     optional = True
     oid = OID.ADF_USIMopt_not_by_default_v2
@@ -622,8 +623,8 @@ class FilesUsimOptionalV2(ProfileTemplate):
         FileTemplate(0x6ffd, 'EF.MudMidCfgdata','BT', None, None,2, None, None, True, ['size'], ass_serv=[134]),
     ]
 
-# Section 9.5.2.3 v3.3.1
 class FilesUsimOptionalV3(ProfileTemplate):
+    """Optional Files at ADF.USIM as per Section 9.5.2.3 v3.3.1"""
     created_by_default = False
     optional = True
     oid = OID.ADF_USIMopt_not_by_default_v3
@@ -633,16 +634,16 @@ class FilesUsimOptionalV3(ProfileTemplate):
         FileTemplate(0x6f01, 'EF.eAKA', 'TR', None, 1, 3, None, None, True, ['size'], ass_serv=[134]),
     ]
 
-# Section 9.5.3
 class FilesUsimDfPhonebook(ProfileTemplate):
+    """DF.PHONEBOOK Files at ADF.USIM as per Section 9.5.3"""
     created_by_default = False
     oid = OID.DF_PHONEBOOK_ADF_USIM
     base_path = Path('ADF.USIM')
     files = df_pb_files
 
 
-# Section 9.5.4
 class FilesUsimDfGsmAccess(ProfileTemplate):
+    """DF.GSM-ACCESS Files at ADF.USIM as per Section 9.5.4"""
     created_by_default = False
     oid = OID.DF_GSM_ACCESS_ADF_USIM
     base_path = Path('ADF.USIM')
@@ -656,8 +657,8 @@ class FilesUsimDfGsmAccess(ProfileTemplate):
     ]
 
 
-# Section 9.5.11 v2.3.1
 class FilesUsimDf5GS(ProfileTemplate):
+    """DF.5GS Files at ADF.USIM as per Section 9.5.11 v2.3.1"""
     created_by_default = False
     oid = OID.DF_5GS
     base_path = Path('ADF.USIM')
@@ -677,8 +678,8 @@ class FilesUsimDf5GS(ProfileTemplate):
     ]
 
 
-# Section 9.5.11.2
 class FilesUsimDf5GSv2(ProfileTemplate):
+    """DF.5GS Files at ADF.USIM as per Section 9.5.11.2"""
     created_by_default = False
     oid = OID.DF_5GS_v2
     base_path = Path('ADF.USIM')
@@ -700,8 +701,8 @@ class FilesUsimDf5GSv2(ProfileTemplate):
     ]
 
 
-# Section 9.5.11.3
 class FilesUsimDf5GSv3(ProfileTemplate):
+    """DF.5GS Files at ADF.USIM as per Section 9.5.11.3"""
     created_by_default = False
     oid = OID.DF_5GS_v3
     base_path = Path('ADF.USIM')
@@ -724,8 +725,8 @@ class FilesUsimDf5GSv3(ProfileTemplate):
         FileTemplate(0x4f0c, 'EF.TN3GPPSNN',         'TR', None,    1,   2, 0x0c, '00', False, ass_serv=[135]),
     ]
 
-# Section 9.5.11.4
 class FilesUsimDf5GSv4(ProfileTemplate):
+    """DF.5GS Files at ADF.USIM as per Section 9.5.11.4"""
     created_by_default = False
     oid = OID.DF_5GS_v4
     base_path = Path('ADF.USIM')
@@ -756,8 +757,8 @@ class FilesUsimDf5GSv4(ProfileTemplate):
     ]
 
 
-# Section 9.5.12
 class FilesUsimDfSaip(ProfileTemplate):
+    """DF.SAIP Files at ADF.USIM as per Section 9.5.12"""
     created_by_default = False
     oid = OID.DF_SAIP
     base_path = Path('ADF.USIM')
@@ -767,8 +768,8 @@ class FilesUsimDfSaip(ProfileTemplate):
         FileTemplate(0x4f01, 'EF.SUCICalcInfo','TR', None, None, 3, None, 'FF...FF', False, ['size'], ass_serv=[125], pe_name='ef-suci-calc-info-usim'),
     ]
 
-# Section 9.5.13
 class FilesDfSnpn(ProfileTemplate):
+    """DF.SNPN Files at ADF.USIM as per Section 9.5.13"""
     created_by_default = False
     oid = OID.DF_SNPN
     base_path = Path('ADF.USIM')
@@ -778,8 +779,8 @@ class FilesDfSnpn(ProfileTemplate):
         FileTemplate(0x4f01, 'EF.PWS_SNPN',     'TR', None,    1,  10, None, None, True, ass_serv=[143]),
     ]
 
-# Section 9.5.14
 class FilesDf5GProSe(ProfileTemplate):
+    """DF.ProSe Files at ADF.USIM as per Section 9.5.14"""
     created_by_default = False
     oid = OID.DF_5GProSe
     base_path = Path('ADF.USIM')
@@ -794,8 +795,8 @@ class FilesDf5GProSe(ProfileTemplate):
         FileTemplate(0x4f06, 'EF.5G_PROSE_UIR',   'TR', None,   32,   2, 0x06, None,  True, ass_serv=[139,1005]),
     ]
 
-# Section 9.6.1
 class FilesIsimMandatory(ProfileTemplate):
+    """Mandatory Files at ADF.ISIM as per Section 9.6.1"""
     created_by_default = True
     oid = OID.ADF_ISIM_by_default
     files = [
@@ -809,8 +810,8 @@ class FilesIsimMandatory(ProfileTemplate):
     ]
 
 
-# Section 9.6.2 v2.3.1
 class FilesIsimOptional(ProfileTemplate):
+    """Optional Files at ADF.ISIM as per Section 9.6.2 of v2.3.1"""
     created_by_default = False
     optional = True
     oid = OID.ADF_ISIMopt_not_by_default
@@ -829,8 +830,8 @@ class FilesIsimOptional(ProfileTemplate):
     ]
 
 
-# Section 9.6.2
 class FilesIsimOptionalv2(ProfileTemplate):
+    """Optional Files at ADF.ISIM as per Section 9.6.2"""
     created_by_default = False
     optional = True
     oid = OID.ADF_ISIMopt_not_by_default_v2
@@ -857,8 +858,8 @@ class FilesIsimOptionalv2(ProfileTemplate):
 # TODO: CSIM
 
 
-# Section 9.8
 class FilesEap(ProfileTemplate):
+    """Files at DF.EAP as per Section 9.8"""
     created_by_default = False
     oid = OID.DF_EAP
     files = [

pySim/esim/saip/validation.py

@@ -1,5 +1,5 @@
-# Implementation of SimAlliance/TCA Interoperable Profile handling
-#
+"""Implementation of SimAlliance/TCA Interoperable Profile validation."""
+
 # (C) 2023-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -19,16 +19,21 @@
 from pySim.esim.saip import *
 
 class ProfileError(Exception):
+    """Raised when a ProfileConstraintChecker finds an error in a file [structure]."""
     pass
 
 class ProfileConstraintChecker:
+    """Base class of a constraint checker for a ProfileElementSequence."""
     def check(self, pes: ProfileElementSequence):
+        """Execute all the check_* methods of the ProfileConstraintChecker against the given
+        ProfileElementSequence"""
         for name in dir(self):
             if name.startswith('check_'):
                 method = getattr(self, name)
                 method(pes)
 
 class CheckBasicStructure(ProfileConstraintChecker):
+    """ProfileConstraintChecker for the basic profile structure constraints."""
     def _is_after_if_exists(self, pes: ProfileElementSequence, opt:str, after:str):
         opt_pe = pes.get_pe_for_type(opt)
         if opt_pe:
@@ -38,6 +43,7 @@ class CheckBasicStructure(ProfileConstraintChecker):
             # FIXME: check order
 
     def check_start_and_end(self, pes: ProfileElementSequence):
+        """Check for mandatory header and end ProfileElements at the right position."""
         if pes.pe_list[0].type != 'header':
             raise ProfileError('first element is not header')
         if pes.pe_list[1].type != 'mf':
@@ -47,6 +53,7 @@ class CheckBasicStructure(ProfileConstraintChecker):
             raise ProfileError('last element is not end')
 
     def check_number_of_occurrence(self, pes: ProfileElementSequence):
+        """Check The number of occurrence of various ProfileElements."""
         # check for invalid number of occurrences
         if len(pes.get_pes_for_type('header')) != 1:
             raise ProfileError('multiple ProfileHeader')
@@ -60,6 +67,7 @@ class CheckBasicStructure(ProfileConstraintChecker):
                 raise ProfileError('multiple PE-%s' % tn.upper())
 
     def check_optional_ordering(self, pes: ProfileElementSequence):
+        """Check the ordering of optional PEs following the respective mandatory ones."""
         # ordering and required depenencies
         self._is_after_if_exists(pes,'opt-usim', 'usim')
         self._is_after_if_exists(pes,'opt-isim', 'isim')
@@ -104,17 +112,21 @@ class CheckBasicStructure(ProfileConstraintChecker):
 FileChoiceList = List[Tuple]
 
 class FileError(ProfileError):
+    """Raised when a FileConstraintChecker finds an error in a file [structure]."""
     pass
 
 class FileConstraintChecker:
     def check(self, l: FileChoiceList):
+        """Execute all the check_* methods of the FileConstraintChecker against the given FileChoiceList"""
         for name in dir(self):
             if name.startswith('check_'):
                 method = getattr(self, name)
                 method(l)
 
 class FileCheckBasicStructure(FileConstraintChecker):
+    """Validator for the basic structure of a decoded file."""
     def check_seqence(self, l: FileChoiceList):
+        """Check the sequence/ordering."""
         by_type = {}
         for k, v in l:
             if k in by_type:

pySim/esim/x509_cert.py

@@ -1,6 +1,5 @@
-# Implementation of X.509 certificate handling in GSMA eSIM
-# as per SGP22 v3.0
-#
+"""Implementation of X.509 certificate handling in GSMA eSIM as per SGP22 v3.0"""
+
 # (C) 2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
@@ -26,12 +25,13 @@ from cryptography.hazmat.primitives.serialization import load_pem_private_key, E
 from cryptography.hazmat.primitives.asymmetric.utils import decode_dss_signature
 
 from pySim.utils import b2h
+from . import x509_err
 
 def check_signed(signed: x509.Certificate, signer: x509.Certificate) -> bool:
     """Verify if 'signed' certificate was signed using 'signer'."""
     # this code only works for ECDSA, but this is all we need for GSMA eSIM
     pkey = signer.public_key()
-    # this 'signed.signature_algorithm_parameters' below requires cryptopgraphy 41.0.0 :(
+    # this 'signed.signature_algorithm_parameters' below requires cryptography 41.0.0 :(
     pkey.verify(signed.signature, signed.tbs_certificate_bytes, signed.signature_algorithm_parameters)
 
 def cert_get_subject_key_id(cert: x509.Certificate) -> bytes:
@@ -65,9 +65,6 @@ class oid:
     id_rspRole_ds_tls_v2 = x509.ObjectIdentifier(ID_RSP_ROLE + '.6')
     id_rspRole_ds_auth_v2 = x509.ObjectIdentifier(ID_RSP_ROLE + '.7')
 
-class VerifyError(Exception):
-    """An error during certificate verification,"""
-
 class CertificateSet:
     """A set of certificates consisting of a trusted [self-signed] CA root certificate,
     and an optional number of intermediate certificates.  Can be used to verify the certificate chain
@@ -136,7 +133,7 @@ class CertificateSet:
             # we cannot check if there's no CRL
             return
         if self.crl.get_revoked_certificate_by_serial_number(cert.serial_nr):
-            raise VerifyError('Certificate is present in CRL, verification failed')
+            raise x509_err.CertificateRevoked()
 
     def verify_cert_chain(self, cert: x509.Certificate, max_depth: int = 100):
         """Verify if a given certificate's signature chain can be traced back to the root CA of this
@@ -150,14 +147,14 @@ class CertificateSet:
                 check_signed(c, self.root_cert)
                 return
             parent_cert = self.intermediate_certs.get(aki, None)
-            if not aki:
-                raise VerifyError('Could not find intermediate certificate for AuthKeyId %s' % b2h(aki))
+            if not parent_cert:
+                raise x509_err.MissingIntermediateCert(b2h(aki))
             check_signed(c, parent_cert)
             # if we reach here, we passed (no exception raised)
             c = parent_cert
             depth += 1
             if depth > max_depth:
-                raise VerifyError('Maximum depth %u exceeded while verifying certificate chain' % max_depth)
+                raise x509_err.MaxDepthExceeded(max_depth, depth)
 
 
 def ecdsa_dss_to_tr03111(sig: bytes) -> bytes:
@@ -189,7 +186,7 @@ class CertAndPrivkey:
     def ecdsa_sign(self, plaintext: bytes) -> bytes:
         """Sign some input-data using an ECDSA signature compliant with SGP.22,
         which internally refers to Global Platform 2.2 Annex E, which in turn points
-        to BSI TS-03111 which states "concatengated raw R + S values". """
+        to BSI TS-03111 which states "concatenated raw R + S values". """
         sig = self.priv_key.sign(plaintext, ec.ECDSA(hashes.SHA256()))
         # convert from DER format to BSI TR-03111; first get long integers; then convert those to bytes
         return ecdsa_dss_to_tr03111(sig)

pySim/esim/x509_err.py

@@ -0,0 +1,58 @@
+"""X.509 certificate verification exceptions for GSMA eSIM."""
+
+class VerifyError(Exception):
+    """Base class for certificate verification errors."""
+    pass
+
+
+class MissingIntermediateCert(VerifyError):
+    """Raised when an intermediate certificate in the chain cannot be found."""
+    def __init__(self, auth_key_id: str):
+        self.auth_key_id = auth_key_id
+        super().__init__(f'Could not find intermediate certificate for AuthKeyId {auth_key_id}')
+
+
+class CertificateRevoked(VerifyError):
+    """Raised when a certificate is found in the CRL."""
+    def __init__(self, cert_serial: str = None):
+        self.cert_serial = cert_serial
+        msg = 'Certificate is present in CRL, verification failed'
+        if cert_serial:
+            msg += f' (serial: {cert_serial})'
+        super().__init__(msg)
+
+
+class MaxDepthExceeded(VerifyError):
+    """Raised when certificate chain depth exceeds the maximum allowed."""
+    def __init__(self, max_depth: int, actual_depth: int):
+        self.max_depth = max_depth
+        self.actual_depth = actual_depth
+        super().__init__(f'Maximum depth {max_depth} exceeded while verifying certificate chain (actual: {actual_depth})')
+
+
+class SignatureVerification(VerifyError):
+    """Raised when certificate signature verification fails."""
+    def __init__(self, cert_subject: str = None, signer_subject: str = None):
+        self.cert_subject = cert_subject
+        self.signer_subject = signer_subject
+        msg = 'Certificate signature verification failed'
+        if cert_subject and signer_subject:
+            msg += f': {cert_subject} not signed by {signer_subject}'
+        super().__init__(msg)
+
+
+class InvalidCertificate(VerifyError):
+    """Raised when a certificate is invalid (missing required fields, wrong type, etc)."""
+    def __init__(self, reason: str):
+        self.reason = reason
+        super().__init__(f'Invalid certificate: {reason}')
+
+
+class CertificateExpired(VerifyError):
+    """Raised when a certificate has expired."""
+    def __init__(self, cert_subject: str = None):
+        self.cert_subject = cert_subject
+        msg = 'Certificate has expired'
+        if cert_subject:
+            msg += f': {cert_subject}'
+        super().__init__(msg)

pySim/euicc.py

@@ -120,7 +120,7 @@ class SetDefaultDpAddress(BER_TLV_IE, tag=0xbf3f, nested=[DefaultDpAddress, SetD
 
 # SGP.22 Section 5.7.7: GetEUICCChallenge
 class EuiccChallenge(BER_TLV_IE, tag=0x80):
-    _construct = HexAdapter(Bytes(16))
+    _construct = Bytes(16)
 class GetEuiccChallenge(BER_TLV_IE, tag=0xbf2e, nested=[EuiccChallenge]):
     pass
 
@@ -128,7 +128,7 @@ class GetEuiccChallenge(BER_TLV_IE, tag=0xbf2e, nested=[EuiccChallenge]):
 class SVN(BER_TLV_IE, tag=0x82):
     _construct = VersionType
 class SubjectKeyIdentifier(BER_TLV_IE, tag=0x04):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class EuiccCiPkiListForVerification(BER_TLV_IE, tag=0xa9, nested=[SubjectKeyIdentifier]):
     pass
 class EuiccCiPkiListForSigning(BER_TLV_IE, tag=0xaa, nested=[SubjectKeyIdentifier]):
@@ -140,15 +140,15 @@ class ProfileVersion(BER_TLV_IE, tag=0x81):
 class EuiccFirmwareVer(BER_TLV_IE, tag=0x83):
     _construct = VersionType
 class ExtCardResource(BER_TLV_IE, tag=0x84):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class UiccCapability(BER_TLV_IE, tag=0x85):
-    _construct = HexAdapter(GreedyBytes) # FIXME
+    _construct = GreedyBytes # FIXME
 class TS102241Version(BER_TLV_IE, tag=0x86):
     _construct = VersionType
 class GlobalPlatformVersion(BER_TLV_IE, tag=0x87):
     _construct = VersionType
 class RspCapability(BER_TLV_IE, tag=0x88):
-    _construct = HexAdapter(GreedyBytes) # FIXME
+    _construct = GreedyBytes # FIXME
 class EuiccCategory(BER_TLV_IE, tag=0x8b):
     _construct = Enum(Int8ub, other=0, basicEuicc=1, mediumEuicc=2, contactlessEuicc=3)
 class PpVersion(BER_TLV_IE, tag=0x04):
@@ -211,7 +211,7 @@ class TagList(BER_TLV_IE, tag=0x5c):
 class ProfileInfoListReq(BER_TLV_IE, tag=0xbf2d, nested=[TagList]): # FIXME: SearchCriteria
     pass
 class IsdpAid(BER_TLV_IE, tag=0x4f):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class ProfileState(BER_TLV_IE, tag=0x9f70):
     _construct = Enum(Int8ub, disabled=0, enabled=1)
 class ProfileNickname(BER_TLV_IE, tag=0x90):
@@ -268,9 +268,20 @@ class DeleteProfileReq(BER_TLV_IE, tag=0xbf33, nested=[IsdpAid, Iccid]):
 class DeleteProfileResp(BER_TLV_IE, tag=0xbf33, nested=[DeleteResult]):
     pass
 
+# SGP.22 Section 5.7.19: EuiccMemoryReset
+class ResetOptions(BER_TLV_IE, tag=0x82):
+    _construct = FlagsEnum(Byte, deleteOperationalProfiles=0x80, deleteFieldLoadedTestProfiles=0x40,
+                           resetDefaultSmdpAddress=0x20)
+class ResetResult(BER_TLV_IE, tag=0x80):
+    _construct = Enum(Int8ub, ok=0, nothingToDelete=1, undefinedError=127)
+class EuiccMemoryResetReq(BER_TLV_IE, tag=0xbf34, nested=[ResetOptions]):
+    pass
+class EuiccMemoryResetResp(BER_TLV_IE, tag=0xbf34, nested=[ResetResult]):
+    pass
+
 # SGP.22 Section 5.7.20 GetEID
 class EidValue(BER_TLV_IE, tag=0x5a):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class GetEuiccData(BER_TLV_IE, tag=0xbf3e, nested=[TagList, EidValue]):
     pass
 
@@ -362,7 +373,7 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
         ged_cmd = GetEuiccData(children=[TagList(decoded=[0x5A])])
         ged = CardApplicationISDR.store_data_tlv(scc, ged_cmd, GetEuiccData)
         d = ged.to_dict()
-        return flatten_dict_lists(d['get_euicc_data'])['eid_value']
+        return b2h(flatten_dict_lists(d['get_euicc_data'])['eid_value'])
 
     def decode_select_response(self, data_hex: Hexstr) -> object:
         t = FciTemplate()
@@ -504,6 +515,30 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             d = dp.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['delete_profile_resp']))
 
+        mem_res_parser = argparse.ArgumentParser()
+        mem_res_parser.add_argument('--delete-operational', action='store_true',
+                                    help='Delete all operational profiles')
+        mem_res_parser.add_argument('--delete-test-field-installed', action='store_true',
+                                    help='Delete all test profiles, except pre-installed ones')
+        mem_res_parser.add_argument('--reset-smdp-address', action='store_true',
+                                    help='Reset the SM-DP+ address')
+
+        @cmd2.with_argparser(mem_res_parser)
+        def do_euicc_memory_reset(self, opts):
+            """Perform an ES10c eUICCMemoryReset function. This will permanently delete the selected subset of
+            profiles from the eUICC."""
+            flags = {}
+            if opts.delete_operational:
+                flags['deleteOperationalProfiles'] = True
+            if opts.delete_test_field_installed:
+                flags['deleteFieldLoadedTestProfiles'] = True
+            if opts.reset_smdp_address:
+                flags['resetDefaultSmdpAddress'] = True
+
+            mr_cmd = EuiccMemoryResetReq(children=[ResetOptions(decoded=flags)])
+            mr = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, mr_cmd, EuiccMemoryResetResp)
+            d = mr.to_dict()
+            self._cmd.poutput_json(flatten_dict_lists(d['euicc_memory_reset_resp']))
 
         def do_get_eid(self, _opts):
             """Perform an ES10c GetEID function."""

pySim/filesystem.py

@@ -301,7 +301,7 @@ class CardFile:
 
     @staticmethod
     def export(as_json: bool, lchan):
-        """
+        r"""
         Export file contents in the form of commandline script. This method is meant to be overloaded by a subclass in
         case any exportable contents are present. The generated script may contain multiple command lines separated by
         line breaks ("\n"), where the last commandline shall have no line break at the end
@@ -661,7 +661,7 @@ class TransparentEF(CardEF):
                 filename = '%s/file' % dirname
                 # write existing data as JSON to file
                 with open(filename, 'w') as text_file:
-                    json.dump(orig_json, text_file, indent=4)
+                    json.dump(orig_json, text_file, indent=4, cls=JsonEncoder)
                 # run a text editor
                 self._cmd.run_editor(filename)
                 with open(filename, 'r') as text_file:
@@ -963,7 +963,7 @@ class LinFixedEF(CardEF):
                 filename = '%s/file' % dirname
                 # write existing data as JSON to file
                 with open(filename, 'w') as text_file:
-                    json.dump(orig_json, text_file, indent=4)
+                    json.dump(orig_json, text_file, indent=4, cls=JsonEncoder)
                 # run a text editor
                 self._cmd.run_editor(filename)
                 with open(filename, 'r') as text_file:
@@ -1224,6 +1224,13 @@ class TransRecEF(TransparentEF):
         Returns:
             abstract_data; dict representing the decoded data
         """
+
+        # The record data length should always be equal or at least greater than the record length defined for the
+        # TransRecEF. Short records may be occur when the length of the underlying TransparentEF is not a multiple
+        # of the TransRecEF record length.
+        if len(raw_hex_data) // 2 < self.__get_rec_len():
+            return {'raw': raw_hex_data}
+
         method = getattr(self, '_decode_record_hex', None)
         if callable(method):
             return method(raw_hex_data)
@@ -1251,6 +1258,11 @@ class TransRecEF(TransparentEF):
         Returns:
             abstract_data; dict representing the decoded data
         """
+
+        # See comment in decode_record_hex (above)
+        if len(raw_bin_data) < self.__get_rec_len():
+            return {'raw': b2h(raw_bin_data)}
+
         method = getattr(self, '_decode_record_bin', None)
         if callable(method):
             return method(raw_bin_data)
@@ -1530,8 +1542,7 @@ class CardModel(abc.ABC):
         """Test if given card matches this model."""
         card_atr = scc.get_atr()
         for atr in cls._atrs:
-            atr_bin = toBytes(atr)
-            if atr_bin == card_atr:
+            if atr == card_atr:
                 print("Detected CardModel:", cls.__name__)
                 return True
         return False

pySim/global_platform/__init__.py

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+import io
 from copy import deepcopy
 from typing import Optional, List, Dict, Tuple
 from construct import Optional as COptional
@@ -29,9 +30,11 @@ from osmocom.construct import *
 from pySim.utils import ResTuple
 from pySim.card_key_provider import card_key_provider_get_field
 from pySim.global_platform.scp import SCP02, SCP03
+from pySim.global_platform.install_param import gen_install_parameters
 from pySim.filesystem import *
 from pySim.profile import CardProfile
 from pySim.ota import SimFileAccessAndToolkitAppSpecParams
+from pySim.javacard import CapFile
 
 # GPCS Table 11-48 Load Parameter Tags
 class NonVolatileCodeMinMemoryReq(BER_TLV_IE, tag=0xC6):
@@ -315,7 +318,7 @@ class CurrentSecurityLevel(BER_TLV_IE, tag=0xd3):
 
 # GlobalPlatform v2.3.1 Section 11.3.3.1.3
 class ApplicationAID(BER_TLV_IE, tag=0x4f):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 class ApplicationTemplate(BER_TLV_IE, tag=0x61, ntested=[ApplicationAID]):
     pass
 class ListOfApplications(BER_TLV_IE, tag=0x2f00, nested=[ApplicationTemplate]):
@@ -422,10 +425,10 @@ class FciTemplate(BER_TLV_IE, tag=0x6f, nested=FciTemplateNestedList):
     pass
 
 class IssuerIdentificationNumber(BER_TLV_IE, tag=0x42):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 class CardImageNumber(BER_TLV_IE, tag=0x45):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 class SequenceCounterOfDefaultKvn(BER_TLV_IE, tag=0xc1):
     _construct = GreedyInteger()
@@ -464,8 +467,7 @@ class LifeCycleState(BER_TLV_IE, tag=0x9f70):
 
 # Section 11.4.3.1 Table 11-36 + Section 11.1.2
 class Privileges(BER_TLV_IE, tag=0xc5):
-    # we only support 3-byte encoding. Can't use StripTrailerAdapter as length==2 is not permitted. sigh.
-    _construct = FlagsEnum(Int24ub,
+    _construct = FlagsEnum(StripTrailerAdapter(GreedyBytes, 3, steps = [1, 3]),
                            security_domain=0x800000, dap_verification=0x400000,
                            delegated_management=0x200000, card_lock=0x100000, card_terminate=0x080000,
                            card_reset=0x040000, cvm_management=0x020000,
@@ -485,7 +487,7 @@ class ImplicitSelectionParameter(BER_TLV_IE, tag=0xcf):
 
 # Section 11.4.3.1 Table 11-36
 class ExecutableLoadFileAID(BER_TLV_IE, tag=0xc4):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # Section 11.4.3.1 Table 11-36
 class ExecutableLoadFileVersionNumber(BER_TLV_IE, tag=0xce):
@@ -493,15 +495,15 @@ class ExecutableLoadFileVersionNumber(BER_TLV_IE, tag=0xce):
     # specification. It shall consist of the version information contained in the original Load File: on a
     # Java Card based card, this version number represents the major and minor version attributes of the
     # original Load File Data Block.
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # Section 11.4.3.1 Table 11-36
 class ExecutableModuleAID(BER_TLV_IE, tag=0x84):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # Section 11.4.3.1 Table 11-36
 class AssociatedSecurityDomainAID(BER_TLV_IE, tag=0xcc):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # Section 11.4.3.1 Table 11-36
 class GpRegistryRelatedData(BER_TLV_IE, tag=0xe3, nested=[ApplicationAID, LifeCycleState, Privileges,
@@ -638,8 +640,8 @@ class ADF_SD(CardADF):
 
         # Table 11-68: Key Data Field - Format 1 (Basic Format)
         KeyDataBasic = GreedyRange(Struct('key_type'/KeyType,
-                                          'kcb'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
-                                          'kcv'/HexAdapter(Prefixed(Int8ub, GreedyBytes))))
+                                          'kcb'/Prefixed(Int8ub, GreedyBytes),
+                                          'kcv'/Prefixed(Int8ub, GreedyBytes)))
 
         def put_key(self, old_kvn:int, kvn: int, kid: int, key_dict: dict) -> bytes:
             """Perform the GlobalPlatform PUT KEY command in order to store a new key on the card.
@@ -702,7 +704,7 @@ class ADF_SD(CardADF):
         def set_status(self, scope:str, status:str, aid:Hexstr = ''):
             SetStatus = Struct(Const(0x80, Byte), Const(0xF0, Byte),
                                'scope'/SetStatusScope, 'status'/CLifeCycleState,
-                               'aid'/HexAdapter(Prefixed(Int8ub, COptional(GreedyBytes))))
+                               'aid'/Prefixed(Int8ub, COptional(GreedyBytes)))
             apdu = build_construct(SetStatus, {'scope':scope, 'status':status, 'aid':aid})
             _data, _sw = self._cmd.lchan.scc.send_apdu_checksw(b2h(apdu))
 
@@ -724,7 +726,7 @@ class ADF_SD(CardADF):
         inst_inst_parser.add_argument('--application-aid', type=is_hexstr, required=True,
                                       help='Application AID')
         inst_inst_parser.add_argument('--install-parameters', type=is_hexstr, default='',
-                                      help='Install Parameters')
+                                      help='Install Parameters (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
         inst_inst_parser.add_argument('--privilege', action='append', dest='privileges', default=[],
                                       choices=list(Privileges._construct.flags.keys()),
                                       help='Privilege granted to newly installed Application')
@@ -736,12 +738,12 @@ class ADF_SD(CardADF):
         @cmd2.with_argparser(inst_inst_parser)
         def do_install_for_install(self, opts):
             """Perform GlobalPlatform INSTALL [for install] command in order to install an application."""
-            InstallForInstallCD = Struct('load_file_aid'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
-                                         'module_aid'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
-                                         'application_aid'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
+            InstallForInstallCD = Struct('load_file_aid'/Prefixed(Int8ub, GreedyBytes),
+                                         'module_aid'/Prefixed(Int8ub, GreedyBytes),
+                                         'application_aid'/Prefixed(Int8ub, GreedyBytes),
                                          'privileges'/Prefixed(Int8ub, Privileges._construct),
-                                         'install_parameters'/HexAdapter(Prefixed(Int8ub, GreedyBytes)),
-                                         'install_token'/HexAdapter(Prefixed(Int8ub, GreedyBytes)))
+                                         'install_parameters'/Prefixed(Int8ub, GreedyBytes),
+                                         'install_token'/Prefixed(Int8ub, GreedyBytes))
             p1 = 0x04
             if opts.make_selectable:
                 p1 |= 0x08
@@ -751,6 +753,31 @@ class ADF_SD(CardADF):
             ifi_bytes = build_construct(InstallForInstallCD, decoded)
             self.install(p1, 0x00, b2h(ifi_bytes))
 
+        inst_load_parser = argparse.ArgumentParser()
+        inst_load_parser.add_argument('--load-file-aid', type=is_hexstr, required=True,
+                                      help='AID of the loded file')
+        inst_load_parser.add_argument('--security-domain-aid', type=is_hexstr, default='',
+                                      help='AID of the Security Domain into which the file shalle be added')
+        inst_load_parser.add_argument('--load-file-hash', type=is_hexstr, default='',
+                                      help='Load File Data Block Hash (GPC_SPE_034, section C.2)')
+        inst_load_parser.add_argument('--load-parameters', type=is_hexstr, default='',
+                                      help='Load Parameters (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        inst_load_parser.add_argument('--load-token', type=is_hexstr, default='',
+                                      help='Load Token (GPC_SPE_034, section C.4.1)')
+
+        @cmd2.with_argparser(inst_load_parser)
+        def do_install_for_load(self, opts):
+            """Perform GlobalPlatform INSTALL [for load] command in order to prepare to load an application."""
+            if opts.load_token != '' and opts.load_file_hash == '':
+                raise ValueError('Load File Data Block Hash is mandatory if a Load Token is present')
+            InstallForLoadCD = Struct('load_file_aid'/Prefixed(Int8ub, GreedyBytes),
+                                      'security_domain_aid'/Prefixed(Int8ub, GreedyBytes),
+                                      'load_file_hash'/Prefixed(Int8ub, GreedyBytes),
+                                      'load_parameters'/Prefixed(Int8ub, GreedyBytes),
+                                      'load_token'/Prefixed(Int8ub, GreedyBytes))
+            ifl_bytes = build_construct(InstallForLoadCD, vars(opts))
+            self.install(0x02, 0x00, b2h(ifl_bytes))
+
         def install(self, p1:int, p2:int, data:Hexstr) -> ResTuple:
             cmd_hex = "80E6%02x%02x%02x%s00" % (p1, p2, len(data)//2, data)
             return self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)
@@ -791,11 +818,101 @@ class ADF_SD(CardADF):
             self.delete(0x00, p2, cmd)
 
         def delete(self, p1:int, p2:int, data:Hexstr) -> ResTuple:
-            cmd_hex = "80E4%02x%02x%02x%s" % (p1, p2, len(data)//2, data)
+            cmd_hex = "80E4%02x%02x%02x%s00" % (p1, p2, len(data)//2, data)
             return self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)
 
+        load_parser = argparse.ArgumentParser()
+        load_parser_from_grp = load_parser.add_mutually_exclusive_group(required=True)
+        load_parser_from_grp.add_argument('--from-hex', type=is_hexstr, help='load from hex string')
+        load_parser_from_grp.add_argument('--from-file', type=argparse.FileType('rb', 0), help='load from binary file')
+        load_parser_from_grp.add_argument('--from-cap-file', type=argparse.FileType('rb', 0), help='load from JAVA-card CAP file')
+
+        @cmd2.with_argparser(load_parser)
+        def do_load(self, opts):
+            """Perform a GlobalPlatform LOAD command. (We currently only support loading without DAP and
+            without ciphering.)"""
+            if opts.from_hex is not None:
+                self.load(h2b(opts.from_hex))
+            elif opts.from_file is not None:
+                self.load(opts.from_file.read())
+            elif opts.from_cap_file is not None:
+                cap = CapFile(opts.from_cap_file)
+                self.load(cap.get_loadfile())
+            else:
+                raise ValueError('load source not specified!')
+
+        def load(self, contents:bytes, chunk_len:int = 240):
+            # TODO:tune chunk_len based on the overhead of the used SCP?
+            # build TLV according to GPC_SPE_034 section 11.6.2.3 / Table 11-58 for unencrypted case
+            remainder = b'\xC4' + bertlv_encode_len(len(contents)) + contents
+            # transfer this in vaious chunks to the card
+            total_size = len(remainder)
+            block_nr = 0
+            while len(remainder):
+                block = remainder[:chunk_len]
+                remainder = remainder[chunk_len:]
+                # build LOAD command APDU according to GPC_SPE_034 section 11.6.2 / Table 11-56
+                p1 = 0x00 if len(remainder) else 0x80
+                p2 = block_nr % 256
+                block_nr += 1
+                cmd_hex = "80E8%02x%02x%02x%s00" % (p1, p2, len(block), b2h(block))
+                _rsp_hex, _sw = self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)
+            self._cmd.poutput("Loaded a total of %u bytes in %u blocks. Don't forget install_for_install (and make selectable) now!" % (total_size, block_nr))
+
+        install_cap_parser = argparse.ArgumentParser()
+        install_cap_parser.add_argument('cap_file', type=str, metavar='FILE',
+                                        help='JAVA-CARD CAP file to install')
+        install_cap_parser_inst_prm_g = install_cap_parser.add_mutually_exclusive_group()
+        install_cap_parser_inst_prm_g.add_argument('--install-parameters', type=is_hexstr, default=None,
+                                                   help='install Parameters (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        install_cap_parser_inst_prm_g_grp = install_cap_parser_inst_prm_g.add_argument_group()
+        install_cap_parser_inst_prm_g_grp.add_argument('--install-parameters-volatile-memory-quota',
+                                                       type=int, default=None,
+                                                       help='volatile memory quota (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        install_cap_parser_inst_prm_g_grp.add_argument('--install-parameters-non-volatile-memory-quota',
+                                                       type=int, default=None,
+                                                       help='non volatile memory quota (GPC_SPE_034, section 11.5.2.3.7, table 11-49)')
+        install_cap_parser_inst_prm_g_grp.add_argument('--install-parameters-stk',
+                                                       type=is_hexstr, default=None,
+                                                       help='Load Parameters (ETSI TS 102 226, section 8.2.1.3.2.1)')
+
+        @cmd2.with_argparser(install_cap_parser)
+        def do_install_cap(self, opts):
+            """Perform a .cap file installation using GlobalPlatform LOAD and INSTALL commands."""
+
+            self._cmd.poutput("loading cap file: %s ..." % opts.cap_file)
+            cap = CapFile(opts.cap_file)
+
+            security_domain_aid = self._cmd.lchan.selected_file.aid
+            load_file = cap.get_loadfile()
+            load_file_aid = cap.get_loadfile_aid()
+            module_aid = cap.get_applet_aid()
+            application_aid = module_aid
+            if opts.install_parameters:
+                install_parameters = opts.install_parameters;
+            else:
+                install_parameters = gen_install_parameters(opts.install_parameters_non_volatile_memory_quota,
+                                                            opts.install_parameters_volatile_memory_quota,
+                                                            opts.install_parameters_stk)
+            self._cmd.poutput("parameters:")
+            self._cmd.poutput(" security-domain-aid: %s" % security_domain_aid)
+            self._cmd.poutput(" load-file: %u bytes" % len(load_file))
+            self._cmd.poutput(" load-file-aid: %s" % load_file_aid)
+            self._cmd.poutput(" module-aid: %s" % module_aid)
+            self._cmd.poutput(" application-aid: %s" % application_aid)
+            self._cmd.poutput(" install-parameters: %s" % install_parameters)
+
+            self._cmd.poutput("step #1: install for load...")
+            self.do_install_for_load("--load-file-aid %s --security-domain-aid %s" % (load_file_aid, security_domain_aid))
+            self._cmd.poutput("step #2: load...")
+            self.load(load_file)
+            self._cmd.poutput("step #3: install_for_install (and make selectable)...")
+            self.do_install_for_install("--load-file-aid %s --module-aid %s --application-aid %s --install-parameters %s --make-selectable" %
+                                        (load_file_aid, module_aid, application_aid, install_parameters))
+            self._cmd.poutput("done.")
+
         est_scp02_parser = argparse.ArgumentParser()
-        est_scp02_parser.add_argument('--key-ver', type=auto_uint8, required=True, help='Key Version Number (KVN)')
+        est_scp02_parser.add_argument('--key-ver', type=auto_uint8, default=0, help='Key Version Number (KVN)')
         est_scp02_parser.add_argument('--host-challenge', type=is_hexstr,
                                       help='Hard-code the host challenge; default: random')
         est_scp02_parser.add_argument('--security-level', type=auto_uint8, default=0x01,
@@ -897,17 +1014,12 @@ class CardApplicationISD(CardApplicationSD):
         super().__init__(aid=aid, name='ADF.ISD', desc='Issuer Security Domain')
         self.adf.scp_key_identity = 'ICCID'
 
-#class CardProfileGlobalPlatform(CardProfile):
-#    ORDER = 23
-#
-#    def __init__(self, name='GlobalPlatform'):
-#        super().__init__(name, desc='GlobalPlatfomr 2.1.1', cla=['00','80','84'], sw=sw_table)
-
-
 class GpCardKeyset:
     """A single set of GlobalPlatform card keys and the associated KVN."""
     def __init__(self, kvn: int, enc: bytes, mac: bytes, dek: bytes):
-        assert 0 < kvn < 256
+        # The Key Version Number is an 8 bit integer number, where 0 refers to the first available key,
+        # see also: GPC_SPE_034, section E.5.1.3
+        assert 0 <= kvn < 256
         assert len(enc) == len(mac) == len(dek)
         self.kvn = kvn
         self.enc = enc

pySim/global_platform/http.py

@@ -17,9 +17,10 @@ Also known as SCP81 for SIM/USIM/UICC/eUICC/eSIM OTA.
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from construct import Struct, Int8ub, Int16ub, Bytes, GreedyBytes, GreedyString, BytesInteger
+from construct import Struct, Int8ub, Int16ub, GreedyString, BytesInteger
 from construct import this, len_, Rebuild, Const
 from construct import Optional as COptional
+from osmocom.construct import Bytes, GreedyBytes
 from osmocom.tlv import BER_TLV_IE
 
 from pySim import cat

pySim/global_platform/install_param.py

@@ -0,0 +1,72 @@
+# GlobalPlatform install parameter generator
+#
+# (C) 2024 by Sysmocom s.f.m.c. GmbH
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from osmocom.construct import *
+from osmocom.utils import *
+from osmocom.tlv import *
+
+class AppSpecificParams(BER_TLV_IE, tag=0xC9):
+    # GPD_SPE_013, table 11-49
+    _construct = GreedyBytes
+
+class VolatileMemoryQuota(BER_TLV_IE, tag=0xC7):
+    # GPD_SPE_013, table 11-49
+    _construct = StripHeaderAdapter(GreedyBytes, 4, steps = [2,4])
+
+class NonVolatileMemoryQuota(BER_TLV_IE, tag=0xC8):
+    # GPD_SPE_013, table 11-49
+    _construct = StripHeaderAdapter(GreedyBytes, 4, steps = [2,4])
+
+class StkParameter(BER_TLV_IE, tag=0xCA):
+    # GPD_SPE_013, table 11-49
+    # ETSI TS 102 226, section 8.2.1.3.2.1
+    _construct = GreedyBytes
+
+class SystemSpecificParams(BER_TLV_IE, tag=0xEF, nested=[VolatileMemoryQuota, NonVolatileMemoryQuota, StkParameter]):
+    # GPD_SPE_013 v1.1 Table 6-5
+    pass
+
+class InstallParams(TLV_IE_Collection, nested=[AppSpecificParams, SystemSpecificParams]):
+    # GPD_SPE_013, table 11-49
+    pass
+
+def gen_install_parameters(non_volatile_memory_quota:int, volatile_memory_quota:int, stk_parameter:str):
+
+    # GPD_SPE_013, table 11-49
+
+    #Mandatory
+    install_params = InstallParams()
+    install_params_dict = [{'app_specific_params': None}]
+
+    #Conditional
+    if non_volatile_memory_quota and volatile_memory_quota and stk_parameter:
+        system_specific_params = []
+        #Optional
+        if non_volatile_memory_quota:
+            system_specific_params += [{'non_volatile_memory_quota': non_volatile_memory_quota}]
+        #Optional
+        if volatile_memory_quota:
+            system_specific_params += [{'volatile_memory_quota': volatile_memory_quota}]
+        #Optional
+        if stk_parameter:
+            system_specific_params += [{'stk_parameter': stk_parameter}]
+        install_params_dict += [{'system_specific_params': system_specific_params}]
+
+    install_params.from_dict(install_params_dict)
+    return b2h(install_params.to_bytes())

pySim/global_platform/scp.py

@@ -20,8 +20,9 @@ import logging
 from typing import Optional
 from Cryptodome.Cipher import DES3, DES
 from Cryptodome.Util.strxor import strxor
-from construct import Struct, Bytes, Int8ub, Int16ub, Const
+from construct import Struct, Int8ub, Int16ub, Const
 from construct import Optional as COptional
+from osmocom.construct import Bytes
 from osmocom.utils import b2h
 from osmocom.tlv import bertlv_parse_len, bertlv_encode_len
 from pySim.utils import parse_command_apdu
@@ -113,7 +114,39 @@ CLA_SM = 0x04
 class SCP(SecureChannel, abc.ABC):
     """Abstract base class containing some common interface + functionality for SCP protocols."""
     def __init__(self, card_keys: 'GpCardKeyset', lchan_nr: int = 0):
-        if hasattr(self, 'kvn_range'):
+
+        # Spec references that explain KVN ranges:
+        # TS 102 225 Annex A.1 states KVN 0x01..0x0F shall be used for SCP80
+        # GPC_GUI_003 states
+        #   * For the Issuer Security Domain, this is initially Key Version Number 'FF' which has been deliberately
+        #     chosen to be outside of the allowable range ('01' to '7F') for a Key Version Number.
+        #   * It is logical that the initial keys in the Issuer Security Domain be replaced by an initial issuer Key
+        #     Version Number in the range '01' to '6F'.
+        #   * Key Version Numbers '70' to '72' and '74' to '7F' are reserved for future use.
+        #   * On an implementation supporting Supplementary Security Domains, the RSA public key with a Key Version
+        #     Number '73' and a Key Identifier of '01' has the following functionality in a Supplementary Security
+        #     Domain with the DAP Verification privilege [...]
+        # GPC_GUI_010 V1.0.1 Section 6 states
+        #   * Key Version number range ('20' to '2F') is reserved for SCP02
+        #   * Key Version 'FF' is reserved for use by an Issuer Security Domain supporting SCP02, and cannot be used
+        #     for SCP80. This initial key set shall be replaced by a key set with a Key Version Number in the
+        #     ('20' to '2F') range.
+        #   * Key Version number range ('01' to '0F') is reserved for SCP80
+        #   * Key Version number '70' with Key Identifier '01' is reserved for the Token Key, which is either a RSA
+        #     public key or a DES key
+        #   * Key Version number '71' with Key Identifier '01' is reserved for the Receipt Key, which is a DES key
+        #   * Key Version Number '11' is reserved for DAP as specified in ETSI TS 102 226 [2]
+        #   * Key Version Number '73' with Key Identifier '01' is reserved for the DAP verification key as specified
+        #     in sections 3.3.3 and 4 of [4], which is either an RSA public key or DES key
+        #   * Key Version Number '74' is reserved for the CASD Keys (cf. section 9.2)
+        #   * Key Version Number '75' with Key Identifier '01' is reserved for the key used to decipher the Ciphered
+        #     Load File Data Block described in section 4.8 of [5].
+
+        if card_keys.kvn == 0:
+            # Key Version Number 0x00 refers to the first available key, so we won't carry out
+            # a range check in this case. See also: GPC_SPE_034, section E.5.1.3
+            pass
+        elif hasattr(self, 'kvn_range'):
             if not card_keys.kvn in range(self.kvn_range[0], self.kvn_range[1]+1):
                 raise ValueError('%s cannot be used with KVN outside range 0x%02x..0x%02x' %
                                  (self.__class__.__name__, self.kvn_range[0], self.kvn_range[1]))
@@ -224,8 +257,9 @@ class SCP02(SCP):
 
     constr_iur = Struct('key_div_data'/Bytes(10), 'key_ver'/Int8ub, Const(b'\x02'),
                         'seq_counter'/Int16ub, 'card_challenge'/Bytes(6), 'card_cryptogram'/Bytes(8))
-    # The 0x70 is a non-spec special-case of sysmoISIM-SJA2/SJA5 and possibly more sysmocom products
-    kvn_ranges = [[0x20, 0x2f], [0x70, 0x70]]
+    # Key Version Number 0x70 is a non-spec special-case of sysmoISIM-SJA2/SJA5 and possibly more sysmocom products
+    # Key Version Number 0x01 is a non-spec special-case of sysmoUSIM-SJS1
+    kvn_ranges = [[0x01, 0x01], [0x20, 0x2f], [0x70, 0x70]]
 
     def __init__(self, *args, **kwargs):
         self.overhead = 8

pySim/gsm_r.py

@@ -26,7 +26,7 @@ order to describe the files specified in UIC Reference P38 T 9001 5.0 "FFFIS for
 
 from pySim.utils import *
 from struct import pack, unpack
-from construct import Struct, Bytes, Int8ub, Int16ub, Int24ub, Int32ub, FlagsEnum
+from construct import Struct, Int8ub, Int16ub, Int24ub, Int32ub, FlagsEnum
 from construct import Optional as COptional
 from osmocom.construct import *
 
@@ -184,13 +184,13 @@ class EF_CallconfI(LinFixedEF):
 class EF_Shunting(TransparentEF):
     """Section 7.6"""
     _test_de_encode = [
-        ( "03f8ffffff000000", { "common_gid": 3, "shunting_gid": "f8ffffff000000" } ),
+        ( "03f8ffffff000000", { "common_gid": 3, "shunting_gid": h2b("f8ffffff000000") } ),
     ]
     def __init__(self):
         super().__init__(fid='6ff4', sfid=None,
                          name='EF.Shunting', desc='Shunting', size=(8, 8))
         self._construct = Struct('common_gid'/Int8ub,
-                                 'shunting_gid'/HexAdapter(Bytes(7)))
+                                 'shunting_gid'/Bytes(7))
 
 
 class EF_GsmrPLMN(LinFixedEF):
@@ -199,13 +199,13 @@ class EF_GsmrPLMN(LinFixedEF):
         ( "22f860f86f8d6f8e01", { "plmn": "228-06", "class_of_network": {
                                     "supported": { "vbs": True, "vgcs": True, "emlpp": True,
                                     "fn": True, "eirene": True }, "preference": 0 },
-                                  "ic_incoming_ref_tbl": "6f8d", "outgoing_ref_tbl": "6f8e",
-                                  "ic_table_ref": "01" } ),
+                                  "ic_incoming_ref_tbl": h2b("6f8d"), "outgoing_ref_tbl": h2b("6f8e"),
+                                  "ic_table_ref": h2b("01") } ),
         ( "22f810416f8d6f8e02", { "plmn": "228-01", "class_of_network": {
                                     "supported": { "vbs": False, "vgcs": False, "emlpp": False,
                                     "fn": True, "eirene": False }, "preference": 1 },
-                                  "ic_incoming_ref_tbl": "6f8d", "outgoing_ref_tbl": "6f8e",
-                                  "ic_table_ref": "02" } ),
+                                  "ic_incoming_ref_tbl": h2b("6f8d"), "outgoing_ref_tbl": h2b("6f8e"),
+                                  "ic_table_ref": h2b("02") } ),
     ]
     def __init__(self):
         super().__init__(fid='6ff5', sfid=None, name='EF.GsmrPLMN',
@@ -213,24 +213,24 @@ class EF_GsmrPLMN(LinFixedEF):
         self._construct = Struct('plmn'/PlmnAdapter(Bytes(3)),
                                  'class_of_network'/BitStruct('supported'/FlagsEnum(BitsInteger(5), vbs=1, vgcs=2, emlpp=4, fn=8, eirene=16),
                                                               'preference'/BitsInteger(3)),
-                                 'ic_incoming_ref_tbl'/HexAdapter(Bytes(2)),
-                                 'outgoing_ref_tbl'/HexAdapter(Bytes(2)),
-                                 'ic_table_ref'/HexAdapter(Bytes(1)))
+                                 'ic_incoming_ref_tbl'/Bytes(2),
+                                 'outgoing_ref_tbl'/Bytes(2),
+                                 'ic_table_ref'/Bytes(1))
 
 
 class EF_IC(LinFixedEF):
     """Section 7.8"""
     _test_de_encode = [
-        ( "f06f8e40f10001", { "next_table_type": "decision", "id_of_next_table": "6f8e",
+        ( "f06f8e40f10001", { "next_table_type": "decision", "id_of_next_table": h2b("6f8e"),
                               "ic_decision_value": "041f", "network_string_table_index": 1 } ),
-        ( "ffffffffffffff", { "next_table_type": "empty", "id_of_next_table": "ffff",
+        ( "ffffffffffffff", { "next_table_type": "empty", "id_of_next_table": h2b("ffff"),
                               "ic_decision_value": "ffff", "network_string_table_index": 65535 } ),
     ]
     def __init__(self):
         super().__init__(fid='6f8d', sfid=None, name='EF.IC',
                          desc='International Code', rec_len=(7, 7))
         self._construct = Struct('next_table_type'/NextTableType,
-                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'id_of_next_table'/Bytes(2),
                                  'ic_decision_value'/BcdAdapter(Bytes(2)),
                                  'network_string_table_index'/Int16ub)
 
@@ -252,18 +252,18 @@ class EF_NW(LinFixedEF):
 class EF_Switching(LinFixedEF):
     """Section 8.4"""
     _test_de_encode = [
-        ( "f26f87f0ff00", { "next_table_type": "num_dial_digits", "id_of_next_table": "6f87",
+        ( "f26f87f0ff00", { "next_table_type": "num_dial_digits", "id_of_next_table": h2b("6f87"),
                             "decision_value": "0fff", "string_table_index": 0 } ),
-        ( "f06f8ff1ff01", { "next_table_type": "decision", "id_of_next_table": "6f8f",
+        ( "f06f8ff1ff01", { "next_table_type": "decision", "id_of_next_table": h2b("6f8f"),
                             "decision_value": "1fff", "string_table_index": 1 } ),
-        ( "f16f89f5ff05", { "next_table_type": "predefined", "id_of_next_table": "6f89",
+        ( "f16f89f5ff05", { "next_table_type": "predefined", "id_of_next_table": h2b("6f89"),
                             "decision_value": "5fff", "string_table_index": 5 } ),
     ]
     def __init__(self, fid='1234', name='Switching', desc=None):
         super().__init__(fid=fid, sfid=None,
                          name=name, desc=desc, rec_len=(6, 6))
         self._construct = Struct('next_table_type'/NextTableType,
-                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'id_of_next_table'/Bytes(2),
                                  'decision_value'/BcdAdapter(Bytes(2)),
                                  'string_table_index'/Int8ub)
 
@@ -271,12 +271,12 @@ class EF_Switching(LinFixedEF):
 class EF_Predefined(LinFixedEF):
     """Section 8.5"""
     _test_de_encode = [
-        ( "f26f85", 1, { "next_table_type": "num_dial_digits", "id_of_next_table": "6f85" } ),
+        ( "f26f85", 1, { "next_table_type": "num_dial_digits", "id_of_next_table": h2b("6f85") } ),
         ( "f0ffc8", 2, { "predefined_value1": "0fff", "string_table_index1": 200 } ),
     ]
     # header and other records have different structure. WTF !?!
     construct_first = Struct('next_table_type'/NextTableType,
-                             'id_of_next_table'/HexAdapter(Bytes(2)))
+                             'id_of_next_table'/Bytes(2))
     construct_others = Struct('predefined_value1'/BcdAdapter(Bytes(2)),
                               'string_table_index1'/Int8ub)
 
@@ -301,13 +301,13 @@ class EF_Predefined(LinFixedEF):
 class EF_DialledVals(TransparentEF):
     """Section 8.6"""
     _test_de_encode = [
-        ( "ffffff22", { "next_table_type": "empty", "id_of_next_table": "ffff", "dialed_digits": "22" } ),
-        ( "f16f8885", { "next_table_type": "predefined", "id_of_next_table": "6f88", "dialed_digits": "58" }),
+        ( "ffffff22", { "next_table_type": "empty", "id_of_next_table": h2b("ffff"), "dialed_digits": "22" } ),
+        ( "f16f8885", { "next_table_type": "predefined", "id_of_next_table": h2b("6f88"), "dialed_digits": "58" }),
     ]
     def __init__(self, fid='1234', name='DialledVals', desc=None):
         super().__init__(fid=fid, sfid=None, name=name, desc=desc, size=(4, 4))
         self._construct = Struct('next_table_type'/NextTableType,
-                                 'id_of_next_table'/HexAdapter(Bytes(2)),
+                                 'id_of_next_table'/Bytes(2),
                                  'dialed_digits'/BcdAdapter(Bytes(1)))
 
 

pySim/iso7816_4.py

@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
-from construct import GreedyBytes, GreedyString
+from construct import GreedyString
 from osmocom.tlv import *
 from osmocom.construct import *
 

pySim/javacard.py

@@ -1,19 +1,142 @@
 # JavaCard related utilities
-
+#
+# (C) 2024 by Sysmocom s.f.m.c. GmbH
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
 import zipfile
 import struct
 import sys
 import io
+from osmocom.utils import b2h, Hexstr
+from construct import Struct, Array, this, Int32ub, Int16ub, Int8ub
+from osmocom.construct import *
+from osmocom.tlv import *
+from construct import Optional as COptional
 
 def ijc_to_cap(in_file: io.IOBase, out_zip: zipfile.ZipFile, p : str = "foo"):
-    """Convert an ICJ (Interoperable Java Card) file [back] to a CAP file."""
-    TAGS = ["Header", "Directory", "Applet", "Import", "ConstantPool", "Class", "Method", "StaticField", "RefLocation", "Export", "Descriptor", "Debug"]
+    """Convert an ICJ (Interoperable Java Card) file [back] to a CAP file.
+    example usage:
+        with io.open(sys.argv[1],"rb") as f, zipfile.ZipFile(sys.argv[2], "wb") as z:
+        ijc_to_cap(f, z)
+    """
+    TAGS = ["Header", "Directory", "Applet", "Import", "ConstantPool", "Class", "Method", "StaticField", "RefLocation",
+            "Export", "Descriptor", "Debug"]
     b = in_file.read()
     while len(b):
         tag, size = struct.unpack('!BH', b[0:3])
         out_zip.writestr(p+"/javacard/"+TAGS[tag-1]+".cap", b[0:3+size])
         b = b[3+size:]
 
-# example usage:
-# with io.open(sys.argv[1],"rb") as f, zipfile.ZipFile(sys.argv[2], "wb") as z:
-#     ijc_to_cap(f, z)
+class CapFile():
+
+    # Java Card Platform Virtual Machine Specification, v3.2, section 6.4
+    __header_component_compact = Struct('tag'/Int8ub,
+                                        'size'/Int16ub,
+                                        'magic'/Int32ub,
+                                        'minor_version'/Int8ub,
+                                        'major_version'/Int8ub,
+                                        'flags'/Int8ub,
+                                        'package'/Struct('minor_version'/Int8ub,
+                                                         'major_version'/Int8ub,
+                                                         'AID'/LV),
+                                        'package_name'/COptional(LV)) #since CAP format 2.2
+
+    # Java Card Platform Virtual Machine Specification, v3.2, section 6.6
+    __applet_component_compact = Struct('tag'/Int8ub,
+                                        'size'/Int16ub,
+                                        'count'/Int8ub,
+                                        'applets'/Array(this.count, Struct('AID'/LV,
+                                                                           'install_method_offset'/Int16ub)),
+                                       )
+
+    def __init__(self, filename:str):
+
+        # In this dictionary we will keep all nested .cap file components by their file names (without .cap suffix)
+        # See also: Java Card Platform Virtual Machine Specification, v3.2, section 6.2.1
+        self.__component = {}
+
+        # Extract the nested .cap components from the .cap file
+        # See also: Java Card Platform Virtual Machine Specification, v3.2, section 6.2.1
+        cap = zipfile.ZipFile(filename)
+        cap_namelist = cap.namelist()
+        for i, filename in enumerate(cap_namelist):
+            if filename.lower().endswith('.capx') and not filename.lower().endswith('.capx'):
+                #TODO: At the moment we only support the compact .cap format, add support for the extended .cap format.
+                raise ValueError("incompatible .cap file, extended .cap format not supported!")
+
+            if filename.lower().endswith('.cap'):
+                key = filename.split('/')[-1].removesuffix('.cap')
+                self.__component[key] = cap.read(filename)
+
+        # Make sure that all mandatory components are present
+        # See also: Java Card Platform Virtual Machine Specification, v3.2, section 6.2
+        required_components = {'Header' : 'COMPONENT_Header',
+                               'Directory' : 'COMPONENT_Directory',
+                               'Import' : 'COMPONENT_Import',
+                               'ConstantPool' : 'COMPONENT_ConstantPool',
+                               'Class' : 'COMPONENT_Class',
+                               'Method' : 'COMPONENT_Method',
+                               'StaticField' : 'COMPONENT_StaticField',
+                               'RefLocation' : 'COMPONENT_ReferenceLocation',
+                               'Descriptor' : 'COMPONENT_Descriptor'}
+        for component in required_components:
+            if component not in self.__component.keys():
+                raise ValueError("invalid cap file, %s missing!" % required_components[component])
+
+    def get_loadfile(self) -> bytes:
+        """Get the executeable loadfile as hexstring"""
+        # Concatenate all cap file components in the specified order
+        # see also: Java Card Platform Virtual Machine Specification, v3.2, section 6.3
+        loadfile = self.__component['Header']
+        loadfile += self.__component['Directory']
+        loadfile += self.__component['Import']
+        if 'Applet' in self.__component.keys():
+            loadfile += self.__component['Applet']
+        loadfile += self.__component['Class']
+        loadfile += self.__component['Method']
+        loadfile += self.__component['StaticField']
+        if 'Export' in self.__component.keys():
+            loadfile += self.__component['Export']
+        loadfile += self.__component['ConstantPool']
+        loadfile += self.__component['RefLocation']
+        if 'Descriptor' in self.__component.keys():
+            loadfile += self.__component['Descriptor']
+        return loadfile
+
+    def get_loadfile_aid(self) -> Hexstr:
+        """Get the loadfile AID as hexstring"""
+        header = self.__header_component_compact.parse(self.__component['Header'])
+        magic = header['magic'] or 0
+        if magic != 0xDECAFFED:
+            raise ValueError("invalid cap file, COMPONENT_Header lacks magic number (0x%08X!=0xDECAFFED)!" % magic)
+        #TODO: check cap version and make sure we are compatible with it
+        return header['package']['AID']
+
+    def get_applet_aid(self, index:int = 0) -> Hexstr:
+        """Get the applet AID as hexstring"""
+        #To get the module AID, we must look into COMPONENT_Applet. Unfortunately, even though this component should
+        #be present in any .cap file, it is defined as an optional component.
+        if 'Applet' not in self.__component.keys():
+            raise ValueError("can't get the AID, this cap file lacks the optional COMPONENT_Applet component!")
+
+        applet = self.__applet_component_compact.parse(self.__component['Applet'])
+
+        if index > applet['count']:
+            raise ValueError("can't get the AID for applet with index=%u, this .cap file only has %u applets!" %
+                             (index, applet['count']))
+
+        return applet['applets'][index]['AID']
+

pySim/legacy/cards.py

@@ -3,7 +3,6 @@
 ################################################################################
 
 import abc
-from smartcard.util import toBytes
 from pytlv.TLV import *
 
 from pySim.cards import SimCardBase, UiccCardBase
@@ -781,7 +780,7 @@ class SysmoSIMgr1(GrcardSim):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 99 18 00 11 88 22 33 44 55 66 77 60"):
+            if scc.get_atr() == "3b991800118822334455667760":
                 return kls(scc)
         except:
             return None
@@ -826,7 +825,7 @@ class SysmoSIMgr2(SimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 7D 94 00 00 55 55 53 0A 74 86 93 0B 24 7C 4D 54 68"):
+            if scc.get_atr() == "3b7d9400005555530a7486930b247c4d5468":
                 return kls(scc)
         except:
             return None
@@ -904,7 +903,7 @@ class SysmoUSIMSJS1(UsimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 9F 96 80 1F C7 80 31 A0 73 BE 21 13 67 43 20 07 18 00 00 01 A5"):
+            if scc.get_atr() == "3b9f96801fc78031a073be21136743200718000001a5":
                 return kls(scc)
         except:
             return None
@@ -1032,7 +1031,7 @@ class FairwavesSIM(UsimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 9F 96 80 1F C7 80 31 A0 73 BE 21 13 67 44 22 06 10 00 00 01 A9"):
+            if scc.get_atr() == "3b9f96801fc78031a073be21136744220610000001a9":
                 return kls(scc)
         except:
             return None
@@ -1166,7 +1165,7 @@ class OpenCellsSim(SimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 9F 95 80 1F C3 80 31 E0 73 FE 21 13 57 86 81 02 86 98 44 18 A8"):
+            if scc.get_atr() == "3b9f95801fc38031e073fe21135786810286984418a8":
                 return kls(scc)
         except:
             return None
@@ -1215,7 +1214,7 @@ class WavemobileSim(UsimCard):
     def autodetect(kls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes("3B 9F 95 80 1F C7 80 31 E0 73 F6 21 13 67 4D 45 16 00 43 01 00 8F"):
+            if scc.get_atr() == "3b9f95801fc78031e073f62113674d4516004301008f":
                 return kls(scc)
         except:
             return None
@@ -1305,18 +1304,18 @@ class SysmoISIMSJA2(UsimCard, IsimCard):
     def autodetect(kls, scc):
         try:
             # Try card model #1
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 30 34 05 4B A9"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a4c753034054ba9"
+            if scc.get_atr() == atr:
                 return kls(scc)
 
             # Try card model #2
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 31 33 02 51 B2"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a4c7531330251b2"
+            if scc.get_atr() == atr:
                 return kls(scc)
 
             # Try card model #3
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 52 75 31 04 51 D5"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a4c5275310451d5"
+            if scc.get_atr() == atr:
                 return kls(scc)
         except:
             return None
@@ -1554,16 +1553,16 @@ class SysmoISIMSJA5(SysmoISIMSJA2):
     def autodetect(kls, scc):
         try:
             # Try card model #1 (9FJ)
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 51 CC"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a357530350251cc"
+            if scc.get_atr() == atr:
                 return kls(scc)
             # Try card model #2 (SLM17)
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 65 F8"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a357530350265f8"
+            if scc.get_atr() == atr:
                 return kls(scc)
             # Try card model #3 (9FV)
-            atr = "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 59 C4"
-            if scc.get_atr() == toBytes(atr):
+            atr = "3b9f96801f878031e073fe211b674a357530350259c4"
+            if scc.get_atr() == atr:
                 return kls(scc)
         except:
             return None
@@ -1592,7 +1591,7 @@ class GialerSim(UsimCard):
     def autodetect(cls, scc):
         try:
             # Look for ATR
-            if scc.get_atr() == toBytes('3B 9F 95 80 1F C7 80 31 A0 73 B6 A1 00 67 CF 32 15 CA 9C D7 09 20'):
+            if scc.get_atr() == '3b9f95801fc78031a073b6a10067cf3215ca9cd70920':
                 return cls(scc)
         except:
             return None

pySim/ota.py

@@ -19,7 +19,7 @@ import zlib
 import abc
 import struct
 from typing import Optional, Tuple
-from construct import Enum, Int8ub, Int16ub, Struct, Bytes, GreedyBytes, BitsInteger, BitStruct
+from construct import Enum, Int8ub, Int16ub, Struct, BitsInteger, BitStruct
 from construct import Flag, Padding, Switch, this, PrefixedArray, GreedyRange
 from osmocom.construct import *
 from osmocom.utils import b2h
@@ -410,6 +410,7 @@ class OtaDialectSms(OtaDialect):
             ciph = encoded[2+8:]
             envelope_data = otak.crypt.decrypt(ciph)
         else:
+            cpl = None # FIXME this line was just added to silence pylint possibly-used-before-assignment
             part_head = encoded[:8]
             envelope_data = encoded[8:]
 

pySim/runtime.py

@@ -53,6 +53,7 @@ class RuntimeState:
         # this is a dict of card identities which different parts of the code might populate,
         # typically with something like ICCID, EID, ATR, ...
         self.identity = {}
+        self.adm_verified = False
 
         # make sure the class and selection control bytes, which are specified
         # by the card profile are used
@@ -139,7 +140,8 @@ class RuntimeState:
             if lchan_nr == 0:
                 continue
             del self.lchan[lchan_nr]
-        atr = i2h(self.card.reset())
+        self.adm_verified = False
+        atr = self.card.reset()
         if cmd_app:
             cmd_app.lchan = self.lchan[0]
         # select MF to reset internal state and to verify card really works

pySim/sms.py

@@ -20,10 +20,10 @@
 import typing
 import abc
 from bidict import bidict
-from construct import Int8ub, Byte, Bytes, Bit, Flag, BitsInteger
+from construct import Int8ub, Byte, Bit, Flag, BitsInteger
 from construct import Struct, Enum, Tell, BitStruct, this, Padding
-from construct import Prefixed, GreedyRange, GreedyBytes
-from osmocom.construct import HexAdapter, BcdAdapter, TonNpi
+from construct import Prefixed, GreedyRange
+from osmocom.construct import BcdAdapter, TonNpi, Bytes, GreedyBytes
 from osmocom.utils import Hexstr, h2b, b2h
 
 from smpp.pdu import pdu_types, operations
@@ -253,6 +253,49 @@ class SMS_DELIVER(SMS_TPDU):
             }
         return cls(**d)
 
+    @classmethod
+    def from_submit(cls, submit: 'SMS_SUBMIT') -> 'SMS_DELIVER':
+        """Construct a SMS_DELIVER instance from a SMS_SUBMIT instance."""
+        d = {
+            # common fields (SMS_TPDU base class) which exist in submit, so we can copy them
+            'tp_mti': submit.tp_mti,
+            'tp_rp': submit.tp_rp,
+            'tp_udhi': submit.tp_udhi,
+            'tp_pid': submit.tp_pid,
+            'tp_dcs': submit.tp_dcs,
+            'tp_udl': submit.tp_udl,
+            'tp_ud': submit.tp_ud,
+            # SMS_DELIVER specific fields
+            'tp_lp': False,
+            'tp_mms': False,
+            'tp_oa': None,
+            'tp_scts': h2b('22705200000000'), # FIXME
+            'tp_sri': False,
+            }
+        return cls(**d)
+
+    def to_smpp(self) -> pdu_types.PDU:
+        """Translate a SMS_DELIVER instance to a smpp.pdu.operations.DeliverSM instance."""
+        # we only deal with binary SMS here:
+        if self.tp_dcs != 0xF6:
+            raise ValueError('Unsupported DCS: We only support DCS=0xF6 for now')
+        dcs = pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED)
+        esm_class = pdu_types.EsmClass(pdu_types.EsmClassMode.DEFAULT, pdu_types.EsmClassType.DEFAULT,
+                                       gsmFeatures=[pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET])
+        if self.tp_oa:
+            oa_digits, oa_ton, oa_npi = self.tp_oa.to_smpp()
+        else:
+            oa_digits, oa_ton, oa_npi = None, None, None
+        return operations.DeliverSM(source_addr=oa_digits,
+                                    source_addr_ton=oa_ton,
+                                    source_addr_npi=oa_npi,
+                                    #destination_addr=ESME_MSISDN,
+                                    esm_class=esm_class,
+                                    protocol_id=self.tp_pid,
+                                    data_coding=dcs,
+                                    short_message=self.tp_ud)
+
 
 
 class SMS_SUBMIT(SMS_TPDU):

pySim/sysmocom_sja2.py

@@ -18,7 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
 from struct import unpack
-from construct import FlagsEnum, Byte, Struct, Int8ub, Bytes, Mapping, Enum, Padding, BitsInteger
+from construct import FlagsEnum, Byte, Struct, Int8ub, Mapping, Enum, Padding, BitsInteger
 from construct import Bit, this, Int32ub, Int16ub, Nibble, BytesInteger, GreedyRange, Const
 from construct import Optional as COptional
 from osmocom.utils import *
@@ -51,13 +51,13 @@ class EF_PIN(TransparentEF):
         ( 'f1030331323334ffffffff0a0a3132333435363738',
           { 'state': { 'valid': True, 'change_able': True, 'unblock_able': True, 'disable_able': True,
                        'not_initialized': False, 'disabled': True },
-           'attempts_remaining': 3, 'maximum_attempts': 3, 'pin': '31323334',
-           'puk': { 'attempts_remaining': 10, 'maximum_attempts': 10, 'puk': '3132333435363738' }
+           'attempts_remaining': 3, 'maximum_attempts': 3, 'pin': b'1234',
+           'puk': { 'attempts_remaining': 10, 'maximum_attempts': 10, 'puk': b'12345678' }
           } ),
         ( 'f003039999999999999999',
           { 'state': { 'valid': True, 'change_able': True, 'unblock_able': True, 'disable_able': True,
                        'not_initialized': False, 'disabled': False },
-           'attempts_remaining': 3, 'maximum_attempts': 3, 'pin': '9999999999999999',
+           'attempts_remaining': 3, 'maximum_attempts': 3, 'pin': h2b('9999999999999999'),
            'puk': None } ),
     ]
     def __init__(self, fid='6f01', name='EF.CHV1'):
@@ -66,29 +66,32 @@ class EF_PIN(TransparentEF):
                                     change_able=0x40, valid=0x80)
         PukStruct = Struct('attempts_remaining'/Int8ub,
                            'maximum_attempts'/Int8ub,
-                           'puk'/HexAdapter(Rpad(Bytes(8))))
+                           'puk'/Rpad(Bytes(8)))
         self._construct = Struct('state'/StateByte,
                                  'attempts_remaining'/Int8ub,
                                  'maximum_attempts'/Int8ub,
-                                 'pin'/HexAdapter(Rpad(Bytes(8))),
+                                 'pin'/Rpad(Bytes(8)),
                                  'puk'/COptional(PukStruct))
 
 
 class EF_MILENAGE_CFG(TransparentEF):
     _test_de_encode = [
         ( '40002040600000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000020000000000000000000000000000000400000000000000000000000000000008',
-          {"r1": 64, "r2": 0, "r3": 32, "r4": 64, "r5": 96, "c1": "00000000000000000000000000000000", "c2":
-           "00000000000000000000000000000001", "c3": "00000000000000000000000000000002", "c4":
-           "00000000000000000000000000000004", "c5": "00000000000000000000000000000008"} ),
+          {"r1": 64, "r2": 0, "r3": 32, "r4": 64, "r5": 96,
+           "c1": h2b("00000000000000000000000000000000"),
+           "c2": h2b("00000000000000000000000000000001"),
+           "c3": h2b("00000000000000000000000000000002"),
+           "c4": h2b("00000000000000000000000000000004"),
+           "c5": h2b("00000000000000000000000000000008")} ),
       ]
     def __init__(self, fid='6f21', name='EF.MILENAGE_CFG', desc='Milenage connfiguration'):
         super().__init__(fid, name=name, desc=desc)
         self._construct = Struct('r1'/Int8ub, 'r2'/Int8ub, 'r3'/Int8ub, 'r4'/Int8ub, 'r5'/Int8ub,
-                                 'c1'/HexAdapter(Bytes(16)),
-                                 'c2'/HexAdapter(Bytes(16)),
-                                 'c3'/HexAdapter(Bytes(16)),
-                                 'c4'/HexAdapter(Bytes(16)),
-                                 'c5'/HexAdapter(Bytes(16)))
+                                 'c1'/Bytes(16),
+                                 'c2'/Bytes(16),
+                                 'c3'/Bytes(16),
+                                 'c4'/Bytes(16),
+                                 'c5'/Bytes(16))
 
 
 class EF_0348_KEY(LinFixedEF):
@@ -102,18 +105,18 @@ class EF_0348_KEY(LinFixedEF):
         self._construct = Struct('security_domain'/Int8ub,
                                  'key_set_version'/Int8ub,
                                  'key_len_and_type'/KeyLenAndType,
-                                 'key'/HexAdapter(Bytes(this.key_len_and_type.key_length)))
+                                 'key'/Bytes(this.key_len_and_type.key_length))
 
 
 class EF_0348_COUNT(LinFixedEF):
     _test_de_encode = [
-        ( 'fe010000000000', {"sec_domain": 254, "key_set_version": 1, "counter": "0000000000"} ),
+        ( 'fe010000000000', {"sec_domain": 254, "key_set_version": 1, "counter": h2b("0000000000")} ),
       ]
     def __init__(self, fid='6f23', name='EF.0348_COUNT', desc='TS 03.48 OTA Counters'):
         super().__init__(fid, name=name, desc=desc, rec_len=(7, 7))
         self._construct = Struct('sec_domain'/Int8ub,
                                  'key_set_version'/Int8ub,
-                                 'counter'/HexAdapter(Bytes(5)))
+                                 'counter'/Bytes(5))
 
 
 class EF_SIM_AUTH_COUNTER(TransparentEF):
@@ -145,8 +148,9 @@ class EF_GP_DIV_DATA(LinFixedEF):
 class EF_SIM_AUTH_KEY(TransparentEF):
     _test_de_encode = [
         ( '14000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f',
-          {"cfg": {"sres_deriv_func": 1, "use_opc_instead_of_op": True, "algorithm": "milenage"}, "key":
-           "000102030405060708090a0b0c0d0e0f", "op_opc": "101112131415161718191a1b1c1d1e1f"} ),
+          {"cfg": {"sres_deriv_func": 1, "use_opc_instead_of_op": True, "algorithm": "milenage"},
+           "key": h2b("000102030405060708090a0b0c0d0e0f"),
+           "op_opc": h2b("101112131415161718191a1b1c1d1e1f")} ),
       ]
     def __init__(self, fid='6f20', name='EF.SIM_AUTH_KEY'):
         super().__init__(fid, name=name, desc='USIM authentication key')
@@ -155,8 +159,8 @@ class EF_SIM_AUTH_KEY(TransparentEF):
                             'use_opc_instead_of_op'/Flag,
                             'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/HexAdapter(Bytes(16)),
-                                 'op_opc' /HexAdapter(Bytes(16)))
+                                 'key'/Bytes(16),
+                                 'op_opc' /Bytes(16))
 
 
 class DF_SYSTEM(CardDF):
@@ -209,13 +213,13 @@ class EF_USIM_AUTH_KEY(TransparentEF):
     _test_de_encode = [
         ( '141898d827f70120d33b3e7462ee5fd6fe6ca53d7a0a804561646816d7b0c702fb',
           { "cfg": { "only_4bytes_res_in_3g": False, "sres_deriv_func_in_2g": 1, "use_opc_instead_of_op": True, "algorithm": "milenage" },
-            "key": "1898d827f70120d33b3e7462ee5fd6fe", "op_opc": "6ca53d7a0a804561646816d7b0c702fb" } ),
+            "key": h2b("1898d827f70120d33b3e7462ee5fd6fe"), "op_opc": h2b("6ca53d7a0a804561646816d7b0c702fb") } ),
         ( '160a04101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f000102030405060708090a0b0c0d0e0f',
          { "cfg" : { "algorithm" : "tuak", "key_length" : 128, "sres_deriv_func_in_2g" : 1, "use_opc_instead_of_op" : True },
            "tuak_cfg" : { "ck_and_ik_size" : 128, "mac_size" : 128, "res_size" : 128 },
            "num_of_keccak_iterations" : 4,
-           "k" : "000102030405060708090a0b0c0d0e0f",
-           "op_opc" : "101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f"
+           "k" : h2b("000102030405060708090a0b0c0d0e0f"),
+           "op_opc" : h2b("101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f")
          } ),
       ]
     def __init__(self, fid='af20', name='EF.USIM_AUTH_KEY'):
@@ -226,8 +230,8 @@ class EF_USIM_AUTH_KEY(TransparentEF):
                             'use_opc_instead_of_op'/Mapping(Bit, {False:0, True:1}),
                             'algorithm'/Algorithm)
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/HexAdapter(Bytes(16)),
-                                 'op_opc' /HexAdapter(Bytes(16)))
+                                 'key'/Bytes(16),
+                                 'op_opc'/Bytes(16))
         # TUAK has a rather different layout for the data, so we define a different
         # construct below and use explicit _{decode,encode}_bin() methods for separating
         # the TUAK and non-TUAK situation
@@ -243,8 +247,8 @@ class EF_USIM_AUTH_KEY(TransparentEF):
         self._constr_tuak = Struct('cfg'/CfgByteTuak,
                                    'tuak_cfg'/TuakCfgByte,
                                    'num_of_keccak_iterations'/Int8ub,
-                                   'op_opc'/HexAdapter(Bytes(32)),
-                                   'k'/HexAdapter(Bytes(this.cfg.key_length//8)))
+                                   'op_opc'/Bytes(32),
+                                   'k'/Bytes(this.cfg.key_length//8))
 
     def _decode_bin(self, raw_bin_data: bytearray) -> dict:
         if raw_bin_data[0] & 0x0F == 0x06:
@@ -263,8 +267,9 @@ class EF_USIM_AUTH_KEY_2G(TransparentEF):
     _test_de_encode = [
         ( '14000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f',
           {"cfg": {"only_4bytes_res_in_3g": False, "sres_deriv_func_in_2g": 1, "use_opc_instead_of_op": True,
-                   "algorithm": "milenage"}, "key": "000102030405060708090a0b0c0d0e0f", "op_opc":
-           "101112131415161718191a1b1c1d1e1f"} ),
+                   "algorithm": "milenage"},
+           "key": h2b("000102030405060708090a0b0c0d0e0f"),
+           "op_opc": h2b("101112131415161718191a1b1c1d1e1f")} ),
       ]
     def __init__(self, fid='af22', name='EF.USIM_AUTH_KEY_2G'):
         super().__init__(fid, name=name, desc='USIM authentication key in 2G context')
@@ -273,8 +278,8 @@ class EF_USIM_AUTH_KEY_2G(TransparentEF):
                             'use_opc_instead_of_op'/Flag,
                             'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3, xor=14))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/HexAdapter(Bytes(16)),
-                                 'op_opc' /HexAdapter(Bytes(16)))
+                                 'key'/Bytes(16),
+                                 'op_opc'/Bytes(16))
 
 
 class EF_GBA_SK(TransparentEF):
@@ -298,9 +303,9 @@ class EF_GBA_INT_KEY(LinFixedEF):
 
 
 class SysmocomSJA2(CardModel):
-    _atrs = ["3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 30 34 05 4B A9",
-             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 75 31 33 02 51 B2",
-             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 4C 52 75 31 04 51 D5"]
+    _atrs = ["3b9f96801f878031e073fe211b674a4c753034054ba9",
+             "3b9f96801f878031e073fe211b674a4c7531330251b2",
+             "3b9f96801f878031e073fe211b674a4c5275310451d5"]
 
     @classmethod
     def add_files(cls, rs: RuntimeState):
@@ -329,9 +334,9 @@ class SysmocomSJA2(CardModel):
             isim_adf.add_files(files_adf_isim)
 
 class SysmocomSJA5(CardModel):
-    _atrs = ["3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 51 CC",
-             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 65 F8",
-             "3B 9F 96 80 1F 87 80 31 E0 73 FE 21 1B 67 4A 35 75 30 35 02 59 C4"]
+    _atrs = ["3b9f96801f878031e073fe211b674a357530350251cc",
+             "3b9f96801f878031e073fe211b674a357530350265f8",
+             "3b9f96801f878031e073fe211b674a357530350259c4"]
 
     @classmethod
     def add_files(cls, rs: RuntimeState):

pySim/transport/__init__.py

@@ -119,6 +119,11 @@ class LinkBase(abc.ABC):
         """Connect to a card immediately
         """
 
+    @abc.abstractmethod
+    def get_atr(self) -> Hexstr:
+        """Retrieve card ATR
+        """
+
     @abc.abstractmethod
     def disconnect(self):
         """Disconnect from card
@@ -159,8 +164,8 @@ class LinkBase(abc.ABC):
         if self.apdu_tracer:
             self.apdu_tracer.trace_response(apdu, sw, data)
 
-        # The APDU case (See aso ISO/IEC 7816-3, table 12) dictates if we should receive a response or not. If we
-        # receive a response in an APDU case that does not allow the reception of a respnse we print a warning to
+        # The APDU case (See also ISO/IEC 7816-3, table 12) dictates if we should receive a response or not. If we
+        # receive a response in an APDU case that does not allow the reception of a response we print a warning to
         # make the user/caller aware of the problem. Since the transaction is over at this point and data was received
         # we count it as a successful transaction anyway, even though the spec was violated. The problem is most likely
         # caused by a missing Le field in the APDU. This is an error that the caller/user should correct to avoid

pySim/transport/calypso.py

@@ -123,6 +123,9 @@ class CalypsoSimLink(LinkBaseTpdu):
     def connect(self):
         self.reset_card()
 
+    def get_atr(self) -> Hexstr:
+        return "3b00" # Dummy ATR
+
     def disconnect(self):
         pass  # Nothing to do really ...
 

pySim/transport/modem_atcmd.py

@@ -139,6 +139,9 @@ class ModemATCommandLink(LinkBaseTpdu):
     def connect(self):
         pass  # Nothing to do really ...
 
+    def get_atr(self) -> Hexstr:
+        return "3b00" # Dummy ATR
+
     def disconnect(self):
         pass  # Nothing to do really ...
 

pySim/transport/pcsc.py

@@ -103,7 +103,7 @@ class PcscSimLink(LinkBaseTpdu):
             raise NoCardError() from exc
 
     def get_atr(self) -> Hexstr:
-        return self._con.getATR()
+        return i2h(self._con.getATR())
 
     def disconnect(self):
         self._con.disconnect()

pySim/transport/serial.py

@@ -21,7 +21,7 @@ import os
 import argparse
 from typing import Optional
 import serial
-from osmocom.utils import h2b, b2h, Hexstr
+from osmocom.utils import h2b, b2h, i2h, Hexstr
 
 from pySim.exceptions import NoCardError, ProtocolError
 from pySim.transport import LinkBaseTpdu
@@ -96,7 +96,7 @@ class SerialSimLink(LinkBaseTpdu):
         self.reset_card()
 
     def get_atr(self) -> Hexstr:
-        return self._atr
+        return i2h(self._atr)
 
     def disconnect(self):
         pass  # Nothing to do really ...

pySim/ts_102_221.py

@@ -119,11 +119,11 @@ class FileDescriptor(BER_TLV_IE, tag=0x82):
 
 # ETSI TS 102 221 11.1.1.4.4
 class FileIdentifier(BER_TLV_IE, tag=0x83):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # ETSI TS 102 221 11.1.1.4.5
 class DfName(BER_TLV_IE, tag=0x84):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 # ETSI TS 102 221 11.1.1.4.6.1
 class UiccCharacteristics(BER_TLV_IE, tag=0x80):
@@ -217,7 +217,7 @@ class SecurityAttribExpanded(BER_TLV_IE, tag=0xab):
 # ETSI TS 102 221 11.1.1.4.7.3
 class SecurityAttribReferenced(BER_TLV_IE, tag=0x8b):
     # TODO: longer format with SEID
-    _construct = Struct('ef_arr_file_id'/HexAdapter(Bytes(2)), 'ef_arr_record_nr'/Int8ub)
+    _construct = Struct('ef_arr_file_id'/Bytes(2), 'ef_arr_record_nr'/Int8ub)
 
 # ETSI TS 102 221 11.1.1.4.8
 class ShortFileIdentifier(BER_TLV_IE, tag=0x88):

pySim/ts_102_222.py

@@ -25,6 +25,16 @@ from osmocom.utils import b2h, auto_uint8, auto_uint16, is_hexstr
 
 from pySim.ts_102_221 import *
 
+def expand_pattern(pattern: bytes, repeat: bool, size: int) -> bytes:
+    """Expand the fill/repeat pattern as per TS 102 222 Section 6.3.2.2.2 Tags C1/C2."""
+    if not repeat:
+        pad_len = size - len(pattern)
+        return pattern + pattern[-1:] * pad_len
+    else:
+        count = size // len(pattern)
+        part_len = size - count * len(pattern)
+        return pattern * count + pattern[:part_len]
+
 @with_default_category('TS 102 222 Administrative Commands')
 class Ts102222Commands(CommandSet):
     """Administrative commands for telecommunication applications."""

pySim/ts_102_310.py

@@ -27,9 +27,9 @@ from pySim.filesystem import CardDF, TransparentEF
 # TS102 310 Section 7.1
 class EF_EAPKEYS(TransparentEF):
     class Msk(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
     class Emsk(BER_TLV_IE, tag=0x81):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
     class MskCollection(TLV_IE_Collection, nested=[EF_EAPKEYS.Msk, EF_EAPKEYS.Emsk]):
         pass
 

pySim/ts_31_102.py

@@ -217,7 +217,7 @@ EF_EST_map = {
 
 # 3gPP TS 31.102 Section 7.5.2.1
 class SUCI_TlvDataObject(BER_TLV_IE, tag=0xA1):
-    _construct = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
 
 ######################################################################
 # ADF.USIM
@@ -230,7 +230,7 @@ class EF_5GS3GPPNSC(LinFixedEF):
         _construct = Int8ub
 
     class K_AMF(BER_TLV_IE, tag=0x81):
-        _construct = HexAdapter(Bytes(32))
+        _construct = Bytes(32)
 
     class UplinkNASCount(BER_TLV_IE, tag=0x82):
         _construct = Int32ub
@@ -260,10 +260,10 @@ class EF_5GS3GPPNSC(LinFixedEF):
 # 3GPP TS 31.102 Section 4.4.11.6
 class EF_5GAUTHKEYS(TransparentEF):
     class K_AUSF(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class K_SEAF(BER_TLV_IE, tag=0x81):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class FiveGAuthKeys(TLV_IE_Collection, nested=[K_AUSF, K_SEAF]):
         pass
@@ -281,9 +281,9 @@ class EF_SUCI_Calc_Info(TransparentEF):
                                                                                       "identifier": 0,
                                                                                       "key_index": 0}],
            "hnet_pubkey_list": [{"hnet_pubkey_identifier": 10, "hnet_pubkey":
-                                 "4e858c4d49d1343e6181284c47ca721730c98742cb7c6182d2e8126e08088d36"},
+                                 h2b("4e858c4d49d1343e6181284c47ca721730c98742cb7c6182d2e8126e08088d36")},
                                 {"hnet_pubkey_identifier": 11, "hnet_pubkey":
-                                 "d1bc365f4997d17ce4374e72181431cbfeba9e1b98d7618f79d48561b144672a"}]} ),
+                                 h2b("d1bc365f4997d17ce4374e72181431cbfeba9e1b98d7618f79d48561b144672a")}]} ),
     ]
     # 3GPP TS 31.102 Section 4.4.11.8
     class ProtSchemeIdList(BER_TLV_IE, tag=0xa0):
@@ -298,7 +298,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
 
     class HnetPubkey(BER_TLV_IE, tag=0x81):
         # contents according to RFC 7748 / RFC 5480
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class HnetPubkeyList(BER_TLV_IE, tag=0xa1, nested=[HnetPubkeyIdentifier, HnetPubkey]):
         pass
@@ -425,7 +425,7 @@ class EF_Keys(TransparentEF):
                  desc='Ciphering and Integrity Keys'):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         self._construct = Struct(
-            'ksi'/Int8ub, 'ck'/HexAdapter(Bytes(16)), 'ik'/HexAdapter(Bytes(16)))
+            'ksi'/Int8ub, 'ck'/Bytes(16), 'ik'/Bytes(16))
 
 # TS 31.102 Section 4.2.6
 class EF_HPPLMN(TransparentEF):
@@ -536,15 +536,15 @@ class EF_ECC(LinFixedEF):
 class EF_LOCI(TransparentEF):
     _test_de_encode = [
         ( '47d1264a62f21037211e00',
-          { "tmsi": "47d1264a", "lai": { "mcc_mnc": "262-01", "lac": "3721" },
+          { "tmsi": h2b("47d1264a"), "lai": { "mcc_mnc": "262-01", "lac": h2b("3721") },
             "rfu": 30, "lu_status": 0 } ),
         ( 'ffffffff62f2200000ff01',
-          {"tmsi": "ffffffff", "lai": {"mcc_mnc": "262-02", "lac": "0000"}, "rfu": 255, "lu_status": 1} ),
+          {"tmsi": h2b("ffffffff"), "lai": {"mcc_mnc": "262-02", "lac": h2b("0000") }, "rfu": 255, "lu_status": 1} ),
     ]
     def __init__(self, fid='6f7e', sfid=0x0b, name='EF.LOCI', desc='Location information', size=(11, 11)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-        Lai = Struct('mcc_mnc'/PlmnAdapter(Bytes(3)), 'lac'/HexAdapter(Bytes(2)))
-        self._construct = Struct('tmsi'/HexAdapter(Bytes(4)), 'lai'/Lai, 'rfu'/Int8ub, 'lu_status'/Int8ub)
+        Lai = Struct('mcc_mnc'/PlmnAdapter(Bytes(3)), 'lac'/Bytes(2))
+        self._construct = Struct('tmsi'/Bytes(4), 'lai'/Lai, 'rfu'/Int8ub, 'lu_status'/Int8ub)
 
 # TS 31.102 Section 4.2.18
 class EF_AD(TransparentEF):
@@ -585,15 +585,15 @@ class EF_AD(TransparentEF):
 class EF_PSLOCI(TransparentEF):
     def __init__(self, fid='6f73', sfid=0x0c, name='EF.PSLOCI', desc='PS Location information', size=(14, 14)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-        self._construct = Struct('ptmsi'/HexAdapter(Bytes(4)), 'ptmsi_sig'/HexAdapter(Bytes(3)),
-                                 'rai'/HexAdapter(Bytes(6)), 'rau_status'/Int8ub)
+        self._construct = Struct('ptmsi'/Bytes(4), 'ptmsi_sig'/Bytes(3),
+                                 'rai'/Bytes(6), 'rau_status'/Int8ub)
 
 # TS 31.102 Section 4.2.33
 class EF_ICI(CyclicEF):
     def __init__(self, fid='6f80', sfid=0x14, name='EF.ICI', rec_len=(28, 48),
                  desc='Incoming Call Information', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
-        self._construct = Struct('alpha_id'/HexAdapter(Bytes(this._.total_len-28)),
+        self._construct = Struct('alpha_id'/Bytes(this._.total_len-28),
                                  'len_of_bcd_contents'/Int8ub,
                                  'ton_npi'/Int8ub,
                                  'call_number'/BcdAdapter(Bytes(10)),
@@ -602,14 +602,14 @@ class EF_ICI(CyclicEF):
                                  'date_and_time'/BcdAdapter(Bytes(7)),
                                  'duration'/Int24ub,
                                  'status'/Byte,
-                                 'link_to_phonebook'/HexAdapter(Bytes(3)))
+                                 'link_to_phonebook'/Bytes(3))
 
 # TS 31.102 Section 4.2.34
 class EF_OCI(CyclicEF):
     def __init__(self, fid='6f81', sfid=0x15, name='EF.OCI', rec_len=(27, 47),
                  desc='Outgoing Call Information', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
-        self._construct = Struct('alpha_id'/HexAdapter(Bytes(this._.total_len-27)),
+        self._construct = Struct('alpha_id'/Bytes(this._.total_len-27),
                                  'len_of_bcd_contents'/Int8ub,
                                  'ton_npi'/Int8ub,
                                  'call_number'/BcdAdapter(Bytes(10)),
@@ -617,7 +617,7 @@ class EF_OCI(CyclicEF):
                                  'ext5_record_id'/Int8ub,
                                  'date_and_time'/BcdAdapter(Bytes(7)),
                                  'duration'/Int24ub,
-                                 'link_to_phonebook'/HexAdapter(Bytes(3)))
+                                 'link_to_phonebook'/Bytes(3))
 
 # TS 31.102 Section 4.2.35
 class EF_ICT(CyclicEF):
@@ -655,7 +655,7 @@ class EF_ACL(TransparentEF):
     def __init__(self, fid='6f57', sfid=None, name='EF.ACL', size=(32, None),
                  desc='Access Point Name Control List', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
-        self._construct = Struct('num_of_apns'/Int8ub, 'tlvs'/HexAdapter(GreedyBytes))
+        self._construct = Struct('num_of_apns'/Int8ub, 'tlvs'/GreedyBytes)
 
 # TS 31.102 Section 4.2.51
 class EF_START_HFN(TransparentEF):
@@ -705,16 +705,16 @@ class EF_MSK(LinFixedEF):
     def __init__(self, fid='6fd7', sfid=None, name='EF.MSK', desc='MBMS Service Key List', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=(20, None), **kwargs)
         msk_ts_constr = Struct('msk_id'/Int32ub, 'timestamp_counter'/Int32ub)
-        self._construct = Struct('key_domain_id'/HexAdapter(Bytes(3)),
+        self._construct = Struct('key_domain_id'/Bytes(3),
                                  'num_msk_id'/Int8ub,
                                  'msk_ids'/msk_ts_constr[this.num_msk_id])
 # TS 31.102 Section 4.2.81
 class EF_MUK(LinFixedEF):
     class MUK_Idr(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class MUK_Idi(BER_TLV_IE, tag=0x82):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class MUK_ID(BER_TLV_IE, tag=0xA0, nested=[MUK_Idr, MUK_Idi]):
         pass
@@ -732,10 +732,10 @@ class EF_MUK(LinFixedEF):
 # TS 31.102 Section 4.2.83
 class EF_GBANL(LinFixedEF):
     class NAF_ID(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class B_TID(BER_TLV_IE, tag=0x81):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class EF_GBANL_Collection(BER_TLV_IE, nested=[NAF_ID, B_TID]):
         pass
@@ -759,7 +759,7 @@ class EF_EHPLMNPI(TransparentEF):
 # TS 31.102 Section 4.2.87
 class EF_NAFKCA(LinFixedEF):
     class NAF_KeyCentreAddress(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
     def __init__(self, fid='6fdd', sfid=None, name='EF.NAFKCA', rec_len=(None, None),
                  desc='NAF Key Centre Address', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
@@ -770,11 +770,11 @@ class EF_NCP_IP(LinFixedEF):
     class DataDestAddrRange(TLV_IE, tag=0x83):
         _construct = Struct('type_of_address'/Enum(Byte, IPv4=0x21, IPv6=0x56),
                             'prefix_length'/Int8ub,
-                            'prefix'/HexAdapter(GreedyBytes))
+                            'prefix'/GreedyBytes)
 
     class AccessPointName(TLV_IE, tag=0x80):
         # coded as per TS 23.003
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class Login(TLV_IE, tag=0x81):
         # as per SMS DCS TS 23.038
@@ -803,8 +803,8 @@ class EF_EPSLOCI(TransparentEF):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         upd_status_constr = Enum(
             Byte, updated=0, not_updated=1, roaming_not_allowed=2)
-        self._construct = Struct('guti'/HexAdapter(Bytes(12)),
-                                 'last_visited_registered_tai'/HexAdapter(Bytes(5)),
+        self._construct = Struct('guti'/Bytes(12),
+                                 'last_visited_registered_tai'/Bytes(5),
                                  'eps_update_status'/upd_status_constr)
 
 # TS 31.102 Section 4.2.92
@@ -813,7 +813,7 @@ class EF_EPSNSC(LinFixedEF):
         _construct = Int8ub
 
     class K_ASME(BER_TLV_IE, tag=0x81):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class UplinkNASCount(BER_TLV_IE, tag=0x82):
         _construct = Int32ub
@@ -822,7 +822,7 @@ class EF_EPSNSC(LinFixedEF):
         _construct = Int32ub
 
     class IDofNASAlgorithms(BER_TLV_IE, tag=0x84):
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class EPS_NAS_Security_Context(BER_TLV_IE, tag=0xa0,
                                    nested=[KSI_ASME, K_ASME, UplinkNASCount, DownlinkNASCount,
@@ -875,7 +875,8 @@ class EF_ePDGId(TransparentEF):
 class EF_ePDGSelection(TransparentEF):
     _test_de_encode = [
         ( '800600f110000100', {'e_pdg_selection': [{'plmn': '001-01', 'epdg_priority': 1, 'epdg_fqdn_format': 'operator_identified' }] }),
-        ( '800600011000a001', {'e_pdg_selection': [{'plmn': '001-001', 'epdg_priority': 160, 'epdg_fqdn_format': 'location_based' }] }),
+        ( '800600110000a001', {'e_pdg_selection': [{'plmn': '001-001', 'epdg_priority': 160, 'epdg_fqdn_format': 'location_based' }] }),
+        ( '800600011000a001', {'e_pdg_selection': [{'plmn': '001-010', 'epdg_priority': 160, 'epdg_fqdn_format': 'location_based' }] }),
     ]
     class ePDGSelection(BER_TLV_IE, tag=0x80):
         _construct = GreedyRange(Struct('plmn'/PlmnAdapter(Bytes(3)),
@@ -1061,8 +1062,8 @@ class EF_5GS3GPPLOCI(TransparentEF):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         upd_status_constr = Enum(
             Byte, updated=0, not_updated=1, roaming_not_allowed=2)
-        self._construct = Struct('5g_guti'/HexAdapter(Bytes(13)),
-                                 'last_visited_registered_tai_in_5gs'/HexAdapter(Bytes(6)),
+        self._construct = Struct('5g_guti'/Bytes(13),
+                                 'last_visited_registered_tai_in_5gs'/Bytes(6),
                                  '5gs_update_status'/upd_status_constr)
 
 # TS 31.102 Section 4.4.11.7 (Rel 15)
@@ -1082,8 +1083,8 @@ class EF_UAC_AIC(TransparentEF):
 class EF_OPL5G(LinFixedEF):
     def __init__(self, fid='4f08', sfid=0x08, name='EF.OPL5G', desc='5GS Operator PLMN List', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=(10, None), **kwargs)
-        Tai = Struct('mcc_mnc'/PlmnAdapter(Bytes(3)), 'tac_min'/HexAdapter(Bytes(3)),
-                     'tac_max'/HexAdapter(Bytes(3)))
+        Tai = Struct('mcc_mnc'/PlmnAdapter(Bytes(3)), 'tac_min'/Bytes(3),
+                     'tac_max'/Bytes(3))
         self._construct = Struct('tai'/Tai, 'pnn_record_id'/Int8ub)
 
 # TS 31.102 Section 4.4.11.10 (Rel 15)
@@ -1118,7 +1119,7 @@ class EF_Routing_Indicator(TransparentEF):
         # operator decides to assign less than 4 digits to Routing Indicator, the remaining digits
         # shall be coded as "1111" to fill the 4 digits coding of Routing Indicator
         self._construct = Struct('routing_indicator'/Rpad(BcdAdapter(Bytes(2)), 'f', 2),
-                                 'rfu'/HexAdapter(Bytes(2)))
+                                 'rfu'/Bytes(2))
 
 # TS 31.102 Section 4.4.11.13 (Rel 16)
 class EF_TN3GPPSNN(TransparentEF):
@@ -1134,14 +1135,14 @@ class EF_CAG(TransparentEF):
     def __init__(self, fid='4f0d', sfid=0x0d, name='EF.CAG',
                  desc='Pre-configured CAG information list EF', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
-        self._construct = HexAdapter(GreedyBytes)
+        self._construct = GreedyBytes
 
 # TS 31.102 Section 4.4.11.15 (Rel 17)
 class EF_SOR_CMCI(TransparentEF):
     def __init__(self, fid='4f0e', sfid=0x0e, name='EF.SOR-CMCI',
                  desc='Steering Of Roaming - Connected Mode Control Information', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
-        self._construct = HexAdapter(GreedyBytes)
+        self._construct = GreedyBytes
 
 # TS 31.102 Section 4.4.11.17 (Rel 17)
 class EF_DRI(TransparentEF):
@@ -1152,9 +1153,9 @@ class EF_DRI(TransparentEF):
                                  'parameters_indicator_status'/FlagsEnum(Byte, roaming_wait_range=1,
                                                                          return_wait_range=2,
                                                                          applicability_indicator=3),
-                                 'roaming_wait_range'/HexAdapter(Bytes(2)),
-                                 'return_wait_range'/HexAdapter(Bytes(2)),
-                                 'applicability_indicator'/HexAdapter(Byte))
+                                 'roaming_wait_range'/Bytes(2),
+                                 'return_wait_range'/Bytes(2),
+                                 'applicability_indicator'/Byte)
 
 # TS 31.102 Section 4.4.12.2 (Rel 17)
 class EF_PWS_SNPN(TransparentEF):
@@ -1172,7 +1173,7 @@ class EF_NID(LinFixedEF):
         self._construct = Struct('assignment_mode'/Enum(Byte, coordinated_ass_opt1=0,
                                                               self_ass=1,
                                                               coordinated_ass_opt2=2),
-                                 'network_identifier'/HexAdapter(Bytes(5)))
+                                 'network_identifier'/Bytes(5))
 
 # TS 31.102 Section 4.4.12 (Rel 17)
 class DF_SNPN(CardDF):
@@ -1412,7 +1413,7 @@ class EF_5MBSUECONFIG(TransparentEF):
                             'nid'/COptional(Bytes(6)))
     class Tmgi(BER_TLV_IE, tag=0x81):
         TmgiEntry = Struct('tmgi'/Bytes(6),
-                           'usd_fid'/HexAdapter(Bytes(2)),
+                           'usd_fid'/Bytes(2),
                            'service_type'/FlagsEnum(Byte, mbs_service_announcement=1, mbs_user_service=2))
         _construct = GreedyRange(TmgiEntry)
     class NrArfcnList(BER_TLV_IE, tag=0x82):
@@ -1481,7 +1482,7 @@ class EF_KAUSF_DERIVATION(TransparentEF):
     def __init__(self, fid='4f16', sfid=0x16, name='EF.KAUSF_DERIVATION',
                  desc='K_AUSF derivation configuration', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
-        self._construct = Struct('k_ausf_deriv_cfg'/FlagsEnum(Byte, use_msk=1), 'rfu'/HexAdapter(GreedyBytes))
+        self._construct = Struct('k_ausf_deriv_cfg'/FlagsEnum(Byte, use_msk=1), 'rfu'/GreedyBytes)
 
 # TS 31.102 Section 4.4.5
 class DF_WLAN(CardDF):
@@ -1825,7 +1826,7 @@ class ADF_USIM(CardADF):
             do = SUCI_TlvDataObject()
             do.from_tlv(h2b(data))
             do_d = do.to_dict()
-            self._cmd.poutput('SUCI TLV Data Object: %s' % do_d['suci__tlv_data_object'])
+            self._cmd.poutput('SUCI TLV Data Object: %s' % b2h(do_d['suci__tlv_data_object']))
 
 
 # TS 31.102 Section 7.3

pySim/ts_31_103.py

@@ -22,7 +22,7 @@ Various constants from 3GPP TS 31.103 V18.1.0
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from construct import Struct, Switch, this, Bytes, GreedyString
+from construct import Struct, Switch, this, GreedyString
 from osmocom.utils import *
 from osmocom.tlv import *
 from osmocom.construct import *
@@ -167,7 +167,7 @@ class EF_GBABP(TransparentEF):
 class EF_GBANL(LinFixedEF):
     class NAF_ID(BER_TLV_IE, tag=0x80):
         _construct = Struct('fqdn'/Utf8Adapter(Bytes(this._.total_len-5)),
-                            'ua_spi'/HexAdapter(Bytes(5)))
+                            'ua_spi'/Bytes(5))
     class B_TID(BER_TLV_IE, tag=0x81):
         _construct = Utf8Adapter(GreedyBytes)
     # pylint: disable=undefined-variable

pySim/ts_31_103_shared.py

@@ -19,9 +19,9 @@ hence need to be in a separate python module to avoid circular dependencies.
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from construct import Struct, Switch, Bytes, GreedyString, GreedyBytes, Int8ub, Prefixed, Enum, Byte
+from construct import Struct, Switch, GreedyString, Int8ub, Prefixed, Enum, Byte
 from osmocom.tlv import BER_TLV_IE, TLV_IE_Collection
-from osmocom.construct import HexAdapter, Utf8Adapter
+from osmocom.construct import Bytes, Utf8Adapter, GreedyBytes
 from pySim.filesystem import *
 
 # TS 31.103 Section 4.2.16
@@ -36,7 +36,7 @@ class EF_UICCIARI(LinFixedEF):
 # TS 31.103 Section 4.2.18
 class EF_IMSConfigData(BerTlvEF):
     class ImsConfigDataEncoding(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(Bytes(1))
+        _construct = Bytes(1)
     class ImsConfigData(BER_TLV_IE, tag=0x81):
         _construct = GreedyString
     # pylint: disable=undefined-variable
@@ -103,7 +103,7 @@ class EF_WebRTCURI(LinFixedEF):
 # TS 31.103 Section 4.2.21
 class EF_MuDMiDConfigData(BerTlvEF):
     class MudMidConfigDataEncoding(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(Bytes(1))
+        _construct = Bytes(1)
     class MudMidConfigData(BER_TLV_IE, tag=0x81):
         _construct = GreedyString
     # pylint: disable=undefined-variable

pySim/ts_51_011.py

@@ -250,7 +250,7 @@ class EF_SMSP(LinFixedEF):
             "tp_sc_addr": { "length": 255, "ton_npi": { "ext": True, "type_of_number": "reserved_for_extension",
                                                         "numbering_plan_id": "reserved_for_extension" },
                             "call_number": "" },
-            "tp_pid": "00", "tp_dcs": "00", "tp_vp_minutes": 1440 } ),
+            "tp_pid": b"\x00", "tp_dcs": b"\x00", "tp_vp_minutes": 1440 } ),
     ]
     _test_no_pad = True
     class ValidityPeriodAdapter(Adapter):
@@ -286,8 +286,8 @@ class EF_SMSP(LinFixedEF):
                                  'tp_dest_addr'/ScAddr,
                                  'tp_sc_addr'/ScAddr,
 
-                                 'tp_pid'/HexAdapter(Bytes(1)),
-                                 'tp_dcs'/HexAdapter(Bytes(1)),
+                                 'tp_pid'/Bytes(1),
+                                 'tp_dcs'/Bytes(1),
                                  'tp_vp_minutes'/EF_SMSP.ValidityPeriodAdapter(Byte))
 
 # TS 51.011 Section 10.5.7
@@ -309,14 +309,14 @@ class EF_SMSR(LinFixedEF):
     def __init__(self, fid='6f47', sfid=None, name='EF.SMSR', desc='SMS status reports', rec_len=(30, 30), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
-            'sms_record_id'/Int8ub, 'sms_status_report'/HexAdapter(Bytes(29)))
+            'sms_record_id'/Int8ub, 'sms_status_report'/Bytes(29))
 
 
 class EF_EXT(LinFixedEF):
     def __init__(self, fid, sfid=None, name='EF.EXT', desc='Extension', rec_len=(13, 13), **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
-            'record_type'/Int8ub, 'extension_data'/HexAdapter(Bytes(11)), 'identifier'/Int8ub)
+            'record_type'/Int8ub, 'extension_data'/Bytes(11), 'identifier'/Int8ub)
 
 # TS 51.011 Section 10.5.16
 class EF_CMI(LinFixedEF):
@@ -589,11 +589,11 @@ class EF_ACC(TransparentEF):
 class EF_LOCI(TransparentEF):
     _test_de_encode = [
             ( "7802570222f81009780000",
-              { "tmsi": "78025702", "lai": "22f8100978", "tmsi_time": 0, "lu_status": "updated" } ),
+              { "tmsi": h2b("78025702"), "lai": h2b("22f8100978"), "tmsi_time": 0, "lu_status": "updated" } ),
         ]
     def __init__(self, fid='6f7e', sfid=None, name='EF.LOCI', desc='Location Information', size=(11, 11)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-        self._construct = Struct('tmsi'/HexAdapter(Bytes(4)), 'lai'/HexAdapter(Bytes(5)), 'tmsi_time'/Int8ub,
+        self._construct = Struct('tmsi'/Bytes(4), 'lai'/Bytes(5), 'tmsi_time'/Int8ub,
                                  'lu_status'/Enum(Byte, updated=0, not_updated=1, plmn_not_allowed=2,
                                                   location_area_not_allowed=3))
 
@@ -751,22 +751,22 @@ class EF_NIA(LinFixedEF):
 # TS 51.011 Section 10.3.32
 class EF_Kc(TransparentEF):
     _test_de_encode = [
-        ( "837d783609a3858f05", { "kc": "837d783609a3858f", "cksn": 5 } ),
+        ( "837d783609a3858f05", { "kc": h2b("837d783609a3858f"), "cksn": 5 } ),
     ]
     def __init__(self, fid='6f20', sfid=None, name='EF.Kc', desc='Ciphering key Kc', size=(9, 9), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
-        self._construct = Struct('kc'/HexAdapter(Bytes(8)), 'cksn'/Int8ub)
+        self._construct = Struct('kc'/Bytes(8), 'cksn'/Int8ub)
 
 # TS 51.011 Section 10.3.33
 class EF_LOCIGPRS(TransparentEF):
     _test_de_encode = [
         ( "ffffffffffffff22f8990000ff01",
-          { "ptmsi": "ffffffff", "ptmsi_sig": "ffffff", "rai": "22f8990000ff", "rau_status": "not_updated" } ),
+          { "ptmsi": h2b("ffffffff"), "ptmsi_sig": h2b("ffffff"), "rai": h2b("22f8990000ff"), "rau_status": "not_updated" } ),
     ]
     def __init__(self, fid='6f53', sfid=None, name='EF.LOCIGPRS', desc='GPRS Location Information', size=(14, 14)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-        self._construct = Struct('ptmsi'/HexAdapter(Bytes(4)), 'ptmsi_sig'/HexAdapter(Bytes(3)),
-                                 'rai'/HexAdapter(Bytes(6)),
+        self._construct = Struct('ptmsi'/Bytes(4), 'ptmsi_sig'/Bytes(3),
+                                 'rai'/Bytes(6),
                                  'rau_status'/Enum(Byte, updated=0, not_updated=1, plmn_not_allowed=2,
                                                    routing_area_not_allowed=3))
 
@@ -867,12 +867,12 @@ class EF_PNN(LinFixedEF):
     class FullNameForNetwork(BER_TLV_IE, tag=0x43):
         # TS 24.008 10.5.3.5a
         # TODO: proper decode
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class ShortNameForNetwork(BER_TLV_IE, tag=0x45):
         # TS 24.008 10.5.3.5a
         # TODO: proper decode
-        _construct = HexAdapter(GreedyBytes)
+        _construct = GreedyBytes
 
     class NetworkNameCollection(TLV_IE_Collection, nested=[FullNameForNetwork, ShortNameForNetwork]):
         pass
@@ -885,12 +885,14 @@ class EF_PNN(LinFixedEF):
 class EF_OPL(LinFixedEF):
     _test_de_encode = [
         ( '62f2100000fffe01',
-          { "lai": { "mcc_mnc": "262-01", "lac_min": "0000", "lac_max": "fffe" }, "pnn_record_id": 1 } ),
+          { "lai": { "mcc_mnc": "262-01", "lac_min": h2b("0000"), "lac_max": h2b("fffe") }, "pnn_record_id": 1 } ),
+        ( '216354789abcde12',
+          { "lai": { "mcc_mnc": "123-456", "lac_min": h2b("789a"), "lac_max": h2b("bcde") }, "pnn_record_id": 18 } ),
     ]
     def __init__(self, fid='6fc6', sfid=None, name='EF.OPL', rec_len=(8, 8), desc='Operator PLMN List', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct('lai'/Struct('mcc_mnc'/PlmnAdapter(Bytes(3)),
-                                 'lac_min'/HexAdapter(Bytes(2)), 'lac_max'/HexAdapter(Bytes(2))), 'pnn_record_id'/Int8ub)
+                                 'lac_min'/Bytes(2), 'lac_max'/Bytes(2)), 'pnn_record_id'/Int8ub)
 
 # TS 51.011 Section 10.3.44 + TS 31.102 4.2.62
 class EF_MBI(LinFixedEF):
@@ -939,8 +941,8 @@ class EF_SPDI(TransparentEF):
 class EF_MMSN(LinFixedEF):
     def __init__(self, fid='6fce', sfid=None, name='EF.MMSN', rec_len=(4, 20), desc='MMS Notification', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
-        self._construct = Struct('mms_status'/HexAdapter(Bytes(2)), 'mms_implementation'/HexAdapter(Bytes(1)),
-                                 'mms_notification'/HexAdapter(Bytes(this._.total_len-4)), 'ext_record_nr'/Byte)
+        self._construct = Struct('mms_status'/Bytes(2), 'mms_implementation'/Bytes(1),
+                                 'mms_notification'/Bytes(this._.total_len-4), 'ext_record_nr'/Byte)
 
 # TS 51.011 Annex K.1
 class MMS_Implementation(BER_TLV_IE, tag=0x80):

requirements.txt

@@ -5,7 +5,7 @@ cmd2>=1.5
 jsonpath-ng
 construct>=2.10.70
 bidict
-pyosmocom>=0.0.6
+pyosmocom>=0.0.9
 pyyaml>=5.1
 termcolor
 colorlog
@@ -14,3 +14,4 @@ cryptography
 git+https://github.com/osmocom/asn1tools
 packaging
 git+https://github.com/hologram-io/smpp.pdu
+smpp.twisted3 @ git+https://github.com/jookies/smpp.twisted

setup.py

@@ -8,6 +8,7 @@ setup(
         'pySim.apdu',
         'pySim.apdu_source',
         'pySim.esim',
+        'pySim.esim.saip',
         'pySim.global_platform',
         'pySim.legacy',
         'pySim.transport',
@@ -24,18 +25,36 @@ setup(
         "jsonpath-ng",
         "construct >= 2.10.70",
         "bidict",
-        "pyosmocom >= 0.0.6",
+        "pyosmocom >= 0.0.9",
         "pyyaml >= 5.1",
         "termcolor",
         "colorlog",
         "pycryptodomex",
         "packaging",
         "smpp.pdu @ git+https://github.com/hologram-io/smpp.pdu",
+        "asn1tools",
+        "smpp.twisted3 @ git+https://github.com/jookies/smpp.twisted",
     ],
     scripts=[
         'pySim-prog.py',
         'pySim-read.py',
         'pySim-shell.py',
         'pySim-trace.py',
-    ]
+        'pySim-smpp2sim.py',
+    ],
+    package_data={
+        'pySim.esim':
+            [
+                'asn1/rsp/*.asn',
+                'asn1/saip/*.asn',
+            ],
+    },
+    extras_require={
+        "smdpp": [
+            "klein",
+            "service-identity",
+            "pyopenssl",
+            "requests",
+        ]
+    },
 )

smdpp-data/certs/DPtls/CERT_S_SM_DP_TLS_BRP.pem

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICgjCCAimgAwIBAgIBCTAKBggqhkjOPQQDAjBEMRAwDgYDVQQDDAdUZXN0IENJ
+MREwDwYDVQQLDAhURVNUQ0VSVDEQMA4GA1UECgwHUlNQVEVTVDELMAkGA1UEBhMC
+SVQwHhcNMjQwNzA5MTUyOTM2WhcNMjUwODExMTUyOTM2WjAzMQ0wCwYDVQQKDARB
+Q01FMSIwIAYDVQQDDBl0ZXN0c21kcHBsdXMxLmV4YW1wbGUuY29tMFowFAYHKoZI
+zj0CAQYJKyQDAwIIAQEHA0IABEwizNgsjQIh+dhUO3LhB7zJ/ZBU1mx1wOt0p73n
+MOdhjvZbJwteguQ6eW+N7guvivvrilNiU3oC/WXHnkEZa7WjggEaMIIBFjAOBgNV
+HQ8BAf8EBAMCB4AwIAYDVR0lAQH/BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMBQG
+A1UdIAQNMAswCQYHZ4ESAQIBAzAdBgNVHQ4EFgQUPTMJg/OfzFvS5K1ophmnR0iu
+i50wHwYDVR0jBBgwFoAUwLxwujaSnUO0Z/9XVwUw5Xq4/NgwKQYDVR0RBCIwIIIZ
+dGVzdHNtZHBwbHVzMS5leGFtcGxlLmNvbYgDiDcKMGEGA1UdHwRaMFgwKqAooCaG
+JGh0dHA6Ly9jaS50ZXN0LmV4YW1wbGUuY29tL0NSTC1BLmNybDAqoCigJoYkaHR0
+cDovL2NpLnRlc3QuZXhhbXBsZS5jb20vQ1JMLUIuY3JsMAoGCCqGSM49BAMCA0cA
+MEQCIHHmXEy9mgudh/VbK0hJwmX7eOgbvHLnlujrpQzvUd4uAiBFVJgSdzYvrmJ9
+5yeIvmjHwxSMBgQp2dde7OtdVEK8Kw==
+-----END CERTIFICATE-----

smdpp-data/certs/DPtls/CERT_S_SM_DP_TLS_NIST.pem

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICgzCCAiigAwIBAgIBCTAKBggqhkjOPQQDAjBEMRAwDgYDVQQDDAdUZXN0IENJ
+MREwDwYDVQQLDAhURVNUQ0VSVDEQMA4GA1UECgwHUlNQVEVTVDELMAkGA1UEBhMC
+SVQwHhcNMjQwNzA5MTUyODMzWhcNMjUwODExMTUyODMzWjAzMQ0wCwYDVQQKDARB
+Q01FMSIwIAYDVQQDDBl0ZXN0c21kcHBsdXMxLmV4YW1wbGUuY29tMFkwEwYHKoZI
+zj0CAQYIKoZIzj0DAQcDQgAEKCQwdc6O/R+uZ2g5QH2ybkzLQ3CUYhybOWEz8bJL
+tQG4/k6yTT4NOS8lP28blGJws8opLjTbb3qHs6X2rJRfCKOCARowggEWMA4GA1Ud
+DwEB/wQEAwIHgDAgBgNVHSUBAf8EFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwFAYD
+VR0gBA0wCzAJBgdngRIBAgEDMB0GA1UdDgQWBBQn/vHyKRh+x4Pt9uApZKRRjVfU
+qTAfBgNVHSMEGDAWgBT1QXK9+YqV1ly+uIo4ocEdgAqFwzApBgNVHREEIjAgghl0
+ZXN0c21kcHBsdXMxLmV4YW1wbGUuY29tiAOINwowYQYDVR0fBFowWDAqoCigJoYk
+aHR0cDovL2NpLnRlc3QuZXhhbXBsZS5jb20vQ1JMLUEuY3JsMCqgKKAmhiRodHRw
+Oi8vY2kudGVzdC5leGFtcGxlLmNvbS9DUkwtQi5jcmwwCgYIKoZIzj0EAwIDSQAw
+RgIhAL1qQ/cnrCZC7UnnLJ8WeK+0aWUJFWh1cOlBEzw0NlTVAiEA25Vf4WHzwmJi
+zkARzxJ1qB0qfBofuJrtfPM4gNJ4Quw=
+-----END CERTIFICATE-----

smdpp-data/sm-dp-sessions-BRP

@@ -0,0 +1 @@
+/tmp/sm-dp-sessions-BRP

smdpp-data/sm-dp-sessions-NIST

@@ -0,0 +1 @@
+/tmp/sm-dp-sessions-NIST

tests/pySim-shell_test/ara_m/adf_ara-m.cfg.ok

@@ -110,7 +110,7 @@
                     {
                         "ref_do": [
                             {
-                                "aid_ref_do": "ffffffffffdd"
+                                "aid_ref_do": null
                             },
                             {
                                 "dev_app_id_ref_do": "a1234567890123bb1f140de987aaa891bbbf0bdd"

tests/pySim-shell_test/ara_m/adf_ara-m.script.ok

@@ -7,7 +7,7 @@
 # Decoded FCP Template: None
 select MF/ADF.ARA-M
 aram_delete_all
-aram_store_ref_ar_do --aid ffffffffffdd --device-app-id a1234567890123bb1f140de987aaa891bbbf0bdd --apdu-filter aabbccdd010203041122334405060708 --nfc-never --android-permissions 0000000000000004
+aram_store_ref_ar_do --aid "" --device-app-id a1234567890123bb1f140de987aaa891bbbf0bdd --apdu-filter aabbccdd010203041122334405060708 --nfc-never --android-permissions 0000000000000004
 aram_store_ref_ar_do --aid ffffffffffcc --device-app-id a1234567890aaabb1f140de987657891a04f0bdd --apdu-filter aabbccdd01020304 --nfc-always --android-permissions 0000000000000004
 aram_store_ref_ar_do --aid ffffffffffbb --device-app-id aa6872f28b340b2345678905d5c2bbd5a04f0bdd --apdu-always --nfc-always --android-permissions 0000000000000004
 aram_store_ref_ar_do --aid ffffffffffaa --device-app-id aa6872787654334567840de535c2bbd5a04f0baa --apdu-never --nfc-never --android-permissions 0000000000000004

tests/pySim-shell_test/ara_m/test.script

@@ -10,7 +10,7 @@ aram_delete_all
 aram_store_ref_ar_do --aid ffffffffffaa --device-app-id aa6872787654334567840de535c2bbd5a04f0baa --apdu-never --nfc-never --android-permissions 0000000000000004
 aram_store_ref_ar_do --aid ffffffffffbb --device-app-id aa6872f28b340b2345678905d5c2bbd5a04f0bdd --apdu-always --nfc-always --android-permissions 0000000000000004
 aram_store_ref_ar_do --aid ffffffffffcc --device-app-id a1234567890aaabb1f140de987657891a04f0bdd --apdu-filter aabbccdd01020304 --nfc-always --android-permissions 0000000000000004
-aram_store_ref_ar_do --aid ffffffffffdd --device-app-id a1234567890123bb1f140de987aaa891bbbf0bdd --apdu-filter aabbccdd010203041122334405060708 --nfc-never --android-permissions 0000000000000004
+aram_store_ref_ar_do --aid "" --device-app-id a1234567890123bb1f140de987aaa891bbbf0bdd --apdu-filter aabbccdd010203041122334405060708 --nfc-never --android-permissions 0000000000000004
 
 # Export ADF.ARA-M to a temporary script file
 export --filename ADF.ARA-M > adf_ara-m.script.tmp

tests/unittests/test_globalplatform.py

@@ -21,6 +21,7 @@ from osmocom.utils import b2h, h2b
 
 from pySim.global_platform import *
 from pySim.global_platform.scp import *
+from pySim.global_platform.install_param import gen_install_parameters
 
 KIC = h2b('100102030405060708090a0b0c0d0e0f') # enc
 KID = h2b('101102030405060708090a0b0c0d0e0f') # MAC
@@ -289,5 +290,13 @@ class SCP03_KCV_Test(unittest.TestCase):
         self.assertEqual(compute_kcv('aes', KEYSET_AES128.dek), h2b('840DE5'))
 
 
+class Install_param_Test(unittest.TestCase):
+    def test_gen_install_parameters(self):
+        load_parameters = gen_install_parameters(256, 256, '010001001505000000000000000000000000')
+        self.assertEqual(load_parameters, 'c900ef1cc8020100c7020100ca12010001001505000000000000000000000000')
+
+        load_parameters = gen_install_parameters(None, None, '')
+        self.assertEqual(load_parameters, 'c900')
+
 if __name__ == "__main__":
 	unittest.main()

tests/unittests/test_javacard.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import os
+import unittest
+from pySim.javacard import *
+
+class TestJavacard(unittest.TestCase):
+
+    def test_CapFile(self):
+        loadfile="01000fdecaffed010204000105d07002ca4402001f000f001f000c002800420018006d00320017000000a900040002002203010004002803020107a0000000620101060210a0000000090003ffffffff8910710002000107a000000062000103000c0108d07002ca44900101002006001843800301ff0007020000002f00398002008101010881000007006d000911188c00048d00012c18197b0002037b00029210240303038b000388007a02318f00053d8c00062e1b8b00077a0120188b000860037a7a02228d00092d1d10076b101a8b000a321fae006b06188c000b7a06118d000c2c1903077b000d037b000d928b000e198b000f3b7a08003200040002000203001857656c636f6d6520746f20546f6f7243616d70203230313203000a48656c6c6f2c2053544b0000000005004200100200000006810900050000020381090b0680030001000000060000010380030103800303068108000381080c0600005306810a000500000003810a1303810a1609001700021e2d0011050306040908040507090a0a06070404040b00a901000100000300030005800281018100ff080000000028ff08000002002800020000008003ff820001002f001d0000000000090020003f000d000000000701002f0042000800000000080100390046001800000000ff020053002f0018000000000010002200240028002a002fffff002f002f003100330022002f00370028003b002201300568109001b008b44323430110012005681080056810a00633b44104b431066800a10231"
+        cap = CapFile(os.path.dirname(os.path.abspath(__file__)) + "/test_javacard.cap")
+        self.assertTrue(b2h(cap.get_loadfile()) == loadfile)
+        self.assertTrue(cap.get_loadfile_aid() == "d07002ca44")
+        self.assertTrue(cap.get_applet_aid() == "d07002ca44900101")
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/unittests/test_utils.py

@@ -36,8 +36,8 @@ class DecTestCase(unittest.TestCase):
 			{'priority': 1, 'identifier': 1, 'key_index': 2},
 			{'priority': 2, 'identifier': 0, 'key_index': 0}],
 		'hnet_pubkey_list': [
-			{'hnet_pubkey_identifier': 27, 'hnet_pubkey': hnet_pubkey_profile_b.lower()}, # because h2b/b2h returns all lower-case
-			{'hnet_pubkey_identifier': 30, 'hnet_pubkey': hnet_pubkey_profile_a.lower()}]
+			{'hnet_pubkey_identifier': 27, 'hnet_pubkey': h2b(hnet_pubkey_profile_b)}, # because h2b/b2h returns all lower-case
+			{'hnet_pubkey_identifier': 30, 'hnet_pubkey': h2b(hnet_pubkey_profile_a)}]
 	}
 
 	def testSplitHexStringToListOf5ByteEntries(self):