pySim.commands: Move away from pytlv for FCP parsing

This is the last pytlv use in non-legacy code, let's replace our
open-coded raw FCP parsing with the proper FCP parser we have in
ts_102_2221.py.

Change-Id: Ic5c5eec3b654b25e70c23c020910353cc5919aa9

.github/FUNDING.yml

@@ -0,0 +1 @@
+open_collective: osmocom

.gitignore

@@ -3,3 +3,7 @@
 
 /docs/_*
 /docs/generated
+/.cache
+/.local
+/build
+/pySim.egg-info

README.md

@@ -77,7 +77,7 @@ Please install the following dependencies:
  - cmd2 >= 1.5.0
  - colorlog
  - construct >= 2.9.51
- - gsm0338
+ - pyosmocom
  - jsonpath-ng
  - packaging
  - pycryptodomex
@@ -123,19 +123,34 @@ sudo pacman -Rs python-pysim-git
 ```
 
 
+Forum
+-----
+
+We welcome any pySim related discussions in the
+[SIM Card Technology](https://discourse.osmocom.org/c/sim-card-technology/)
+section of the osmocom discourse (web based Forum).
+
+
 Mailing List
 ------------
 
 There is no separate mailing list for this project. However,
-discussions related to pysim-prog are happening on the
-<openbsc@lists.osmocom.org> mailing list, please see
-<https://lists.osmocom.org/mailman/listinfo/openbsc> for subscription
+discussions related to pySim are happening on the simtrace
+<simtrace@lists.osmocom.org> mailing list, please see
+<https://lists.osmocom.org/mailman/listinfo/simtrace> for subscription
 options and the list archive.
 
 Please observe the [Osmocom Mailing List
 Rules](https://osmocom.org/projects/cellular-infrastructure/wiki/Mailing_List_Rules)
 when posting.
 
+Issue Tracker
+-------------
+
+We use the [issue tracker of the pysim project on osmocom.org](https://osmocom.org/projects/pysim/issues) for
+tracking the state of bug reports and feature requests.  Feel free to submit any issues you may find, or help
+us out by resolving existing issues.
+
 
 Contributing
 ------------

contrib/csv-encrypt-columns.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+# Utility program to perform column-based encryption of a CSV file holding SIM/UICC
+# related key materials.
+#
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import csv
+import argparse
+from Cryptodome.Cipher import AES
+from osmocom.utils import h2b, b2h, Hexstr
+
+from pySim.card_key_provider import CardKeyProviderCsv
+
+def dict_keys_to_upper(d: dict) -> dict:
+    return {k.upper():v for k,v in d.items()}
+
+class CsvColumnEncryptor:
+    def __init__(self, filename: str, transport_keys: dict):
+        self.filename = filename
+        self.transport_keys = dict_keys_to_upper(transport_keys)
+
+    def encrypt_col(self, colname:str, value: str) -> Hexstr:
+        key = self.transport_keys[colname]
+        cipher = AES.new(h2b(key), AES.MODE_CBC, CardKeyProviderCsv.IV)
+        return b2h(cipher.encrypt(h2b(value)))
+
+    def encrypt(self) -> None:
+        with open(self.filename, 'r') as infile:
+            cr = csv.DictReader(infile)
+            cr.fieldnames = [field.upper() for field in cr.fieldnames]
+
+            with open(self.filename + '.encr', 'w') as outfile:
+                cw = csv.DictWriter(outfile, dialect=csv.unix_dialect, fieldnames=cr.fieldnames)
+                cw.writeheader()
+
+                for row in cr:
+                    for key_colname in self.transport_keys:
+                        if key_colname in row:
+                            row[key_colname] = self.encrypt_col(key_colname, row[key_colname])
+                    cw.writerow(row)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('CSVFILE', help="CSV file name")
+    parser.add_argument('--csv-column-key', action='append', required=True,
+                        help='per-CSV-column AES transport key')
+
+    opts = parser.parse_args()
+
+    csv_column_keys = {}
+    for par in opts.csv_column_key:
+        name, key = par.split(':')
+        csv_column_keys[name] = key
+
+    if len(csv_column_keys) == 0:
+        print("You must specify at least one key!")
+        sys.exit(1)
+
+    csv_column_keys = CardKeyProviderCsv.process_transport_keys(csv_column_keys)
+    for name, key in csv_column_keys.items():
+        print("Encrypting column %s using AES key %s" % (name, key))
+
+    cce = CsvColumnEncryptor(opts.CSVFILE, csv_column_keys)
+    cce.encrypt()

contrib/es9p_client.py

@@ -0,0 +1,318 @@
+#!/usr/bin/env python3
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import argparse
+import logging
+import hashlib
+
+from typing import List
+from urllib.parse import urlparse
+
+from cryptography import x509
+from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
+from cryptography.hazmat.primitives.asymmetric import ec
+
+from osmocom.utils import h2b, b2h, swap_nibbles, is_hexstr
+from osmocom.tlv import bertlv_parse_one_rawtag, bertlv_return_one_rawtlv
+
+import pySim.esim.rsp as rsp
+from pySim.esim import es9p, PMO
+from pySim.esim.x509_cert import CertAndPrivkey
+from pySim.esim.es8p import BoundProfilePackage
+
+logging.basicConfig(level=logging.DEBUG)
+
+parser = argparse.ArgumentParser(description="""
+Utility to manually issue requests against the ES9+ API of an SM-DP+ according to GSMA SGP.22.""")
+parser.add_argument('--url', required=True, help='Base URL of ES9+ API endpoint')
+parser.add_argument('--server-ca-cert', help="""X.509 CA certificates acceptable for the server side. In
+                    production use cases, this would be the GSMA Root CA (CI) certificate.""")
+parser.add_argument('--certificate-path', default='.',
+                    help="Path in which to look for certificate and key files.")
+parser.add_argument('--euicc-certificate', default='CERT_EUICC_ECDSA_NIST.der',
+                    help="File name of DER-encoded eUICC certificate file.")
+parser.add_argument('--euicc-private-key', default='SK_EUICC_ECDSA_NIST.pem',
+                    help="File name of PEM-format eUICC secret key file.")
+parser.add_argument('--eum-certificate', default='CERT_EUM_ECDSA_NIST.der',
+                    help="File name of DER-encoded EUM certificate file.")
+parser.add_argument('--ci-certificate', default='CERT_CI_ECDSA_NIST.der',
+                    help="File name of DER-encoded CI certificate file.")
+
+subparsers = parser.add_subparsers(dest='command',help="The command (API function) to call", required=True)
+
+# download
+parser_dl = subparsers.add_parser('download', help="ES9+ download")
+parser_dl.add_argument('--matchingId', required=True,
+                       help='MatchingID that shall be used by profile download')
+parser_dl.add_argument('--output-path', default='.',
+                       help="Path to which the output files will be written.")
+parser_dl.add_argument('--confirmation-code',
+                       help="Confirmation Code for the eSIM download")
+
+# notification
+parser_ntf = subparsers.add_parser('notification', help='ES9+ (other) notification')
+parser_ntf.add_argument('operation', choices=['enable','disable','delete'],
+                        help='Profile Management Opreation whoise occurrence shall be notififed')
+parser_ntf.add_argument('--sequence-nr', type=int, required=True,
+                        help='eUICC global notification sequence number')
+parser_ntf.add_argument('--notification-address', help='notificationAddress, if different from URL')
+parser_ntf.add_argument('--iccid', type=is_hexstr, help='ICCID to which the notification relates')
+
+# notification-install
+parser_ntfi = subparsers.add_parser('notification-install', help='ES9+ installation notification')
+parser_ntfi.add_argument('--sequence-nr', type=int, required=True,
+                         help='eUICC global notification sequence number')
+parser_ntfi.add_argument('--transaction-id', required=True,
+                         help='transactionId of previous ES9+ download')
+parser_ntfi.add_argument('--notification-address', help='notificationAddress, if different from URL')
+parser_ntfi.add_argument('--iccid', type=is_hexstr, help='ICCID to which the notification relates')
+parser_ntfi.add_argument('--smdpp-oid', required=True, help='SM-DP+ OID (as in CERT.DPpb.ECDSA)')
+parser_ntfi.add_argument('--isdp-aid', type=is_hexstr, required=True,
+                         help='AID of the ISD-P of the installed profile')
+parser_ntfi.add_argument('--sima-response', type=is_hexstr, required=True,
+                         help='hex digits of BER-encoded SAIP EUICCResponse')
+
+class Es9pClient:
+    def __init__(self, opts):
+        self.opts = opts
+        self.cert_and_key = CertAndPrivkey()
+        self.cert_and_key.cert_from_der_file(os.path.join(opts.certificate_path, opts.euicc_certificate))
+        self.cert_and_key.privkey_from_pem_file(os.path.join(opts.certificate_path, opts.euicc_private_key))
+
+        with open(os.path.join(opts.certificate_path, opts.eum_certificate), 'rb') as f:
+            self.eum_cert = x509.load_der_x509_certificate(f.read())
+
+        with open(os.path.join(opts.certificate_path, opts.ci_certificate), 'rb') as f:
+            self.ci_cert = x509.load_der_x509_certificate(f.read())
+            subject_exts = list(filter(lambda x: isinstance(x.value, x509.SubjectKeyIdentifier), self.ci_cert.extensions))
+            subject_pkid = subject_exts[0].value
+            self.ci_pkid = subject_pkid.key_identifier
+
+        print("EUICC: %s" % self.cert_and_key.cert.subject)
+        print("EUM: %s" % self.eum_cert.subject)
+        print("CI: %s" % self.ci_cert.subject)
+
+        self.eid = self.cert_and_key.cert.subject.get_attributes_for_oid(x509.oid.NameOID.SERIAL_NUMBER)[0].value
+        print("EID: %s" % self.eid)
+        print("CI PKID: %s" % b2h(self.ci_pkid))
+        print()
+
+        self.peer = es9p.Es9pApiClient(opts.url, server_cert_verify=opts.server_ca_cert)
+
+
+    def do_notification(self):
+
+        ntf_metadata = {
+            'seqNumber': self.opts.sequence_nr,
+            'profileManagementOperation': PMO(self.opts.operation).to_bitstring(),
+            'notificationAddress': self.opts.notification_address or urlparse(self.opts.url).netloc,
+        }
+        if opts.iccid:
+            ntf_metadata['iccid'] = h2b(swap_nibbles(opts.iccid))
+
+        if self.opts.operation == 'download':
+            pird = {
+                'transactionId': self.opts.transaction_id,
+                'notificationMetadata': ntf_metadata,
+                'smdpOid': self.opts.smdpp_oid,
+                'finalResult': ('successResult', {
+                    'aid': self.opts.isdp_aid,
+                    'simaResponse': self.opts.sima_response,
+                    }),
+            }
+            pird_bin = rsp.asn1.encode('ProfileInstallationResultData', pird)
+            signature = self.cert_and_key.ecdsa_sign(pird_bin)
+            pn_dict = ('profileInstallationResult', {
+                'profileInstallationResultData': pird,
+                'euiccSignPIR': signature,
+            })
+        else:
+            ntf_bin = rsp.asn1.encode('NotificationMetadata', ntf_metadata)
+            signature = self.cert_and_key.ecdsa_sign(ntf_bin)
+            pn_dict = ('otherSignedNotification', {
+                'tbsOtherNotification': ntf_metadata,
+                'euiccNotificationSignature': signature,
+                'euiccCertificate': rsp.asn1.decode('Certificate', self.cert_and_key.get_cert_as_der()),
+                'eumCertificate': rsp.asn1.decode('Certificate', self.eum_cert.public_bytes(Encoding.DER)),
+            })
+
+        data = {
+            'pendingNotification': pn_dict,
+            }
+        #print(data)
+        res = self.peer.call_handleNotification(data)
+
+
+    def do_download(self):
+
+        print("Step 1: InitiateAuthentication...")
+
+        euiccInfo1 = {
+                'svn': b'\x02\x04\x00',
+                'euiccCiPKIdListForVerification': [
+                    self.ci_pkid,
+                ],
+                'euiccCiPKIdListForSigning': [
+                    self.ci_pkid,
+                ],
+          }
+
+        data = {
+            'euiccChallenge': os.urandom(16),
+            'euiccInfo1': euiccInfo1,
+            'smdpAddress': urlparse(self.opts.url).netloc,
+          }
+        init_auth_res = self.peer.call_initiateAuthentication(data)
+        print(init_auth_res)
+
+        print("Step 2: AuthenticateClient...")
+
+        #res['serverSigned1']
+        #res['serverSignature1']
+        print("TODO: verify serverSignature1 over serverSigned1")
+        #res['transactionId']
+        print("TODO: verify transactionId matches the signed one in serverSigned1")
+        #res['euiccCiPKIdToBeUsed']
+        # TODO: select eUICC certificate based on CI
+        #res['serverCertificate']
+        # TODO: verify server certificate against CI
+
+        euiccInfo2 = {
+            'profileVersion': b'\x02\x03\x01',
+            'svn': euiccInfo1['svn'],
+            'euiccFirmwareVer': b'\x23\x42\x00',
+            'extCardResource': b'\x81\x01\x00\x82\x04\x00\x04\x9ch\x83\x02"#',
+            'uiccCapability': (b'k6\xd3\xc3', 32),
+            'javacardVersion': b'\x11\x02\x00',
+            'globalplatformVersion': b'\x02\x03\x00',
+            'rspCapability': (b'\x9c', 6),
+            'euiccCiPKIdListForVerification': euiccInfo1['euiccCiPKIdListForVerification'],
+            'euiccCiPKIdListForSigning': euiccInfo1['euiccCiPKIdListForSigning'],
+            #'euiccCategory':
+            #'forbiddenProfilePolicyRules':
+            'ppVersion': b'\x01\x00\x00',
+            'sasAcreditationNumber': 'OSMOCOM-TEST-1', #TODO: make configurable
+            #'certificationDataObject':
+        }
+
+        euiccSigned1 = {
+            'transactionId': h2b(init_auth_res['transactionId']),
+            'serverAddress': init_auth_res['serverSigned1']['serverAddress'],
+            'serverChallenge': init_auth_res['serverSigned1']['serverChallenge'],
+            'euiccInfo2': euiccInfo2,
+            'ctxParams1':
+                ('ctxParamsForCommonAuthentication', {
+                    'matchingId': self.opts.matchingId,
+                    'deviceInfo': {
+                        'tac': b'\x35\x23\x01\x45', # same as lpac
+                        'deviceCapabilities': {},
+                        #imei:
+                    }
+                }),
+        }
+        euiccSigned1_bin = rsp.asn1.encode('EuiccSigned1', euiccSigned1)
+        euiccSignature1 = self.cert_and_key.ecdsa_sign(euiccSigned1_bin)
+        auth_clnt_req = {
+            'transactionId': init_auth_res['transactionId'],
+            'authenticateServerResponse':
+                ('authenticateResponseOk', {
+                    'euiccSigned1': euiccSigned1,
+                    'euiccSignature1': euiccSignature1,
+                    'euiccCertificate': rsp.asn1.decode('Certificate', self.cert_and_key.get_cert_as_der()),
+                    'eumCertificate': rsp.asn1.decode('Certificate', self.eum_cert.public_bytes(Encoding.DER))
+                 })
+        }
+        auth_clnt_res = self.peer.call_authenticateClient(auth_clnt_req)
+        print(auth_clnt_res)
+        #auth_clnt_res['transactionId']
+        print("TODO: verify transactionId matches previous ones")
+        #auth_clnt_res['profileMetadata']
+        # TODO: what's in here?
+        #auth_clnt_res['smdpSigned2']['bppEuiccOtpk']
+        #auth_clnt_res['smdpSignature2']
+        print("TODO: verify serverSignature2 over smdpSigned2")
+
+        smdp_cert = x509.load_der_x509_certificate(auth_clnt_res['smdpCertificate'])
+
+        print("Step 3: GetBoundProfilePackage...")
+        # Generate a one-time ECKA key pair (ot{PK,SK}.DP.ECKA) using the curve indicated by the Key Parameter
+        # Reference value of CERT.DPpb.ECDSA
+        euicc_ot = ec.generate_private_key(smdp_cert.public_key().public_numbers().curve)
+
+        # extract the public key in (hopefully) the right format for the ES8+ interface
+        euicc_otpk = euicc_ot.public_key().public_bytes(Encoding.X962, PublicFormat.UncompressedPoint)
+
+        euiccSigned2 = {
+            'transactionId': h2b(auth_clnt_res['transactionId']),
+            'euiccOtpk': euicc_otpk,
+            #hashCC
+        }
+        # check for smdpSigned2 ccRequiredFlag, and send it in PrepareDownloadRequest hashCc
+        if auth_clnt_res['smdpSigned2']['ccRequiredFlag']:
+            if not self.opts.confirmation_code:
+                raise ValueError('Confirmation Code required but not provided')
+            cc_hash = hashlib.sha256(self.opts.confirmation_code.encode('ascii')).digest()
+            euiccSigned2['hashCc'] = hashlib.sha256(cc_hash + euiccSigned2['transactionId']).digest()
+        euiccSigned2_bin = rsp.asn1.encode('EUICCSigned2', euiccSigned2)
+        euiccSignature2 = self.cert_and_key.ecdsa_sign(euiccSigned2_bin + auth_clnt_res['smdpSignature2'])
+        gbp_req = {
+            'transactionId': auth_clnt_res['transactionId'],
+            'prepareDownloadResponse':
+                ('downloadResponseOk', {
+                    'euiccSigned2': euiccSigned2,
+                    'euiccSignature2': euiccSignature2,
+                 })
+        }
+        gbp_res = self.peer.call_getBoundProfilePackage(gbp_req)
+        print(gbp_res)
+        #gbp_res['transactionId']
+        # TODO: verify transactionId
+        print("TODO: verify transactionId matches previous ones")
+        bpp_bin = gbp_res['boundProfilePackage']
+        print("TODO: verify boundProfilePackage smdpSignature")
+
+        bpp = BoundProfilePackage()
+        upp_bin = bpp.decode(euicc_ot, self.eid, bpp_bin)
+
+        iccid = swap_nibbles(b2h(bpp.storeMetadataRequest['iccid']))
+        base_name = os.path.join(self.opts.output_path, '%s' % iccid)
+
+        print("SUCCESS: Storing files as %s.*.der" % base_name)
+
+        # write various output files
+        with open(base_name+'.upp.der', 'wb') as f:
+            f.write(bpp.upp)
+        with open(base_name+'.isdp.der', 'wb') as f:
+            f.write(bpp.encoded_configureISDPRequest)
+        with open(base_name+'.smr.der', 'wb') as f:
+            f.write(bpp.encoded_storeMetadataRequest)
+
+
+if __name__ == '__main__':
+    opts = parser.parse_args()
+
+    c = Es9pClient(opts)
+
+    if opts.command == 'download':
+        c.do_download()
+    elif opts.command == 'notification':
+        c.do_notification()
+    elif opts.command == 'notification-install':
+        opts.operation = 'install'
+        c.do_notification()

contrib/fsdump-diff-apply.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+
+# The purpose of this script is to
+# * load two SIM card 'fsdump' files
+# * determine which file contents in "B" differs from that of "A"
+# * create a pySim-shell script to update the contents of "A" to match that of "B"
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import argparse
+
+# Files that we should not update
+FILES_TO_SKIP = [
+    "MF/EF.ICCID",
+    #"MF/DF.GSM/EF.IMSI",
+    #"MF/ADF.USIM/EF.IMSI",
+    ]
+
+# Files that need zero-padding at the end, not ff-padding
+FILES_PAD_ZERO = [
+    "DF.GSM/EF.SST",
+    "MF/ADF.USIM/EF.UST",
+    "MF/ADF.USIM/EF.EST",
+    "MF/ADF.ISIM/EF.IST",
+    ]
+
+def pad_file(path, instr, byte_len):
+    if path in FILES_PAD_ZERO:
+        pad = '0'
+    else:
+        pad = 'f'
+    return pad_hexstr(instr, byte_len, pad)
+
+def pad_hexstr(instr, byte_len:int, pad='f'):
+    """Pad given hex-string to the number of bytes given in byte_len, using ff as padding."""
+    if len(instr) == byte_len*2:
+        return instr
+    elif len(instr) > byte_len*2:
+        raise ValueError('Cannot pad string of length %u to smaller length %u' % (len(instr)/2, byte_len))
+    else:
+        return instr + pad * (byte_len*2 - len(instr))
+
+def is_all_ff(instr):
+    """Determine if the entire input hex-string consists of f-digits."""
+    if all([x == 'f' for x in instr.lower()]):
+        return True
+    else:
+        return False
+
+parser = argparse.ArgumentParser()
+parser.add_argument('file_a')
+parser.add_argument('file_b')
+
+
+if __name__ == '__main__':
+    opts = parser.parse_args()
+
+    with open(opts.file_a, 'r') as file_a:
+        json_a = json.loads(file_a.read())
+    with open(opts.file_b, 'r') as file_b:
+        json_b = json.loads(file_b.read())
+
+    for path in json_b.keys():
+        print()
+        print("# %s" % path)
+
+        if not path in json_a:
+            raise ValueError("%s doesn't exist in file_a!" % path)
+
+        if path in FILES_TO_SKIP:
+            print("# skipped explicitly as it is in FILES_TO_SKIP")
+            continue
+
+        if not 'body' in json_b[path]:
+            print("# file doesn't exist in B so we cannot possibly need to modify A")
+            continue
+
+        if not 'body' in json_a[path]:
+            # file was not readable in original (permissions? deactivated?)
+            print("# ERROR: %s not readable in A; please fix that" % path)
+            continue
+
+        body_a = json_a[path]['body']
+        body_b = json_b[path]['body']
+        if body_a == body_b:
+            print("# file body is identical")
+            continue
+
+        file_size_a = json_a[path]['fcp']['file_size']
+        file_size_b = json_b[path]['fcp']['file_size']
+
+        cmds = []
+        structure = json_b[path]['fcp']['file_descriptor']['file_descriptor_byte']['structure']
+        if structure == 'transparent':
+            val_a = body_a
+            val_b = body_b
+            if file_size_a < file_size_b:
+                if not is_all_ff(val_b[2*file_size_a:]):
+                    print("# ERROR: file_size_a (%u) < file_size_b (%u); please fix!" % (file_size_a, file_size_b))
+                    continue
+                else:
+                    print("# WARN: file_size_a (%u) < file_size_b (%u); please fix!" % (file_size_a, file_size_b))
+                    # truncate val_b to fit in A
+                    val_b = val_b[:file_size_a*2]
+
+            elif file_size_a != file_size_b:
+                print("# NOTE: file_size_a (%u) != file_size_b (%u)" % (file_size_a, file_size_b))
+
+            # Pad to file_size_a
+            val_b = pad_file(path, val_b, file_size_a)
+            if val_b != val_a:
+                cmds.append("update_binary %s" % val_b)
+            else:
+                print("# padded file body is identical")
+        elif structure in ['linear_fixed', 'cyclic']:
+            record_len_a = json_a[path]['fcp']['file_descriptor']['record_len']
+            record_len_b = json_b[path]['fcp']['file_descriptor']['record_len']
+            if record_len_a < record_len_b:
+                print("# ERROR: record_len_a (%u) < record_len_b (%u); please fix!" % (file_size_a, file_size_b))
+                continue
+            elif record_len_a != record_len_b:
+                print("# NOTE: record_len_a (%u) != record_len_b (%u)" % (record_len_a, record_len_b))
+
+            num_rec_a = file_size_a // record_len_a
+            num_rec_b = file_size_b // record_len_b
+            if num_rec_a < num_rec_b:
+                if not all([is_all_ff(x) for x in body_b[num_rec_a:]]):
+                    print("# ERROR: num_rec_a (%u) < num_rec_b (%u); please fix!" % (num_rec_a, num_rec_b))
+                    continue
+                else:
+                    print("# WARN: num_rec_a (%u) < num_rec_b (%u); but they're empty" % (num_rec_a, num_rec_b))
+            elif num_rec_a != num_rec_b:
+                print("# NOTE: num_rec_a (%u) != num_rec_b (%u)" % (num_rec_a, num_rec_b))
+
+            i = 0
+            for r in body_b:
+                if i < len(body_a):
+                    break
+                val_a = body_a[i]
+                # Pad to record_len_a
+                val_b = pad_file(path, body_b[i], record_len_a)
+                if val_a != val_b:
+                    cmds.append("update_record %u %s" % (i+1, val_b))
+                i = i + 1
+            if len(cmds) == 0:
+                print("# padded file body is identical")
+        elif structure == 'ber_tlv':
+            print("# FIXME: Implement BER-TLV")
+        else:
+            raise ValueError('Unsupported structure %s' % structure)
+
+        if len(cmds):
+            print("select %s" % path)
+            for cmd in cmds:
+                print(cmd)

contrib/jenkins.sh

@@ -4,18 +4,23 @@
 # environment variables:
 # * WITH_MANUALS: build manual PDFs if set to "1"
 # * PUBLISH: upload manuals after building if set to "1" (ignored without WITH_MANUALS = "1")
-# * JOB_TYPE: one of 'test', 'pylint', 'docs'
+# * JOB_TYPE: one of 'test', 'distcheck', 'pylint', 'docs'
+# * SKIP_CLEAN_WORKSPACE: don't run osmo-clean-workspace.sh (for pyosmocom CI)
 #
 
 export PYTHONUNBUFFERED=1
 
-if [ ! -d "./pysim-testdata/" ] ; then
+if [ ! -d "./tests/" ] ; then
 	echo "###############################################"
 	echo "Please call from pySim-prog top directory"
 	echo "###############################################"
 	exit 1
 fi
 
+if [ -z "$SKIP_CLEAN_WORKSPACE" ]; then
+	osmo-clean-workspace.sh
+fi
+
 case "$JOB_TYPE" in
 "test")
 	virtualenv -p python3 venv --system-site-packages
@@ -25,16 +30,39 @@ case "$JOB_TYPE" in
 	pip install pyshark
 
 	# Execute automatically discovered unit tests first
-	python -m unittest discover -v -s tests/
+	python -m unittest discover -v -s tests/unittests
 
-	# Run the test with physical cards
-	cd pysim-testdata
-	../tests/pySim-prog_test.sh
-	../tests/pySim-trace_test.sh
+	# Run pySim-prog integration tests (requires physical cards)
+	cd tests/pySim-prog_test/
+        ./pySim-prog_test.sh
+	cd ../../
+
+	# Run pySim-trace test
+	tests/pySim-trace_test/pySim-trace_test.sh
+
+	# Run pySim-shell integration tests (requires physical cards)
+	python3 -m unittest discover -v -s ./tests/pySim-shell_test/
+	;;
+"distcheck")
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
+
+	pip install .
+	pip install pyshark
+
+	for prog in venv/bin/pySim-*.py; do
+		$prog --help > /dev/null
+	done
 	;;
 "pylint")
 	# Print pylint version
 	pip3 freeze | grep pylint
+
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
+
+	pip install .
+
 	# Run pylint to find potential errors
 	# Ignore E1102: not-callable
 	#   pySim/filesystem.py: E1102: method is not callable (not-callable)
@@ -45,10 +73,15 @@ case "$JOB_TYPE" in
 		--disable E1102 \
 		--disable E0401 \
 		--enable W0301 \
-		pySim tests/*.py *.py \
-		contrib/es2p_client.py
+		pySim tests/unittests/*.py *.py \
+		contrib/*.py
 	;;
 "docs")
+	virtualenv -p python3 venv --system-site-packages
+	. venv/bin/activate
+
+	pip install -r requirements.txt
+
 	rm -rf docs/_build
 	make -C "docs" html latexpdf
 
@@ -61,3 +94,5 @@ case "$JOB_TYPE" in
 	echo "ERROR: JOB_TYPE has unexpected value '$JOB_TYPE'."
 	exit 1
 esac
+
+osmo-clean-workspace.sh

contrib/saip-tool.py

@@ -0,0 +1,258 @@
+#!/usr/bin/env python3
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import argparse
+import logging
+import zipfile
+from pathlib import Path as PlPath
+from typing import List
+from osmocom.utils import h2b, b2h, swap_nibbles
+
+from pySim.esim.saip import *
+from pySim.esim.saip.validation import CheckBasicStructure
+from pySim import javacard
+from pySim.pprint import HexBytesPrettyPrinter
+
+pp = HexBytesPrettyPrinter(indent=4,width=500)
+
+parser = argparse.ArgumentParser(description="""
+Utility program to work with eSIM SAIP (SimAlliance Interoperable Profile) files.""")
+parser.add_argument('INPUT_UPP', help='Unprotected Profile Package Input file')
+parser.add_argument("--loglevel", dest="loglevel", choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
+                    default='INFO', help="Set the logging level")
+parser.add_argument('--debug', action='store_true', help='Enable DEBUG logging')
+subparsers = parser.add_subparsers(dest='command', help="The command to perform", required=True)
+
+parser_split = subparsers.add_parser('split', help='Split PE-Sequence into individual PEs')
+parser_split.add_argument('--output-prefix', default='.', help='Prefix path/filename for output files')
+
+parser_dump = subparsers.add_parser('dump', help='Dump information on PE-Sequence')
+parser_dump.add_argument('mode', choices=['all_pe', 'all_pe_by_type', 'all_pe_by_naa'])
+parser_dump.add_argument('--dump-decoded', action='store_true', help='Dump decoded PEs')
+
+parser_check = subparsers.add_parser('check', help='Run constraint checkers on PE-Sequence')
+
+parser_rpe = subparsers.add_parser('remove-pe', help='Remove specified PEs from PE-Sequence')
+parser_rpe.add_argument('--output-file', required=True, help='Output file name')
+parser_rpe.add_argument('--identification', type=int, action='append', help='Remove PEs matching specified identification')
+
+parser_rn = subparsers.add_parser('remove-naa', help='Remove speciifed NAAs from PE-Sequence')
+parser_rn.add_argument('--output-file', required=True, help='Output file name')
+parser_rn.add_argument('--naa-type', required=True, choices=NAAs.keys(), help='Network Access Application type to remove')
+# TODO: add an --naa-index or the like, so only one given instance can be removed
+
+parser_info = subparsers.add_parser('info', help='Display information about the profile')
+
+parser_eapp = subparsers.add_parser('extract-apps', help='Extract applications as loadblock file')
+parser_eapp.add_argument('--output-dir', default='.', help='Output directory (where to store files)')
+parser_eapp.add_argument('--format', default='cap', choices=['ijc', 'cap'], help='Data format of output files')
+
+parser_info = subparsers.add_parser('tree', help='Display the filesystem tree')
+
+def do_split(pes: ProfileElementSequence, opts):
+    i = 0
+    for pe in pes.pe_list:
+        basename = PlPath(opts.INPUT_UPP).stem
+        if not pe.identification:
+            fname = '%s-%02u-%s.der' % (basename, i, pe.type)
+        else:
+            fname = '%s-%02u-%05u-%s.der' % (basename, i, pe.identification, pe.type)
+        print("writing single PE to file '%s'" % fname)
+        with open(os.path.join(opts.output_prefix, fname), 'wb') as outf:
+            outf.write(pe.to_der())
+        i += 1
+
+def do_dump(pes: ProfileElementSequence, opts):
+    def print_all_pe(pes: ProfileElementSequence, dump_decoded:bool = False):
+        # iterate over each pe in the pes (using its __iter__ method)
+        for pe in pes:
+            print("="*70 + " " + pe.type)
+            if dump_decoded:
+                pp.pprint(pe.decoded)
+
+    def print_all_pe_by_type(pes: ProfileElementSequence, dump_decoded:bool = False):
+        # sort by PE type and show all PE within that type
+        for pe_type in pes.pe_by_type.keys():
+            print("="*70 + " " + pe_type)
+            for pe in pes.pe_by_type[pe_type]:
+                pp.pprint(pe)
+                if dump_decoded:
+                    pp.pprint(pe.decoded)
+
+    def print_all_pe_by_naa(pes: ProfileElementSequence, dump_decoded:bool = False):
+        for naa in pes.pes_by_naa:
+            i = 0
+            for naa_instance in pes.pes_by_naa[naa]:
+                print("="*70 + " " + naa + str(i))
+                i += 1
+                for pe in naa_instance:
+                    pp.pprint(pe.type)
+                    if dump_decoded:
+                        for d in pe.decoded:
+                            print("    %s" % d)
+
+    if opts.mode == 'all_pe':
+        print_all_pe(pes, opts.dump_decoded)
+    elif opts.mode == 'all_pe_by_type':
+        print_all_pe_by_type(pes, opts.dump_decoded)
+    elif opts.mode == 'all_pe_by_naa':
+        print_all_pe_by_naa(pes, opts.dump_decoded)
+
+def do_check(pes: ProfileElementSequence, opts):
+    print("Checking PE-Sequence structure...")
+    checker = CheckBasicStructure()
+    checker.check(pes)
+    print("All good!")
+
+def do_remove_pe(pes: ProfileElementSequence, opts):
+    new_pe_list = []
+    for pe in pes.pe_list:
+        identification = pe.identification
+        if identification:
+            if identification in opts.identification:
+                print("Removing PE %s (id=%u) from Sequence..." % (pe, identification))
+                continue
+        new_pe_list.append(pe)
+
+    pes.pe_list = new_pe_list
+    pes._process_pelist()
+    print("Writing %u PEs to file '%s'..." % (len(pes.pe_list), opts.output_file))
+    with open(opts.output_file, 'wb') as f:
+        f.write(pes.to_der())
+
+def do_remove_naa(pes: ProfileElementSequence, opts):
+    if not opts.naa_type in NAAs:
+        raise ValueError('unsupported NAA type %s' % opts.naa_type)
+    naa = NAAs[opts.naa_type]
+    print("Removing NAAs of type '%s' from Sequence..." % opts.naa_type)
+    pes.remove_naas_of_type(naa)
+    print("Writing %u PEs to file '%s'..." % (len(pes.pe_list), opts.output_file))
+    with open(opts.output_file, 'wb') as f:
+        f.write(pes.to_der())
+
+def do_info(pes: ProfileElementSequence, opts):
+    def get_naa_count(pes: ProfileElementSequence) -> dict:
+        """return a dict with naa-type (usim, isim) as key and the count of NAA instances as value."""
+        ret = {}
+        for naa_type in pes.pes_by_naa:
+            ret[naa_type] = len(pes.pes_by_naa[naa_type])
+        return ret
+
+    pe_hdr_dec = pes.pe_by_type['header'][0].decoded
+    print()
+    print("SAIP Profile Version: %u.%u" % (pe_hdr_dec['major-version'], pe_hdr_dec['minor-version']))
+    print("Profile Type: '%s'" % pe_hdr_dec['profileType'])
+    print("ICCID: %s" % b2h(pe_hdr_dec['iccid']))
+    print("Mandatory Services: %s" % ', '.join(pe_hdr_dec['eUICC-Mandatory-services'].keys()))
+    print()
+    naa_strs = ["%s[%u]" % (k, v) for k, v in get_naa_count(pes).items()]
+    print("NAAs: %s" % ', '.join(naa_strs))
+    for naa_type in pes.pes_by_naa:
+        for naa_inst in pes.pes_by_naa[naa_type]:
+            first_pe = naa_inst[0]
+            adf_name = ''
+            if hasattr(first_pe, 'adf_name'):
+                adf_name = '(' + first_pe.adf_name + ')'
+            print("NAA %s %s" % (first_pe.type, adf_name))
+            if hasattr(first_pe, 'imsi'):
+                print("\tIMSI: %s" % first_pe.imsi)
+
+    # applications
+    print()
+    apps = pes.pe_by_type.get('application', [])
+    print("Number of applications: %u" % len(apps))
+    for app_pe in apps:
+        print("App Load Package AID: %s" % b2h(app_pe.decoded['loadBlock']['loadPackageAID']))
+        print("\tMandated: %s" % ('mandated' in app_pe.decoded['app-Header']))
+        print("\tLoad Block Size: %s" % len(app_pe.decoded['loadBlock']['loadBlockObject']))
+        for inst in app_pe.decoded.get('instanceList', []):
+            print("\tInstance AID: %s" % b2h(inst['instanceAID']))
+
+    # security domains
+    print()
+    sds = pes.pe_by_type.get('securityDomain', [])
+    print("Number of security domains: %u" % len(sds))
+    for sd in sds:
+        print("Security domain Instance AID: %s" % b2h(sd.decoded['instance']['instanceAID']))
+        # FIXME: 'applicationSpecificParametersC9' parsing to figure out enabled SCP
+        for key in sd.keys:
+            print("\tKVN=0x%02x, KID=0x%02x, %s" % (key.key_version_number, key.key_identifier, key.key_components))
+
+    # RFM
+    print()
+    rfms = pes.pe_by_type.get('rfm', [])
+    print("Number of RFM instances: %u" % len(rfms))
+    for rfm in rfms:
+        inst_aid = rfm.decoded['instanceAID']
+        print("RFM instanceAID: %s" % b2h(inst_aid))
+        print("\tMSL: 0x%02x" % rfm.decoded['minimumSecurityLevel'][0])
+        adf = rfm.decoded.get('adfRFMAccess', None)
+        if adf:
+            print("\tADF AID: %s" % b2h(adf['adfAID']))
+        tar_list = rfm.decoded.get('tarList', [inst_aid[-3:]])
+        for tar in tar_list:
+            print("\tTAR: %s" % b2h(tar))
+
+def do_extract_apps(pes:ProfileElementSequence, opts):
+    apps = pes.pe_by_type.get('application', [])
+    for app_pe in apps:
+        package_aid = b2h(app_pe.decoded['loadBlock']['loadPackageAID'])
+
+        fname = os.path.join(opts.output_dir, '%s-%s.%s' % (pes.iccid, package_aid, opts.format))
+        load_block_obj = app_pe.decoded['loadBlock']['loadBlockObject']
+        print("Writing Load Package AID: %s to file %s" % (package_aid, fname))
+        if opts.format == 'ijc':
+            with open(fname, 'wb') as f:
+                f.write(load_block_obj)
+        else:
+            with io.BytesIO(load_block_obj) as f, zipfile.ZipFile(fname, 'w') as z:
+                javacard.ijc_to_cap(f, z, package_aid)
+
+def do_tree(pes:ProfileElementSequence, opts):
+    pes.mf.print_tree()
+
+if __name__ == '__main__':
+    opts = parser.parse_args()
+
+    if opts.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.getLevelName(opts.loglevel))
+
+    with open(opts.INPUT_UPP, 'rb') as f:
+        pes = ProfileElementSequence.from_der(f.read())
+
+    print("Read %u PEs from file '%s'" % (len(pes.pe_list), opts.INPUT_UPP))
+
+    if opts.command == 'split':
+        do_split(pes, opts)
+    elif opts.command == 'dump':
+        do_dump(pes, opts)
+    elif opts.command == 'check':
+        do_check(pes, opts)
+    elif opts.command == 'remove-pe':
+        do_remove_pe(pes, opts)
+    elif opts.command == 'remove-naa':
+        do_remove_naa(pes, opts)
+    elif opts.command == 'info':
+        do_info(pes, opts)
+    elif opts.command == 'extract-apps':
+        do_extract_apps(pes, opts)
+    elif opts.command == 'tree':
+        do_tree(pes, opts)

contrib/sim-rest-client.py

@@ -162,6 +162,7 @@ def main(argv):
     parser.add_argument("-v", "--verbose", help="increase output verbosity", action='count', default=0)
     parser.add_argument("-n", "--slot-nr", help="SIM slot number", type=int, default=0)
     subp = parser.add_subparsers()
+    subp.required = True
 
     auth_p = subp.add_parser('auth', help='UMTS AKA Authentication')
     auth_p.add_argument("-c", "--count", help="Auth count", type=int, default=10)

contrib/unber.py

@@ -6,7 +6,8 @@
 import sys
 import argparse
 
-from pySim.utils import bertlv_parse_one, bertlv_encode_tag, b2h, h2b
+from osmocom.utils import b2h, h2b
+from osmocom.tlv import bertlv_parse_one, bertlv_encode_tag
 
 def process_one_level(content: bytes, indent: int):
     remainder = content
@@ -35,5 +36,8 @@ if __name__ == '__main__':
             content = f.read()
     elif opts.hex:
         content = h2b(opts.hex)
+    else:
+        # avoid pylint "(possibly-used-before-assignment)" below
+        sys.exit(2)
 
     process_one_level(content, 0)

contrib/wsrc_card_client.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+"""Connect smartcard to a remote server, so the remote server can take control and
+perform commands on it."""
+
+import sys
+import json
+import logging
+import argparse
+from osmocom.utils import b2h
+import websockets
+
+from pySim.transport import init_reader, argparse_add_reader_args, LinkBase
+from pySim.commands import SimCardCommands
+from pySim.wsrc.client_blocking import WsClientBlocking
+from pySim.exceptions import NoCardError
+from pySim.wsrc import WSRC_DEFAULT_PORT_CARD
+
+logging.basicConfig(format="[%(levelname)s] %(asctime)s %(message)s", level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class CardWsClientBlocking(WsClientBlocking):
+    """Implementation of the card (reader) client of the WSRC (WebSocket Remote Card) protocol"""
+
+    def __init__(self, ws_uri, tp: LinkBase):
+        super().__init__('card', ws_uri)
+        self.tp = tp
+
+    def perform_outbound_hello(self):
+        hello_data = {
+            'atr': b2h(self.tp.get_atr()),
+            # TODO: include various card information in the HELLO message
+        }
+        super().perform_outbound_hello(hello_data)
+
+    def handle_rx_c_apdu(self, rx: dict):
+        """handle an inbound APDU transceive command"""
+        data, sw = self.tp.send_apdu(rx['command'])
+        tx = {
+            'response': data,
+            'sw': sw,
+        }
+        self.tx_json('r_apdu', tx)
+
+    def handle_rx_disconnect(self, rx: dict):
+        """server tells us to disconnect"""
+        self.tx_json('disconnect_ack')
+        # FIXME: tear down connection and/or terminate entire program
+
+    def handle_rx_state_notification(self, rx: dict):
+        logger.info("State Notification: %s" % rx['new_state'])
+
+    def handle_rx_print(self, rx: dict):
+        """print a message (text) given by server to the local console/log"""
+        logger.info("SERVER MSG: %s" % rx['message'])
+        # no response
+
+    def handle_rx_reset_req(self, rx: dict):
+        """server tells us to reset the card"""
+        self.tp.reset_card()
+        self.tx_json('reset_resp', {'atr': b2h(self.tp.get_atr())})
+
+
+parser = argparse.ArgumentParser()
+argparse_add_reader_args(parser)
+parser.add_argument("--uri", default="ws://localhost:%u/" % (WSRC_DEFAULT_PORT_CARD),
+                    help="URI of the sever to which to connect")
+
+
+if __name__ == '__main__':
+    opts = parser.parse_args()
+
+    # open the card reader / slot
+    logger.info("Initializing Card Reader...")
+    try:
+        tp = init_reader(opts)
+    except Exception as e:
+        logger.fatal("Error opening reader: %s" % e)
+        sys.exit(1)
+
+    logger.info("Connecting to Card...")
+    try:
+        tp.connect()
+    except NoCardError as e:
+        logger.fatal("Error opening card! Is a card inserted in the reader?")
+        sys.exit(1)
+
+    scc = SimCardCommands(transport=tp)
+    logger.info("Detected Card with ATR: %s" % b2h(tp.get_atr()))
+
+    # TODO: gather various information about the card; print it
+
+    # create + connect the client to the server
+    cl = CardWsClientBlocking(opts.uri, tp)
+    logger.info("Connecting to remote server...")
+    try:
+        cl.connect()
+        logger.info("Successfully connected to Server")
+    except ConnectionRefusedError as e:
+        logger.fatal(e)
+        sys.exit(1)
+
+    try:
+        while True:
+            # endless loop: wait for inbound command from server + execute it
+            cl.rx_and_execute_cmd()
+    except websockets.exceptions.ConnectionClosedOK as e:
+        print(e)
+        sys.exit(1)
+    except KeyboardInterrupt as e:
+        print(e.__class__.__name__)
+        sys.exit(2)

contrib/wsrc_server.py

@@ -0,0 +1,354 @@
+#!/usr/bin/env python
+
+import json
+import asyncio
+import logging
+import argparse
+from typing import Optional, Tuple
+from websockets.asyncio.server import serve
+from websockets.exceptions import ConnectionClosedError
+from osmocom.utils import Hexstr, swap_nibbles
+
+from pySim.utils import SwMatchstr, ResTuple, sw_match, dec_iccid
+from pySim.exceptions import SwMatchError
+from pySim.wsrc import WSRC_DEFAULT_PORT_USER, WSRC_DEFAULT_PORT_CARD
+
+logging.basicConfig(format="[%(levelname)s] %(asctime)s %(message)s", level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+card_clients = set()
+user_clients = set()
+
+class WsClientLogAdapter(logging.LoggerAdapter):
+    """LoggerAdapter adding context (for now: remote IP/Port of client) to log"""
+    def process(self, msg, kwargs):
+        return '%s([%s]:%u) %s' % (self.extra['type'], self.extra['remote_addr'][0],
+                                   self.extra['remote_addr'][1], msg), kwargs
+
+class WsClient:
+    def __init__(self, websocket, hello: dict):
+        self.websocket = websocket
+        self.hello = hello
+        self.identity = {}
+        self.logger = WsClientLogAdapter(logger, {'type': self.__class__.__name__,
+                                                  'remote_addr': websocket.remote_address})
+
+    def __str__(self):
+        return '%s([%s]:%u)' % (self.__class__.__name__, self.websocket.remote_address[0],
+                                self.websocket.remote_address[1])
+
+    async def rx_json(self):
+        rx = await self.websocket.recv()
+        rx_js = json.loads(rx)
+        self.logger.debug("Rx: %s", rx_js)
+        assert 'msg_type' in rx
+        return rx_js
+
+    async def tx_json(self, msg_type:str, d: dict = {}):
+        """Transmit a json-serializable dict to the peer"""
+        d['msg_type'] = msg_type
+        d_js = json.dumps(d)
+        self.logger.debug("Tx: %s", d_js)
+        await self.websocket.send(d_js)
+
+    async def tx_hello_ack(self):
+        await self.tx_json('hello_ack')
+
+    async def xceive_json(self, msg_type:str, d:dict = {}, exp_msg_type:Optional[str] = None) -> dict:
+        await self.tx_json(msg_type, d)
+        rx = await self.rx_json()
+        if exp_msg_type:
+            assert rx['msg_type'] == exp_msg_type
+        return rx;
+
+    async def tx_error(self, message: str):
+        """Transmit an error message to the peer"""
+        event = {
+            "message": message,
+        }
+        self.logger.error("Transmitting error message: '%s'" % message)
+        await self.tx_json('error', event)
+
+#    async def ws_hdlr(self):
+#        """kind of a 'main' function for the websocket client: wait for incoming message,
+#        and handle it."""
+#        try:
+#            async for message in self.websocket:
+#                method = getattr(self, 'handle_rx_%s' % message['msg_type'], None)
+#                if not method:
+#                    await self.tx_error("Unknonw msg_type: %s" % message['msg_type'])
+#                else:
+#                    method(message)
+#        except ConnectionClosedError:
+#            # we handle this in the outer loop
+#            pass
+
+class CardClient(WsClient):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.identity['ATR'] = self.hello['atr']
+        self.state = 'init'
+
+    def __str__(self):
+        eid = self.identity.get('EID', None)
+        if eid:
+            return '%s(EID=%s)' % (self.__class__.__name__, eid)
+        iccid = self.identity.get('ICCID', None)
+        if iccid:
+            return '%s(ICCID=%s)' % (self.__class__.__name__, iccid)
+        return super().__str__()
+
+    def listing_entry(self) -> dict:
+        return {'remote_addr': self.websocket.remote_address,
+                'identities': self.identity,
+                'state': self.state}
+
+    """A websocket client that represents a reader/card. This is what we use to talk to a card"""
+    async def xceive_apdu_raw(self, cmd: Hexstr) -> ResTuple:
+        """transceive a single APDU with the card"""
+        message = await self.xceive_json('c_apdu', {'command': cmd}, 'r_apdu')
+        return message['response'], message['sw']
+
+    async def xceive_apdu(self, pdu: Hexstr) -> ResTuple:
+        """transceive an APDU with the card, handling T=0 GET_RESPONSE cases"""
+        prev_pdu = pdu
+        data, sw = await self.xceive_apdu_raw(pdu)
+
+        if sw is not None:
+            while (sw[0:2] in ['9f', '61', '62', '63']):
+                # SW1=9F: 3GPP TS 51.011 9.4.1, Responses to commands which are correctly executed
+                # SW1=61: ISO/IEC 7816-4, Table 5 — General meaning of the interindustry values of SW1-SW2
+                # SW1=62: ETSI TS 102 221 7.3.1.1.4 Clause 4b): 62xx, 63xx, 9xxx != 9000
+                pdu_gr = pdu[0:2] + 'c00000' + sw[2:4]
+                prev_pdu = pdu_gr
+                d, sw = await self.xceive_apdu_raw(pdu_gr)
+                data += d
+            if sw[0:2] == '6c':
+                # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
+                pdu_gr = prev_pdu[0:8] + sw[2:4]
+                data, sw = await self.xceive_apdu_raw(pdu_gr)
+
+        return data, sw
+
+    async def xceive_apdu_checksw(self, pdu: Hexstr, sw: SwMatchstr = "9000") -> ResTuple:
+        """like xceive_apdu, but checking the status word matches the expected pattern"""
+        rv = await self.xceive_apdu(pdu)
+        last_sw = rv[1]
+
+        if not sw_match(rv[1], sw):
+            raise SwMatchError(rv[1], sw.lower())
+        return rv
+
+    async def card_reset(self):
+        """reset the card"""
+        rx = await self.xceive_json('reset_req', exp_msg_type='reset_resp')
+        self.identity['ATR'] = rx['atr']
+
+    async def notify_state(self):
+        """notify the card client of a state [change]"""
+        await self.tx_json('state_notification', {'new_state': self.state})
+
+    async def get_iccid(self):
+        """high-level method to obtain the ICCID of the card"""
+        await self.xceive_apdu_checksw('00a40000023f00') # SELECT MF
+        await self.xceive_apdu_checksw('00a40000022fe2') # SELECT EF.ICCID
+        res, sw = await self.xceive_apdu_checksw('00b0000000') # READ BINARY
+        return dec_iccid(res)
+
+    async def get_eid_sgp22(self):
+        """high-level method to obtain the EID of a SGP.22 consumer eUICC"""
+        await self.xceive_apdu_checksw('00a4040410a0000005591010ffffffff8900000100')
+        res, sw = await self.xceive_apdu_checksw('80e2910006bf3e035c015a')
+        return res[-32:]
+
+    async def set_state(self, new_state:str):
+        self.logger.info("Card now in '%s' state" % new_state)
+        self.state = new_state
+        await self.notify_state()
+
+    async def identify(self):
+        # identify the card by asking for its EID and/or ICCID
+        try:
+            eid = await self.get_eid_sgp22()
+            self.logger.debug("EID: %s", eid)
+            self.identity['EID'] = eid
+        except SwMatchError:
+            pass
+        try:
+            iccid = await self.get_iccid()
+            self.logger.debug("ICCID: %s", iccid)
+            self.identity['ICCID'] = iccid
+        except SwMatchError:
+            pass
+        await self.set_state('ready')
+
+    async def associate_user(self, user):
+        assert self.state == 'ready'
+        self.user = user
+        await self.set_state('associated')
+
+    async def disassociate_user(self):
+        assert self.user
+        assert self.state == 'associated'
+        await self.set_state('ready')
+
+    @staticmethod
+    def find_client_for_id(id_type: str, id_str: str) -> Optional['CardClient']:
+        for c in card_clients:
+            if c.state != 'ready':
+                continue
+            c_id = c.identity.get(id_type.upper(), None)
+            if c_id and c_id.lower() == id_str.lower():
+                return c
+        return None
+
+class UserClient(WsClient):
+    """A websocket client representing a user application like pySim-shell."""
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.state = 'init'
+        self.card = None
+
+    async def associate_card(self, card: CardClient):
+        assert self.state == 'init'
+        self.card = card
+        self.state = 'associated'
+
+    async def disassociate_card(self):
+        assert self.state == 'associated'
+        self.card = None
+        self.state = 'init'
+
+    async def state_init(self):
+        """Wait for incoming 'select_card' and process it."""
+        while True:
+            rx = await self.rx_json()
+            if rx['msg_type'] == 'select_card':
+                # look-up if the card can be found
+                card = CardClient.find_client_for_id(rx['id_type'], rx['id_str'])
+                if not card:
+                    await self.tx_error('No CardClient found for %s == %s' % (rx['id_type'], rx['id_str']))
+                    continue
+                # transition to next statee
+                await card.associate_user(self)
+                await self.associate_card(card)
+                await self.tx_json('select_card_ack', {'identities': card.identity})
+                break
+            elif rx['msg_type'] == 'list_cards':
+                res = [x.listing_entry() for x in card_clients]
+                await self.tx_json('list_cards_ack', {'cards': res})
+            else:
+                self.tx_error('Unknown message type %s' % rx['msg_type'])
+
+    async def state_selected(self):
+        while True:
+            rx = await self.rx_json()
+            if rx['msg_type'] == 'c_apdu':
+                rsp, sw = await self.card.xceive_apdu_raw(rx['command'])
+                await self.tx_json('r_apdu', {'response': rsp, 'sw': sw})
+            elif rx['msg_type'] == 'reset_req':
+                await self.card.card_reset()
+                await self.tx_json('reset_resp')
+            else:
+                self.logger.warning("Unknown/unsupported command '%s' received" % rx['msg_type'])
+
+
+
+async def card_conn_hdlr(websocket):
+    """Handler for incoming connection to 'card' port."""
+    # receive first message, which should be a 'hello'
+    rx_raw = await websocket.recv()
+    rx = json.loads(rx_raw)
+    assert rx['msg_type'] == 'hello'
+    client_type = rx['client_type']
+
+    if client_type != 'card':
+        logger.error("Rejecting client (unknown type %s) connection", client_type)
+        raise ValueError("client_type '%s' != expected 'card'" % client_type)
+
+    card = CardClient(websocket, rx)
+    card.logger.info("connection accepted")
+    # first go through identity phase
+    async with asyncio.timeout(30):
+        await card.tx_hello_ack()
+        card.logger.info("hello-handshake completed")
+        card_clients.add(card)
+        # first obtain the identity of the card
+        await card.identify()
+    # then go into the "main loop"
+    try:
+        # wait 'indefinitely'. We cannot call websocket.recv() here, as we will call another
+        # recv() while waiting for the R-APDU after forwarding one from the user client.
+        await websocket.wait_closed()
+    finally:
+        card.logger.info("connection closed")
+        card_clients.remove(card)
+
+
+async def user_conn_hdlr(websocket):
+    """Handler for incoming connection to 'user' port."""
+    # receive first message, which should be a 'hello'
+    rx_raw = await websocket.recv()
+    rx = json.loads(rx_raw)
+    assert rx['msg_type'] == 'hello'
+    client_type = rx['client_type']
+
+    if client_type != 'user':
+        logger.error("Rejecting client (unknown type %s) connection", client_type)
+        raise ValueError("client_type '%s' != expected 'card'" % client_type)
+
+    user = UserClient(websocket, rx)
+    user.logger.info("connection accepted")
+    # first go through hello phase
+    async with asyncio.timeout(10):
+        await user.tx_hello_ack()
+        user.logger.info("hello-handshake completed")
+        user_clients.add(user)
+        # first wait for the user to specify the select the card
+        try:
+            await user.state_init()
+        except ConnectionClosedError:
+            user.logger.info("connection closed")
+            user_clients.remove(user)
+            return
+        except asyncio.exceptions.CancelledError: # direct cause of TimeoutError
+            user.logger.error("User failed to transition to 'associated' state within 10s")
+            user_clients.remove(user)
+            return
+        except Exception as e:
+            user.logger.error("Unknown exception %s", e)
+            raise e
+            user_clients.remove(user)
+            return
+    # then perform APDU exchanges without any time limit
+    try:
+        await user.state_selected()
+    except ConnectionClosedError:
+        pass
+    finally:
+        user.logger.info("connection closed")
+        await user.card.disassociate_user()
+        await user.disassociate_card()
+        user_clients.remove(user)
+
+
+parser = argparse.ArgumentParser(description="""Osmocom WebSocket Remote Card server""")
+parser.add_argument('--user-bind-port', type=int, default=WSRC_DEFAULT_PORT_USER,
+                    help="port number to bind for connections from users")
+parser.add_argument('--user-bind-host', type=str, default='localhost',
+                    help="local Host/IP to bind for connections from users")
+parser.add_argument('--card-bind-port', type=int, default=WSRC_DEFAULT_PORT_CARD,
+                    help="port number to bind for connections from cards")
+parser.add_argument('--card-bind-host', type=str, default='localhost',
+                    help="local Host/IP to bind for connections from cards")
+
+if __name__ == '__main__':
+    opts = parser.parse_args()
+
+    async def main():
+        # we use different ports for user + card connections to ensure they can
+        # have different packet filter rules apply to them
+        async with serve(card_conn_hdlr, opts.card_bind_host, opts.card_bind_port), serve(user_conn_hdlr, opts.user_bind_host, opts.user_bind_port):
+            await asyncio.get_running_loop().create_future() # run forever
+
+    asyncio.run(main())

docs/Makefile

@@ -4,7 +4,7 @@
 # You can set these variables from the command line, and also
 # from the environment for the first two.
 SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
+SPHINXBUILD   ?= python3 -m sphinx.cmd.build
 SOURCEDIR     = .
 BUILDDIR      = _build
 

docs/card-key-provider.rst

@@ -0,0 +1,126 @@
+
+Retrieving card-individual keys via CardKeyProvider
+==================================================
+
+When working with a batch of cards, or more than one card in general, it
+is a lot of effort to manually retrieve the card-specific PIN (like
+ADM1) or key material (like SCP02/SCP03 keys).
+
+To increase productivity in that regard, pySim has a concept called the
+`CardKeyProvider`.  This is a generic mechanism by which different parts
+of the pySim[-shell] code can programmatically request card-specific key material
+from some data source (*provider*).
+
+For example, when you want to verify the ADM1 PIN using the `verify_adm`
+command without providing an ADM1 value yourself, pySim-shell will
+request the ADM1 value for the ICCID of the card via the
+CardKeyProvider.
+
+There can in theory be multiple different CardKeyProviders.  You can for
+example develop your own CardKeyProvider that queries some kind of
+database for the key material, or that uses a key derivation function to
+derive card-specific key material from a global master key.
+
+The only actual CardKeyProvider implementation included in pySim is the
+`CardKeyProviderCsv` which retrieves the key material from a
+[potentially encrypted] CSV file.
+
+
+The CardKeyProviderCsv
+----------------------
+
+The `CardKeyProviderCsv` allows you to retrieve card-individual key
+material from a CSV (comma separated value) file that is accessible to pySim.
+
+The CSV file must have the expected column names, for example `ICCID`
+and `ADM1` in case you would like to use that CSV to obtain the
+card-specific ADM1 PIN when using the `verify_adm` command.
+
+You can specify the CSV file to use via the `--csv` command-line option
+of pySim-shell.  If you do not specify a CSV file, pySim will attempt to
+open a CSV file from the default location at
+`~/.osmocom/pysim/card_data.csv`, and use that, if it exists.
+
+Column-Level CSV encryption
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+pySim supports column-level CSV encryption.  This feature will make sure
+that your key material is not stored in plaintext in the CSV file.
+
+The encryption mechanism uses AES in CBC mode.  You can use any key
+length permitted by AES (128/192/256 bit).
+
+Following GSMA FS.28, the encryption works on column level.  This means
+different columns can be decrypted using different key material.  This
+means that leakage of a column encryption key for one column or set of
+columns (like a specific security domain) does not compromise various
+other keys that might be stored in other columns.
+
+You can specify column-level decryption keys using the
+`--csv-column-key` command line argument.  The syntax is
+`FIELD:AES_KEY_HEX`, for example:
+
+`pySim-shell.py --csv-column-key SCP03_ENC_ISDR:000102030405060708090a0b0c0d0e0f`
+
+In order to avoid having to repeat the column key for each and every
+column of a group of keys within a keyset, there are pre-defined column
+group aliases, which will make sure that the specified key will be used
+by all columns of the set:
+
+* `UICC_SCP02` is a group alias for `UICC_SCP02_KIC1`, `UICC_SCP02_KID1`, `UICC_SCP02_KIK1`
+* `UICC_SCP03` is a group alias for `UICC_SCP03_KIC1`, `UICC_SCP03_KID1`, `UICC_SCP03_KIK1`
+* `SCP03_ECASD` is a group alias for `SCP03_ENC_ECASD`, `SCP03_MAC_ECASD`, `SCP03_DEK_ECASD`
+* `SCP03_ISDA` is a group alias for `SCP03_ENC_ISDA`, `SCP03_MAC_ISDA`, `SCP03_DEK_ISDA`
+* `SCP03_ISDR` is a group alias for `SCP03_ENC_ISDR`, `SCP03_MAC_ISDR`, `SCP03_DEK_ISDR`
+
+
+Field naming
+------------
+
+* For look-up of UICC/SIM/USIM/ISIM or eSIM profile specific key
+  material, pySim uses the `ICCID` field as lookup key.
+
+* For look-up of eUICC specific key material (like SCP03 keys for the
+  ISD-R, ECASD), pySim uses the `EID` field as lookup key.
+
+As soon as the CardKeyProviderCsv finds a line (row) in your CSV where
+the ICCID or EID match, it looks for the column containing the requested
+data.
+
+
+ADM PIN
+~~~~~~~
+
+The `verify_adm` command will attempt to look up the `ADM1` column
+indexed by the ICCID of the SIM/UICC.
+
+
+SCP02 / SCP03
+~~~~~~~~~~~~~
+
+SCP02 and SCP03 each use key triplets consisting if ENC, MAC and DEK
+keys.  For more details, see the applicable GlobalPlatform
+specifications.
+
+If you do not want to manually enter the key material for each specific
+card as arguments to the `establish_scp02` or `establish_scp03`
+commands, you can make use of the `--key-provider-suffix` option.  pySim
+uses this suffix to compose the column names for the CardKeyProvider as
+follows.
+
+* `SCP02_ENC_` + suffix for the SCP02 ciphering key
+* `SCP02_MAC_` + suffix for the SCP02 MAC key
+* `SCP02_DEK_` + suffix for the SCP02 DEK key
+* `SCP03_ENC_` + suffix for the SCP03 ciphering key
+* `SCP03_MAC_` + suffix for the SCP03 MAC key
+* `SCP03_DEK_` + suffix for the SCP03 DEK key
+
+So for example, if you are using a command like `establish_scp03
+--key-provider-suffix ISDR`, then the column names for the key material
+look-up are `SCP03_ENC_ISDR`, `SCP03_MAC_ISDR` and `SCP03_DEK_ISDR`,
+respectively.
+
+The identifier used for look-up is determined by the definition of the
+Security Domain.  For example, the eUICC ISD-R and ECASD will use the EID
+of the eUICC.  On the other hand, the ISD-P of an eSIM or the ISD of an
+UICC will use the ICCID.

docs/index.rst

@@ -43,6 +43,7 @@ pySim consists of several parts:
    legacy
    library
    osmo-smdpp
+   wsrc
 
 
 Indices and tables

docs/legacy.rst

@@ -1,20 +1,20 @@
-Legacy tools
+Legacy tools
 ============
 
 *legacy tools* are the classic ``pySim-prog`` and ``pySim-read`` programs that
 existed long before ``pySim-shell``.
 
-These days, you should primarily use ``pySim-shell`` instead of these
+These days, it is highly recommended to use ``pySim-shell`` instead of these
 legacy tools.
 
 pySim-prog
 ----------
 
-``pySim-prog`` was the first part of the pySim software suite.  It started as
-a tool to write ICCID, IMSI, MSISDN and Ki to very simplistic SIM cards, and
-was later extended to a variety of other cards.  As the number of features supported
-became no longer bearable to express with command-line arguments, `pySim-shell` was
-created.
+``pySim-prog`` was the first part of the pySim software suite. It started as a
+tool to write ICCID, IMSI, MSISDN and Ki to very simplistic SIM cards, and was
+later extended to a variety of other cards. As the number of features supported
+became no longer bearable to express with command-line arguments, `pySim-shell`
+was created.
 
 Basic use cases can still use `pySim-prog`.
 
@@ -22,36 +22,180 @@ Program customizable SIMs
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 Two modes are possible:
 
-  - one where you specify every parameter manually :
+  - one where the user specifies every parameter manually:
 
-``./pySim-prog.py -n 26C3 -c 49 -x 262 -y 42 -i <IMSI> -s <ICCID>``
+    This is the most common way to use ``pySim-prog``. The user will specify all relevant parameters directly via the
+    commandline. A typical commandline would look like this:
+
+    ``pySim-prog.py -p <pcsc_reader> --ki <ki_value> --opc <opc_value> --mcc <mcc_value> --mnc <mnc_value>
+    --country <country_code> --imsi <imsi_value> --iccid <iccid_value> --pin-adm <adm_pin>``
+
+    Please note, that this already lengthy commandline still only contains the most common card parameters. For a full
+    list of all possible parameters, use the ``--help`` option of ``pySim-prog``. It is also important to mention
+    that not all parameters are supported by all card types. In particular, very simple programmable SIM cards will only
+    support a very basic set of parameters, such as MCC, MNC, IMSI and KI values.
+
+  - one where the parameters are generated from a minimal set:
+
+    It is also possible to leave the generation of certain parameters to ``pySim-prog``. This is in particular helpful
+    when a large number of cards should be initialized with randomly generated key material.
+
+    ``pySim-prog.py -p <pcsc_reader> --mcc <mcc_value> --mnc <mnc_value> --secret <random_secret> --num <card_number> --pin-adm <adm_pin>``
+
+    The parameter ``--secret`` specifies a random seed that is used to generate the card individual parameters. (IMSI).
+    The secret should contain enough randomness to avoid conflicts. It is also recommended to store the secret safely,
+    in case cards have to be re-generated or the current card batch has to be extended later. For security reasons, the
+    key material, which is also card individual, will not be derived from the random seed. Instead a new random set of
+    Ki and OPc will be generated during each programming cycle. This means fresh keys are generated, even when the
+    ``--num`` remains unchanged.
+
+    The parameter ``--num`` specifies a card individual number. This number will be manged into the random seed so that
+    it serves as an identifier for a particular set of randomly generated parameters.
+
+    In the example above the parameters ``--mcc``, and ``--mnc`` are specified as well, since they identify the GSM
+    network where the cards should operate in, it is absolutely required to keep them static. ``pySim-prog`` will use
+    those parameters to generate a valid IMSI that thas the specified MCC/MNC at the beginning and a random tail.
+
+Specifying the card type:
+
+``pySim-prog`` usually autodetects the card type. In case auto detection does not work, it is possible to specify
+the parameter ``--type``. The following card types are supported:
+
+ * Fairwaves-SIM
+ * fakemagicsim
+ * gialersim
+ * grcardsim
+ * magicsim
+ * OpenCells-SIM
+ * supersim
+ * sysmoISIM-SJA2
+ * sysmoISIM-SJA5
+ * sysmosim-gr1
+ * sysmoSIM-GR2
+ * sysmoUSIM-SJS1
+ * Wavemobile-SIM
+
+Specifying the card reader:
+
+It is most common to use ``pySim-prog`` together whith a PCSC reader. The PCSC reader number is specified via the
+``--pcsc-device`` or ``-p`` option. However, other reader types (such as serial readers and modems) are supported. Use
+the ``--help`` option of ``pySim-prog`` for more information.
 
 
-  - one where they are generated from some minimal set :
+Card programming using CSV files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``./pySim-prog.py -n 26C3 -c 49 -x 262 -y 42 -z <random_string_of_choice> -j <card_num>``
+To simplify the card programming process, ``pySim-prog`` also allows to read
+the card parameters from a CSV file. When a CSV file is used as input, the
+user does not have to craft an individual commandline for each card. Instead
+all card related parameters are automatically drawn from the CSV file.
 
-    With <random_string_of_choice> and <card_num>, the soft will generate
-    'predictable' IMSI and ICCID, so make sure you choose them so as not to
-    conflict with anyone. (for eg. your name as <random_string_of_choice> and
-    0 1 2 ... for <card num>).
+A CSV files may hold rows for multiple (hundreds or even thousands) of
+cards. ``pySim-prog`` is able to identify the rows either by ICCID
+(recommended as ICCIDs are normally not changed) or IMSI.
 
-  You also need to enter some parameters to select the device :
-   -t TYPE : type of card (supersim, magicsim, fakemagicsim or try 'auto')
-   -d DEV  : Serial port device (default /dev/ttyUSB0)
-   -b BAUD : Baudrate (default 9600)
+The CSV file format is a flexible format with mandatory and optional columns,
+here the same rules as for the commandline parameters apply. The column names
+match the command line options. The CSV file may also contain columns that are
+unknown to pySim-prog, such as inventory numbers, nicknames or parameters that
+are unrelated to the card programming process. ``pySim-prog`` will silently
+ignore all unknown columns.
+
+A CSV file may contain the following columns:
+
+* name
+* iccid (typically used as key)
+* mcc
+* mnc
+* imsi (may be used as key, but not recommended)
+* smsp
+* ki
+* opc
+* acc
+* pin_adm, adm1 or pin_adm_hex (must be present)
+* msisdn
+* epdgid
+* epdgSelection
+* pcscf
+* ims_hdomain
+* impi
+* impu
+* opmode
+* fplmn
+
+Due to historical reasons, and to maintain the compatibility between multiple different CSV file formats, the ADM pin
+may be stored in three different columns. Only one of the three columns must be available.
+
+* adm1: This column contains the ADM pin in numeric ASCII digit format. This format is the most common.
+* pin_adm: Same as adm1, only the column name is different
+* pin_adm_hex: If the ADM pin consists of raw HEX digits, rather then of numerical ASCII digits, then the ADM pin
+  can also be provided as HEX string using this column.
+
+The following example shows a typical minimal example
+::
+
+   "imsi","iccid","acc","ki","opc","adm1"
+   "999700000053010","8988211000000530108","0001","51ACE8BD6313C230F0BFE1A458928DF0","E5A00E8DE427E21B206526B5D1B902DF","65942330"
+   "999700000053011","8988211000000530116","0002","746AAFD7F13CFED3AE626B770E53E860","38F7CE8322D2A7417E0BBD1D7B1190EC","13445792"
+   "999700123053012","8988211000000530124","0004","D0DA4B7B150026ADC966DC637B26429C","144FD3AEAC208DFFF4E2140859BAE8EC","53540383"
+   "999700000053013","8988211000000530132","0008","52E59240ABAC6F53FF5778715C5CE70E","D9C988550DC70B95F40342298EB84C5E","26151368"
+   "999700000053014","8988211000000530140","0010","3B4B83CB9C5F3A0B41EBD17E7D96F324","D61DCC160E3B91F284979552CC5B4D9F","64088605"
+   "999700000053015","8988211000000530157","0020","D673DAB320D81039B025263610C2BBB3","4BCE1458936B338067989A06E5327139","94108841"
+   "999700000053016","8988211000000530165","0040","89DE5ACB76E06D14B0F5D5CD3594E2B1","411C4B8273FD7607E1885E59F0831906","55184287"
+   "999700000053017","8988211000000530173","0080","977852F7CEE83233F02E69E211626DE1","2EC35D48DBF2A99C07D4361F19EF338F","70284674"
+
+The following commandline will instruct ``pySim-prog`` to use the provided CSV file as parameter source and the
+ICCID (read from the card before programming) as a key to identify the card. To use the IMSI as a key, the parameter
+``--read-imsi`` can be used instead of ``--read-iccid``. However, this option is only recommended to be used in very
+specific corner cases.
+
+``pySim-prog.py -p <pcsc_reader> --read-csv <path_to_csv_file> --source csv --read-iccid``
+
+It is also possible to pick a row from the CSV file by manually providing an ICCID (option ``--iccid``) or an IMSI
+(option ``--imsi``) that is then used as a key to find the matching row in the CSV file.
+
+``pySim-prog.py -p <pcsc_reader> --read-csv <path_to_csv_file> --source csv --iccid <iccid_value>``
+
+
+Writing CSV files
+~~~~~~~~~~~~~~~~~
+``pySim-prog`` is also able to generate CSV files that contain a subset of the parameters it has generated or received
+from some other source (commandline, CSV-File). The generated file will be header-less and contain the following
+columns:
+
+* name
+* iccid
+* mcc
+* mnc
+* imsi
+* smsp
+* ki
+* opc
+
+A commandline that makes use of the CSV write feature would look like this:
+
+``pySim-prog.py -p <pcsc_reader> --read-csv <path_to_input_csv_file> --read-iccid --source csv --write-csv <path_to_output_csv_file>``
+
+
+Batch programming
+~~~~~~~~~~~~~~~~~
+
+In case larger card batches need to be programmed, it is possible to use the ``--batch`` parameter to run ``pySim-prog`` in batch mode.
+
+The batch mode will prompt the user to insert a card. Once a card is detected in the reader, the programming is carried out. The user may then remove the card again and the process starts over. This allows for a quick and efficient card programming without permanent commandline interaction.
 
 
 pySim-read
 ----------
 
-``pySim-read`` allows you to read some data from a SIM card.  It will only some files
-of the card, and will only read files accessible to a normal user (without any special authentication)
+``pySim-read`` allows to read some of the most important data items from a SIM
+card. This means it will only read some files of the card, and will only read
+files accessible to a normal user (without any special authentication)
 
-These days, you should use the ``export`` command of ``pySim-shell``
-instead. It performs a much more comprehensive export of all of the
-[standard] files that can be found on the card.  To get a human-readable
-decode instead of the raw hex export, you can use ``export --json``.
+These days, it is recommended to use the ``export`` command of ``pySim-shell``
+instead. It performs a much more comprehensive export of all of the [standard]
+files that can be found on the card. To get a human-readable decode instead of
+the raw hex export, you can use ``export --json``.
 
 Specifically, pySim-read will dump the following:
 

docs/library.rst

@@ -74,18 +74,6 @@ at 9600 bps.  These readers are sometimes called `Phoenix`.
    :members:
 
 
-pySim construct utilities
--------------------------
-
-.. automodule:: pySim.construct
-   :members:
-
-pySim TLV utilities
--------------------
-
-.. automodule:: pySim.tlv
-   :members:
-
 pySim utility functions
 -----------------------
 

docs/osmo-smdpp.rst

@@ -19,16 +19,21 @@ support for profile personalization yet.
 
 osmo-smdpp currently
 
-* uses test certificates copied from GSMA SGP.26 into `./smdpp-data/certs`, assuming that your osmo-smdppp
-  would be running at the host name `testsmdpplus1.example.com`
+* [by default] uses test certificates copied from GSMA SGP.26 into `./smdpp-data/certs`, assuming that your
+  osmo-smdpp would be running at the host name `testsmdpplus1.example.com`. You can of course replace those
+  certificates with your own, whether SGP.26 derived or part of a *private root CA* setup with mathcing eUICCs.
 * doesn't understand profile state. Any profile can always be downloaded any number of times, irrespective
-  of the EID or whether it was donwloaded before
-* doesn't perform any personalization, so the IMSI/ICCID etc. are always identical
+  of the EID or whether it was donwloaded before.  This is actually very useful for R&D and testing, as it
+  doesn't require you to generate new profiles all the time.  This logic of course is unsuitable for
+  production usage.
+* doesn't perform any personalization, so the IMSI/ICCID etc. are always identical (the ones that are stored in
+  the respective UPP `.der` files)
 * **is absolutely insecure**, as it
 
- * does not perform any certificate verification
- * does not evaluate/consider any *Matching ID* or *Confirmation Code*
- * stores the sessions in an unencrypted _python shelve_ and is hence leaking one-time key materials
+ * does not perform all of the mandatory certificate verification (it checks the certificate chain, but not
+   the expiration dates nor any CRL)
+ * does not evaluate/consider any *Confirmation Code*
+ * stores the sessions in an unencrypted *python shelve* and is hence leaking one-time key materials
    used for profile encryption and signing.
 
 
@@ -74,18 +79,22 @@ If you use `nginx` as web server, you can use the following configuration snippe
 You can of course achieve a similar functionality with apache, lighttpd or many other web server
 software.
 
+supplementary files
+~~~~~~~~~~~~~~~~~~~
 
-osmo-smdpp
-~~~~~~~~~~
+The `smdpp-data/certs`` directory contains the DPtls, DPauth and DPpb as well as CI certificates
+used; they are copied from GSMA SGP.26 v2.  You can of course replace them with custom certificates
+if you're operating eSIM with a *private root CA*.
+
+The `smdpp-data/upp` directory contains the UPP (Unprotected Profile Package) used.  The file names (without
+.der suffix) are looked up by the matchingID parameter from the activation code presented by the LPA.
+
+commandline options
+~~~~~~~~~~~~~~~~~~~
 
 osmo-smdpp currently doesn't have any configuration file or command line options.  You just run it,
 and it will bind its plain-HTTP ES9+ interface to local TCP port 8000.
 
-The `smdpp-data/certs`` directory contains the DPtls, DPauth and DPpb as well as CI certificates
-used; they are copied from GSMA SGP.26 v2.
-
-The `smdpp-data/upp` directory contains the UPP (Unprotected Profile Package) used.  The file names (without
-.der suffix) are looked up by the matchingID parameter from the activation code presented by the LPA.
 
 
 DNS setup for your LPA

docs/remote-access.rst

@@ -0,0 +1,46 @@
+
+Remote access to an UICC/eUICC
+==============================
+
+To access a card with pySim-shell, it is not strictly necessary to have physical
+access to it. There are solutions that allow remote access to UICC/eUICC cards.
+In this section we will give a brief overview.
+
+
+osmo-remsim
+-----------
+
+osmo-remsim is a suite of software programs enabling physical/geographic
+separation of a cellular phone (or modem) on the one hand side and the
+UICC/eUICC card on the other side.
+
+Using osmo-remsim, you can operate an entire fleet of modems/phones, as well as
+banks of SIM cards and dynamically establish or remove the connections between
+modems/phones and cards.
+
+To access remote cards with pySim-shell via osmo-remseim (RSPRO), the
+provided libifd_remsim_client would be used to provide a virtual PC/SC reader
+on the local machine. pySim-shell can then access this reader like any other
+PC/SC reader.
+
+More information on osmo-remsim can be found under:
+ * https://osmocom.org/projects/osmo-remsim/wiki
+ * https://ftp.osmocom.org/docs/osmo-remsim/master/osmo-remsim-usermanual.pdf
+
+
+Android APDU proxy
+------------------
+
+Android APDU proxy is an Android app that provides a bridge between a host
+computer and the UICC/eUICC slot of an Android smartphone.
+
+The APDU proxy connects to VPCD server that runs on the remote host (in this
+case the local machine where pySim-shell is running). The VPCD server then
+provides a virtual PC/SC reader, that pySim-shell can access like any other
+PC/SC reader.
+
+On the Android side the UICC/eUICC is accessed via OMAPI (Open Mobile API),
+which is available in Android since API level Android 8 (API level 29).
+
+More information Android APDU proxy can be found under:
+ * https://gitea.osmocom.org/sim-card/android-apdu-proxy

docs/shell.rst

@@ -20,6 +20,9 @@ The pySim-shell interactive shell provides commands for
 
 * if your card supports it, and you have the related privileges: resizing, creating, enabling and disabling of
   files
+* performing GlobalPlatform operations, including establishment of Secure Channel Protocol (SCP), Installing
+  applications, installing key material, etc.
+* listing/enabling/disabling/deleting eSIM profiles on Consumer eUICC
 
 By means of using the python ``cmd2`` module, various useful features improve usability:
 
@@ -66,6 +69,15 @@ Usage Examples
    suci-tutorial
 
 
+Advanced Topics
+---------------
+.. toctree::
+   :maxdepth: 1
+   :caption: Advanced pySIM-shell topics
+
+   card-key-provider
+   remote-access
+
 cmd2 basics
 -----------
 
@@ -133,6 +145,32 @@ optional files in some later 3GPP release) were not found on the card, or were i
 trying to SELECT them.
 
 
+fsdump
+~~~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: PySimCommands.fsdump_parser
+
+Please note that `fsdump` works relative to the current working
+directory, so if you are in `MF`, then the dump will contain all known
+files on the card.  However, if you are in `ADF.ISIM`, only files below
+that ADF will be part of the dump.
+
+Furthermore, it is strongly advised to first enter the ADM1 pin
+(`verify_adm`) to maximize the chance of having permission to read
+all/most files.
+
+One use case for this is to systematically analyze the differences between the contents of two
+cards.  To do this, you can create fsdumps of the two cards, and then use some general-purpose JSON
+diffing tool like `jycm --show` (see https://github.com/eggachecat/jycm).
+
+Example:
+::
+
+  pySIM-shell (00:MF)> fsdump > /tmp/fsdump.json
+  pySIM-shell (00:MF)>
+
+
 tree
 ~~~~
 Display a tree of the card filesystem.  It is important to note that this displays a tree
@@ -505,6 +543,9 @@ read_record_decoded
    :module: pySim.filesystem
    :func: LinFixedEF.ShellCommands.read_rec_dec_parser
 
+If this command fails, it means that the record is not decodable, and you should use the :ref:`read_record`
+command and proceed with manual decoding of the contents.
+
 
 read_records
 ~~~~~~~~~~~~
@@ -519,6 +560,9 @@ read_records_decoded
    :module: pySim.filesystem
    :func: LinFixedEF.ShellCommands.read_recs_dec_parser
 
+If this command fails, it means that the record[s] are not decodable, and you should use the :ref:`read_records`
+command and proceed with manual decoding of the contents.
+
 
 update_record
 ~~~~~~~~~~~~~
@@ -533,6 +577,9 @@ update_record_decoded
    :module: pySim.filesystem
    :func: LinFixedEF.ShellCommands.upd_rec_dec_parser
 
+If this command fails, it means that the record is not encodable; please check your input and/or use the raw
+:ref:`update_record` command.
+
 
 edit_record_decoded
 ~~~~~~~~~~~~~~~~~~~
@@ -551,6 +598,12 @@ back to the record on the SIM card.
 
 This allows for easy interactive modification of records.
 
+If this command fails before the editor is spawned, it means that the current record contents is not decodable,
+and you should use the :ref:`update_record_decoded` or :ref:`update_record` command.
+
+If this command fails after making your modificatiosn in the editor, it means that the new file contents is not
+encodable; please check your input and/or us the raw :ref:`update_record` comamdn.
+
 
 decode_hex
 ~~~~~~~~~~
@@ -579,6 +632,8 @@ read_binary_decoded
    :module: pySim.filesystem
    :func: TransparentEF.ShellCommands.read_bin_dec_parser
 
+If this command fails, it means that the file is not decodable, and you should use the :ref:`read_binary`
+command and proceed with manual decoding of the contents.
 
 update_binary
 ~~~~~~~~~~~~~
@@ -632,6 +687,10 @@ The below example demonstrates this by modifying the ciphering indicator field w
       "extensions": "ff"
   }
 
+If this command fails, it means that the file is not encodable; please check your input and/or use the raw
+:ref:`update_binary` command.
+
+
 edit_binary_decoded
 ~~~~~~~~~~~~~~~~~~~
 This command will read the selected binary EF, decode it to its JSON representation, save
@@ -645,6 +704,12 @@ to the SIM card.
 
 This allows for easy interactive modification of file contents.
 
+If this command fails before the editor is spawned, it means that the current file contents is not decodable,
+and you should use the :ref:`update_binary_decoded` or :ref:`update_binary` command.
+
+If this command fails after making your modificatiosn in the editor, it means that the new file contents is not
+encodable; please check your input and/or us the raw :ref:`update_binary` comamdn.
+
 
 decode_hex
 ~~~~~~~~~~
@@ -937,7 +1002,7 @@ aram_delete_all
 ~~~~~~~~~~~~~~~
 This command will request deletion of all access rules stored within the
 ARA-M applet.  Use it with caution, there is no undo.  Any rules later
-intended must be manually inserted again using `aram_store_ref_ar_do`
+intended must be manually inserted again using :ref:`aram_store_ref_ar_do`
 
 
 GlobalPlatform commands

docs/suci-tutorial.rst

@@ -1,40 +1,56 @@
 
 Guide: Enabling 5G SUCI
-========================
+=======================
 
 SUPI/SUCI Concealment is a feature of 5G-Standalone (SA) to encrypt the
 IMSI/SUPI with a network operator public key.  3GPP Specifies two different
 variants for this:
 
-* SUCI calculation *in the UE*, using data from the SIM
+* SUCI calculation *in the UE*, using key data from the SIM
 * SUCI calculation *on the card itself*
 
-pySIM supports writing the 5G-specific files for *SUCI calculation in the UE* on USIM cards, assuming that
-your cards contain the required files, and you have the privileges/credentials to write to them.  This is
-the case using sysmocom sysmoISIM-SJA2 cards (or successor products).
+pySim supports writing the 5G-specific files for *SUCI calculation in the UE* on USIM cards, assuming
+that your cards contain the required files, and you have the privileges/credentials to write to them.
+This is the case using sysmocom sysmoISIM-SJA2 or any flavor of sysmoISIM-SJA5.
 
-In short, you can enable SUCI with these steps:
+There is no 3GPP/ETSI standard method for configuring *SUCI calculation on the card*; pySim currently
+supports the vendor-specific method for the sysmoISIM-SJA5-S17).
 
-* activate USIM **Service 124**
-* make sure USIM **Service 125** is disabled
-* store the public keys in **SUCI_Calc_Info**
-* set the **Routing Indicator** (required)
+This document describes both methods.
 
-If you want to disable the feature, you can just disable USIM Service 124 (and 125).
 
 Technical References
 ~~~~~~~~~~~~~~~~~~~~
 
 This guide covers the basic workflow of provisioning SIM cards with the 5G SUCI feature. For detailed information on the SUCI feature and file contents, the following documents are helpful:
 
-* USIM files and structure: `TS 31.102 <https://www.etsi.org/deliver/etsi_ts/131100_131199/131102/16.06.00_60/ts_131102v160600p.pdf>`__
-* USIM tests (incl. file content examples) `TS 31.121 <https://www.etsi.org/deliver/etsi_ts/131100_131199/131121/16.01.00_60/ts_131121v160100p.pdf>`__
+* USIM files and structure: `3GPP TS 31.102 <https://www.etsi.org/deliver/etsi_ts/131100_131199/131102/16.06.00_60/ts_131102v160600p.pdf>`__
+* USIM tests (incl. file content examples): `3GPP TS 31.121 <https://www.etsi.org/deliver/etsi_ts/131100_131199/131121/16.01.00_60/ts_131121v160100p.pdf>`__
+* Test keys for SUCI calculation: `3GPP TS 33.501 <https://www.etsi.org/deliver/etsi_ts/133500_133599/133501/16.05.00_60/ts_133501v160500p.pdf>`__
 
-For specific information on sysmocom SIM cards, refer to Section 9.1 of the `sysmoUSIM User
-Manual <https://www.sysmocom.de/manuals/sysmousim-manual.pdf>`__.
+For specific information on sysmocom SIM cards, refer to
+
+* the `sysmoISIM-SJA5 User Manual <https://sysmocom.de/manuals/sysmoisim-sja5-manual.pdf>`__ for the curent
+  sysmoISIM-SJA5 product
+* the `sysmoISIM-SJA2 User Manual <https://sysmocom.de/manuals/sysmousim-manual.pdf>`__ for the older
+  sysmoISIM-SJA2 product
 
 --------------
 
+
+Enabling 5G SUCI *calculated in the UE*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In short, you can enable *SUCI calculation in the UE* with these steps:
+
+* activate USIM **Service 124**
+* make sure USIM **Service 125** is disabled
+* store the public keys in **EF.SUCI_Calc_Info**
+* set the **Routing Indicator** (required)
+
+If you want to disable the feature, you can just disable USIM Service 124 (and 125) in `EF.UST`.
+
+
 Admin PIN
 ---------
 
@@ -83,8 +99,8 @@ By default, the file is present but empty:
    missing Protection Scheme Identifier List data object tag
    9000: ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff -> {}
 
-The following JSON config defines the testfile from `TS 31.121 <https://www.etsi.org/deliver/etsi_ts/131100_131199/131121/16.01.00_60/ts_131121v160100p.pdf>`__ Section 4.9.4 with
-test keys from `TS 33.501 <hhttps://www.etsi.org/deliver/etsi_ts/133500_133599/133501/16.05.00_60/ts_133501v160500p.pdf>`__ Annex C.4. Highest priority (``0``) has a
+The following JSON config defines the testfile from 3GPP TS 31.121, Section 4.9.4 with
+test keys from 3GPP TS 33.501, Annex C.4. Highest priority (``0``) has a
 Profile-B (``identifier: 2``) key in key slot ``1``, which means the key
 with ``hnet_pubkey_identifier: 27``.
 
@@ -97,7 +113,7 @@ with ``hnet_pubkey_identifier: 27``.
            {"priority": 2, "identifier": 0, "key_index": 0}],
         "hnet_pubkey_list": [
            {"hnet_pubkey_identifier": 27,
-            "hnet_pubkey": "0272DA71976234CE833A6907425867B82E074D44EF907DFB4B3E21C1C2256EBCD1"},
+            "hnet_pubkey": "0472DA71976234CE833A6907425867B82E074D44EF907DFB4B3E21C1C2256EBCD15A7DED52FCBB097A4ED250E036C7B9C8C7004C4EEDC4F068CD7BF8D3F900E3B4"},
            {"hnet_pubkey_identifier": 30,
             "hnet_pubkey": "5A8D38864820197C3394B92613B20B91633CBD897119273BF8E4A6F4EEC0A650"}]
    }
@@ -106,7 +122,7 @@ Write the config to file (must be single-line input as for now):
 
 ::
 
-   pySIM-shell (00:MF/ADF.USIM/DF.5GS/EF.SUCI_Calc_Info)> update_binary_decoded '{ "prot_scheme_id_list": [ {"priority": 0, "identifier": 2, "key_index": 1}, {"priority": 1, "identifier": 1, "key_index": 2}, {"priority": 2, "identifier": 0, "key_index": 0}], "hnet_pubkey_list": [ {"hnet_pubkey_identifier": 27, "hnet_pubkey": "0272DA71976234CE833A6907425867B82E074D44EF907DFB4B3E21C1C2256EBCD1"}, {"hnet_pubkey_identifier": 30, "hnet_pubkey": "5A8D38864820197C3394B92613B20B91633CBD897119273BF8E4A6F4EEC0A650"}]}'
+   pySIM-shell (00:MF/ADF.USIM/DF.5GS/EF.SUCI_Calc_Info)> update_binary_decoded '{ "prot_scheme_id_list": [ {"priority": 0, "identifier": 2, "key_index": 1}, {"priority": 1, "identifier": 1, "key_index": 2}, {"priority": 2, "identifier": 0, "key_index": 0}], "hnet_pubkey_list": [ {"hnet_pubkey_identifier": 27, "hnet_pubkey": "0472DA71976234CE833A6907425867B82E074D44EF907DFB4B3E21C1C2256EBCD15A7DED52FCBB097A4ED250E036C7B9C8C7004C4EEDC4F068CD7BF8D3F900E3B4"}, {"hnet_pubkey_identifier": 30, "hnet_pubkey": "5A8D38864820197C3394B92613B20B91633CBD897119273BF8E4A6F4EEC0A650"}]}'
 
 WARNING: These are TEST KEYS with publicly known/specified private keys, and hence unsafe for live/secure
 deployments! For use in production networks, you need to generate your own set[s] of keys.
@@ -150,7 +166,7 @@ First, check out the USIM Service Table (UST):
    pySIM-shell (00:MF/ADF.USIM/EF.UST)> read_binary_decoded
    9000: beff9f9de73e0408400170730000002e00000000 -> [2, 3, 4, 5, 6, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 25, 27, 28, 29, 33, 34, 35, 38, 39, 42, 43, 44, 45, 46, 51, 60, 71, 73, 85, 86, 87, 89, 90, 93, 94, 95, 122, 123, 124, 126]
 
-.. list-table:: From TS31.102
+.. list-table:: From 3GPP TS 31.102
    :widths: 15 40
    :header-rows: 1
 
@@ -184,7 +200,7 @@ be disabled.
 USIM Error with 5G and sysmoISIM
 --------------------------------
 
-sysmoISIMs come 5GS-enabled. By default however, the configuration stored
+sysmoISIM-SJA2 come 5GS-enabled. By default however, the configuration stored
 in the card file-system is **not valid** for 5G networks: Service 124 is enabled,
 but EF.SUCI_Calc_Info and EF.Routing_Indicator are empty files (hence
 do not contain valid data).
@@ -193,3 +209,62 @@ At least for Qualcomm’s X55 modem, this results in an USIM error and the
 whole modem shutting 5G down. If you don’t need SUCI concealment but the
 smartphone refuses to connect to any 5G network, try to disable the UST
 service 124.
+
+sysmoISIM-SJA5 are shipped with a more forgiving default, with valid EF.Routing_Indicator
+contents and disabled Service 124
+
+
+SUCI calculation by the USIM
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The SUCI calculation can also be performed by the USIM application on the UICC
+directly. The UE then uses the GET IDENTITY command (see also 3GPP TS 31.102,
+section 7.5) to retrieve a SUCI value.
+
+The sysmoISIM-SJA5-S17 supports *SUCI calculation by the USIM*. The configuration
+is not much different to the above described configuration of *SUCI calculation
+in the UE*.
+
+The main difference is how the key provisioning is done. When the SUCI
+calculation is done by the USIM, then the key material is not accessed by the
+UE. The specification (see also 3GPP TS 31.102, section 7.5.1.1), also does not
+specify any file or file format to store the key material. This means the exact
+way to perform the key provisioning is an implementation detail of the USIM
+card application.
+
+In the case of sysmoISIM-SJA5-S17, the key material for *SUCI calculation by the USIM* is stored in
+`ADF.USIM/DF.SAIP/EF.SUCI_Calc_Info` (**not** in `ADF.USIM/DF.5GS/EF.SUCI_Calc_Info`!).
+
+::
+
+   pySIM-shell (00:MF)> select MF
+   pySIM-shell (00:MF)> select ADF.USIM
+   pySIM-shell (00:MF/ADF.USIM)> select DF.SAIP
+   pySIM-shell (00:MF/ADF.USIM/DF.SAIP)> select EF.SUCI_Calc_Info
+
+The file format is exactly the same as specified in 3GPP TS 31.102, section
+4.4.11.8. This means the above described key provisioning procedure can be
+applied without any changes, except that the file location is different.
+
+To signal to the UE that the USIM is setup up for SUCI calculation, service
+125 must be enabled in addition to service 124 (see also 3GPP TS 31.102,
+section 5.3.48)
+
+::
+
+   pySIM-shell (00:MF/ADF.USIM/EF.UST)> ust_service_activate 124
+   pySIM-shell (00:MF/ADF.USIM/EF.UST)> ust_service_activate 125
+
+To verify that the SUCI calculation works as expected, it is possible to issue
+a GET IDENTITY command using pySim-shell:
+
+::
+
+   select ADF.USIM
+   get_identity
+
+The USIM should then return a SUCI TLV Data object that looks like this:
+
+::
+
+   SUCI TLV Data Object: 0199f90717ff021b027a2c58ce1c6b89df088a9eb4d242596dd75746bb5f3503d2cf58a7461e4fd106e205c86f76544e9d732226a4e1

docs/wsrc.rst

@@ -0,0 +1,57 @@
+WebSocket Remote Card (WSRC)
+============================
+
+WSRC (*Web Socket Remote Card*) is a mechanism by which card readers can be made remotely available
+via a computer network.  The transport mechanism is (as the name implies) a WebSocket.  This transport
+method was chosen to be as firewall/NAT friendly as possible.
+
+WSRC Network Architecture
+-------------------------
+
+In a WSRC network, there are three major elements:
+
+* The **WSRC Card Client** which exposes a locally attached smart card (usually via a Smart Card Reader)
+  to a remote *WSRC Server*
+* The **WSRC Server** manges incoming connections from both *WSRC Card Clients* as well as *WSRC User Clients*
+* The **WSRC User Client** is a user application, like for example pySim-shell, which is accessing a remote
+  card by connecting to the *WSRC Server* which relays the information to the selected *WSRC Card Client*
+
+WSRC Protocol
+-------------
+
+The WSRC protocl consits of JSON objects being sent over a websocket.  The websocket communication itself
+is based on HTTP and should usually operate via TLS for security reasons.
+
+The detailed protocol is currently still WIP. The plan is to document it here.
+
+
+pySim implementations
+---------------------
+
+
+wsrc_server
+~~~~~~~~~~~~~~~~
+.. argparse::
+   :filename: ../contrib/wsrc_server.py
+   :func: parser
+   :prog: contrib/wsrc_server.py
+
+
+wsrc_card_client
+~~~~~~~~~~~~~~~~
+.. argparse::
+   :filename: ../contrib/wsrc_card_client.py
+   :func: parser
+   :prog: contrib/wsrc_card_client.py
+
+pySim-shell
+~~~~~~~~~~~
+
+pySim-shell can talk to a remote card via WSRC if you use the *wsrc transport*, for example like this:
+
+::
+
+    ./pySim-shell.py --wsrc-eid 89882119900000000000000000007280 --wsrc-serer-url ws://localhost:4220/
+
+
+You can specify `--wsrc-eid` or `--wsrc-iccid` to identify the remote eUICC or UICC you would like to select.

osmo-smdpp.py

@@ -32,10 +32,10 @@ from klein import Klein
 from twisted.web.iweb import IRequest
 import asn1tools
 
-from pySim.utils import h2b, b2h, swap_nibbles
+from osmocom.utils import h2b, b2h, swap_nibbles
 
 import pySim.esim.rsp as rsp
-from pySim.esim import saip
+from pySim.esim import saip, PMO
 from pySim.esim.es8p import *
 from pySim.esim.x509_cert import oid, cert_policy_has_oid, cert_get_auth_key_id
 from pySim.esim.x509_cert import CertAndPrivkey, CertificateSet, cert_get_subject_key_id, VerifyError
@@ -186,7 +186,9 @@ class SmDppHttpServer:
             print("Rx JSON: %s" % json.dumps(content))
             set_headers(request)
 
-            output = func(self, request, content) or {}
+            output = func(self, request, content)
+            if output == None:
+                return ''
 
             build_resp_header(output)
             print("Tx JSON: %s" % json.dumps(output))
@@ -239,7 +241,7 @@ class SmDppHttpServer:
 
         # Generate a TransactionID which is used to identify the ongoing RSP session. The TransactionID
         # SHALL be unique within the scope and lifetime of each SM-DP+.
-        transactionId = uuid.uuid4().hex
+        transactionId = uuid.uuid4().hex.upper()
         assert not transactionId in self.rss
 
         # Generate a serverChallenge for eUICC authentication attached to the ongoing RSP session.
@@ -292,8 +294,7 @@ class SmDppHttpServer:
 
         r_ok = authenticateServerResp[1]
         euiccSigned1 = r_ok['euiccSigned1']
-        # TODO: use original data, don't re-encode?
-        euiccSigned1_bin = rsp.asn1.encode('EuiccSigned1', euiccSigned1)
+        euiccSigned1_bin = rsp.extract_euiccSigned1(authenticateServerResp_bin)
         euiccSignature1_bin = r_ok['euiccSignature1']
         euiccCertificate_dec = r_ok['euiccCertificate']
         # TODO: use original data, don't re-encode?
@@ -326,14 +327,14 @@ class SmDppHttpServer:
         try:
             cs.verify_cert_chain(euicc_cert)
         except VerifyError:
-            raise ApiError('8.1.3', '6.1', 'Verification failed')
+            raise ApiError('8.1.3', '6.1', 'Verification failed (certificate chain)')
         #   raise ApiError('8.1.3', '6.3', 'Expired')
 
 
         # Verify euiccSignature1 over euiccSigned1 using pubkey from euiccCertificate.
         # Otherwise, the SM-DP+ SHALL return a status code "eUICC - Verification failed"
         if not self._ecdsa_verify(euicc_cert, euiccSignature1_bin, euiccSigned1_bin):
-            raise ApiError('8.1', '6.1', 'Verification failed')
+            raise ApiError('8.1', '6.1', 'Verification failed (euiccSignature1 over euiccSigned1)')
 
         # TODO: verify EID of eUICC cert is  within permitted range of EUM cert
 
@@ -344,7 +345,7 @@ class SmDppHttpServer:
         # serverChallenge returned by the eUICC. Otherwise, the SM-DP+ SHALL return a status code "eUICC -
         # Verification failed".
         if euiccSigned1['serverChallenge'] != ss.serverChallenge:
-            raise ApiError('8.1', '6.1', 'Verification failed')
+            raise ApiError('8.1', '6.1', 'Verification failed (serverChallenge)')
 
         # If ctxParams1 contains a ctxParamsForCommonAuthentication data object, the SM-DP+ Shall [...]
         # TODO: We really do a very simplistic job here, this needs to be properly implemented later,
@@ -367,11 +368,17 @@ class SmDppHttpServer:
                 with open(path, 'rb') as f:
                     pes = saip.ProfileElementSequence.from_der(f.read())
                     iccid_str = b2h(pes.get_pe_for_type('header').decoded['iccid'])
+        else:
+            # there's currently no other option in the ctxParams1 choice, so this cannot happen
+            raise ApiError('1.3.1', '2.2', 'ctxParams1 missing mandatory ctxParamsForCommonAuthentication')
 
         # FIXME: we actually want to perform the profile binding herr, and read the profile metadat from the profile
 
         # Put together profileMetadata + _bin
         ss.profileMetadata = ProfileMetadata(iccid_bin=h2b(swap_nibbles(iccid_str)), spn="OsmocomSPN", profile_name=matchingId)
+        # enable notifications for all operations
+        for event in ['enable', 'disable', 'delete']:
+            ss.profileMetadata.add_notification(event, self.server_hostname)
         profileMetadata_bin = ss.profileMetadata.gen_store_metadata_request()
 
         # Put together smdpSigned2 + _bin
@@ -419,8 +426,7 @@ class SmDppHttpServer:
 
         # Verify the euiccSignature2 computed over euiccSigned2 and smdpSignature2 using the PK.EUICC.SIG attached to the ongoing RSP session
         euiccSigned2 = r_ok['euiccSigned2']
-        # TODO: use original data, don't re-encode?
-        euiccSigned2_bin = rsp.asn1.encode('EUICCSigned2', euiccSigned2)
+        euiccSigned2_bin = rsp.extract_euiccSigned2(prepDownloadResp_bin)
         if not self._ecdsa_verify(ss.euicc_cert, r_ok['euiccSignature2'], euiccSigned2_bin + ss.smdpSignature2_do):
             raise ApiError('8.1', '6.1', 'eUICC signature is invalid')
 
@@ -443,7 +449,7 @@ class SmDppHttpServer:
 
         ss.host_id = b'mahlzeit'
 
-        # Generate Session Keys using the CRT, opPK.eUICC.ECKA and otSK.DP.ECKA according to annex G
+        # Generate Session Keys using the CRT, otPK.eUICC.ECKA and otSK.DP.ECKA according to annex G
         euicc_public_key = ec.EllipticCurvePublicKey.from_encoded_point(ss.smdp_ot.curve, ss.euicc_otpk)
         ss.shared_secret = ss.smdp_ot.exchange(ec.ECDH(), euicc_public_key)
         print("shared_secret: %s" % b2h(ss.shared_secret))
@@ -475,6 +481,9 @@ class SmDppHttpServer:
     @rsp_api_wrapper
     def handleNotification(self, request: IRequest, content: dict) -> dict:
         """See ES9+ HandleNotification in SGP.22 Section 5.6.4"""
+        # SGP.22 Section 6.3: "A normal notification function execution status (MEP Notification)
+        # SHALL be indicated by the HTTP status code '204' (No Content) with an empty HTTP response body"
+        request.setResponseCode(204)
         pendingNotification_bin = b64decode(content['pendingNotification'])
         pendingNotification = rsp.asn1.decode('PendingNotification', pendingNotification_bin)
         print("Rx %s: %s" % pendingNotification)
@@ -491,13 +500,33 @@ class SmDppHttpServer:
             pird_bin = rsp.asn1.encode('ProfileInstallationResultData', pird)
             # verify eUICC signature
             if not self._ecdsa_verify(ss.euicc_cert, profileInstallRes['euiccSignPIR'], pird_bin):
-                print("Unable to verify eUICC signature")
+                raise Exception('ECDSA signature verification failed on notification')
             print("Profile Installation Final Result: ", pird['finalResult'])
             # remove session state
             del self.rss[transactionId]
         elif pendingNotification[0] == 'otherSignedNotification':
-            # TODO
-            pass
+            otherSignedNotif = pendingNotification[1]
+            # TODO: use some kind of partially-parsed original data, don't re-encode?
+            euiccCertificate_bin = rsp.asn1.encode('Certificate', otherSignedNotif['euiccCertificate'])
+            eumCertificate_bin = rsp.asn1.encode('Certificate', otherSignedNotif['eumCertificate'])
+            euicc_cert = x509.load_der_x509_certificate(euiccCertificate_bin)
+            eum_cert = x509.load_der_x509_certificate(eumCertificate_bin)
+            ci_cert_id = cert_get_auth_key_id(eum_cert)
+            # Verify the validity of the eUICC certificate chain
+            cs = CertificateSet(self.ci_get_cert_for_pkid(ci_cert_id))
+            cs.add_intermediate_cert(eum_cert)
+            # TODO v3: otherCertsInChain
+            cs.verify_cert_chain(euicc_cert)
+            tbs_bin = rsp.asn1.encode('NotificationMetadata', otherSignedNotif['tbsOtherNotification'])
+            if not self._ecdsa_verify(euicc_cert, otherSignedNotif['euiccNotificationSignature'], tbs_bin):
+                raise Exception('ECDSA signature verification failed on notification')
+            other_notif = otherSignedNotif['tbsOtherNotification']
+            pmo = PMO.from_bitstring(other_notif['profileManagementOperation'])
+            eid = euicc_cert.subject.get_attributes_for_oid(x509.oid.NameOID.SERIAL_NUMBER)[0].value
+            iccid = other_notif.get('iccid', None)
+            if iccid:
+                iccid = swap_nibbles(b2h(iccid))
+            print("handleNotification: EID %s: %s of %s" % (eid, pmo, iccid))
         else:
             raise ValueError(pendingNotification)
 
@@ -558,7 +587,7 @@ def main(argv):
 
     args = parser.parse_args()
 
-    hs = SmDppHttpServer(HOSTNAME, os.path.join(DATA_DIR, 'certs', 'CertificateIssuer'), use_brainpool=True)
+    hs = SmDppHttpServer(HOSTNAME, os.path.join(DATA_DIR, 'certs', 'CertificateIssuer'), use_brainpool=False)
     #hs.app.run(endpoint_description="ssl:port=8000:dhParameters=dh_param_2048.pem")
     hs.app.run("localhost", 8000)
 

pySim-prog.py

@@ -25,7 +25,7 @@
 #
 
 import hashlib
-from optparse import OptionParser
+import argparse
 import os
 import random
 import re
@@ -33,12 +33,13 @@ import sys
 import traceback
 import json
 import csv
+from osmocom.utils import h2b, swap_nibbles, rpad
 
 from pySim.commands import SimCardCommands
-from pySim.transport import init_reader
+from pySim.transport import init_reader, argparse_add_reader_args
 from pySim.legacy.cards import _cards_classes, card_detect
-from pySim.utils import h2b, swap_nibbles, rpad, derive_milenage_opc, calculate_luhn, dec_iccid
-from pySim.ts_51_011 import EF_AD
+from pySim.utils import derive_milenage_opc, calculate_luhn, dec_iccid
+from pySim.profile.ts_51_011 import EF_AD
 from pySim.legacy.ts_51_011 import EF
 from pySim.card_handler import *
 from pySim.utils import *
@@ -46,169 +47,146 @@ from pySim.utils import *
 
 def parse_options():
 
-    parser = OptionParser(usage="usage: %prog [options]")
+    parser = argparse.ArgumentParser()
+    argparse_add_reader_args(parser)
 
-    parser.add_option("-d", "--device", dest="device", metavar="DEV",
-                      help="Serial Device for SIM access [default: %default]",
-                      default="/dev/ttyUSB0",
-                      )
-    parser.add_option("-b", "--baud", dest="baudrate", type="int", metavar="BAUD",
-                      help="Baudrate used for SIM access [default: %default]",
-                      default=9600,
-                      )
-    parser.add_option("-p", "--pcsc-device", dest="pcsc_dev", type='int', metavar="PCSC",
-                      help="Which PC/SC reader number for SIM access",
-                      default=None,
-                      )
-    parser.add_option("--modem-device", dest="modem_dev", metavar="DEV",
-                      help="Serial port of modem for Generic SIM Access (3GPP TS 27.007)",
-                      default=None,
-                      )
-    parser.add_option("--modem-baud", dest="modem_baud", type="int", metavar="BAUD",
-                      help="Baudrate used for modem's port [default: %default]",
-                      default=115200,
-                      )
-    parser.add_option("--osmocon", dest="osmocon_sock", metavar="PATH",
-                      help="Socket path for Calypso (e.g. Motorola C1XX) based reader (via OsmocomBB)",
-                      default=None,
-                      )
-    parser.add_option("-t", "--type", dest="type",
-                      help="Card type (user -t list to view) [default: %default]",
+    parser.add_argument("-t", "--type", dest="type",
+                      help="Card type (user -t list to view) [default: %(default)s]",
                       default="auto",
                       )
-    parser.add_option("-T", "--probe", dest="probe",
+    parser.add_argument("-T", "--probe", dest="probe",
                       help="Determine card type",
                       default=False, action="store_true"
                       )
-    parser.add_option("-a", "--pin-adm", dest="pin_adm",
+    parser.add_argument("-a", "--pin-adm", dest="pin_adm",
                       help="ADM PIN used for provisioning (overwrites default)",
                       )
-    parser.add_option("-A", "--pin-adm-hex", dest="pin_adm_hex",
+    parser.add_argument("-A", "--pin-adm-hex", dest="pin_adm_hex",
                       help="ADM PIN used for provisioning, as hex string (16 characters long",
                       )
-    parser.add_option("-e", "--erase", dest="erase", action='store_true',
-                      help="Erase beforehand [default: %default]",
+    parser.add_argument("-e", "--erase", dest="erase", action='store_true',
+                      help="Erase beforehand [default: %(default)s]",
                       default=False,
                       )
 
-    parser.add_option("-S", "--source", dest="source",
-                      help="Data Source[default: %default]",
+    parser.add_argument("-S", "--source", dest="source",
+                      help="Data Source[default: %(default)s]",
                       default="cmdline",
                       )
 
     # if mode is "cmdline"
-    parser.add_option("-n", "--name", dest="name",
-                      help="Operator name [default: %default]",
+    parser.add_argument("-n", "--name", dest="name",
+                      help="Operator name [default: %(default)s]",
                       default="Magic",
                       )
-    parser.add_option("-c", "--country", dest="country", type="int", metavar="CC",
-                      help="Country code [default: %default]",
+    parser.add_argument("-c", "--country", dest="country", type=int, metavar="CC",
+                      help="Country code [default: %(default)s]",
                       default=1,
                       )
-    parser.add_option("-x", "--mcc", dest="mcc", type="string",
-                      help="Mobile Country Code [default: %default]",
+    parser.add_argument("-x", "--mcc", dest="mcc",
+                      help="Mobile Country Code [default: %(default)s]",
                       default="901",
                       )
-    parser.add_option("-y", "--mnc", dest="mnc", type="string",
-                      help="Mobile Network Code [default: %default]",
+    parser.add_argument("-y", "--mnc", dest="mnc",
+                      help="Mobile Network Code [default: %(default)s]",
                       default="55",
                       )
-    parser.add_option("--mnclen", dest="mnclen", type="choice",
-                      help="Length of Mobile Network Code [default: %default]",
+    parser.add_argument("--mnclen", dest="mnclen",
+                      help="Length of Mobile Network Code [default: %(default)s]",
                       default="auto",
                       choices=["2", "3", "auto"],
                       )
-    parser.add_option("-m", "--smsc", dest="smsc",
+    parser.add_argument("-m", "--smsc", dest="smsc",
                       help="SMSC number (Start with + for international no.) [default: '00 + country code + 5555']",
                       )
-    parser.add_option("-M", "--smsp", dest="smsp",
+    parser.add_argument("-M", "--smsp", dest="smsp",
                       help="Raw SMSP content in hex [default: auto from SMSC]",
                       )
 
-    parser.add_option("-s", "--iccid", dest="iccid", metavar="ID",
+    parser.add_argument("-s", "--iccid", dest="iccid", metavar="ID",
                       help="Integrated Circuit Card ID",
                       )
-    parser.add_option("-i", "--imsi", dest="imsi",
+    parser.add_argument("-i", "--imsi", dest="imsi",
                       help="International Mobile Subscriber Identity",
                       )
-    parser.add_option("--msisdn", dest="msisdn",
+    parser.add_argument("--msisdn", dest="msisdn",
                       help="Mobile Subscriber Integrated Services Digital Number",
                       )
-    parser.add_option("-k", "--ki", dest="ki",
+    parser.add_argument("-k", "--ki", dest="ki",
                       help="Ki (default is to randomize)",
                       )
-    parser.add_option("-o", "--opc", dest="opc",
+    parser.add_argument("-o", "--opc", dest="opc",
                       help="OPC (default is to randomize)",
                       )
-    parser.add_option("--op", dest="op",
+    parser.add_argument("--op", dest="op",
                       help="Set OP to derive OPC from OP and KI",
                       )
-    parser.add_option("--acc", dest="acc",
+    parser.add_argument("--acc", dest="acc",
                       help="Set ACC bits (Access Control Code). not all card types are supported",
                       )
-    parser.add_option("--opmode", dest="opmode", type="choice",
+    parser.add_argument("--opmode", dest="opmode",
                       help="Set UE Operation Mode in EF.AD (Administrative Data)",
                       default=None,
                       choices=['{:02X}'.format(int(m)) for m in EF_AD.OP_MODE],
                       )
-    parser.add_option("-f", "--fplmn", dest="fplmn", action="append",
+    parser.add_argument("-f", "--fplmn", dest="fplmn", action="append",
                       help="Set Forbidden PLMN. Add multiple time for multiple FPLMNS",
                       )
-    parser.add_option("--epdgid", dest="epdgid",
+    parser.add_argument("--epdgid", dest="epdgid",
                       help="Set Home Evolved Packet Data Gateway (ePDG) Identifier. (Only FQDN format supported)",
                       )
-    parser.add_option("--epdgSelection", dest="epdgSelection",
+    parser.add_argument("--epdgSelection", dest="epdgSelection",
                       help="Set PLMN for ePDG Selection Information. (Only Operator Identifier FQDN format supported)",
                       )
-    parser.add_option("--pcscf", dest="pcscf",
+    parser.add_argument("--pcscf", dest="pcscf",
                       help="Set Proxy Call Session Control Function (P-CSCF) Address. (Only FQDN format supported)",
                       )
-    parser.add_option("--ims-hdomain", dest="ims_hdomain",
+    parser.add_argument("--ims-hdomain", dest="ims_hdomain",
                       help="Set IMS Home Network Domain Name in FQDN format",
                       )
-    parser.add_option("--impi", dest="impi",
+    parser.add_argument("--impi", dest="impi",
                       help="Set IMS private user identity",
                       )
-    parser.add_option("--impu", dest="impu",
+    parser.add_argument("--impu", dest="impu",
                       help="Set IMS public user identity",
                       )
-    parser.add_option("--read-imsi", dest="read_imsi", action="store_true",
+    parser.add_argument("--read-imsi", dest="read_imsi", action="store_true",
                       help="Read the IMSI from the CARD", default=False
                       )
-    parser.add_option("--read-iccid", dest="read_iccid", action="store_true",
+    parser.add_argument("--read-iccid", dest="read_iccid", action="store_true",
                       help="Read the ICCID from the CARD", default=False
                       )
-    parser.add_option("-z", "--secret", dest="secret", metavar="STR",
+    parser.add_argument("-z", "--secret", dest="secret", metavar="STR",
                       help="Secret used for ICCID/IMSI autogen",
                       )
-    parser.add_option("-j", "--num", dest="num", type=int,
+    parser.add_argument("-j", "--num", dest="num", type=int,
                       help="Card # used for ICCID/IMSI autogen",
                       )
-    parser.add_option("--batch", dest="batch_mode",
-                      help="Enable batch mode [default: %default]",
+    parser.add_argument("--batch", dest="batch_mode",
+                      help="Enable batch mode [default: %(default)s]",
                       default=False, action='store_true',
                       )
-    parser.add_option("--batch-state", dest="batch_state", metavar="FILE",
+    parser.add_argument("--batch-state", dest="batch_state", metavar="FILE",
                       help="Optional batch state file",
                       )
 
     # if mode is "csv"
-    parser.add_option("--read-csv", dest="read_csv", metavar="FILE",
+    parser.add_argument("--read-csv", dest="read_csv", metavar="FILE",
                       help="Read parameters from CSV file rather than command line")
 
-    parser.add_option("--write-csv", dest="write_csv", metavar="FILE",
+    parser.add_argument("--write-csv", dest="write_csv", metavar="FILE",
                       help="Append generated parameters in CSV file",
                       )
-    parser.add_option("--write-hlr", dest="write_hlr", metavar="FILE",
+    parser.add_argument("--write-hlr", dest="write_hlr", metavar="FILE",
                       help="Append generated parameters to OpenBSC HLR sqlite3",
                       )
-    parser.add_option("--dry-run", dest="dry_run",
+    parser.add_argument("--dry-run", dest="dry_run",
                       help="Perform a 'dry run', don't actually program the card",
                       default=False, action="store_true")
-    parser.add_option("--card_handler", dest="card_handler_config", metavar="FILE",
+    parser.add_argument("--card_handler", dest="card_handler_config", metavar="FILE",
                       help="Use automatic card handling machine")
 
-    (options, args) = parser.parse_args()
+    options = parser.parse_args()
 
     if options.type == 'list':
         for kls in _cards_classes:
@@ -219,15 +197,13 @@ def parse_options():
         return options
 
     if options.source == 'csv':
-        if (options.imsi is None) and (options.batch_mode is False) and (options.read_imsi is False) and (options.read_iccid is False):
-            parser.error(
-                "CSV mode needs either an IMSI, --read-imsi, --read-iccid or batch mode")
+        if (options.imsi is None) and (options.iccid is None)  and (options.read_imsi is False) and (options.read_iccid is False):
+           parser.error("CSV mode requires one additional parameter: --read-iccid, --read-imsi, --iccid or --imsi")
         if options.read_csv is None:
             parser.error("CSV mode requires a CSV input file")
     elif options.source == 'cmdline':
         if ((options.imsi is None) or (options.iccid is None)) and (options.num is None):
-            parser.error(
-                "If either IMSI or ICCID isn't specified, num is required")
+            parser.error("If either IMSI or ICCID isn't specified, num is required")
     else:
         parser.error("Only `cmdline' and `csv' sources supported")
 
@@ -242,9 +218,6 @@ def parse_options():
             parser.error(
                 "Can't give ICCID/IMSI for batch mode, need to use automatic parameters ! see --num and --secret for more information")
 
-    if args:
-        parser.error("Extraneous arguments")
-
     return options
 
 
@@ -649,6 +622,9 @@ def read_params_csv(opts, imsi=None, iccid=None):
 
 def write_params_hlr(opts, params):
     # SQLite3 OpenBSC HLR
+    # FIXME: The format of the osmo-hlr database has evolved, so that the code below will no longer work.
+    print("Warning: the database format of recent OsmoHLR versions is not compatible with pySim-prog!")
+
     if opts.write_hlr:
         import sqlite3
         conn = sqlite3.connect(opts.write_hlr)
@@ -749,16 +725,18 @@ def process_card(scc, opts, first, ch):
             card.erase()
             card.reset()
 
+    cp = None
+
     # Generate parameters
     if opts.source == 'cmdline':
         cp = gen_parameters(opts)
     elif opts.source == 'csv':
-        imsi = None
-        iccid = None
         if opts.read_iccid:
             (res, _) = scc.read_binary(['3f00', '2fe2'], length=10)
             iccid = dec_iccid(res)
-        elif opts.read_imsi:
+        else:
+            iccid = opts.iccid
+        if opts.read_imsi:
             (res, _) = scc.read_binary(EF['IMSI'])
             imsi = swap_nibbles(res)[3:]
         else:

pySim-read.py

@@ -29,7 +29,9 @@ import random
 import re
 import sys
 
-from pySim.ts_51_011 import EF_SST_map, EF_AD
+from osmocom.utils import h2b, h2s, swap_nibbles, rpad
+
+from pySim.profile.ts_51_011 import EF_SST_map, EF_AD
 from pySim.legacy.ts_51_011 import EF, DF
 from pySim.ts_31_102 import EF_UST_map
 from pySim.legacy.ts_31_102 import EF_USIM_ADF_map
@@ -40,8 +42,8 @@ from pySim.commands import SimCardCommands
 from pySim.transport import init_reader, argparse_add_reader_args
 from pySim.exceptions import SwMatchError
 from pySim.legacy.cards import card_detect, SimCard, UsimCard, IsimCard
-from pySim.utils import h2b, h2s, swap_nibbles, rpad, dec_imsi, dec_iccid, dec_msisdn
-from pySim.legacy.utils import format_xplmn_w_act, dec_st
+from pySim.utils import dec_imsi, dec_iccid
+from pySim.legacy.utils import format_xplmn_w_act, dec_st, dec_msisdn
 
 option_parser = argparse.ArgumentParser(description='Legacy tool for reading some parts of a SIM card',
                                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -86,7 +88,7 @@ if __name__ == '__main__':
     scc.sel_ctrl = "0004"
 
     # Testing for Classic SIM or UICC
-    (res, sw) = sl.send_apdu(scc.cla_byte + "a4" + scc.sel_ctrl + "02" + "3f00")
+    (res, sw) = sl.send_apdu(scc.cla_byte + "a4" + scc.sel_ctrl + "02" + "3f00" + "00")
     if sw == '6e00':
         # Just a Classic SIM
         scc.cla_byte = "a0"

pySim-shell.py

@@ -21,6 +21,7 @@ from typing import List, Optional
 
 import json
 import traceback
+import re
 
 import cmd2
 from packaging import version
@@ -47,16 +48,18 @@ from io import StringIO
 
 from pprint import pprint as pp
 
+from osmocom.utils import h2b, b2h, i2h, swap_nibbles, rpad, JsonEncoder, is_hexstr, is_decimal
+from osmocom.utils import is_hexstr_or_decimal, Hexstr
+from osmocom.tlv import bertlv_parse_one
+
 from pySim.exceptions import *
 from pySim.transport import init_reader, ApduTracer, argparse_add_reader_args, ProactiveHandler
-from pySim.utils import h2b, b2h, i2h, swap_nibbles, rpad, JsonEncoder, bertlv_parse_one, sw_match
-from pySim.utils import sanitize_pin_adm, tabulate_str_list, boxed_heading_str, Hexstr, dec_iccid
-from pySim.utils import is_hexstr_or_decimal, is_hexstr, is_decimal
+from pySim.utils import sanitize_pin_adm, tabulate_str_list, boxed_heading_str, dec_iccid, sw_match
 from pySim.card_handler import CardHandler, CardHandlerAuto
 
-from pySim.filesystem import CardMF, CardDF, CardADF
+from pySim.filesystem import CardMF, CardEF, CardDF, CardADF, LinFixedEF, TransparentEF, BerTlvEF
+from pySim.ts_102_221 import pin_names
 from pySim.ts_102_222 import Ts102222Commands
-from pySim.gsm_r import DF_EIRENE
 from pySim.cat import ProactiveCommand
 
 from pySim.card_key_provider import CardKeyProviderCsv, card_key_provider_register, card_key_provider_get_field
@@ -110,6 +113,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         self.conserve_write = True
         self.json_pretty_print = True
         self.apdu_trace = False
+        self.apdu_strict = False
 
         self.add_settable(Settable2Compat('numeric_path', bool, 'Print File IDs instead of names', self,
                                           onchange_cb=self._onchange_numeric_path))
@@ -118,6 +122,9 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         self.add_settable(Settable2Compat('json_pretty_print', bool, 'Pretty-Print JSON output', self))
         self.add_settable(Settable2Compat('apdu_trace', bool, 'Trace and display APDUs exchanged with card', self,
                                           onchange_cb=self._onchange_apdu_trace))
+        self.add_settable(Settable2Compat('apdu_strict', bool,
+                                          'Enforce APDU responses according to ISO/IEC 7816-3, table 12', self,
+                                          onchange_cb=self._onchange_apdu_strict))
         self.equip(card, rs)
 
     def equip(self, card, rs):
@@ -160,9 +167,9 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
 
             try:
                 self.lchan.select('MF/EF.ICCID', self)
-                self.iccid = dec_iccid(self.lchan.read_binary()[0])
+                rs.identity['ICCID'] = dec_iccid(self.lchan.read_binary()[0])
             except:
-                self.iccid = None
+                rs.identity['ICCID'] = None
 
             self.lchan.select('MF', self)
             rc = True
@@ -194,6 +201,13 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
             else:
                 self.card._scc._tp.apdu_tracer = None
 
+    def _onchange_apdu_strict(self, param_name, old, new):
+        if self.card:
+            if new == True:
+                self.card._scc._tp.apdu_strict = True
+            else:
+                self.card._scc._tp.apdu_strict = False
+
     class Cmd2ApduTracer(ApduTracer):
         def __init__(self, cmd2_app):
             self.cmd2 = cmd2_app
@@ -235,9 +249,10 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         self.equip(card, rs)
 
     apdu_cmd_parser = argparse.ArgumentParser()
-    apdu_cmd_parser.add_argument('APDU', type=is_hexstr, help='APDU as hex string')
     apdu_cmd_parser.add_argument('--expect-sw', help='expect a specified status word', type=str, default=None)
+    apdu_cmd_parser.add_argument('--expect-response-regex', help='match response against regex', type=str, default=None)
     apdu_cmd_parser.add_argument('--raw', help='Bypass the logical channel (and secure channel)', action='store_true')
+    apdu_cmd_parser.add_argument('APDU', type=is_hexstr, help='APDU as hex string')
 
     @cmd2.with_argparser(apdu_cmd_parser)
     def do_apdu(self, opts):
@@ -251,9 +266,9 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         # can be executed without the presence of a runtime state (self.rs) object. However, this also means that
         # self.lchan is also not present (see method equip).
         if opts.raw or self.lchan is None:
-            data, sw = self.card._scc.send_apdu(opts.APDU)
+            data, sw = self.card._scc.send_apdu(opts.APDU, apply_lchan = False)
         else:
-            data, sw = self.lchan.scc.send_apdu(opts.APDU)
+            data, sw = self.lchan.scc.send_apdu(opts.APDU, apply_lchan = False)
         if data:
             self.poutput("SW: %s, RESP: %s" % (sw, data))
         else:
@@ -261,14 +276,21 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         if opts.expect_sw:
             if not sw_match(sw, opts.expect_sw):
                 raise SwMatchError(sw, opts.expect_sw)
+        if opts.expect_response_regex:
+            response_regex_compiled = re.compile(opts.expect_response_regex)
+            if  re.match(response_regex_compiled, data) is None:
+                raise ValueError("RESP does not match regex \'%s\'" % opts.expect_response_regex)
 
     @cmd2.with_category(CUSTOM_CATEGORY)
     def do_reset(self, opts):
         """Reset the Card."""
-        atr = self.card.reset()
-        if self.lchan and self.lchan.scc.scp:
-            self.lchan.scc.scp = None
-        self.poutput('Card ATR: %s' % i2h(atr))
+        if self.rs is None:
+            # In case no runtime state is available we go the direct route
+            self.card._scc.reset_card()
+            atr = b2h(self.card._scc.get_atr())
+        else:
+            atr = self.rs.reset(self)
+        self.poutput('Card ATR: %s' % atr)
         self.update_prompt()
 
     class InterceptStderr(list):
@@ -347,8 +369,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         return -1
 
     bulk_script_parser = argparse.ArgumentParser()
-    bulk_script_parser.add_argument(
-        'script_path', help="path to the script file")
+    bulk_script_parser.add_argument('SCRIPT_PATH', help="path to the script file")
     bulk_script_parser.add_argument('--halt_on_error', help='stop card handling if an exeption occurs',
                                     action='store_true')
     bulk_script_parser.add_argument('--tries', type=int, default=2,
@@ -364,7 +385,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         """Run script on multiple cards (bulk provisioning)"""
 
         # Make sure that the script file exists and that it is readable.
-        if not os.access(opts.script_path, os.R_OK):
+        if not os.access(opts.SCRIPT_PATH, os.R_OK):
             self.poutput("Invalid script file!")
             return
 
@@ -374,7 +395,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         first = True
         while 1:
             # TODO: Count consecutive failures, if more than N consecutive failures occur, then stop.
-            # The ratinale is: There may be a problem with the device, we do want to prevent that
+            # The rationale is: There may be a problem with the device, we do want to prevent that
             # all remaining cards are fired to the error bin. This is only relevant for situations
             # with large stacks, probably we do not need this feature right now.
 
@@ -389,7 +410,7 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
                         os.system(opts.pre_card_action)
 
                     # process the card
-                    rc = self._process_card(first, opts.script_path)
+                    rc = self._process_card(first, opts.SCRIPT_PATH)
                     if rc == 0:
                         success_count = success_count + 1
                         self._show_success_sign()
@@ -441,13 +462,13 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
             first = False
 
     echo_parser = argparse.ArgumentParser()
-    echo_parser.add_argument('string', help="string to echo on the shell", nargs='+')
+    echo_parser.add_argument('STRING', help="string to echo on the shell", nargs='+')
 
     @cmd2.with_argparser(echo_parser)
     @cmd2.with_category(CUSTOM_CATEGORY)
     def do_echo(self, opts):
         """Echo (print) a string on the console"""
-        self.poutput(' '.join(opts.string))
+        self.poutput(' '.join(opts.STRING))
 
     @cmd2.with_category(CUSTOM_CATEGORY)
     def do_version(self, opts):
@@ -496,12 +517,25 @@ class PySimCommands(CommandSet):
         self._cmd.poutput(directory_str)
         self._cmd.poutput("%d files" % len(selectables))
 
-    def walk(self, indent=0, action_ef=None, action_df=None, context=None, **kwargs):
+    def __walk_action(self, action, filename, context, **kwargs):
+        # Changing the currently selected file while walking over the filesystem tree would disturb the
+        # walk, so we memorize the currently selected file here so that we can select it again after
+        # we have executed the action callback.
+        selected_file_before_action = self._cmd.lchan.selected_file
+
+        # Perform action
+        action(filename, context, **kwargs)
+
+        # When the action callback is done, make sure the file that was selected before is selected again.
+        if selected_file_before_action != self._cmd.lchan.selected_file:
+            self._cmd.lchan.select_file(selected_file_before_action, self._cmd)
+
+    def __walk(self, indent=0, action_ef=None, action_df=None, context=None, **kwargs):
         """Recursively walk through the file system, starting at the currently selected DF"""
 
         if isinstance(self._cmd.lchan.selected_file, CardDF):
             if action_df:
-                action_df(context, **kwargs)
+                self.__walk_action(action_df, self._cmd.lchan.selected_file.name, context, **kwargs)
 
         files = self._cmd.lchan.selected_file.get_selectables(
             flags=['FNAMES', 'ANAMES'])
@@ -534,143 +568,45 @@ class PySimCommands(CommandSet):
                 # If the DF was skipped, we never have entered the directory
                 # below, so we must not move up.
                 if skip_df == False:
-                    self.walk(indent + 1, action_ef, action_df, context, **kwargs)
-
-                    parent = self._cmd.lchan.selected_file.parent
-                    df = self._cmd.lchan.selected_file
-                    adf = self._cmd.lchan.selected_adf
-                    if isinstance(parent, CardMF) and (adf and adf.has_fs == False):
-                        # Not every application that may be present on a GlobalPlatform card will support the SELECT
-                        # command as we know it from ETSI TS 102 221, section 11.1.1. In fact the only subset of
-                        # SELECT we may rely on is the OPEN SELECT command as specified in GlobalPlatform Card
-                        # Specification, section 11.9. Unfortunately the OPEN SELECT command only supports the
-                        # "select by name" method, which means we can only select an application and not a file.
-                        # The consequence of this is that we may get trapped in an application that does not have
-                        # ISIM/USIM like file system support and the only way to leave that application is to select
-                        # an ISIM/USIM application in order to get the file system access back.
-                        #
-                        # To automate this escape-route while traversing the file system we will check whether
-                        # the parent file is the MF. When this is the case and the selected ADF has no file system
-                        # support, we will select an arbitrary ADF that has file system support first and from there
-                        # we will then select the MF.
-                        for selectable in parent.get_selectables().items():
-                            if isinstance(selectable[1], CardADF) and selectable[1].has_fs == True:
-                                self._cmd.lchan.select(selectable[1].name, self._cmd)
-                                break
-                        self._cmd.lchan.select(df.get_mf().name, self._cmd)
-                    else:
-                        # Normal DF/ADF selection
-                        fcp_dec = self._cmd.lchan.select("..", self._cmd)
+                    self.__walk(indent + 1, action_ef, action_df, context, **kwargs)
+                    self._cmd.lchan.select_file(self._cmd.lchan.selected_file.parent, self._cmd)
 
             elif action_ef:
-                df_before_action = self._cmd.lchan.selected_file
-                action_ef(f, context, **kwargs)
-                # When walking through the file system tree the action must not
-                # always restore the currently selected file to the file that
-                # was selected before executing the action() callback.
-                if df_before_action != self._cmd.lchan.selected_file:
-                    raise RuntimeError("inconsistent walk, %s is currently selected but expecting %s to be selected"
-                                       % (str(self._cmd.lchan.selected_file), str(df_before_action)))
+                self.__walk_action(action_ef, f, context, **kwargs)
 
     def do_tree(self, opts):
         """Display a filesystem-tree with all selectable files"""
-        self.walk()
+        self.__walk()
 
-    def export_ef(self, filename, context, as_json):
-        """ Select and export a single elementary file (EF) """
+    def __export_file(self, filename, context, as_json):
+        """ Select and export a single file (EF, DF or ADF) """
         context['COUNT'] += 1
-        df = self._cmd.lchan.selected_file
 
-	# The currently selected file (not the file we are going to export)
-	# must always be an ADF or DF. From this starting point we select
-	# the EF we want to export. To maintain consistency we will then
-	# select the current DF again (see comment below).
-        if not isinstance(df, CardDF):
-            raise RuntimeError(
-                "currently selected file %s is not a DF or ADF" % str(df))
+        file = self._cmd.lchan.get_file_by_name(filename)
+        if file:
+            self._cmd.poutput(boxed_heading_str(file.fully_qualified_path_str(True)))
+            self._cmd.poutput("# directory: %s (%s)" % (file.fully_qualified_path_str(True),
+                                                        file.fully_qualified_path_str(False)))
+        else:
+            # If this is called from self.__walk(), then it is ensured that the file exists.
+            raise RuntimeError("cannot export, file %s does not exist in the file system tree" % filename)
 
-        df_path_list = df.fully_qualified_path(True)
-        df_path = df.fully_qualified_path_str(True)
-        df_path_fid = df.fully_qualified_path_str(False)
-
-        file_str = df_path + "/" + str(filename)
-        self._cmd.poutput(boxed_heading_str(file_str))
-
-        self._cmd.poutput("# directory: %s (%s)" % (df_path, df_path_fid))
         try:
-            fcp_dec = self._cmd.lchan.select(filename, self._cmd)
-            self._cmd.poutput("# file: %s (%s)" % (
-                self._cmd.lchan.selected_file.name, self._cmd.lchan.selected_file.fid))
-
-            structure = self._cmd.lchan.selected_file_structure()
-            self._cmd.poutput("# structure: %s" % str(structure))
+            fcp_dec = self._cmd.lchan.select_file(file, self._cmd)
+            self._cmd.poutput("# file: %s (%s)" %
+                              (self._cmd.lchan.selected_file.name, self._cmd.lchan.selected_file.fid))
+            if isinstance(self._cmd.lchan.selected_file, CardEF):
+                self._cmd.poutput("# structure: %s" % str(self._cmd.lchan.selected_file_structure()))
             self._cmd.poutput("# RAW FCP Template: %s" % str(self._cmd.lchan.selected_file_fcp_hex))
             self._cmd.poutput("# Decoded FCP Template: %s" % str(self._cmd.lchan.selected_file_fcp))
-
-            for f in df_path_list:
-                self._cmd.poutput("select " + str(f))
-            self._cmd.poutput("select " + self._cmd.lchan.selected_file.name)
-
-            if structure == 'transparent':
-                if as_json:
-                    result = self._cmd.lchan.read_binary_dec()
-                    self._cmd.poutput("update_binary_decoded '%s'" % json.dumps(result[0], cls=JsonEncoder))
-                else:
-                    result = self._cmd.lchan.read_binary()
-                    self._cmd.poutput("update_binary " + str(result[0]))
-            elif structure == 'cyclic' or structure == 'linear_fixed':
-                # Use number of records specified in select response
-                num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
-                if num_of_rec:
-                    for r in range(1, num_of_rec + 1):
-                        if as_json:
-                            result = self._cmd.lchan.read_record_dec(r)
-                            self._cmd.poutput("update_record_decoded %d '%s'" % (r, json.dumps(result[0], cls=JsonEncoder)))
-                        else:
-                            result = self._cmd.lchan.read_record(r)
-                            self._cmd.poutput("update_record %d %s" % (r, str(result[0])))
-
-                # When the select response does not return the number of records, read until we hit the
-                # first record that cannot be read.
-                else:
-                    r = 1
-                    while True:
-                        try:
-                            if as_json:
-                                result = self._cmd.lchan.read_record_dec(r)
-                                self._cmd.poutput("update_record_decoded %d '%s'" % (r, json.dumps(result[0], cls=JsonEncoder)))
-                            else:
-                                result = self._cmd.lchan.read_record(r)
-                                self._cmd.poutput("update_record %d %s" % (r, str(result[0])))
-                        except SwMatchError as e:
-                            # We are past the last valid record - stop
-                            if e.sw_actual == "9402":
-                                break
-                            # Some other problem occurred
-                            else:
-                                raise e
-                        r = r + 1
-            elif structure == 'ber_tlv':
-                tags = self._cmd.lchan.retrieve_tags()
-                for t in tags:
-                    result = self._cmd.lchan.retrieve_data(t)
-                    (tag, l, val, remainer) = bertlv_parse_one(h2b(result[0]))
-                    self._cmd.poutput("set_data 0x%02x %s" % (t, b2h(val)))
-            else:
-                raise RuntimeError(
-                    'Unsupported structure "%s" of file "%s"' % (structure, filename))
+            self._cmd.poutput("select " + self._cmd.lchan.selected_file.fully_qualified_path_str())
+            self._cmd.poutput(self._cmd.lchan.selected_file.export(as_json, self._cmd.lchan))
         except Exception as e:
-            bad_file_str = df_path + "/" + str(filename) + ", " + str(e)
+            bad_file_str = file.fully_qualified_path_str(True) + "/" + str(file.name) + ", " + str(e)
             self._cmd.poutput("# bad file: %s" % bad_file_str)
             context['ERR'] += 1
             context['BAD'].append(bad_file_str)
 
-        # When reading the file is done, make sure the parent file is
-        # selected again. This will be the usual case, however we need
-        # to check before since we must not select the same DF twice
-        if df != self._cmd.lchan.selected_file:
-            self._cmd.lchan.select(df.fid or df.aid, self._cmd)
-
         self._cmd.poutput("#")
 
     export_parser = argparse.ArgumentParser()
@@ -688,10 +624,10 @@ class PySimCommands(CommandSet):
         exception_str_add = ""
 
         if opts.filename:
-            self.export_ef(opts.filename, context, **kwargs_export)
+            self.__walk_action(self.__export_file, opts.filename, context, **kwargs_export)
         else:
             try:
-                self.walk(0, self.export_ef, None, context, **kwargs_export)
+                self.__walk(0, self.__export_file, self.__export_file, context, **kwargs_export)
             except Exception as e:
                 print("# Stopping early here due to exception: " + str(e))
                 print("#")
@@ -719,41 +655,202 @@ class PySimCommands(CommandSet):
             raise RuntimeError(
                     "unable to export %i dedicated files(s)%s" % (context['ERR'], exception_str_add))
 
+    def __dump_file(self, filename, context, as_json):
+        """ Select and dump a single file (EF, DF or ADF) """
+        file = self._cmd.lchan.get_file_by_name(filename)
+        if file:
+            res = {
+                'path': file.fully_qualified_path(True)
+            }
+        else:
+            # If this is called from self.__walk(), then it is ensured that the file exists.
+            raise RuntimeError("cannot dump, file %s does not exist in the file system tree" % filename)
+
+        try:
+            fcp_dec = self._cmd.lchan.select(filename, self._cmd)
+
+            # File control parameters (common for EF, DF and ADF files)
+            if not self._cmd.lchan.selected_file_fcp_hex:
+                # An application without a real ADF (like ADF.ARA-M) / filesystem
+                return
+
+            res['fcp_raw'] = str(self._cmd.lchan.selected_file_fcp_hex)
+            res['fcp'] = fcp_dec
+
+            # File structure and contents (EF only)
+            if isinstance(self._cmd.lchan.selected_file, CardEF):
+                structure = self._cmd.lchan.selected_file_structure()
+                if structure == 'transparent':
+                    if as_json:
+                        result = self._cmd.lchan.read_binary_dec()
+                        body = result[0]
+                    else:
+                        result = self._cmd.lchan.read_binary()
+                        body = str(result[0])
+                elif structure == 'cyclic' or structure == 'linear_fixed':
+                    body = []
+                    # Use number of records specified in select response
+                    num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
+                    if num_of_rec:
+                        for r in range(1, num_of_rec + 1):
+                            if as_json:
+                                result = self._cmd.lchan.read_record_dec(r)
+                                body.append(result[0])
+                            else:
+                                result = self._cmd.lchan.read_record(r)
+                                body.append(str(result[0]))
+
+                    # When the select response does not return the number of records, read until we hit the
+                    # first record that cannot be read.
+                    else:
+                        r = 1
+                        while True:
+                            try:
+                                if as_json:
+                                    result = self._cmd.lchan.read_record_dec(r)
+                                    body.append(result[0])
+                                else:
+                                    result = self._cmd.lchan.read_record(r)
+                                    body.append(str(result[0]))
+                            except SwMatchError as e:
+                                # We are past the last valid record - stop
+                                if e.sw_actual == "9402":
+                                    break
+                                # Some other problem occurred
+                                raise e
+                            r = r + 1
+                elif structure == 'ber_tlv':
+                    tags = self._cmd.lchan.retrieve_tags()
+                    body = {}
+                    for t in tags:
+                        result = self._cmd.lchan.retrieve_data(t)
+                        (tag, l, val, remainer) = bertlv_parse_one(h2b(result[0]))
+                        body[t] = b2h(val)
+                else:
+                    raise RuntimeError('Unsupported structure "%s" of file "%s"' % (structure, filename))
+                res['body'] = body
+
+        except SwMatchError as e:
+            res['error'] = {
+                'sw_actual': e.sw_actual,
+                'sw_expected': e.sw_expected,
+                'message': e.description,
+            }
+        except Exception as e:
+            raise(e)
+            res['error'] = {
+                'message': str(e)
+            }
+
+        context['result']['files'][file.fully_qualified_path_str(True)] = res
+
+    fsdump_parser = argparse.ArgumentParser()
+    fsdump_parser.add_argument(
+        '--filename', type=str, default=None, help='only export specific (named) file')
+    fsdump_parser.add_argument(
+        '--json', action='store_true', help='export file contents as JSON (less reliable)')
+
+    @cmd2.with_argparser(fsdump_parser)
+    def do_fsdump(self, opts):
+        """Export filesystem metadata and file contents of all files below current DF in
+        machine-readable json format.  This is similar to "export", but much easier to parse by
+        downstream processing tools.  You usually may want to call this from the MF and verify
+        the ADM1 PIN (if available) to maximize the amount of readable files."""
+        result = {
+            'name': self._cmd.card.name,
+            'atr': self._cmd.rs.identity['ATR'],
+            'eid': self._cmd.rs.identity.get('EID', None),
+            'iccid': self._cmd.rs.identity.get('ICCID', None),
+            'aids': {x.aid:{} for x in self._cmd.rs.mf.applications.values()},
+            'files': {},
+        }
+        context = {'result': result, 'DF_SKIP': 0, 'DF_SKIP_REASON': []}
+        kwargs_export = {'as_json': opts.json}
+        exception_str_add = ""
+
+        if opts.filename:
+            self.__walk_action(self.__dump_file, opts.filename, context, **kwargs_export)
+        else:
+            # export an entire subtree
+            try:
+                self.__walk(0, self.__dump_file, self.__dump_file, context, **kwargs_export)
+            except Exception as e:
+                print("# Stopping early here due to exception: " + str(e))
+                print("#")
+                exception_str_add = ", also had to stop early due to exception:" + str(e)
+                #raise e
+
+        self._cmd.poutput_json(context['result'])
+
+
     def do_desc(self, opts):
         """Display human readable file description for the currently selected file"""
         desc = self._cmd.lchan.selected_file.desc
         if desc:
-            self._cmd.poutput(desc)
+            self._cmd.poutput("%s: %s" % (self._cmd.lchan.selected_file, desc))
         else:
-            self._cmd.poutput("no description available")
+            self._cmd.poutput("%s: no description available" % self._cmd.lchan.selected_file)
+        self._cmd.poutput(" file structure: %s" % self._cmd.lchan.selected_file_structure())
+        if isinstance(self._cmd.lchan.selected_file, LinFixedEF):
+            self._cmd.poutput(" record length:")
+            self._cmd.poutput("  minimum_length: %s" % str(self._cmd.lchan.selected_file.rec_len[0]))
+            self._cmd.poutput("  recommended_length: %s" % str(self._cmd.lchan.selected_file.rec_len[1]))
+            self._cmd.poutput("  actual_length: %s" % str(self._cmd.lchan.selected_file_record_len()))
+            self._cmd.poutput(" number of records: %s" % str(self._cmd.lchan.selected_file_num_of_rec()))
+        elif isinstance(self._cmd.lchan.selected_file, TransparentEF):
+            self._cmd.poutput(" file size:")
+            self._cmd.poutput("  minimum_size: %s" % str(self._cmd.lchan.selected_file.size[0]))
+            self._cmd.poutput("  recommended_size: %s" % str(self._cmd.lchan.selected_file.size[1]))
+            self._cmd.poutput("  actual_size: %s" % str(self._cmd.lchan.selected_file_size()))
+        elif isinstance(self._cmd.lchan.selected_file, BerTlvEF):
+            self._cmd.poutput(" file size:")
+            self._cmd.poutput("  minimum_size: %s" % str(self._cmd.lchan.selected_file.size[0]))
+            self._cmd.poutput("  recommended_size: %s" % str(self._cmd.lchan.selected_file.size[1]))
+            self._cmd.poutput("  actual_size: %s" % str(self._cmd.lchan.selected_file_size()))
+            self._cmd.poutput("  reserved_file_size: %s" % str(self._cmd.lchan.selected_file_reserved_file_size()))
+            self._cmd.poutput("  maximum_file_size: %s" % str(self._cmd.lchan.selected_file_maximum_file_size()))
 
     verify_adm_parser = argparse.ArgumentParser()
-    verify_adm_parser.add_argument('ADM1', nargs='?', type=is_hexstr_or_decimal,
-                                   help='ADM1 pin value. If none given, CSV file will be queried')
+    verify_adm_parser.add_argument('--pin-is-hex', action='store_true',
+                                   help='ADM pin value is specified as hex-string (not decimal)')
+    verify_adm_parser.add_argument('--adm-type',
+                                   choices=[x for x in pin_names.values() if x.startswith('ADM')],
+                                   help='Override ADM number. Default is card-model-specific, usually 1')
+    verify_adm_parser.add_argument('ADM', nargs='?', type=is_hexstr_or_decimal,
+                                   help='ADM pin value. If none given, CSV file will be queried')
 
     @cmd2.with_argparser(verify_adm_parser)
     def do_verify_adm(self, opts):
         """Verify the ADM (Administrator) PIN specified as argument.  This is typically needed in order
-to get write/update permissions to most of the files on SIM cards.
-
-Currently only ADM1 is supported."""
-        if opts.ADM1:
-            # use specified ADM-PIN
-            pin_adm = sanitize_pin_adm(opts.ADM1)
+        to get write/update permissions to most of the files on SIM cards.
+        """
+        if opts.adm_type:
+            # pylint: disable=unsubscriptable-object
+            adm_chv_num = pin_names.inverse[opts.adm_type]
         else:
-            # try to find an ADM-PIN if none is specified
-            result = card_key_provider_get_field(
-                'ADM1', key='ICCID', value=self._cmd.iccid)
-            pin_adm = sanitize_pin_adm(result)
-            if pin_adm:
-                self._cmd.poutput(
-                    "found ADM-PIN '%s' for ICCID '%s'" % (result, self._cmd.iccid))
+            adm_chv_num = self._cmd.card._adm_chv_num
+        if opts.ADM:
+            # use specified ADM-PIN
+            if opts.pin_is_hex:
+                pin_adm = sanitize_pin_adm(None, opts.ADM)
             else:
-                raise ValueError(
-                    "cannot find ADM-PIN for ICCID '%s'" % (self._cmd.iccid))
+                pin_adm = sanitize_pin_adm(opts.ADM)
+        else:
+            iccid = self._cmd.rs.identity['ICCID']
+            adm_type = opts.adm_type or 'ADM1'
+            # try to find an ADM-PIN if none is specified
+            result = card_key_provider_get_field(adm_type, key='ICCID', value=iccid)
+            if opts.pin_is_hex or (result and len(result) > 8):
+                pin_adm = sanitize_pin_adm(None, result)
+            else:
+                pin_adm = sanitize_pin_adm(result)
+            if pin_adm:
+                self._cmd.poutput("found %s '%s' for ICCID '%s'" % (adm_type, result, iccid))
+            else:
+                raise ValueError("cannot find %s for ICCID '%s'" % (adm_type, iccid))
 
         if pin_adm:
-            self._cmd.lchan.scc.verify_chv(self._cmd.card._adm_chv_num, h2b(pin_adm))
+            self._cmd.lchan.scc.verify_chv(adm_chv_num, h2b(pin_adm))
         else:
             raise ValueError("error: cannot authenticate, no adm-pin!")
 
@@ -761,13 +858,17 @@ Currently only ADM1 is supported."""
         """Display information about the currently inserted card"""
         self._cmd.poutput("Card info:")
         self._cmd.poutput(" Name: %s" % self._cmd.card.name)
-        self._cmd.poutput(" ATR: %s" % b2h(self._cmd.lchan.scc.get_atr()))
-        self._cmd.poutput(" ICCID: %s" % self._cmd.iccid)
-        self._cmd.poutput(" Class-Byte: %s" % self._cmd.lchan.scc.cla_byte)
-        self._cmd.poutput(" Select-Ctrl: %s" % self._cmd.lchan.scc.sel_ctrl)
-        self._cmd.poutput(" AIDs:")
-        for a in self._cmd.rs.mf.applications:
-                self._cmd.poutput("  %s" % a)
+        self._cmd.poutput(" ATR: %s" % self._cmd.rs.identity['ATR'].lower())
+        eid = self._cmd.rs.identity.get('EID', None)
+        if eid:
+            self._cmd.poutput(" EID: %s" % eid.lower())
+        self._cmd.poutput(" ICCID: %s" % self._cmd.rs.identity['ICCID'].lower())
+        self._cmd.poutput(" Class-Byte: %s" % self._cmd.lchan.scc.cla_byte.lower())
+        self._cmd.poutput(" Select-Ctrl: %s" % self._cmd.lchan.scc.sel_ctrl.lower())
+        if len(self._cmd.rs.mf.applications) > 0:
+            self._cmd.poutput(" AIDs:")
+            for a in self._cmd.rs.mf.applications:
+                self._cmd.poutput("  %s" % a.lower())
 
 @with_default_category('ISO7816 Commands')
 class Iso7816Commands(CommandSet):
@@ -792,69 +893,64 @@ class Iso7816Commands(CommandSet):
         index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
         return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
 
-    def get_code(self, code):
-        """Use code either directly or try to get it from external data source"""
-        auto = ('PIN1', 'PIN2', 'PUK1', 'PUK2')
-
-        if str(code).upper() not in auto:
+    def get_code(self, code, field):
+        """Use code either directly or try to get it from external data source using the provided field name"""
+        if code is not None:
             return sanitize_pin_adm(code)
-
-        result = card_key_provider_get_field(
-            str(code), key='ICCID', value=self._cmd.iccid)
+        iccid = self._cmd.rs.identity['ICCID']
+        result = card_key_provider_get_field(field, key='ICCID', value=iccid)
         result = sanitize_pin_adm(result)
         if result:
-            self._cmd.poutput("found %s '%s' for ICCID '%s'" %
-                              (code.upper(), result, self._cmd.iccid))
+            self._cmd.poutput("found %s '%s' for ICCID '%s'" % (field, result, iccid))
         else:
-            self._cmd.poutput("cannot find %s for ICCID '%s'" %
-                              (code.upper(), self._cmd.iccid))
+            raise RuntimeError("cannot find %s for ICCID '%s'" % (field, iccid))
         return result
 
     verify_chv_parser = argparse.ArgumentParser()
     verify_chv_parser.add_argument(
         '--pin-nr', type=int, default=1, help='PIN Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
-    verify_chv_parser.add_argument(
-        'pin_code', type=is_decimal, help='PIN code digits, \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+    verify_chv_parser.add_argument('PIN', nargs='?', type=is_decimal,
+                                   help='PIN code value. If none given, CSV file will be queried')
 
     @cmd2.with_argparser(verify_chv_parser)
     def do_verify_chv(self, opts):
         """Verify (authenticate) using specified CHV (PIN) code, which is how the specifications
         call it if you authenticate yourself using the specified PIN.  There usually is at least PIN1 and
         PIN2."""
-        pin = self.get_code(opts.pin_code)
+        pin = self.get_code(opts.PIN, "PIN" + str(opts.pin_nr))
         (data, sw) = self._cmd.lchan.scc.verify_chv(opts.pin_nr, h2b(pin))
         self._cmd.poutput("CHV verification successful")
 
     unblock_chv_parser = argparse.ArgumentParser()
     unblock_chv_parser.add_argument(
         '--pin-nr', type=int, default=1, help='PUK Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
-    unblock_chv_parser.add_argument(
-        'puk_code', type=is_decimal, help='PUK code digits \"PUK1\" or \"PUK2\" to get PUK code from external data source')
-    unblock_chv_parser.add_argument(
-        'new_pin_code', type=is_decimal, help='PIN code digits \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+    unblock_chv_parser.add_argument('PUK', nargs='?', type=is_decimal,
+                                   help='PUK code value. If none given, CSV file will be queried')
+    unblock_chv_parser.add_argument('NEWPIN', nargs='?', type=is_decimal,
+                                   help='PIN code value. If none given, CSV file will be queried')
 
     @cmd2.with_argparser(unblock_chv_parser)
     def do_unblock_chv(self, opts):
         """Unblock PIN code using specified PUK code"""
-        new_pin = self.get_code(opts.new_pin_code)
-        puk = self.get_code(opts.puk_code)
+        new_pin = self.get_code(opts.NEWPIN, "PIN" + str(opts.pin_nr))
+        puk = self.get_code(opts.PUK, "PUK" + str(opts.pin_nr))
         (data, sw) = self._cmd.lchan.scc.unblock_chv(
             opts.pin_nr, h2b(puk), h2b(new_pin))
         self._cmd.poutput("CHV unblock successful")
 
     change_chv_parser = argparse.ArgumentParser()
+    change_chv_parser.add_argument('NEWPIN', nargs='?', type=is_decimal,
+                                   help='PIN code value. If none given, CSV file will be queried')
+    change_chv_parser.add_argument('PIN', nargs='?', type=is_decimal,
+                                   help='PIN code value. If none given, CSV file will be queried')
     change_chv_parser.add_argument(
         '--pin-nr', type=int, default=1, help='PUK Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
-    change_chv_parser.add_argument(
-        'pin_code', type=is_decimal, help='PIN code digits \"PIN1\" or \"PIN2\" to get PIN code from external data source')
-    change_chv_parser.add_argument(
-        'new_pin_code', type=is_decimal, help='PIN code digits \"PIN1\" or \"PIN2\" to get PIN code from external data source')
 
     @cmd2.with_argparser(change_chv_parser)
     def do_change_chv(self, opts):
         """Change PIN code to a new PIN code"""
-        new_pin = self.get_code(opts.new_pin_code)
-        pin = self.get_code(opts.pin_code)
+        new_pin = self.get_code(opts.NEWPIN, "PIN" + str(opts.pin_nr))
+        pin = self.get_code(opts.PIN, "PIN" + str(opts.pin_nr))
         (data, sw) = self._cmd.lchan.scc.change_chv(
             opts.pin_nr, h2b(pin), h2b(new_pin))
         self._cmd.poutput("CHV change successful")
@@ -862,26 +958,26 @@ class Iso7816Commands(CommandSet):
     disable_chv_parser = argparse.ArgumentParser()
     disable_chv_parser.add_argument(
         '--pin-nr', type=int, default=1, help='PIN Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
-    disable_chv_parser.add_argument(
-        'pin_code', type=is_decimal, help='PIN code digits, \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+    disable_chv_parser.add_argument('PIN', nargs='?', type=is_decimal,
+                                   help='PIN code value. If none given, CSV file will be queried')
 
     @cmd2.with_argparser(disable_chv_parser)
     def do_disable_chv(self, opts):
         """Disable PIN code using specified PIN code"""
-        pin = self.get_code(opts.pin_code)
+        pin = self.get_code(opts.PIN, "PIN" + str(opts.pin_nr))
         (data, sw) = self._cmd.lchan.scc.disable_chv(opts.pin_nr, h2b(pin))
         self._cmd.poutput("CHV disable successful")
 
     enable_chv_parser = argparse.ArgumentParser()
     enable_chv_parser.add_argument(
         '--pin-nr', type=int, default=1, help='PIN Number, 1=PIN1, 2=PIN2 or custom value (decimal)')
-    enable_chv_parser.add_argument(
-        'pin_code', type=is_decimal, help='PIN code digits, \"PIN1\" or \"PIN2\" to get PIN code from external data source')
+    enable_chv_parser.add_argument('PIN', nargs='?', type=is_decimal,
+                                   help='PIN code value. If none given, CSV file will be queried')
 
     @cmd2.with_argparser(enable_chv_parser)
     def do_enable_chv(self, opts):
         """Enable PIN code using specified PIN code"""
-        pin = self.get_code(opts.pin_code)
+        pin = self.get_code(opts.PIN, "PIN" + str(opts.pin_nr))
         (data, sw) = self._cmd.lchan.scc.enable_chv(opts.pin_nr, h2b(pin))
         self._cmd.poutput("CHV enable successful")
 
@@ -894,14 +990,14 @@ class Iso7816Commands(CommandSet):
     @cmd2.with_argparser(activate_file_parser)
     def do_activate_file(self, opts):
         """Activate the specified EF by sending an ACTIVATE FILE apdu command (used to be called REHABILITATE
-in TS 11.11 for classic SIM).
+        in TS 11.11 for classic SIM).
 
-This command is used to (re-)activate a file that is currently in deactivated (sometimes also called
-"invalidated") state.  You need to call this from the DF above the to-be-activated EF and specify the name or
-FID of the file to activate.
+        This command is used to (re-)activate a file that is currently in deactivated (sometimes also called
+        "invalidated") state.  You need to call this from the DF above the to-be-activated EF and specify the name or
+        FID of the file to activate.
 
-Note that for *deactivation* the to-be-deactivated EF must be selected, but for *activation*, the DF
-above the to-be-activated EF must be selected!"""
+        Note that for *deactivation* the to-be-deactivated EF must be selected, but for *activation*, the DF
+        above the to-be-activated EF must be selected!"""
         (data, sw) = self._cmd.lchan.activate_file(opts.NAME)
 
     def complete_activate_file(self, text, line, begidx, endidx) -> List[str]:
@@ -911,7 +1007,7 @@ above the to-be-activated EF must be selected!"""
 
     open_chan_parser = argparse.ArgumentParser()
     open_chan_parser.add_argument(
-        'chan_nr', type=int, default=0, help='Channel Number')
+        'chan_nr', type=int, default=1, choices=range(1,16), help='Channel Number')
 
     @cmd2.with_argparser(open_chan_parser)
     def do_open_channel(self, opts):
@@ -923,7 +1019,7 @@ above the to-be-activated EF must be selected!"""
 
     close_chan_parser = argparse.ArgumentParser()
     close_chan_parser.add_argument(
-        'chan_nr', type=int, default=0, help='Channel Number')
+        'chan_nr', type=int, default=1, choices=range(1,16), help='Channel Number')
 
     @cmd2.with_argparser(close_chan_parser)
     def do_close_channel(self, opts):
@@ -935,14 +1031,14 @@ above the to-be-activated EF must be selected!"""
 
     switch_chan_parser = argparse.ArgumentParser()
     switch_chan_parser.add_argument(
-        'chan_nr', type=int, default=0, help='Channel Number')
+        'chan_nr', type=int, default=0, choices=range(0,16), help='Channel Number')
 
     @cmd2.with_argparser(switch_chan_parser)
     def do_switch_channel(self, opts):
         """Switch currently active logical channel."""
-        self._cmd.lchan._select_pre(self._cmd)
+        self._cmd.lchan.unregister_cmds(self._cmd)
         self._cmd.lchan = self._cmd.rs.lchan[opts.chan_nr]
-        self._cmd.lchan._select_post(self._cmd)
+        self._cmd.lchan.register_cmds(self._cmd)
         self._cmd.update_prompt()
 
     def do_status(self, opts):
@@ -968,8 +1064,14 @@ global_group.add_argument('--script', metavar='PATH', default=None,
                           help='script with pySim-shell commands to be executed automatically at start-up')
 global_group.add_argument('--csv', metavar='FILE',
                           default=None, help='Read card data from CSV file')
+global_group.add_argument('--csv-column-key', metavar='FIELD:AES_KEY_HEX', default=[], action='append',
+                          help='per-CSV-column AES transport key')
 global_group.add_argument("--card_handler", dest="card_handler_config", metavar="FILE",
                           help="Use automatic card handling machine")
+global_group.add_argument("--noprompt", help="Run in non interactive mode",
+                          action='store_true', default=False)
+global_group.add_argument("--skip-card-init", help="Skip all card/profile initialization",
+                          action='store_true', default=False)
 
 adm_group = global_group.add_mutually_exclusive_group()
 adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM1', dest='pin_adm', default=None,
@@ -977,6 +1079,8 @@ adm_group.add_argument('-a', '--pin-adm', metavar='PIN_ADM1', dest='pin_adm', de
 adm_group.add_argument('-A', '--pin-adm-hex', metavar='PIN_ADM1_HEX', dest='pin_adm_hex', default=None,
                        help='ADM PIN used for provisioning, as hex string (16 characters long)')
 
+option_parser.add_argument('-e', '--execute-command', action='append', default=[],
+                           help='A pySim-shell command that will be executed at startup')
 option_parser.add_argument("command", nargs='?',
                            help="A pySim-shell command that would optionally be executed at startup")
 option_parser.add_argument('command_args', nargs=argparse.REMAINDER,
@@ -985,22 +1089,20 @@ option_parser.add_argument('command_args', nargs=argparse.REMAINDER,
 
 if __name__ == '__main__':
 
-    # Parse options
+    startup_errors = False
     opts = option_parser.parse_args()
 
-    # If a script file is specified, be sure that it actually exists
-    if opts.script:
-        if not os.access(opts.script, os.R_OK):
-            print("Invalid script file!")
-            sys.exit(2)
-
     # Register csv-file as card data provider, either from specified CSV
     # or from CSV file in home directory
+    csv_column_keys = {}
+    for par in opts.csv_column_key:
+        name, key = par.split(':')
+        csv_column_keys[name] = key
     csv_default = str(Path.home()) + "/.osmocom/pysim/card_data.csv"
     if opts.csv:
-        card_key_provider_register(CardKeyProviderCsv(opts.csv))
+        card_key_provider_register(CardKeyProviderCsv(opts.csv, csv_column_keys))
     if os.path.isfile(csv_default):
-        card_key_provider_register(CardKeyProviderCsv(csv_default))
+        card_key_provider_register(CardKeyProviderCsv(csv_default, csv_column_keys))
 
     # Init card reader driver
     sl = init_reader(opts, proactive_handler = Proact())
@@ -1015,20 +1117,19 @@ if __name__ == '__main__':
     # is no card in the reader or the card is unresponsive. PysimApp is
     # able to tolerate and recover from that.
     try:
-        rs, card = init_card(sl)
-        app = PysimApp(card, rs, sl, ch, opts.script)
+        rs, card = init_card(sl, opts.skip_card_init)
+        app = PysimApp(card, rs, sl, ch)
     except:
+        startup_errors = True
         print("Card initialization (%s) failed with an exception:" % str(sl))
         print("---------------------8<---------------------")
         traceback.print_exc()
         print("---------------------8<---------------------")
-        print("(you may still try to recover from this manually by using the 'equip' command.)")
-        print(
-            " it should also be noted that some readers may behave strangely when no card")
-        print(" is inserted.)")
-        print("")
-        if opts.script:
-            print("will not execute startup script due to card initialization errors!")
+        if not opts.noprompt:
+            print("(you may still try to recover from this manually by using the 'equip' command.)")
+            print(" it should also be noted that some readers may behave strangely when no card")
+            print(" is inserted.)")
+            print("")
         app = PysimApp(None, None, sl, ch)
 
     # If the user supplies an ADM PIN at via commandline args authenticate
@@ -1040,9 +1141,38 @@ if __name__ == '__main__':
         try:
             card._scc.verify_chv(card._adm_chv_num, h2b(pin_adm))
         except Exception as e:
+            startup_errors = True
+            print("ADM verification (%s) failed with an exception:" % str(pin_adm))
+            print("---------------------8<---------------------")
             print(e)
+            print("---------------------8<---------------------")
 
+    # Run optional commands
+    for c in opts.execute_command:
+        if not startup_errors:
+            app.onecmd_plus_hooks(c)
+        else:
+            print("Errors during startup, refusing to execute command (%s)" % c)
+
+    # Run optional command
     if opts.command:
-        app.onecmd_plus_hooks('{} {}'.format(opts.command, ' '.join(opts.command_args)))
-    else:
+        if not startup_errors:
+            app.onecmd_plus_hooks('{} {}'.format(opts.command, ' '.join(opts.command_args)))
+        else:
+            print("Errors during startup, refusing to execute command (%s)" % opts.command)
+
+    # Run optional script file
+    if opts.script:
+        if not startup_errors:
+            if not os.access(opts.script, os.R_OK):
+                print("Error: script file (%s) not readable!" % opts.script)
+                startup_errors = True
+            else:
+                app.onecmd_plus_hooks('{} {}'.format('run_script', opts.script), add_to_history = False)
+        else:
+            print("Errors during startup, refusing to execute script (%s)" % opts.script)
+
+    if not opts.noprompt:
         app.cmdloop()
+    elif startup_errors:
+        sys.exit(2)

pySim-trace.py

@@ -8,17 +8,21 @@ from pprint import pprint as pp
 from pySim.apdu import *
 from pySim.runtime import RuntimeState
 
+from osmocom.utils import JsonEncoder
+
 from pySim.cards import UiccCardBase
 from pySim.commands import SimCardCommands
 from pySim.profile import CardProfile
-from pySim.ts_102_221 import CardProfileUICC
+from pySim.profile.ts_102_221 import CardProfileUICC
 from pySim.ts_31_102 import CardApplicationUSIM
 from pySim.ts_31_103 import CardApplicationISIM
+from pySim.euicc import CardApplicationISDR, CardApplicationECASD
 from pySim.transport import LinkBase
 
 from pySim.apdu_source.gsmtap import GsmtapApduSource
 from pySim.apdu_source.pyshark_rspro import PysharkRsproPcap, PysharkRsproLive
 from pySim.apdu_source.pyshark_gsmtap import PysharkGsmtapPcap
+from pySim.apdu_source.tca_loader_log import TcaLoaderLogApduSource
 
 from pySim.apdu.ts_102_221 import UiccSelect, UiccStatus
 
@@ -51,7 +55,7 @@ class DummySimLink(LinkBase):
     def __str__(self):
         return "dummy"
 
-    def _send_apdu_raw(self, pdu):
+    def _send_apdu(self, pdu):
         #print("DummySimLink-apdu: %s" % pdu)
         return [], '9000'
 
@@ -61,7 +65,7 @@ class DummySimLink(LinkBase):
     def disconnect(self):
         pass
 
-    def reset_card(self):
+    def _reset_card(self):
         return 1
 
     def get_atr(self):
@@ -78,6 +82,8 @@ class Tracer:
         profile = CardProfileUICC()
         profile.add_application(CardApplicationUSIM())
         profile.add_application(CardApplicationISIM())
+        profile.add_application(CardApplicationISDR())
+        profile.add_application(CardApplicationECASD())
         scc = SimCardCommands(transport=DummySimLink())
         card = UiccCardBase(scc)
         self.rs = RuntimeState(card, profile)
@@ -93,7 +99,8 @@ class Tracer:
         """Output a single decoded + processed ApduCommand."""
         if self.show_raw_apdu:
             print(apdu)
-        print("%02u %-16s %-35s %-8s %s %s" % (inst.lchan_nr, inst._name, inst.path_str, inst.col_id, inst.col_sw, inst.processed))
+        print("%02u %-16s %-35s %-8s %s %s" % (inst.lchan_nr, inst._name, inst.path_str, inst.col_id,
+                                               inst.col_sw, json.dumps(inst.processed, cls=JsonEncoder)))
         print("===============================")
 
     def format_reset(self, apdu: CardReset):
@@ -178,6 +185,11 @@ parser_rspro_pyshark_live = subparsers.add_parser('rspro-pyshark-live', help="""
 parser_rspro_pyshark_live.add_argument('-i', '--interface', required=True,
                                        help='Name of the network interface to capture on')
 
+parser_tcaloader_log = subparsers.add_parser('tca-loader-log', help="""
+    Read APDUs from a TCA Loader log file.""")
+parser_tcaloader_log.add_argument('-f', '--log-file', required=True,
+                                  help='Name of te log file to be read')
+
 if __name__ == '__main__':
 
     opts = option_parser.parse_args()
@@ -191,6 +203,10 @@ if __name__ == '__main__':
         s = PysharkRsproLive(opts.interface)
     elif opts.source == 'gsmtap-pyshark-pcap':
         s = PysharkGsmtapPcap(opts.pcap_file)
+    elif opts.source == 'tca-loader-log':
+        s = TcaLoaderLogApduSource(opts.log_file)
+    else:
+        raise ValueError("unsupported source %s", opts.source)
 
     tracer = Tracer(source=s, suppress_status=opts.suppress_status, suppress_select=opts.suppress_select,
                     show_raw_apdu=opts.show_raw_apdu)

pySim/apdu/__init__.py

@@ -9,7 +9,7 @@ is far too simplistic, while this decoder can utilize all of the information
 we already know in pySim about the filesystem structure, file encoding, etc.
 """
 
-# (C) 2022 by Harald Welte <laforge@osmocom.org>
+# (C) 2022-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -29,12 +29,11 @@ import abc
 import typing
 from typing import List, Dict, Optional
 from termcolor import colored
-
 from construct import Byte, GreedyBytes
 from construct import Optional as COptional
+from osmocom.construct import *
+from osmocom.utils import *
 
-from pySim.construct import *
-from pySim.utils import *
 from pySim.runtime import RuntimeLchan, RuntimeState, lchan_nr_from_cla
 from pySim.filesystem import CardADF, CardFile, TransparentEF, LinFixedEF
 
@@ -150,8 +149,10 @@ class ApduCommand(Apdu, metaclass=ApduCommandMeta):
     # fall-back constructs if the derived class provides no override
     _construct_p1 = Byte
     _construct_p2 = Byte
-    _construct = HexAdapter(GreedyBytes)
-    _construct_rsp = HexAdapter(GreedyBytes)
+    _construct = GreedyBytes
+    _construct_rsp = GreedyBytes
+    _tlv = None
+    _tlv_rsp = None
 
     def __init__(self, cmd: BytesOrHex, rsp: Optional[BytesOrHex] = None):
         """Instantiate a new ApduCommand from give cmd + resp."""
@@ -270,7 +271,7 @@ class ApduCommand(Apdu, metaclass=ApduCommandMeta):
         """Does the given CLA match the CLA list of the command?."""
         if not isinstance(cla, str):
             cla = '%02X' % cla
-        cla = cla.lower()
+        cla = cla.upper()
         # see https://github.com/PyCQA/pylint/issues/7219
         # pylint: disable=no-member
         for cla_match in cls._cla:
@@ -280,7 +281,7 @@ class ApduCommand(Apdu, metaclass=ApduCommandMeta):
                     cla_masked += 'X'
                 else:
                     cla_masked += cla[i]
-            if cla_masked == cla_match:
+            if cla_masked == cla_match.upper():
                 return True
         return False
 
@@ -290,17 +291,26 @@ class ApduCommand(Apdu, metaclass=ApduCommandMeta):
         if callable(method):
             return method()
         else:
-            r = {}
-            method = getattr(self, '_decode_p1p2', None)
-            if callable(method):
-                r = self._decode_p1p2()
+            return self._cmd_to_dict()
+
+    def _cmd_to_dict(self) -> Dict:
+        """back-end function performing automatic decoding using _construct / _tlv."""
+        r = {}
+        method = getattr(self, '_decode_p1p2', None)
+        if callable(method):
+            r = self._decode_p1p2()
+        else:
+            r['p1'] = parse_construct(self._construct_p1, self.p1.to_bytes(1, 'big'))
+            r['p2'] = parse_construct(self._construct_p2, self.p2.to_bytes(1, 'big'))
+        r['p3'] = self.p3
+        if self.cmd_data:
+            if self._tlv:
+                ie = self._tlv()
+                ie.from_tlv(self.cmd_data)
+                r['body'] = ie.to_dict()
             else:
-                r['p1'] = parse_construct(self._construct_p1, self.p1.to_bytes(1, 'big'))
-                r['p2'] = parse_construct(self._construct_p2, self.p2.to_bytes(1, 'big'))
-            r['p3'] = self.p3
-            if self.cmd_data:
                 r['body'] = parse_construct(self._construct, self.cmd_data)
-            return r
+        return r
 
     def rsp_to_dict(self) -> Dict:
         """Convert the Response part of the APDU to a dict."""
@@ -310,7 +320,12 @@ class ApduCommand(Apdu, metaclass=ApduCommandMeta):
         else:
             r = {}
             if self.rsp_data:
-                r['body'] = parse_construct(self._construct_rsp, self.rsp_data)
+                if self._tlv_rsp:
+                    ie = self._tlv_rsp()
+                    ie.from_tlv(self.rsp_data)
+                    r['body'] = ie.to_dict()
+                else:
+                    r['body'] = parse_construct(self._construct_rsp, self.rsp_data)
             r['sw'] = b2h(self.sw)
             return r
 

pySim/apdu/global_platform.py

@@ -1,7 +1,7 @@
 # coding=utf-8
 """APDU definition/decoder of GlobalPLatform Card Spec (currently 2.1.1)
 
-(C) 2022 by Harald Welte <laforge@osmocom.org>
+(C) 2022-2024 by Harald Welte <laforge@osmocom.org>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -17,7 +17,11 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+from construct import FlagsEnum, Struct
+from osmocom.tlv import flatten_dict_lists
+from osmocom.construct import *
 from pySim.apdu import ApduCommand, ApduCommandSet
+from pySim.global_platform import InstallParameters
 
 class GpDelete(ApduCommand, n='DELETE', ins=0xE4, cla=['8X', 'CX', 'EX']):
     _apdu_case = 4
@@ -40,8 +44,29 @@ class GpGetDataCB(ApduCommand, n='GET DATA', ins=0xCB, cla=['8X', 'CX', 'EX']):
 class GpGetStatus(ApduCommand, n='GET STATUS', ins=0xF2, cla=['8X', 'CX', 'EX']):
     _apdu_case = 4
 
+# GPCS Section 11.5.2
 class GpInstall(ApduCommand, n='INSTALL', ins=0xE6, cla=['8X', 'CX', 'EX']):
     _apdu_case = 4
+    _construct_p1 = FlagsEnum(Byte, more_commands=0x80, for_registry_update=0x40,
+                              for_personalization=0x20, for_extradition=0x10,
+                              for_make_selectable=0x08, for_install=0x04, for_load=0x02)
+    _construct_p2 = Enum(Byte, no_info_provided=0x00, beginning_of_combined=0x01,
+                         end_of_combined=0x03)
+    _construct = Struct('load_file_aid'/Prefixed(Int8ub, GreedyBytes),
+                        'module_aid'/Prefixed(Int8ub, GreedyBytes),
+                        'application_aid'/Prefixed(Int8ub, GreedyBytes),
+                        'privileges'/Prefixed(Int8ub, GreedyBytes),
+                        'install_parameters'/Prefixed(Int8ub, GreedyBytes), # TODO: InstallParameters
+                        'install_token'/Prefixed(Int8ub, GreedyBytes))
+    def _decode_cmd(self):
+        # first use _construct* above
+        res = self._cmd_to_dict()
+        # then do TLV decode of install_parameters
+        ip = InstallParameters()
+        ip.from_tlv(res['body']['install_parameters'])
+        res['body']['install_parameters'] = flatten_dict_lists(ip.to_dict())
+        return res
+
 
 class GpLoad(ApduCommand, n='LOAD', ins=0xE8, cla=['8X', 'CX', 'EX']):
     _apdu_case = 4

pySim/apdu/ts_102_221.py

@@ -1,7 +1,7 @@
 # coding=utf-8
 """APDU definitions/decoders of ETSI TS 102 221, the core UICC spec.
 
-(C) 2022 by Harald Welte <laforge@osmocom.org>
+(C) 2022-2024 by Harald Welte <laforge@osmocom.org>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -22,11 +22,13 @@ import logging
 
 from construct import GreedyRange, Struct
 
-from pySim.construct import *
+from osmocom.utils import i2h
+from osmocom.construct import *
+
 from pySim.filesystem import *
 from pySim.runtime import RuntimeLchan
 from pySim.apdu import ApduCommand, ApduCommandSet
-from pySim.utils import i2h
+from pySim import cat
 
 logger = logging.getLogger(__name__)
 
@@ -98,7 +100,7 @@ class UiccSelect(ApduCommand, n='SELECT', ins=0xA4, cla=['0X', '4X', '6X']):
                 logger.warning('SELECT UNKNOWN FID %s', file_hex)
         elif mode == 'df_name':
             # Select by AID (can be sub-string!)
-            aid = self.cmd_dict['body']
+            aid = b2h(self.cmd_dict['body'])
             sels = lchan.rs.mf.get_app_selectables(['AIDS'])
             adf = self._find_aid_substr(sels, aid)
             if adf:
@@ -114,7 +116,7 @@ class UiccSelect(ApduCommand, n='SELECT', ins=0xA4, cla=['0X', '4X', '6X']):
             self.file = lchan.selected_file
             if 'body' in self.rsp_dict:
                 # not every SELECT is asking for the FCP in response...
-                return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+                return lchan.selected_file.decode_select_response(b2h(self.rsp_dict['body']))
         return None
 
 
@@ -127,7 +129,7 @@ class UiccStatus(ApduCommand, n='STATUS', ins=0xF2, cla=['8X', 'CX', 'EX']):
 
     def process_on_lchan(self, lchan):
         if self.cmd_dict['p2'] == 'response_like_select':
-            return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+            return lchan.selected_file.decode_select_response(b2h(self.rsp_dict['body']))
 
 def _decode_binary_p1p2(p1, p2) -> Dict:
     ret = {}
@@ -458,14 +460,17 @@ class TerminalProfile(ApduCommand, n='TERMINAL PROFILE', ins=0x10, cla=['80']):
 # TS 102 221 Section 11.2.2 / TS 102 223
 class Envelope(ApduCommand, n='ENVELOPE', ins=0xC2, cla=['80']):
     _apdu_case = 4
+    _tlv = cat.EventCollection
 
 # TS 102 221 Section 11.2.3 / TS 102 223
 class Fetch(ApduCommand, n='FETCH', ins=0x12, cla=['80']):
     _apdu_case = 2
+    _tlv_rsp = cat.ProactiveCommand
 
 # TS 102 221 Section 11.2.3 / TS 102 223
 class TerminalResponse(ApduCommand, n='TERMINAL RESPONSE', ins=0x14, cla=['80']):
     _apdu_case = 3
+    _tlv = cat.TerminalResponse
 
 # TS 102 221 Section 11.3.1
 class RetrieveData(ApduCommand, n='RETRIEVE DATA', ins=0xCB, cla=['8X', 'CX', 'EX']):

pySim/apdu/ts_102_222.py

@@ -0,0 +1,60 @@
+# coding=utf-8
+"""APDU definitions/decoders of ETSI TS 102 222.
+
+(C) 2022-2024 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import logging
+
+from construct import Struct
+from osmocom.construct import *
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+from pySim.ts_102_221 import FcpTemplate
+
+logger = logging.getLogger(__name__)
+
+# TS 102 222 Section 6.3
+class CreateFile(ApduCommand, n='CREATE FILE', ins=0xE0, cla=['0X', '4X', 'EX']):
+    _apdu_case = 3
+    _tlv = FcpTemplate
+
+# TS 102 222 Section 6.4
+class DeleteFile(ApduCommand, n='DELETE FILE', ins=0xE4, cla=['0X', '4X']):
+    _apdu_case = 3
+    _construct = Struct('file_id'/Bytes(2))
+
+# TS 102 222 Section 6.7
+class TerminateDF(ApduCommand, n='TERMINATE DF', ins=0xE6, cla=['0X', '4X']):
+    _apdu_case = 1
+
+# TS 102 222 Section 6.8
+class TerminateEF(ApduCommand, n='TERMINATE EF', ins=0xE8, cla=['0X', '4X']):
+    _apdu_case = 1
+
+# TS 102 222 Section 6.9
+class TerminateCardUsage(ApduCommand, n='TERMINATE CARD USAGE', ins=0xFE, cla=['0X', '4X']):
+    _apdu_case = 1
+
+# TS 102 222 Section 6.10
+class ResizeFile(ApduCommand, n='RESIZE FILE', ins=0xD4, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+    _construct_p1 = Enum(Byte, mode_0=0, mode_1=1)
+    _tlv = FcpTemplate
+
+
+ApduCommands = ApduCommandSet('TS 102 222', cmds=[CreateFile, DeleteFile, TerminateDF,
+                                                  TerminateEF, TerminateCardUsage, ResizeFile])

pySim/apdu/ts_31_102.py

@@ -11,9 +11,9 @@ from typing import Dict
 
 from construct import BitStruct, Enum, BitsInteger, Int8ub, Bytes, this, Struct, If, Switch, Const
 from construct import Optional as COptional
+from osmocom.construct import *
 
 from pySim.filesystem import *
-from pySim.construct import *
 from pySim.ts_31_102 import SUCI_TlvDataObject
 from pySim.apdu import ApduCommand, ApduCommandSet
 
@@ -42,28 +42,28 @@ class UsimAuthenticateEven(ApduCommand, n='AUTHENTICATE', ins=0x88, cla=['0X', '
                               BitsInteger(4),
                               'authentication_context'/Enum(BitsInteger(3), gsm=0, umts=1,
                                                             vgcs_vbs=2, gba=4))
-    _cs_cmd_gsm_3g =  Struct('_rand_len'/Int8ub, 'rand'/HexAdapter(Bytes(this._rand_len)),
-                         '_autn_len'/COptional(Int8ub), 'autn'/If(this._autn_len, HexAdapter(Bytes(this._autn_len))))
-    _cs_cmd_vgcs = Struct('_vsid_len'/Int8ub, 'vservice_id'/HexAdapter(Bytes(this._vsid_len)),
-                          '_vkid_len'/Int8ub, 'vk_id'/HexAdapter(Bytes(this._vkid_len)),
-                          '_vstk_rand_len'/Int8ub, 'vstk_rand'/HexAdapter(Bytes(this._vstk_rand_len)))
-    _cmd_gba_bs = Struct('_rand_len'/Int8ub, 'rand'/HexAdapter(Bytes(this._rand_len)),
-                         '_autn_len'/Int8ub, 'autn'/HexAdapter(Bytes(this._autn_len)))
-    _cmd_gba_naf = Struct('_naf_id_len'/Int8ub, 'naf_id'/HexAdapter(Bytes(this._naf_id_len)),
-                          '_impi_len'/Int8ub, 'impi'/HexAdapter(Bytes(this._impi_len)))
+    _cs_cmd_gsm_3g =  Struct('_rand_len'/Int8ub, 'rand'/Bytes(this._rand_len),
+                         '_autn_len'/COptional(Int8ub), 'autn'/If(this._autn_len, Bytes(this._autn_len)))
+    _cs_cmd_vgcs = Struct('_vsid_len'/Int8ub, 'vservice_id'/Bytes(this._vsid_len),
+                          '_vkid_len'/Int8ub, 'vk_id'/Bytes(this._vkid_len),
+                          '_vstk_rand_len'/Int8ub, 'vstk_rand'/Bytes(this._vstk_rand_len))
+    _cmd_gba_bs = Struct('_rand_len'/Int8ub, 'rand'/Bytes(this._rand_len),
+                         '_autn_len'/Int8ub, 'autn'/Bytes(this._autn_len))
+    _cmd_gba_naf = Struct('_naf_id_len'/Int8ub, 'naf_id'/Bytes(this._naf_id_len),
+                          '_impi_len'/Int8ub, 'impi'/Bytes(this._impi_len))
     _cs_cmd_gba = Struct('tag'/Int8ub, 'body'/Switch(this.tag, { 0xDD: 'bootstrap'/_cmd_gba_bs,
                                                                  0xDE: 'naf_derivation'/_cmd_gba_naf }))
-    _cs_rsp_gsm = Struct('_len_sres'/Int8ub, 'sres'/HexAdapter(Bytes(this._len_sres)),
-                         '_len_kc'/Int8ub, 'kc'/HexAdapter(Bytes(this._len_kc)))
-    _rsp_3g_ok = Struct('_len_res'/Int8ub, 'res'/HexAdapter(Bytes(this._len_res)),
-                        '_len_ck'/Int8ub, 'ck'/HexAdapter(Bytes(this._len_ck)),
-                        '_len_ik'/Int8ub, 'ik'/HexAdapter(Bytes(this._len_ik)),
-                        '_len_kc'/COptional(Int8ub), 'kc'/If(this._len_kc, HexAdapter(Bytes(this._len_kc))))
-    _rsp_3g_sync = Struct('_len_auts'/Int8ub, 'auts'/HexAdapter(Bytes(this._len_auts)))
+    _cs_rsp_gsm = Struct('_len_sres'/Int8ub, 'sres'/Bytes(this._len_sres),
+                         '_len_kc'/Int8ub, 'kc'/Bytes(this._len_kc))
+    _rsp_3g_ok = Struct('_len_res'/Int8ub, 'res'/Bytes(this._len_res),
+                        '_len_ck'/Int8ub, 'ck'/Bytes(this._len_ck),
+                        '_len_ik'/Int8ub, 'ik'/Bytes(this._len_ik),
+                        '_len_kc'/COptional(Int8ub), 'kc'/If(this._len_kc, Bytes(this._len_kc)))
+    _rsp_3g_sync = Struct('_len_auts'/Int8ub, 'auts'/Bytes(this._len_auts))
     _cs_rsp_3g = Struct('tag'/Int8ub, 'body'/Switch(this.tag, { 0xDB: 'success'/_rsp_3g_ok,
                                                                 0xDC: 'sync_fail'/_rsp_3g_sync}))
-    _cs_rsp_vgcs = Struct(Const(b'\xDB'), '_vstk_len'/Int8ub, 'vstk'/HexAdapter(Bytes(this._vstk_len)))
-    _cs_rsp_gba_naf = Struct(Const(b'\xDB'), '_ks_ext_naf_len'/Int8ub, 'ks_ext_naf'/HexAdapter(Bytes(this._ks_ext_naf_len)))
+    _cs_rsp_vgcs = Struct(Const(b'\xDB'), '_vstk_len'/Int8ub, 'vstk'/Bytes(this._vstk_len))
+    _cs_rsp_gba_naf = Struct(Const(b'\xDB'), '_ks_ext_naf_len'/Int8ub, 'ks_ext_naf'/Bytes(this._ks_ext_naf_len))
     def _decode_cmd(self) -> Dict:
         r = {}
         r['p1'] = parse_construct(self._construct_p1, self.p1.to_bytes(1, 'big'))

pySim/apdu_source/gsmtap.py

@@ -16,15 +16,16 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-from pySim.gsmtap import GsmtapSource
+from osmocom.gsmtap import GsmtapReceiver
 
 from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_102_222 import ApduCommands as UiccAdmApduCommands
 from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
 from pySim.apdu.global_platform import ApduCommands as GpApduCommands
 
 from . import ApduSource, PacketType, CardReset
 
-ApduCommands = UiccApduCommands + UsimApduCommands + GpApduCommands
+ApduCommands = UiccApduCommands + UiccAdmApduCommands + UsimApduCommands + GpApduCommands
 
 class GsmtapApduSource(ApduSource):
     """ApduSource for handling GSMTAP-SIM messages received via UDP, such as
@@ -40,7 +41,7 @@ class GsmtapApduSource(ApduSource):
             bind_port: UDP port number to which the socket should be bound (default: 4729)
         """
         super().__init__()
-        self.gsmtap = GsmtapSource(bind_ip, bind_port)
+        self.gsmtap = GsmtapReceiver(bind_ip, bind_port)
 
     def read_packet(self) -> PacketType:
         gsmtap_msg, _addr = self.gsmtap.read_packet()

pySim/apdu_source/pyshark_gsmtap.py

@@ -19,17 +19,17 @@
 import logging
 from typing import Tuple
 import pyshark
+from osmocom.gsmtap import GsmtapMessage
 
 from pySim.utils import h2b
-from pySim.gsmtap import GsmtapMessage
-
 from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_102_222 import ApduCommands as UiccAdmApduCommands
 from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
 from pySim.apdu.global_platform import ApduCommands as GpApduCommands
 
 from . import ApduSource, PacketType, CardReset
 
-ApduCommands = UiccApduCommands + UsimApduCommands + GpApduCommands
+ApduCommands = UiccApduCommands + UiccAdmApduCommands + UsimApduCommands + GpApduCommands
 
 logger = logging.getLogger(__name__)
 

pySim/apdu_source/tca_loader_log.py

@@ -0,0 +1,48 @@
+# coding=utf-8
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from pySim.utils import h2b
+
+from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_102_222 import ApduCommands as UiccAdmApduCommands
+from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
+from pySim.apdu.global_platform import ApduCommands as GpApduCommands
+
+from . import ApduSource, PacketType, CardReset
+
+ApduCommands = UiccApduCommands + UiccAdmApduCommands + UsimApduCommands + GpApduCommands
+
+class TcaLoaderLogApduSource(ApduSource):
+    """ApduSource for reading log files created by TCALoader."""
+    def __init__(self, filename:str):
+        super().__init__()
+        self.logfile = open(filename, 'r')
+
+    def read_packet(self) -> PacketType:
+        command = None
+        response = None
+        for line in self.logfile:
+            if line.startswith('Command'):
+                command = line.split()[1]
+                print("Command: '%s'" % command)
+                pass
+            elif command and line.startswith('Response'):
+                response = line.split()[1]
+                print("Response: '%s'" % response)
+                return ApduCommands.parse_cmd_bytes(h2b(command) + h2b(response))
+        raise StopIteration

pySim/app.py

@@ -19,12 +19,13 @@ from typing import Tuple
 from pySim.transport import LinkBase
 from pySim.commands import SimCardCommands
 from pySim.filesystem import CardModel, CardApplication
-from pySim.cards import card_detect, SimCardBase, UiccCardBase
+from pySim.cards import card_detect, SimCardBase, UiccCardBase, CardBase
 from pySim.runtime import RuntimeState
 from pySim.profile import CardProfile
-from pySim.cdma_ruim import CardProfileRUIM
-from pySim.ts_102_221 import CardProfileUICC
+from pySim.profile.cdma_ruim import CardProfileRUIM
+from pySim.profile.ts_102_221 import CardProfileUICC
 from pySim.utils import all_subclasses
+from pySim.exceptions import SwMatchError
 
 # we need to import this module so that the SysmocomSJA2 sub-class of
 # CardModel is created, which will add the ATR-based matching and
@@ -39,9 +40,9 @@ import pySim.ts_31_103
 import pySim.ts_31_104
 import pySim.ara_m
 import pySim.global_platform
-import pySim.euicc
+import pySim.profile.euicc
 
-def init_card(sl: LinkBase) -> Tuple[RuntimeState, SimCardBase]:
+def init_card(sl: LinkBase, skip_card_init: bool = False) -> Tuple[RuntimeState, SimCardBase]:
     """
     Detect card in reader and setup card profile and runtime state. This
     function must be called at least once on startup. The card and runtime
@@ -56,6 +57,12 @@ def init_card(sl: LinkBase) -> Tuple[RuntimeState, SimCardBase]:
     print("Waiting for card...")
     sl.wait_for_card(3)
 
+    # The user may opt to skip all card initialization. In this case only the
+    # most basic card profile is selected. This mode is suitable for blank
+    # cards that need card O/S initialization using APDU scripts first.
+    if skip_card_init:
+        return None, CardBase(scc)
+
     generic_card = False
     card = card_detect(scc)
     if card is None:
@@ -106,4 +113,16 @@ def init_card(sl: LinkBase) -> Tuple[RuntimeState, SimCardBase]:
     # inform the transport that we can do context-specific SW interpretation
     sl.set_sw_interpreter(rs)
 
+    # try to obtain the EID, if any
+    isd_r = rs.mf.applications.get(pySim.euicc.AID_ISD_R.lower(), None)
+    if isd_r:
+        rs.lchan[0].select_file(isd_r)
+        try:
+            rs.identity['EID'] = pySim.euicc.CardApplicationISDR.get_eid(scc)
+        except SwMatchError:
+            # has ISD-R but not a SGP.22/SGP.32 eUICC - maybe SGP.02?
+            pass
+        finally:
+            rs.reset()
+
     return rs, card

pySim/ara_m.py

@@ -28,10 +28,10 @@ Support for the Secure Element Access Control, specifically the ARA-M inside an
 
 from construct import GreedyBytes, GreedyString, Struct, Enum, Int8ub, Int16ub
 from construct import Optional as COptional
-from pySim.construct import *
+from osmocom.construct import *
+from osmocom.tlv import *
+from osmocom.utils import Hexstr
 from pySim.filesystem import *
-from pySim.tlv import *
-from pySim.utils import Hexstr
 import pySim.global_platform
 
 # various BER-TLV encoded Data Objects (DOs)
@@ -76,13 +76,13 @@ class ApduArDO(BER_TLV_IE, tag=0xd0):
         else:
             if len(do) % 8:
                 return ValueError('Invalid non-modulo-8 length of APDU filter: %d' % len(do))
-            self.decoded['apdu_filter'] = []
+            self.decoded = {'apdu_filter': []}
             offset = 0
             while offset < len(do):
-                self.decoded['apdu_filter'] += {'header': b2h(do[offset:offset+4]),
-                                                'mask': b2h(do[offset+4:offset+8])}
-            self.decoded = res
-            return res
+                self.decoded['apdu_filter'] += [{'header': b2h(do[offset:offset+4]),
+                                                'mask': b2h(do[offset+4:offset+8])}]
+                offset += 8 # Move offset to the beginning of the next apdu_filter object
+            return self.decoded
 
     def _to_bytes(self):
         if 'generic_access_rule' in self.decoded:
@@ -228,16 +228,14 @@ class BlockDO(BER_TLV_IE, tag=0xe7):
 class GetCommandDoCollection(TLV_IE_Collection, nested=[RefDO, DeviceConfigDO]):
     pass
 
+
 # SEID v1.1 Table 4-2
-
-
 class GetResponseDoCollection(TLV_IE_Collection, nested=[ResponseAllRefArDO, ResponseArDO,
                                                          ResponseRefreshTagDO, ResponseAramConfigDO]):
     pass
 
+
 # SEID v1.1 Table 5-1
-
-
 class StoreCommandDoCollection(TLV_IE_Collection,
                                nested=[BlockDO, CommandStoreRefArDO, CommandDelete,
                                        CommandUpdateRefreshTagDO, CommandRegisterClientAidsDO,
@@ -264,7 +262,7 @@ class ADF_ARAM(CardADF):
         return pySim.global_platform.decode_select_response(data_hex)
 
     @staticmethod
-    def xceive_apdu_tlv(tp, hdr: Hexstr, cmd_do, resp_cls, exp_sw='9000'):
+    def xceive_apdu_tlv(scc, hdr: Hexstr, cmd_do, resp_cls, exp_sw='9000'):
         """Transceive an APDU with the card, transparently encoding the command data from TLV
         and decoding the response data tlv."""
         if cmd_do:
@@ -276,7 +274,7 @@ class ADF_ARAM(CardADF):
             cmd_do_enc = b''
             cmd_do_len = 0
         c_apdu = hdr + ('%02x' % cmd_do_len) + b2h(cmd_do_enc)
-        (data, _sw) = tp.send_apdu_checksw(c_apdu, exp_sw)
+        (data, _sw) = scc.send_apdu_checksw(c_apdu, exp_sw)
         if data:
             if resp_cls:
                 resp_do = resp_cls()
@@ -287,32 +285,32 @@ class ADF_ARAM(CardADF):
             return None
 
     @staticmethod
-    def store_data(tp, do) -> bytes:
+    def store_data(scc, do) -> bytes:
         """Build the Command APDU for STORE DATA."""
-        return ADF_ARAM.xceive_apdu_tlv(tp, '80e29000', do, StoreResponseDoCollection)
+        return ADF_ARAM.xceive_apdu_tlv(scc, '80e29000', do, StoreResponseDoCollection)
 
     @staticmethod
-    def get_all(tp):
-        return ADF_ARAM.xceive_apdu_tlv(tp, '80caff40', None, GetResponseDoCollection)
+    def get_all(scc):
+        return ADF_ARAM.xceive_apdu_tlv(scc, '80caff40', None, GetResponseDoCollection)
 
     @staticmethod
-    def get_config(tp, v_major=0, v_minor=0, v_patch=1):
+    def get_config(scc, v_major=0, v_minor=0, v_patch=1):
         cmd_do = DeviceConfigDO()
-        cmd_do.from_dict([{'device_interface_version_do': {
-                         'major': v_major, 'minor': v_minor, 'patch': v_patch}}])
-        return ADF_ARAM.xceive_apdu_tlv(tp, '80cadf21', cmd_do, ResponseAramConfigDO)
+        cmd_do.from_val_dict([{'device_interface_version_do': {
+                               'major': v_major, 'minor': v_minor, 'patch': v_patch}}])
+        return ADF_ARAM.xceive_apdu_tlv(scc, '80cadf21', cmd_do, ResponseAramConfigDO)
 
     @with_default_category('Application-Specific Commands')
     class AddlShellCommands(CommandSet):
         def do_aram_get_all(self, _opts):
             """GET DATA [All] on the ARA-M Applet"""
-            res_do = ADF_ARAM.get_all(self._cmd.lchan.scc._tp)
+            res_do = ADF_ARAM.get_all(self._cmd.lchan.scc)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
         def do_aram_get_config(self, _opts):
             """Perform GET DATA [Config] on the ARA-M Applet: Tell it our version and retrieve its version."""
-            res_do = ADF_ARAM.get_config(self._cmd.lchan.scc._tp)
+            res_do = ADF_ARAM.get_config(self._cmd.lchan.scc)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
@@ -334,7 +332,7 @@ class ADF_ARAM(CardADF):
         apdu_grp.add_argument(
             '--apdu-always', action='store_true', help='APDU access is allowed')
         apdu_grp.add_argument(
-            '--apdu-filter', help='APDU filter: 4 byte CLA/INS/P1/P2 followed by 4 byte mask (8 hex bytes)')
+            '--apdu-filter', help='APDU filter: multiple groups of 8 hex bytes (4 byte CLA/INS/P1/P2 followed by 4 byte mask)')
         nfc_grp = store_ref_ar_do_parse.add_mutually_exclusive_group()
         nfc_grp.add_argument('--nfc-always', action='store_true',
                              help='NFC event access is allowed')
@@ -358,12 +356,19 @@ class ADF_ARAM(CardADF):
             # AR
             ar_do_content = []
             if opts.apdu_never:
-                ar_do_content += [{'apdu_ar_od': {'generic_access_rule': 'never'}}]
+                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
             elif opts.apdu_always:
                 ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
             elif opts.apdu_filter:
-                # TODO: multiple filters
-                ar_do_content += [{'apdu_ar_do': {'apdu_filter': [opts.apdu_filter]}}]
+                if len(opts.apdu_filter) % 16:
+                    return ValueError('Invalid non-modulo-16 length of APDU filter: %d' % len(do))
+                offset = 0
+                apdu_filter = []
+                while offset < len(opts.apdu_filter):
+                    apdu_filter += [{'header': opts.apdu_filter[offset:offset+8],
+                                     'mask': opts.apdu_filter[offset+8:offset+16]}]
+                    offset += 16 # Move offset to the beginning of the next apdu_filter object
+                ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter}}]
             if opts.nfc_always:
                 ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
             elif opts.nfc_never:
@@ -372,15 +377,15 @@ class ADF_ARAM(CardADF):
                 ar_do_content += [{'perm_ar_do': {'permissions': opts.android_permissions}}]
             d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
             csrado = CommandStoreRefArDO()
-            csrado.from_dict(d)
-            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc._tp, csrado)
+            csrado.from_val_dict(d)
+            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, csrado)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
         def do_aram_delete_all(self, _opts):
             """Perform STORE DATA [Command-Delete[all]] to delete all access rules."""
             deldo = CommandDelete()
-            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc._tp, deldo)
+            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, deldo)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
@@ -410,3 +415,78 @@ sw_aram = {
 class CardApplicationARAM(CardApplication):
     def __init__(self):
         super().__init__('ARA-M', adf=ADF_ARAM(), sw=sw_aram)
+
+    @staticmethod
+    def __export_get_from_dictlist(key, dictlist):
+        # Data objects are organized in lists that contain dictionaries, usually there is only one dictionary per
+        # list item. This function goes through that list and gets the value of the first dictionary that has the
+        # matching key.
+        if dictlist is None:
+            return None
+        obj = None
+        for d in dictlist:
+            obj = d.get(key, obj)
+        return obj
+
+    @staticmethod
+    def __export_ref_ar_do_list(ref_ar_do_list):
+        export_str = ""
+        ref_do_list = CardApplicationARAM.__export_get_from_dictlist('ref_do', ref_ar_do_list.get('ref_ar_do'))
+        ar_do_list = CardApplicationARAM.__export_get_from_dictlist('ar_do', ref_ar_do_list.get('ref_ar_do'))
+
+        if ref_do_list and ar_do_list:
+            # Get ref_do parameters
+            aid_ref_do = CardApplicationARAM.__export_get_from_dictlist('aid_ref_do', ref_do_list)
+            dev_app_id_ref_do = CardApplicationARAM.__export_get_from_dictlist('dev_app_id_ref_do', ref_do_list)
+            pkg_ref_do = CardApplicationARAM.__export_get_from_dictlist('pkg_ref_do', ref_do_list)
+
+            # Get ar_do parameters
+            apdu_ar_do = CardApplicationARAM.__export_get_from_dictlist('apdu_ar_do', ar_do_list)
+            nfc_ar_do = CardApplicationARAM.__export_get_from_dictlist('nfc_ar_do', ar_do_list)
+            perm_ar_do = CardApplicationARAM.__export_get_from_dictlist('perm_ar_do', ar_do_list)
+
+            # Write command-line
+            export_str += "aram_store_ref_ar_do"
+            if aid_ref_do:
+                export_str += (" --aid %s" % aid_ref_do)
+            else:
+                export_str += " --aid-empty"
+            if dev_app_id_ref_do:
+                export_str += (" --device-app-id %s" % dev_app_id_ref_do)
+            if apdu_ar_do and 'generic_access_rule' in apdu_ar_do:
+                export_str += (" --apdu-%s" % apdu_ar_do['generic_access_rule'])
+            elif apdu_ar_do and 'apdu_filter' in apdu_ar_do:
+                export_str += (" --apdu-filter ")
+                for apdu_filter in apdu_ar_do['apdu_filter']:
+                    export_str += apdu_filter['header']
+                    export_str += apdu_filter['mask']
+            if nfc_ar_do and 'nfc_event_access_rule' in nfc_ar_do:
+                export_str += (" --nfc-%s" % nfc_ar_do['nfc_event_access_rule'])
+            if perm_ar_do:
+                export_str += (" --android-permissions %s" % perm_ar_do['permissions'])
+            if pkg_ref_do:
+                export_str += (" --pkg-ref %s" % pkg_ref_do['package_name_string'])
+            export_str += "\n"
+        return export_str
+
+    @staticmethod
+    def export(as_json: bool, lchan):
+
+        # TODO: Add JSON output as soon as aram_store_ref_ar_do is able to process input in JSON format.
+        if as_json:
+            raise NotImplementedError("res_do encoder not yet implemented. Patches welcome.")
+
+        export_str = ""
+        export_str += "aram_delete_all\n"
+
+        res_do = ADF_ARAM.get_all(lchan.scc)
+        if not res_do:
+            return export_str.strip()
+
+        for res_do_dict in res_do.to_dict():
+            if not res_do_dict.get('response_all_ref_ar_do', False):
+                continue
+            for ref_ar_do_list in res_do_dict['response_all_ref_ar_do']:
+                export_str += CardApplicationARAM.__export_ref_ar_do_list(ref_ar_do_list)
+
+        return export_str.strip()

pySim/card_key_provider.py

@@ -10,10 +10,10 @@ the need of manually entering the related card-individual data on every
 operation with pySim-shell.
 """
 
-# (C) 2021 by Sysmocom s.f.m.c. GmbH
+# (C) 2021-2024 by Sysmocom s.f.m.c. GmbH
 # All Rights Reserved
 #
-# Author: Philipp Maier
+# Author: Philipp Maier, Harald Welte
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -29,18 +29,29 @@ operation with pySim-shell.
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from typing import List, Dict, Optional
+from Cryptodome.Cipher import AES
+from osmocom.utils import h2b, b2h
 
 import abc
 import csv
 
 card_key_providers = []  # type: List['CardKeyProvider']
 
+# well-known groups of columns relate to a given functionality.  This avoids having
+# to specify the same transport key N number of times, if the same key is used for multiple
+# fields of one group, like KIC+KID+KID of one SD.
+CRYPT_GROUPS = {
+    'UICC_SCP02': ['UICC_SCP02_KIC1', 'UICC_SCP02_KID1', 'UICC_SCP02_KIK1'],
+    'UICC_SCP03': ['UICC_SCP03_KIC1', 'UICC_SCP03_KID1', 'UICC_SCP03_KIK1'],
+    'SCP03_ISDR': ['SCP03_ENC_ISDR', 'SCP03_MAC_ISDR', 'SCP03_DEK_ISDR'],
+    'SCP03_ISDA': ['SCP03_ENC_ISDR', 'SCP03_MAC_ISDA', 'SCP03_DEK_ISDA'],
+    'SCP03_ECASD': ['SCP03_ENC_ECASD', 'SCP03_MAC_ECASD', 'SCP03_DEK_ECASD'],
+    }
 
 class CardKeyProvider(abc.ABC):
     """Base class, not containing any concrete implementation."""
 
-    VALID_FIELD_NAMES = ['ICCID', 'ADM1',
-                         'IMSI', 'PIN1', 'PIN2', 'PUK1', 'PUK2']
+    VALID_KEY_FIELD_NAMES = ['ICCID', 'EID', 'IMSI' ]
 
     # check input parameters, but do nothing concrete yet
     def _verify_get_data(self, fields: List[str] = [], key: str = 'ICCID', value: str = "") -> Dict[str, str]:
@@ -53,14 +64,10 @@ class CardKeyProvider(abc.ABC):
         Returns:
                 dictionary of {field, value} strings for each requested field from 'fields'
         """
-        for f in fields:
-            if f not in self.VALID_FIELD_NAMES:
-                raise ValueError("Requested field name '%s' is not a valid field name, valid field names are: %s" %
-                                 (f, str(self.VALID_FIELD_NAMES)))
 
-        if key not in self.VALID_FIELD_NAMES:
+        if key not in self.VALID_KEY_FIELD_NAMES:
             raise ValueError("Key field name '%s' is not a valid field name, valid field names are: %s" %
-                             (key, str(self.VALID_FIELD_NAMES)))
+                             (key, str(self.VALID_KEY_FIELD_NAMES)))
 
         return {}
 
@@ -84,19 +91,47 @@ class CardKeyProvider(abc.ABC):
 
 
 class CardKeyProviderCsv(CardKeyProvider):
-    """Card key provider implementation that allows to query against a specified CSV file"""
+    """Card key provider implementation that allows to query against a specified CSV file.
+    Supports column-based encryption as it is generally a bad idea to store cryptographic key material in
+    plaintext.  Instead, the key material should be encrypted by a "key-encryption key", occasionally also
+    known as "transport key" (see GSMA FS.28)."""
+    IV = b'\x23' * 16
     csv_file = None
     filename = None
 
-    def __init__(self, filename: str):
+    def __init__(self, filename: str, transport_keys: dict):
         """
         Args:
                 filename : file name (path) of CSV file containing card-individual key/data
+                transport_keys : a dict indexed by field name, whose values are hex-encoded AES keys for the
+                                 respective field (column) of the CSV.  This is done so that different fields
+                                 (columns) can use different transport keys, which is strongly recommended by
+                                 GSMA FS.28
         """
         self.csv_file = open(filename, 'r')
         if not self.csv_file:
             raise RuntimeError("Could not open CSV file '%s'" % filename)
         self.filename = filename
+        self.transport_keys = self.process_transport_keys(transport_keys)
+
+    @staticmethod
+    def process_transport_keys(transport_keys: dict):
+        """Apply a single transport key to multiple fields/columns, if the name is a group."""
+        new_dict = {}
+        for name, key in transport_keys.items():
+            if name in CRYPT_GROUPS:
+                for field in CRYPT_GROUPS[name]:
+                    new_dict[field] = key
+            else:
+                new_dict[name] = key
+        return new_dict
+
+    def _decrypt_field(self, field_name: str, encrypted_val: str) -> str:
+        """decrypt a single field, if we have a transport key for the field of that name."""
+        if not field_name in self.transport_keys:
+            return encrypted_val
+        cipher = AES.new(h2b(self.transport_keys[field_name]), AES.MODE_CBC, self.IV)
+        return b2h(cipher.decrypt(h2b(encrypted_val)))
 
     def get(self, fields: List[str], key: str, value: str) -> Dict[str, str]:
         super()._verify_get_data(fields, key, value)
@@ -113,7 +148,7 @@ class CardKeyProviderCsv(CardKeyProvider):
             if row[key] == value:
                 for f in fields:
                     if f in row:
-                        rc.update({f: row[f]})
+                        rc.update({f: self._decrypt_field(f, row[f])})
                     else:
                         raise RuntimeError("CSV-File '%s' lacks column '%s'" %
                                            (self.filename, f))

pySim/cards.py

@@ -23,10 +23,11 @@
 #
 
 from typing import Optional, Tuple
-from pySim.ts_102_221 import EF_DIR
-from pySim.ts_51_011 import DF_GSM
+from osmocom.utils import *
 
-from pySim.utils import *
+from pySim.profile.ts_102_221 import EF_DIR, CardProfileUICC
+from pySim.profile.ts_51_011 import DF_GSM
+from pySim.utils import SwHexstr
 from pySim.commands import Path, SimCardCommands
 
 class CardBase:
@@ -54,10 +55,17 @@ class CardBase:
         print("warning: erasing is not supported for specified card type!")
 
     def file_exists(self, fid: Path) -> bool:
+        """Determine if the file exists (and is not deactivated)."""
         res_arr = self._scc.try_select_path(fid)
         for res in res_arr:
             if res[1] != '9000':
                 return False
+        try:
+            d = CardProfileUICC.decode_select_response(res_arr[-1][0])
+            if d.get('life_cycle_status_integer', 'operational_activated') != 'operational_activated':
+                return False
+        except:
+            pass
         return True
 
     def read_aids(self) -> List[Hexstr]:

pySim/cat.py

@@ -23,38 +23,53 @@ from bidict import bidict
 from construct import Int8ub, Int16ub, Byte, Bytes, BitsInteger
 from construct import Struct, Enum, BitStruct, this
 from construct import GreedyBytes, Switch, GreedyRange, FlagsEnum
-from pySim.tlv import TLV_IE, COMPR_TLV_IE, BER_TLV_IE, TLV_IE_Collection
-from pySim.construct import PlmnAdapter, BcdAdapter, HexAdapter, GsmStringAdapter, TonNpi
-from pySim.utils import b2h, dec_xplmn_w_act
+from osmocom.tlv import TLV_IE, COMPR_TLV_IE, BER_TLV_IE, TLV_IE_Collection
+from osmocom.construct import PlmnAdapter, BcdAdapter, HexAdapter, GsmStringAdapter, TonNpi, GsmString
+from osmocom.utils import b2h
+from pySim.utils import dec_xplmn_w_act
 
 # Tag values as per TS 101 220 Table 7.23
 
 # TS 102 223 Section 8.1
-class Address(COMPR_TLV_IE, tag=0x06):
+class Address(COMPR_TLV_IE, tag=0x86):
     _construct = Struct('ton_npi'/Int8ub,
-                        'call_number'/BcdAdapter(Bytes(this._.total_len-1)))
+                        'call_number'/BcdAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.2
-class AlphaIdentifier(COMPR_TLV_IE, tag=0x05):
+class AlphaIdentifier(COMPR_TLV_IE, tag=0x85):
     # FIXME: like EF.ADN
     pass
 
 # TS 102 223 Section 8.3
-class Subaddress(COMPR_TLV_IE, tag=0x08):
+class Subaddress(COMPR_TLV_IE, tag=0x88):
     pass
 
 # TS 102 223 Section 8.4 + TS 31.111 Section 8.4
-class CapabilityConfigParams(COMPR_TLV_IE, tag=0x07):
+class CapabilityConfigParams(COMPR_TLV_IE, tag=0x87):
     pass
 
 # TS 31.111 Section 8.5
-class CBSPage(COMPR_TLV_IE, tag=0x0C):
+class CBSPage(COMPR_TLV_IE, tag=0x8C):
     pass
 
+# TS 102 223 V15.3.0 Section 9.4
+TypeOfCommand = Enum(Int8ub, refresh=0x01, more_time=0x02, poll_interval=0x03, polling_off=0x04,
+                     set_up_event_list=0x05, set_up_call=0x10, send_ss=0x11, send_ussd=0x12,
+                     send_short_message=0x13, send_dtmf=0x14, launch_browser=0x15, geo_location_req=0x16,
+                     play_tone=0x20, display_text=0x21, get_inkey=0x22, get_input=0x23, select_item=0x24,
+                     set_up_menu=0x25, provide_local_info=0x26, timer_management=0x27,
+                     set_up_idle_mode_text=0x28, perform_card_apdu=0x30, power_on_card=0x31,
+                     power_off_card=0x32, get_reader_status=0x33, run_at_command=0x34,
+                     language_notification=0x35, open_channel=0x40, close_channel=0x41, receive_data=0x42,
+                     send_data=0x43, get_channel_status=0x44, service_search=0x45, get_service_info=0x46,
+                     declare_service=0x47, set_frames=0x50, get_frames_status=0x51, retrieve_mms=0x60,
+                     submit_mms=0x61, display_mms=0x62, activate=0x70, contactless_state_changed=0x71,
+                     command_container=0x72, encapsulated_session_control=0x73)
+
 # TS 102 223 Section 8.6 + TS 31.111 Section 8.6
 class CommandDetails(COMPR_TLV_IE, tag=0x81):
     _construct = Struct('command_number'/Int8ub,
-                        'type_of_command'/Int8ub,
+                        'type_of_command'/TypeOfCommand,
                         'command_qualifier'/Int8ub)
 
 # TS 102 223 Section 8.7
@@ -107,26 +122,26 @@ class DeviceIdentities(COMPR_TLV_IE, tag=0x82):
         return bytes([src, dst])
 
 # TS 102 223 Section 8.8
-class Duration(COMPR_TLV_IE, tag=0x04):
+class Duration(COMPR_TLV_IE, tag=0x84):
     _construct = Struct('time_unit'/Enum(Int8ub, minutes=0, seconds=1, tenths_of_seconds=2),
                         'time_interval'/Int8ub)
 
 # TS 102 223 Section 8.9
-class Item(COMPR_TLV_IE, tag=0x0f):
+class Item(COMPR_TLV_IE, tag=0x8f):
     _construct = Struct('identifier'/Int8ub,
                         'text_string'/GsmStringAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.10
-class ItemIdentifier(COMPR_TLV_IE, tag=0x10):
+class ItemIdentifier(COMPR_TLV_IE, tag=0x90):
     _construct = Struct('identifier'/Int8ub)
 
 # TS 102 223 Section 8.11
-class ResponseLength(COMPR_TLV_IE, tag=0x11):
+class ResponseLength(COMPR_TLV_IE, tag=0x91):
     _construct = Struct('minimum_length'/Int8ub,
                         'maximum_length'/Int8ub)
 
 # TS 102 223 Section 8.12
-class Result(COMPR_TLV_IE, tag=0x03):
+class Result(COMPR_TLV_IE, tag=0x83):
     GeneralResult = Enum(Int8ub,
                          # '0X' and '1X' indicate that the command has been performed
                          performed_successfully=0,
@@ -252,12 +267,15 @@ class SsString(COMPR_TLV_IE, tag=0x89):
 
 
 # TS 102 223 Section 8.15
-class TextString(COMPR_TLV_IE, tag=0x0d):
+class TextString(COMPR_TLV_IE, tag=0x8D):
+    _test_de_encode = [
+        ( '8d090470617373776f7264', {'dcs': 4, 'text_string': '70617373776f7264'} ),
+    ]
     _construct = Struct('dcs'/Int8ub, # TS 03.38
                         'text_string'/HexAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.16
-class Tone(COMPR_TLV_IE, tag=0x0e):
+class Tone(COMPR_TLV_IE, tag=0x8E):
     _construct = Struct('tone'/Enum(Int8ub, dial_tone=0x01,
                                             called_subscriber_busy=0x02,
                                             congestion=0x03,
@@ -288,12 +306,12 @@ class Tone(COMPR_TLV_IE, tag=0x0e):
                                             melody_8=0x47))
 
 # TS 31 111 Section 8.17
-class USSDString(COMPR_TLV_IE, tag=0x0a):
+class USSDString(COMPR_TLV_IE, tag=0x8A):
     _construct = Struct('dcs'/Int8ub,
                         'ussd_string'/HexAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.18
-class FileList(COMPR_TLV_IE, tag=0x12):
+class FileList(COMPR_TLV_IE, tag=0x92):
     FileId=HexAdapter(Bytes(2))
     _construct = Struct('number_of_files'/Int8ub,
                         'files'/GreedyRange(FileId))
@@ -320,7 +338,7 @@ class DefaultText(COMPR_TLV_IE, tag=0x97):
                         'text_string'/HexAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.24
-class ItemsNextActionIndicator(COMPR_TLV_IE, tag=0x18):
+class ItemsNextActionIndicator(COMPR_TLV_IE, tag=0x98):
     _construct = GreedyRange(Int8ub)
 
 class EventList(COMPR_TLV_IE, tag=0x99):
@@ -365,7 +383,7 @@ class LocationStatus(COMPR_TLV_IE, tag=0x9b):
     _construct = Enum(Int8ub, normal_service=0, limited_service=1, no_service=2)
 
 # TS 102 223 Section 8.31
-class IconIdentifier(COMPR_TLV_IE, tag=0x1e):
+class IconIdentifier(COMPR_TLV_IE, tag=0x9e):
     _construct = Struct('icon_qualifier'/FlagsEnum(Int8ub, not_self_explanatory=1),
                         'icon_identifier'/Int8ub)
 
@@ -391,25 +409,48 @@ class AtCommand(COMPR_TLV_IE, tag=0xA8):
     _construct = HexAdapter(GreedyBytes)
 
 # TS 102 223 Section 8.43
-class ImmediateResponse(COMPR_TLV_IE, tag=0x2b):
+class ImmediateResponse(COMPR_TLV_IE, tag=0xAB):
     pass
 
+# TS 102 223 Section 8.44
+class DtmfString(COMPR_TLV_IE, tag=0xAC):
+    _construct = BcdAdapter(GreedyBytes)
+
 # TS 102 223 Section 8.45
 class Language(COMPR_TLV_IE, tag=0xAD):
     _construct = HexAdapter(GreedyBytes)
 
 # TS 31.111 Section 8.46
-class TimingAdvance(COMPR_TLV_IE, tag=0x46):
+class TimingAdvance(COMPR_TLV_IE, tag=0xC6):
     _construct = Struct('me_status'/Enum(Int8ub, in_idle_state=0, not_in_idle_state=1),
                         'timing_advance'/Int8ub)
 
+# TS 31.111 Section 8.47
+class BrowserIdentity(COMPR_TLV_IE, tag=0xB0):
+    _construct = Enum(Int8ub, default=0, wml=1, html=2, xhtml=3, chtml=4)
+
+# TS 31.111 Section 8.48
+class Url(COMPR_TLV_IE, tag=0xB1):
+    _construct = GsmString(GreedyBytes)
+
 # TS 31.111 Section 8.49
 class Bearer(COMPR_TLV_IE, tag=0xB2):
     SingleBearer = Enum(Int8ub, sms=0, csd=1, ussd=2, packet_Service=3)
     _construct = GreedyRange(SingleBearer)
 
+# TS 102 223 Section 8.50
+class ProvisioningFileReference(COMPR_TLV_IE, tag=0xB3):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.51
+class BrowserTerminationCause(COMPR_TLV_IE, tag=0xB4):
+    _construct = Enum(Int8ub, user_termination=0, error_termination=1)
+
 # TS 102 223 Section 8.52
 class BearerDescription(COMPR_TLV_IE, tag=0xB5):
+    _test_de_encode = [
+        ( 'b50103', {'bearer_parameters': '', 'bearer_type': 'default'} ),
+    ]
     # TS 31.111 Section 8.52.1
     BearerParsCs = Struct('data_rate'/Int8ub,
                           'bearer_service'/Int8ub,
@@ -465,25 +506,32 @@ class ChannelDataLength(COMPR_TLV_IE, tag = 0xB7):
 class BufferSize(COMPR_TLV_IE, tag = 0xB9):
     _construct = Int16ub
 
-# TS 31.111 Section 8.56
+# TS 102 223 Section 8.56 + TS 31.111 Section 8.56
 class ChannelStatus(COMPR_TLV_IE, tag = 0xB8):
     # complex decoding, depends on out-of-band context/knowledge :(
-    pass
+    # for default / TCP Client mode: bit 8 of first byte indicates connected, 3 LSB indicate channel nr
+    _construct = HexAdapter(GreedyBytes)
 
 # TS 102 223 Section 8.58
 class OtherAddress(COMPR_TLV_IE, tag = 0xBE):
+    _test_de_encode = [
+        ( 'be052101020304', {'address': '01020304', 'type_of_address': 'ipv4'} ),
+    ]
     _construct = Struct('type_of_address'/Enum(Int8ub, ipv4=0x21, ipv6=0x57),
                         'address'/HexAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.59
 class UiccTransportLevel(COMPR_TLV_IE, tag = 0xBC):
+    _test_de_encode = [
+        ( 'bc03028000', {'port_number': 32768, 'protocol_type': 'tcp_uicc_client_remote'} ),
+    ]
     _construct = Struct('protocol_type'/Enum(Int8ub, udp_uicc_client_remote=1, tcp_uicc_client_remote=2,
                                              tcp_uicc_server=3, udp_uicc_client_local=4,
                                              tcp_uicc_client_local=5, direct_channel=6),
                         'port_number'/Int16ub)
 
 # TS 102 223 Section 8.60
-class Aid(COMPR_TLV_IE, tag=0x2f):
+class Aid(COMPR_TLV_IE, tag=0xAF):
     _construct = Struct('aid'/HexAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.61
@@ -523,10 +571,13 @@ class RemoteEntityAddress(COMPR_TLV_IE, tag=0xC9):
 
 # TS 102 223 Section 8.70
 class NetworkAccessName(COMPR_TLV_IE, tag=0xC7):
+    _test_de_encode = [
+        ( 'c704036e6161', '036e6161' ),
+    ]
     _construct = HexAdapter(GreedyBytes)
 
 # TS 102 223 Section 8.72
-class TextAttribute(COMPR_TLV_IE, tag=0x50):
+class TextAttribute(COMPR_TLV_IE, tag=0xD0):
     pass
 
 # TS 31.111 Section 8.72
@@ -562,7 +613,7 @@ class ItemTextAttributeList(COMPR_TLV_IE, tag=0xD1):
     _construct = GreedyRange(Int8ub)
 
 # TS 102 223 Section 8.80
-class FrameIdentifier(COMPR_TLV_IE, tag=0x68):
+class FrameIdentifier(COMPR_TLV_IE, tag=0xE8):
     _construct = Struct('identifier'/Int8ub)
 
 # TS 102 223 Section 8.82
@@ -669,7 +720,7 @@ class SaTemplate(COMPR_TLV_IE, tag=0xA3):
     _construct = HexAdapter(GreedyBytes)
 
 # TS 102 223 Section 8.103
-class RefreshEnforcementPolicy(COMPR_TLV_IE, tag=0x3A):
+class RefreshEnforcementPolicy(COMPR_TLV_IE, tag=0xBA):
     _construct = FlagsEnum(Byte, even_if_navigating_menus=0, even_if_data_call=1, even_if_voice_call=2)
 
 # TS 102 223 Section 8.104
@@ -683,7 +734,7 @@ class SupportedRadioAccessTechnologies(COMPR_TLV_IE, tag=0xB4):
     _construct = GreedyRange(AccessTechTuple)
 
 # TS 102 223 Section 8.107
-class ApplicationSpecificRefreshData(COMPR_TLV_IE, tag=0x3B):
+class ApplicationSpecificRefreshData(COMPR_TLV_IE, tag=0xBB):
     pass
 
 # TS 31.111 Section 8.108
@@ -717,19 +768,194 @@ class SMSCBDownload(BER_TLV_IE, tag=0xD2,
                     nested=[DeviceIdentities, CBSPage]):
     pass
 
+# TS 101 220 Table 7.17
+class MenuSelection(BER_TLV_IE, tag=0xD3,
+                    nested=[DeviceIdentities, ItemIdentifier, HelpRequest]):
+    pass
 
+class BcRepeatIndicator(BER_TLV_IE, tag=0x2A):
+    pass
+
+# TS 101 220 Table 7.17
+class CallControl(BER_TLV_IE, tag=0xD4,
+                  nested=[DeviceIdentities, Address, CapabilityConfigParams, Subaddress,
+                          LocationInformation, BcRepeatIndicator]):
+    pass
+
+# TS 101 220 Table 7.17
+class MoShortMessageControl(BER_TLV_IE, tag=0xD5):
+    pass
+
+
+# TS 101 220 Table 7.23
+class TransactionIdentifier(BER_TLV_IE, tag=0x1C):
+    pass
+
+# TS 101 220 Table 7.23
+class ImsURI(BER_TLV_IE, tag=0x31):
+    pass
+
+# TS 101 220 Table 7.23
+class UriTruncated(BER_TLV_IE, tag=0x73):
+    pass
+
+# TS 101 220 Table 7.23
+class TrackingAreaIdentification(BER_TLV_IE, tag=0x7D):
+    pass
+
+# TS 101 220 Table 7.23
+class ExtendedRejectionCauseCode(BER_TLV_IE, tag=0x57):
+    pass
+
+# TS 101 220 Table 7.23
+class CsgCellSelectionStatus(BER_TLV_IE, tag=0x55):
+    pass
+
+# TS 101 220 Table 7.23
+class CsgId(BER_TLV_IE, tag=0x56):
+    pass
+
+# TS 101 220 Table 7.23
+class HnbName(BER_TLV_IE, tag=0x57):
+    pass
+
+# TS 101 220 Table 7.23
+class PlmnId(BER_TLV_IE, tag=0x09):
+    pass
+
+# TS 101 220 Table 7.23
+class ImsCallDisconnectionStatus(BER_TLV_IE, tag=0x55):
+    pass
+
+# TS 101 220 Table 7.23
+class Iari(BER_TLV_IE, tag=0x76):
+    pass
+
+# TS 101 220 Table 7.23
+class ImpuList(BER_TLV_IE, tag=0x77):
+    pass
+
+# TS 101 220 Table 7.23
+class ImsStatusCode(BER_TLV_IE, tag=0x77):
+    pass
+
+# TS 101 220 Table 7.23
+class DateTimeAndTimezone(BER_TLV_IE, tag=0x26):
+    pass
+
+# TS 101 220 Table 7.23
+class PdpPdnPduType(BER_TLV_IE, tag=0x0B):
+    pass
+
+# TS 101 220 Table 7.23
+class GadShape(BER_TLV_IE, tag=0x77):
+    pass
+
+# TS 101 220 Table 7.23
+class NmeaSentence(BER_TLV_IE, tag=0x78):
+    pass
+
+# TS 101 220 Table 7.23
+class WlanAccessStatus(BER_TLV_IE, tag=0x4B):
+    pass
+
+# TS 101 220 Table 7.17
+class EventDownload(BER_TLV_IE, tag=0xD6,
+                    nested=[EventList, DeviceIdentities,
+                            # 7.5.1.2 (I-)WLAN Access Status
+                            WlanAccessStatus,
+                            # 7.5.1A.2 MT Call
+                            TransactionIdentifier, Address,
+                            Subaddress, ImsURI, MediaType, UriTruncated,
+                            # 7.5.2.2 Network Rejection
+                            LocationInformation, RoutingAreaIdentification, TrackingAreaIdentification,
+                            AccessTechnology, UpdateAttachRegistrationType, RejectionCauseCode,
+                            ExtendedRejectionCauseCode,
+                            # 7.5.2A.2 Call Connected
+                            # TransactionIdentifier, MediaType
+                            # 7.5.3.2 CSG Cell Selection
+                            # AccessTechnology
+                            CsgCellSelectionStatus, CsgId, HnbName, PlmnId,
+                            # 7.5.3A.2 CAll Disconnected
+                            # TransactionIdentifier, MediaType,
+                            ImsCallDisconnectionStatus,
+                            # TS 102 223 7.5.4 LocationStatusEvent
+                            # TS 102 223 7.5.5 UserActivityEvent
+                            # TS 102 223 7.5.6 IdleScreenAvailableEvent
+                            # TS 102 223 7.5.7 CardReaderStatusEvent
+                            # TS 102 223 7.5.8 LanguageSelectionEvent
+                            # TS 102 223 7.5.9 BrowserTerminationEvent
+                            # TS 102 223 7.5.10 DataAvailableEvent
+                            ChannelStatus, ChannelDataLength,
+                            # TS 102 223 7.5.11 ChannelStatusEvent
+                            # TS 102 223 7.5.12 AccessTechnologyChangeEvent
+                            # TS 102 223 7.5.13 DisplayParametersChangedEvent
+                            # TS 102 223 7.5.14 LocalConnectionEvent
+                            # TS 102 223 7.5.15 NetworkSearchModeChangeEvent
+                            # TS 102 223 7.5.16 BrowsingStatusEvent
+                            # TS 102 223 7.5.17 FramesInformationChangedEvent
+                            # 7.5.20 Incoming IMS Data
+                            Iari,
+                            # 7.5.21 MS Registration Event
+                            ImpuList, ImsStatusCode,
+                            # 7.5.24 / TS 102 223 7.5.22 PollIntervalNegotiation
+                            # 7.5.25 DataConnectionStatusChangeEvent
+                            DataConnectionStatus, DataConnectionType, SmCause,
+                            # TransactionIdentifier, LocationInformation, AccessTechnology
+                            DateTimeAndTimezone, LocationStatus, NetworkAccessName, PdpPdnPduType,
+                            # 7.7 / TS 102 223 7.6 MMS Transfer Status
+                            # 7.8 / TS 102 223 MMS Notification Download
+                            # 7.9 / TS 102 223 8.8 Terminal Applications
+                    ]):
+    pass
+
+# TS 101 220 Table 7.17
+class TimerExpiration(BER_TLV_IE, tag=0xD7):
+    pass
+
+# TS 101 220 Table 7.17 + TS 31.111 7.6.2
 class USSDDownload(BER_TLV_IE, tag=0xD9,
                    nested=[DeviceIdentities, USSDString]):
     pass
 
+# TS 101 220 Table 7.17 + TS 102 223 7.6
+class MmsTransferStatus(BER_TLV_IE, tag=0xDA):
+    pass
 
+# TS 101 220 Table 7.17 + 102 223
+class MmsNotificationDownload(BER_TLV_IE, tag=0xDB):
+    pass
 
+# TS 101 220 Table 7.17 + 102 223 7.8
+class TerminalApplication(BER_TLV_IE, tag=0xDC):
+    pass
 
+# TS 101 220 Table 7.17 + TS 31.111 7.10.2
+class GeographicalLocation(BER_TLV_IE, tag=0xDD,
+                           nested=[DeviceIdentities, GadShape, NmeaSentence]):
+    pass
+
+# TS 101 220 Table 7.17
+class EnvelopeContainer(BER_TLV_IE, tag=0xDE):
+    pass
+
+# TS 101 220 Table 7.17
+class ProSeReport(BER_TLV_IE, tag=0xDF):
+    pass
+
+# TS 101 220 Table 7.17
 class ProactiveCmd(BER_TLV_IE):
     def _compute_tag(self) -> int:
         return 0xD0
 
 
+class EventCollection(TLV_IE_Collection,
+                      nested=[SMSPPDownload, SMSCBDownload,
+                              EventDownload, CallControl, MoShortMessageControl,
+                              USSDDownload, GeographicalLocation, ProSeReport]):
+    pass
+
+
 # TS 101 220 Table 7.17 + 102 223 6.6.13/9.4 + TS 31.111 6.6.13
 class Refresh(ProactiveCmd, tag=0x01,
               nested=[CommandDetails, DeviceIdentities, FileList, Aid, AlphaIdentifier,
@@ -737,20 +963,24 @@ class Refresh(ProactiveCmd, tag=0x01,
                       ApplicationSpecificRefreshData, PlmnWactList, PlmnList]):
     pass
 
+# TS 102 223 Section 6.6.4
 class MoreTime(ProactiveCmd, tag=0x02,
-        nested=[CommandDetails]):
+        nested=[CommandDetails, DeviceIdentities]):
     pass
 
+# TS 102 223 Section 6.6.5
 class PollInterval(ProactiveCmd, tag=0x03,
-        nested=[CommandDetails]):
+        nested=[CommandDetails, DeviceIdentities, Duration]):
     pass
 
+# TS 102 223 Section 6.6.14
 class PollingOff(ProactiveCmd, tag=0x04,
-        nested=[CommandDetails]):
+        nested=[CommandDetails, DeviceIdentities]):
     pass
 
+# TS 102 223 Section 6.6.16
 class SetUpEventList(ProactiveCmd, tag=0x05,
-        nested=[CommandDetails]):
+        nested=[CommandDetails, DeviceIdentities, EventList]):
     pass
 
 # TS 31.111 Section 6.6.12
@@ -778,20 +1008,27 @@ class SendShortMessage(ProactiveCmd, tag=0x13,
                                SMS_TPDU, IconIdentifier, TextAttribute, FrameIdentifier]):
     pass
 
+# TS 102 223 6.6.24
 class SendDTMF(ProactiveCmd, tag=0x14,
-        nested=[CommandDetails]):
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier,
+                DtmfString, IconIdentifier, TextAttribute, FrameIdentifier]):
     pass
 
+# TS 102 223 6.6.26
 class LaunchBrowser(ProactiveCmd, tag=0x15,
-        nested=[CommandDetails]):
+        nested=[CommandDetails, DeviceIdentities, BrowserIdentity, Url, Bearer, ProvisioningFileReference,
+                TextString, AlphaIdentifier, IconIdentifier, TextAttribute, FrameIdentifier,
+                NetworkAccessName]):
     pass
 
 class GeographicalLocationRequest(ProactiveCmd, tag=0x16,
         nested=[CommandDetails]):
     pass
 
+# TS 102 223 6.6.5
 class PlayTone(ProactiveCmd, tag=0x20,
-        nested=[CommandDetails]):
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier,
+                Tone, Duration, IconIdentifier, TextAttribute, FrameIdentifier]):
     pass
 
 # TS 101 220 Table 7.17 + 102 223 6.6.1/9.4 CMD=0x21
@@ -1003,7 +1240,7 @@ class ProactiveCommand(TLV_IE_Collection,
         pcmd.from_tlv(binary)
         cmd_details = pcmd.find_cmd_details()
         # then do a second decode stage for the specific
-        cmd_type = cmd_details.decoded['type_of_command']
+        cmd_type = TypeOfCommand.encmapping[cmd_details.decoded['type_of_command']]
         if cmd_type in self.members_by_tag:
             cls = self.members_by_tag[cmd_type]
             inst = cls()
@@ -1022,6 +1259,15 @@ class ProactiveCommand(TLV_IE_Collection,
     def to_bytes(self, context: dict = {}):
         return self.decoded.to_tlv()
 
+# TS 101 223 Section 6.8.0
+class TerminalResponse(TLV_IE_Collection,
+                       nested=[CommandDetails, DeviceIdentities, Result,
+                               Duration, TextString, ItemIdentifier,
+                               #TODO: LocalInformation and other optional/conditional IEs
+                               ChannelData, ChannelDataLength,
+                               ChannelStatus, BufferSize, BearerDescription,
+                               ]):
+    pass
 
 # reasonable default for playing with OTA
 # 010203040506070809101112131415161718192021222324252627282930313233

pySim/commands.py

@@ -23,14 +23,16 @@
 
 from typing import List, Tuple
 import typing # construct also has a Union, so we do typing.Union below
-
 from construct import Construct, Struct, Const, Select
 from construct import Optional as COptional
-from pySim.construct import LV, filter_dict
-from pySim.utils import rpad, lpad, b2h, h2b, sw_match, bertlv_encode_len, h2i, i2h, str_sanitize, expand_hex, SwMatchstr
-from pySim.utils import Hexstr, SwHexstr, ResTuple
+from osmocom.construct import LV, filter_dict
+from osmocom.utils import rpad, lpad, b2h, h2b, h2i, i2h, str_sanitize, Hexstr
+from osmocom.tlv import bertlv_encode_len
+
+from pySim.utils import sw_match, expand_hex, SwHexstr, ResTuple, SwMatchstr
 from pySim.exceptions import SwMatchError
 from pySim.transport import LinkBase
+from pySim.ts_102_221 import decode_select_response
 
 # A path can be either just a FID or a list of FID
 Path = typing.Union[Hexstr, List[Hexstr]]
@@ -65,7 +67,6 @@ class SimCardCommands:
     byte by the respective instance. """
     def __init__(self, transport: LinkBase, lchan_nr: int = 0):
         self._tp = transport
-        self._cla_byte = None
         self.sel_ctrl = "0000"
         self.lchan_nr = lchan_nr
         # invokes the setter below
@@ -75,15 +76,10 @@ class SimCardCommands:
     def fork_lchan(self, lchan_nr: int) -> 'SimCardCommands':
         """Fork a per-lchan specific SimCardCommands instance off the current instance."""
         ret = SimCardCommands(transport = self._tp, lchan_nr = lchan_nr)
-        ret.cla_byte = self._cla_byte
+        ret.cla_byte = self.cla_byte
         ret.sel_ctrl = self.sel_ctrl
         return ret
 
-    @property
-    def cla_byte(self) -> Hexstr:
-        """Return the (cached) patched default CLA byte for this card."""
-        return self._cla4lchan
-
     @property
     def max_cmd_len(self) -> int:
         """Maximum length of the command apdu data section. Depends on secure channel protocol used."""
@@ -92,58 +88,46 @@ class SimCardCommands:
         else:
             return 255
 
-    @cla_byte.setter
-    def cla_byte(self, new_val: Hexstr):
-        """Set the (raw, without lchan) default CLA value for this card."""
-        self._cla_byte = new_val
-        # compute cached result
-        self._cla4lchan = cla_with_lchan(self._cla_byte, self.lchan_nr)
-
-    def cla4lchan(self, cla: Hexstr) -> Hexstr:
-        """Compute the lchan-patched value of the given CLA value. If no CLA
-        value is provided as argument, the lchan-patched version of the SimCardCommands._cla_byte
-        value is used. Most commands will use the latter, while some wish to override it and
-        can pass it as argument here."""
-        if not cla:
-            # return cached result to avoid re-computing this over and over again
-            return self._cla4lchan
-        else:
-            return cla_with_lchan(cla, self.lchan_nr)
-
-    def send_apdu(self, pdu: Hexstr) -> ResTuple:
+    def send_apdu(self, pdu: Hexstr, apply_lchan:bool = True) -> ResTuple:
         """Sends an APDU and auto fetch response data
 
         Args:
            pdu : string of hexadecimal characters (ex. "A0A40000023F00")
+           apply_lchan : apply the currently selected lchan to the CLA byte before sending
         Returns:
            tuple(data, sw), where
                         data : string (in hex) of returned data (ex. "074F4EFFFF")
                         sw   : string (in hex) of status word (ex. "9000")
         """
+        if apply_lchan:
+            pdu = cla_with_lchan(pdu[0:2], self.lchan_nr) + pdu[2:]
         if self.scp:
             return self.scp.send_apdu_wrapper(self._tp.send_apdu, pdu)
         else:
             return self._tp.send_apdu(pdu)
 
-    def send_apdu_checksw(self, pdu: Hexstr, sw: SwMatchstr = "9000") -> ResTuple:
+    def send_apdu_checksw(self, pdu: Hexstr, sw: SwMatchstr = "9000", apply_lchan:bool = True) -> ResTuple:
         """Sends an APDU and check returned SW
 
         Args:
            pdu : string of hexadecimal characters (ex. "A0A40000023F00")
            sw : string of 4 hexadecimal characters (ex. "9000"). The user may mask out certain
-                        digits using a '?' to add some ambiguity if needed.
+                digits using a '?' to add some ambiguity if needed.
+           apply_lchan : apply the currently selected lchan to the CLA byte before sending
         Returns:
                 tuple(data, sw), where
                         data : string (in hex) of returned data (ex. "074F4EFFFF")
                         sw   : string (in hex) of status word (ex. "9000")
         """
+        if apply_lchan:
+            pdu = cla_with_lchan(pdu[0:2], self.lchan_nr) + pdu[2:]
         if self.scp:
             return self.scp.send_apdu_wrapper(self._tp.send_apdu_checksw, pdu, sw)
         else:
             return self._tp.send_apdu_checksw(pdu, sw)
 
     def send_apdu_constr(self, cla: Hexstr, ins: Hexstr, p1: Hexstr, p2: Hexstr, cmd_constr: Construct,
-                         cmd_data: Hexstr, resp_constr: Construct) -> Tuple[dict, SwHexstr]:
+                         cmd_data: Hexstr, resp_constr: Construct, apply_lchan:bool = True) -> Tuple[dict, SwHexstr]:
         """Build and sends an APDU using a 'construct' definition; parses response.
 
         Args:
@@ -153,14 +137,16 @@ class SimCardCommands:
                 p2 : string (in hex) ISO 7116 Parameter 2 byte
                 cmd_cosntr : defining how to generate binary APDU command data
                 cmd_data : command data passed to cmd_constr
-                resp_cosntr : defining how to decode  binary APDU response data
+                resp_cosntr : defining how to decode binary APDU response data
+                apply_lchan : apply the currently selected lchan to the CLA byte before sending
         Returns:
                 Tuple of (decoded_data, sw)
         """
         cmd = cmd_constr.build(cmd_data) if cmd_data else ''
-        p3 = i2h([len(cmd)])
-        pdu = ''.join([cla, ins, p1, p2, p3, b2h(cmd)])
-        (data, sw) = self.send_apdu(pdu)
+        lc = i2h([len(cmd)]) if cmd_data else ''
+        le = '00' if resp_constr else ''
+        pdu = ''.join([cla, ins, p1, p2, lc, b2h(cmd), le])
+        (data, sw) = self.send_apdu(pdu, apply_lchan = apply_lchan)
         if data:
             # filter the resulting dict to avoid '_io' members inside
             rsp = filter_dict(resp_constr.parse(h2b(data)))
@@ -170,7 +156,7 @@ class SimCardCommands:
 
     def send_apdu_constr_checksw(self, cla: Hexstr, ins: Hexstr, p1: Hexstr, p2: Hexstr,
                                  cmd_constr: Construct, cmd_data: Hexstr, resp_constr: Construct,
-                                 sw_exp: SwMatchstr="9000") -> Tuple[dict, SwHexstr]:
+                                 sw_exp: SwMatchstr="9000", apply_lchan:bool = True) -> Tuple[dict, SwHexstr]:
         """Build and sends an APDU using a 'construct' definition; parses response.
 
         Args:
@@ -185,44 +171,12 @@ class SimCardCommands:
         Returns:
                 Tuple of (decoded_data, sw)
         """
-        (rsp, sw) = self.send_apdu_constr(cla, ins,
-                                          p1, p2, cmd_constr, cmd_data, resp_constr)
+        (rsp, sw) = self.send_apdu_constr(cla, ins, p1, p2, cmd_constr, cmd_data, resp_constr,
+                                          apply_lchan = apply_lchan)
         if not sw_match(sw, sw_exp):
             raise SwMatchError(sw, sw_exp.lower(), self._tp.sw_interpreter)
         return (rsp, sw)
 
-    # Extract a single FCP item from TLV
-    def __parse_fcp(self, fcp: Hexstr):
-        # see also: ETSI TS 102 221, chapter 11.1.1.3.1 Response for MF,
-        # DF or ADF
-        from pytlv.TLV import TLV
-        tlvparser = TLV(['82', '83', '84', 'a5', '8a', '8b',
-                        '8c', '80', 'ab', 'c6', '81', '88'])
-
-        # pytlv is case sensitive!
-        fcp = fcp.lower()
-
-        if fcp[0:2] != '62':
-            raise ValueError(
-                'Tag of the FCP template does not match, expected 62 but got %s' % fcp[0:2])
-
-        # Unfortunately the spec is not very clear if the FCP length is
-        # coded as one or two byte vale, so we have to try it out by
-        # checking if the length of the remaining TLV string matches
-        # what we get in the length field.
-        # See also ETSI TS 102 221, chapter 11.1.1.3.0 Base coding.
-        exp_tlv_len = int(fcp[2:4], 16)
-        if len(fcp[4:]) // 2 == exp_tlv_len:
-            skip = 4
-        else:
-            exp_tlv_len = int(fcp[2:6], 16)
-            if len(fcp[4:]) // 2 == exp_tlv_len:
-                skip = 6
-
-        # Skip FCP tag and length
-        tlv = fcp[skip:]
-        return tlvparser.parse(tlv)
-
     # Tell the length of a record by the card response
     # USIMs respond with an FCP template, which is different
     # from what SIMs responds. See also:
@@ -230,10 +184,8 @@ class SimCardCommands:
     # SIM: GSM 11.11, chapter 9.2.1 SELECT
     def __record_len(self, r) -> int:
         if self.sel_ctrl == "0004":
-            tlv_parsed = self.__parse_fcp(r[-1])
-            file_descriptor = tlv_parsed['82']
-            # See also ETSI TS 102 221, chapter 11.1.1.4.3 File Descriptor
-            return int(file_descriptor[4:8], 16)
+            fcp_parsed = decode_select_response(r[-1])
+            return fcp_parsed['file_descriptor']['record_len']
         else:
             return int(r[-1][28:30], 16)
 
@@ -241,8 +193,8 @@ class SimCardCommands:
     # above.
     def __len(self, r) -> int:
         if self.sel_ctrl == "0004":
-            tlv_parsed = self.__parse_fcp(r[-1])
-            return int(tlv_parsed['80'], 16)
+            fcp_parsed = decode_select_response(r[-1])
+            return fcp_parsed['file_size']
         else:
             return int(r[-1][4:8], 16)
 
@@ -261,7 +213,7 @@ class SimCardCommands:
         if not isinstance(dir_list, list):
             dir_list = [dir_list]
         for i in dir_list:
-            data, sw = self.send_apdu(self.cla_byte + "a4" + self.sel_ctrl + "02" + i)
+            data, sw = self.send_apdu(self.cla_byte + "a4" + self.sel_ctrl + "02" + i + "00")
             rv.append((data, sw))
             if sw != '9000':
                 return rv
@@ -291,11 +243,11 @@ class SimCardCommands:
                 fid : file identifier as hex string
         """
 
-        return self.send_apdu_checksw(self.cla_byte + "a4" + self.sel_ctrl + "02" + fid)
+        return self.send_apdu_checksw(self.cla_byte + "a4" + self.sel_ctrl + "02" + fid + "00")
 
     def select_parent_df(self) -> ResTuple:
         """Execute SELECT to switch to the parent DF """
-        return self.send_apdu_checksw(self.cla_byte + "a4030400")
+        return self.send_apdu_checksw(self.cla_byte + "a40304")
 
     def select_adf(self, aid: Hexstr) -> ResTuple:
         """Execute SELECT a given Applicaiton ADF.
@@ -305,7 +257,7 @@ class SimCardCommands:
         """
 
         aidlen = ("0" + format(len(aid) // 2, 'x'))[-2:]
-        return self.send_apdu_checksw(self.cla_byte + "a4" + "0404" + aidlen + aid)
+        return self.send_apdu_checksw(self.cla_byte + "a4" + "0404" + aidlen + aid + "00")
 
     def read_binary(self, ef: Path, length: int = None, offset: int = 0) -> ResTuple:
         """Execute READD BINARY.
@@ -332,8 +284,8 @@ class SimCardCommands:
             try:
                 data, sw = self.send_apdu_checksw(pdu)
             except Exception as e:
-                raise ValueError('%s, failed to read (offset %d)' %
-                                 (str_sanitize(str(e)), offset)) from e
+                e.add_note('failed to read (offset %d)' % offset)
+                raise e
             total_data += data
             chunk_offset += chunk_len
         return total_data, sw
@@ -392,8 +344,8 @@ class SimCardCommands:
             try:
                 chunk_data, chunk_sw = self.send_apdu_checksw(pdu)
             except Exception as e:
-                raise ValueError('%s, failed to write chunk (chunk_offset %d, chunk_len %d)' %
-                                 (str_sanitize(str(e)), chunk_offset, chunk_len)) from e
+                e.add_note('failed to write chunk (chunk_offset %d, chunk_len %d)' % (chunk_offset, chunk_len))
+                raise e
             total_data += data
             chunk_offset += chunk_len
         if verify:
@@ -508,9 +460,9 @@ class SimCardCommands:
     # TS 102 221 Section 11.3.1 low-level helper
     def _retrieve_data(self, tag: int, first: bool = True) -> ResTuple:
         if first:
-            pdu = self.cla4lchan('80') + 'cb008001%02x' % (tag)
+            pdu = '80cb008001%02x00' % (tag)
         else:
-            pdu = self.cla4lchan('80') + 'cb000000'
+            pdu = '80cb0000'
         return self.send_apdu_checksw(pdu)
 
     def retrieve_data(self, ef: Path, tag: int) -> ResTuple:
@@ -540,7 +492,7 @@ class SimCardCommands:
             p1 = 0x00
         if isinstance(data, (bytes, bytearray)):
             data = b2h(data)
-        pdu = self.cla4lchan('80') + 'db00%02x%02x%s' % (p1, len(data)//2, data)
+        pdu = '80db00%02x%02x%s' % (p1, len(data)//2, data)
         return self.send_apdu_checksw(pdu)
 
     def set_data(self, ef, tag: int, value: str, verify: bool = False, conserve: bool = False) -> ResTuple:
@@ -583,7 +535,7 @@ class SimCardCommands:
         if len(rand) != 32:
             raise ValueError('Invalid rand')
         self.select_path(['3f00', '7f20'])
-        return self.send_apdu_checksw(self.cla4lchan('a0') + '88000010' + rand, sw='9000')
+        return self.send_apdu_checksw('a088000010' + rand + '00', sw='9000')
 
     def authenticate(self, rand: Hexstr, autn: Hexstr, context: str = '3g') -> ResTuple:
         """Execute AUTHENTICATE (USIM/ISIM).
@@ -604,6 +556,8 @@ class SimCardCommands:
             p2 = '81'
         elif context == 'gsm':
             p2 = '80'
+        else:
+            raise ValueError("Unsupported context '%s'" % context)
         (data, sw) = self.send_apdu_constr_checksw(
             self.cla_byte, '88', '00', p2, AuthCmd3G, cmd_data, AuthResp3G)
         if 'auts' in data:
@@ -614,7 +568,7 @@ class SimCardCommands:
 
     def status(self) -> ResTuple:
         """Execute a STATUS command as per TS 102 221 Section 11.1.2."""
-        return self.send_apdu_checksw(self.cla4lchan('80') + 'F20000ff')
+        return self.send_apdu_checksw('80F20000')
 
     def deactivate_file(self) -> ResTuple:
         """Execute DECATIVATE FILE command as per TS 102 221 Section 11.1.14."""
@@ -629,12 +583,12 @@ class SimCardCommands:
         return self.send_apdu_checksw(self.cla_byte + '44000002' + fid)
 
     def create_file(self, payload: Hexstr) -> ResTuple:
-        """Execute CREEATE FILE command as per TS 102 222 Section 6.3"""
+        """Execute CREATE FILE command as per TS 102 222 Section 6.3"""
         return self.send_apdu_checksw(self.cla_byte + 'e00000%02x%s' % (len(payload)//2, payload))
 
     def resize_file(self, payload: Hexstr) -> ResTuple:
         """Execute RESIZE FILE command as per TS 102 222 Section 6.10"""
-        return self.send_apdu_checksw(self.cla4lchan('80') + 'd40000%02x%s' % (len(payload)//2, payload))
+        return self.send_apdu_checksw('80d40000%02x%s' % (len(payload)//2, payload))
 
     def delete_file(self, fid: Hexstr) -> ResTuple:
         """Execute DELETE FILE command as per TS 102 222 Section 6.4"""
@@ -663,7 +617,7 @@ class SimCardCommands:
             p1 = 0x80
         else:
             p1 = 0x00
-        pdu = self.cla_byte + '70%02x%02x00' % (p1, lchan_nr)
+        pdu = self.cla_byte + '70%02x%02x' % (p1, lchan_nr)
         return self.send_apdu_checksw(pdu)
 
     def reset_card(self) -> Hexstr:
@@ -675,7 +629,7 @@ class SimCardCommands:
             raise RuntimeError('Failed to %s chv_no 0x%02X with code 0x%s, %i tries left.' %
                                (op_name, chv_no, b2h(pin_code).upper(), int(sw[3])))
         if sw != '9000':
-            raise SwMatchError(sw, '9000')
+            raise SwMatchError(sw, '9000', self._tp.sw_interpreter)
 
     def verify_chv(self, chv_no: int, code: Hexstr) -> ResTuple:
         """Verify a given CHV (Card Holder Verification == PIN)
@@ -746,7 +700,7 @@ class SimCardCommands:
         Args:
                 payload : payload as hex string
         """
-        return self.send_apdu_checksw('80c20000%02x%s' % (len(payload)//2, payload))
+        return self.send_apdu_checksw('80c20000%02x%s' % (len(payload)//2, payload), apply_lchan = False)
 
     def terminal_profile(self, payload: Hexstr) -> ResTuple:
         """Send TERMINAL PROFILE to card
@@ -755,7 +709,7 @@ class SimCardCommands:
                 payload : payload as hex string
         """
         data_length = len(payload) // 2
-        data, sw = self.send_apdu(('80100000%02x' % data_length) + payload)
+        data, sw = self.send_apdu_checksw(('80100000%02x' % data_length) + payload, apply_lchan = False)
         return (data, sw)
 
     # ETSI TS 102 221 11.1.22
@@ -793,7 +747,7 @@ class SimCardCommands:
             raise ValueError('Time unit must be 0x00..0x04')
         min_dur_enc = encode_duration(min_len_secs)
         max_dur_enc = encode_duration(max_len_secs)
-        data, sw = self.send_apdu_checksw('8076000004' + min_dur_enc + max_dur_enc)
+        data, sw = self.send_apdu_checksw('8076000004' + min_dur_enc + max_dur_enc, apply_lchan = False)
         negotiated_duration_secs = decode_duration(data[:4])
         resume_token = data[4:]
         return (negotiated_duration_secs, resume_token, sw)
@@ -803,11 +757,12 @@ class SimCardCommands:
         """Send SUSPEND UICC (resume) to the card."""
         if len(h2b(token)) != 8:
             raise ValueError("Token must be 8 bytes long")
-        data, sw = self.send_apdu_checksw('8076010008' + token)
+        data, sw = self.send_apdu_checksw('8076010008' + token, apply_lchan = False)
         return (data, sw)
 
+    # GPC_SPE_034 11.3
     def get_data(self, tag: int, cla: int = 0x00):
-        data, sw = self.send_apdu('%02xca%04x00' % (cla, tag))
+        data, sw = self.send_apdu_checksw('%02xca%04x00' % (cla, tag))
         return (data, sw)
 
     # TS 31.102 Section 7.5.2

pySim/construct.py

@@ -1,587 +0,0 @@
-"""Utility code related to the integration of the 'construct' declarative parser."""
-
-import typing
-import codecs
-import ipaddress
-
-import gsm0338
-
-from construct.lib.containers import Container, ListContainer
-from construct.core import EnumIntegerString
-from construct import Adapter, Prefixed, Int8ub, GreedyBytes, Default, Flag, Byte, Construct, Enum
-from construct import BitsInteger, BitStruct, Bytes, StreamError, stream_read_entire, stream_write
-from construct import SizeofError, IntegerError, swapbytes
-from construct.core import evaluate
-from construct.lib import integertypes
-
-from pySim.utils import b2h, h2b, swap_nibbles
-
-# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-
-class HexAdapter(Adapter):
-    """convert a bytes() type to a string of hex nibbles."""
-
-    def _decode(self, obj, context, path):
-        return b2h(obj)
-
-    def _encode(self, obj, context, path):
-        return h2b(obj)
-
-class Utf8Adapter(Adapter):
-    """convert a bytes() type that contains utf8 encoded text to human readable text."""
-
-    def _decode(self, obj, context, path):
-        # In case the string contains only 0xff bytes we interpret it as an empty string
-        if obj == b'\xff' * len(obj):
-            return ""
-        return codecs.decode(obj, "utf-8")
-
-    def _encode(self, obj, context, path):
-        return codecs.encode(obj, "utf-8")
-
-class GsmOrUcs2Adapter(Adapter):
-    """Try to encode into a GSM 03.38 string; if that fails, fall back to UCS-2 as described
-    in TS 102 221 Annex A."""
-    def _decode(self, obj, context, path):
-        # In case the string contains only 0xff bytes we interpret it as an empty string
-        if obj == b'\xff' * len(obj):
-            return ""
-        # one of the magic bytes of TS 102 221 Annex A
-        if obj[0] in [0x80, 0x81, 0x82]:
-            ad = Ucs2Adapter(GreedyBytes)
-        else:
-            ad = GsmString(GreedyBytes)
-        return ad._decode(obj, context, path)
-
-    def _encode(self, obj, context, path):
-        # first try GSM 03.38; then fall back to TS 102 221 Annex A UCS-2
-        try:
-            ad = GsmString(GreedyBytes)
-            return ad._encode(obj, context, path)
-        except:
-            ad = Ucs2Adapter(GreedyBytes)
-            return ad._encode(obj, context, path)
-
-class Ucs2Adapter(Adapter):
-    """convert a bytes() type that contains UCS2 encoded characters encoded as defined in TS 102 221
-    Annex A to normal python string representation (and back)."""
-    def _decode(self, obj, context, path):
-        # In case the string contains only 0xff bytes we interpret it as an empty string
-        if obj == b'\xff' * len(obj):
-            return ""
-        if obj[0] == 0x80:
-            # TS 102 221 Annex A Variant 1
-            return codecs.decode(obj[1:], 'utf_16_be')
-        elif obj[0] == 0x81:
-            # TS 102 221 Annex A Variant 2
-            out = ""
-            # second byte contains a value indicating the number of characters
-            num_of_chars = obj[1]
-            # the third byte contains an 8 bit number which defines bits 15 to 8 of a 16 bit base
-            # pointer, where bit 16 is set to zero, and bits 7 to 1 are also set to zero. These
-            # sixteen bits constitute a base pointer to a "half-page" in the UCS2 code space
-            base_ptr = obj[2] << 7
-            for ch in obj[3:3+num_of_chars]:
-                # if bit 8 of the byte is set to zero, the remaining 7 bits of the byte contain a
-                # GSM Default Alphabet character, whereas if bit 8 of the byte is set to one, then
-                # the remaining seven bits are an offset value added to the 16 bit base pointer
-                # defined earlier, and the resultant 16 bit value is a UCS2 code point
-                if ch & 0x80:
-                    codepoint = (ch & 0x7f) + base_ptr
-                    out += codecs.decode(codepoint.to_bytes(2, byteorder='big'), 'utf_16_be')
-                else:
-                    out += codecs.decode(bytes([ch]), 'gsm03.38')
-            return out
-        elif obj[0] == 0x82:
-            # TS 102 221 Annex A Variant 3
-            out = ""
-            # second byte contains a value indicating the number of characters
-            num_of_chars = obj[1]
-            # third and fourth bytes contain a 16 bit number which defines the complete 16 bit base
-            # pointer to a half-page in the UCS2 code space, for use with some or all of the
-            # remaining bytes in the string
-            base_ptr = obj[2] << 8 | obj[3]
-            for ch in obj[4:4+num_of_chars]:
-                # if bit 8 of the byte is set to zero, the remaining 7 bits of the byte contain a
-                # GSM Default Alphabet character, whereas if bit 8 of the byte is set to one, the
-                # remaining seven bits are an offset value added to the base pointer defined in
-                # bytes three and four, and the resultant 16 bit value is a UCS2 code point, else: #
-                # GSM default alphabet
-                if ch & 0x80:
-                    codepoint = (ch & 0x7f) + base_ptr
-                    out += codecs.decode(codepoint.to_bytes(2, byteorder='big'), 'utf_16_be')
-                else:
-                    out += codecs.decode(bytes([ch]), 'gsm03.38')
-            return out
-        else:
-            raise ValueError('First byte of TS 102 221 UCS-2 must be 0x80, 0x81 or 0x82')
-
-    def _encode(self, obj, context, path):
-        def encodable_in_gsm338(instr: str) -> bool:
-            """Determine if given input string is encode-ale in gsm03.38."""
-            try:
-                # TODO: figure out if/how we can constrain to default alphabet.  The gsm0338
-                # library seems to include the spanish lock/shift table
-                codecs.encode(instr, 'gsm03.38')
-            except ValueError:
-                return False
-            return True
-
-        def codepoints_not_in_gsm338(instr: str) -> typing.List[int]:
-            """Return an integer list of UCS2 codepoints for all characters of 'inster'
-            which are not representable in the GSM 03.38 default alphabet."""
-            codepoint_list = []
-            for c in instr:
-                if encodable_in_gsm338(c):
-                    continue
-                c_codepoint = int.from_bytes(codecs.encode(c, 'utf_16_be'), byteorder='big')
-                codepoint_list.append(c_codepoint)
-            return codepoint_list
-
-        def diff_between_min_and_max_of_list(inlst: typing.List) -> int:
-            return max(inlst) - min(inlst)
-
-        def encodable_in_variant2(instr: str) -> bool:
-            codepoint_prefix = None
-            for c in instr:
-                if encodable_in_gsm338(c):
-                    continue
-                c_codepoint = int.from_bytes(codecs.encode(c, 'utf_16_be'), byteorder='big')
-                if c_codepoint >= 0x8000:
-                    return False
-                c_prefix = c_codepoint >> 7
-                if codepoint_prefix is None:
-                    codepoint_prefix = c_prefix
-                else:
-                    if c_prefix != codepoint_prefix:
-                        return False
-            return True
-
-        def encodable_in_variant3(instr: str) -> bool:
-            codepoint_list = codepoints_not_in_gsm338(instr)
-            # compute delta between max and min; check if it's encodable in 7 bits
-            if diff_between_min_and_max_of_list(codepoint_list) >= 0x80:
-                return False
-            return True
-
-        def _encode_variant1(instr: str) -> bytes:
-            """Encode according to TS 102 221 Annex A Variant 1"""
-            return b'\x80' + codecs.encode(instr, 'utf_16_be')
-
-        def _encode_variant2(instr: str) -> bytes:
-            """Encode according to TS 102 221 Annex A Variant 2"""
-            codepoint_prefix = None
-            # second byte contains a value indicating the number of characters
-            hdr = b'\x81' + len(instr).to_bytes(1, byteorder='big')
-            chars = b''
-            for c in instr:
-                try:
-                    enc = codecs.encode(c, 'gsm03.38')
-                except ValueError:
-                    c_codepoint = int.from_bytes(codecs.encode(c, 'utf_16_be'), byteorder='big')
-                    c_prefix = c_codepoint >> 7
-                    if codepoint_prefix is None:
-                        codepoint_prefix = c_prefix
-                    assert codepoint_prefix == c_prefix
-                    enc = (0x80 + (c_codepoint & 0x7f)).to_bytes(1, byteorder='big')
-                chars += enc
-            if codepoint_prefix is None:
-                codepoint_prefix = 0
-            return hdr + codepoint_prefix.to_bytes(1, byteorder='big') + chars
-
-        def _encode_variant3(instr: str) -> bytes:
-            """Encode according to TS 102 221 Annex A Variant 3"""
-            # second byte contains a value indicating the number of characters
-            hdr = b'\x82' + len(instr).to_bytes(1, byteorder='big')
-            chars = b''
-            codepoint_list = codepoints_not_in_gsm338(instr)
-            codepoint_base = min(codepoint_list)
-            for c in instr:
-                try:
-                    # if bit 8 of the byte is set to zero, the remaining 7 bits of the byte contain a GSM
-                    # Default # Alphabet character
-                    enc = codecs.encode(c, 'gsm03.38')
-                except ValueError:
-                    # if bit 8 of the byte is set to one, the remaining seven bits are an offset
-                    # value added to the base pointer defined in bytes three and four, and the
-                    # resultant 16 bit value is a UCS2 code point
-                    c_codepoint = int.from_bytes(codecs.encode(c, 'utf_16_be'), byteorder='big')
-                    c_codepoint_delta = c_codepoint - codepoint_base
-                    assert c_codepoint_delta < 0x80
-                    enc = (0x80 + c_codepoint_delta).to_bytes(1, byteorder='big')
-                chars += enc
-            # third and fourth bytes contain a 16 bit number which defines the complete 16 bit base
-            # pointer to a half-page in the UCS2 code space
-            return hdr + codepoint_base.to_bytes(2, byteorder='big') + chars
-
-        if encodable_in_variant2(obj):
-            return _encode_variant2(obj)
-        elif encodable_in_variant3(obj):
-            return _encode_variant3(obj)
-        else:
-            return _encode_variant1(obj)
-
-class BcdAdapter(Adapter):
-    """convert a bytes() type to a string of BCD nibbles."""
-
-    def _decode(self, obj, context, path):
-        return swap_nibbles(b2h(obj))
-
-    def _encode(self, obj, context, path):
-        return h2b(swap_nibbles(obj))
-
-class PlmnAdapter(BcdAdapter):
-    """convert a bytes(3) type to BCD string like 262-02 or 262-002."""
-    def _decode(self, obj, context, path):
-        bcd = super()._decode(obj, context, path)
-        if bcd[3] == 'f':
-            return '-'.join([bcd[:3], bcd[4:]])
-        else:
-            return '-'.join([bcd[:3], bcd[3:]])
-
-    def _encode(self, obj, context, path):
-        l = obj.split('-')
-        if len(l[1]) == 2:
-            bcd = l[0] + 'f' + l[1]
-        else:
-            bcd = l[0] + l[1]
-        return super()._encode(bcd, context, path)
-
-class InvertAdapter(Adapter):
-    """inverse logic (false->true, true->false)."""
-    @staticmethod
-    def _invert_bool_in_obj(obj):
-        for k,v in obj.items():
-            # skip all private entries
-            if k.startswith('_'):
-                continue
-            if v is False:
-                obj[k] = True
-            elif v is True:
-                obj[k] = False
-        return obj
-
-    def _decode(self, obj, context, path):
-        return self._invert_bool_in_obj(obj)
-
-    def _encode(self, obj, context, path):
-        return self._invert_bool_in_obj(obj)
-
-class Rpad(Adapter):
-    """
-    Encoder appends padding bytes (b'\\xff') or characters up to target size.
-    Decoder removes trailing padding bytes/characters.
-
-    Parameters:
-        subcon: Subconstruct as defined by construct library
-        pattern: set padding pattern (default: b'\\xff')
-        num_per_byte: number of 'elements' per byte. E.g. for hex nibbles: 2
-    """
-
-    def __init__(self, subcon, pattern=b'\xff', num_per_byte=1):
-        super().__init__(subcon)
-        self.pattern = pattern
-        self.num_per_byte = num_per_byte
-
-    def _decode(self, obj, context, path):
-        return obj.rstrip(self.pattern)
-
-    def _encode(self, obj, context, path):
-        target_size = self.sizeof() * self.num_per_byte
-        if len(obj) > target_size:
-            raise SizeofError("Input ({}) exceeds target size ({})".format(
-                len(obj), target_size))
-        return obj + self.pattern * (target_size - len(obj))
-
-class MultiplyAdapter(Adapter):
-    """
-    Decoder multiplies by multiplicator
-    Encoder divides by multiplicator
-
-    Parameters:
-        subcon: Subconstruct as defined by construct library
-        multiplier: Multiplier to apply to raw encoded value
-    """
-
-    def __init__(self, subcon, multiplicator):
-        super().__init__(subcon)
-        self.multiplicator = multiplicator
-
-    def _decode(self, obj, context, path):
-        return obj * 8
-
-    def _encode(self, obj, context, path):
-        return obj // 8
-
-
-class GsmStringAdapter(Adapter):
-    """Convert GSM 03.38 encoded bytes to a string."""
-
-    def __init__(self, subcon, codec='gsm03.38', err='strict'):
-        super().__init__(subcon)
-        self.codec = codec
-        self.err = err
-
-    def _decode(self, obj, context, path):
-        return obj.decode(self.codec)
-
-    def _encode(self, obj, context, path):
-        return obj.encode(self.codec, self.err)
-
-class Ipv4Adapter(Adapter):
-    """
-    Encoder converts from 4 bytes to string representation (A.B.C.D).
-    Decoder converts from string representation (A.B.C.D) to four bytes.
-    """
-    def _decode(self, obj, context, path):
-        ia = ipaddress.IPv4Address(obj)
-        return ia.compressed
-
-    def _encode(self, obj, context, path):
-        ia = ipaddress.IPv4Address(obj)
-        return ia.packed
-
-class Ipv6Adapter(Adapter):
-    """
-    Encoder converts from 16 bytes to string representation.
-    Decoder converts from string representation to 16 bytes.
-    """
-    def _decode(self, obj, context, path):
-        ia = ipaddress.IPv6Address(obj)
-        return ia.compressed
-
-    def _encode(self, obj, context, path):
-        ia = ipaddress.IPv6Address(obj)
-        return ia.packed
-
-class StripTrailerAdapter(Adapter):
-    """
-    Encoder removes all trailing bytes matching the default_value
-    Decoder pads input data up to total_length with default_value
-
-    This is used in constellations like "FlagsEnum(StripTrailerAdapter(GreedyBytes, 3), ..."
-    where you have a bit-mask that may have 1, 2 or 3 bytes, depending on whether or not any
-    of the LSBs are actually set.
-    """
-    def __init__(self, subcon, total_length:int, default_value=b'\x00', min_len=1):
-        super().__init__(subcon)
-        assert len(default_value) == 1
-        self.total_length = total_length
-        self.default_value = default_value
-        self.min_len = min_len
-
-    def _decode(self, obj, context, path):
-        assert isinstance(obj, bytes)
-        # pad with suppressed/missing bytes
-        if len(obj) < self.total_length:
-            obj += self.default_value * (self.total_length - len(obj))
-        return int.from_bytes(obj, 'big')
-
-    def _encode(self, obj, context, path):
-        assert isinstance(obj, int)
-        obj = obj.to_bytes(self.total_length, 'big')
-        # remove trailing bytes if they are zero
-        while len(obj) > self.min_len and obj[-1] == self.default_value[0]:
-            obj = obj[:-1]
-        return obj
-
-
-def filter_dict(d, exclude_prefix='_'):
-    """filter the input dict to ensure no keys starting with 'exclude_prefix' remain."""
-    if not isinstance(d, dict):
-        return d
-    res = {}
-    for (key, value) in d.items():
-        if key.startswith(exclude_prefix):
-            continue
-        if isinstance(value, dict):
-            res[key] = filter_dict(value)
-        else:
-            res[key] = value
-    return res
-
-
-def normalize_construct(c, exclude_prefix: str = '_'):
-    """Convert a construct specific type to a related base type, mostly useful
-    so we can serialize it."""
-    # we need to include the filter_dict as we otherwise get elements like this
-    # in the dict: '_io': <_io.BytesIO object at 0x7fdb64e05860> which we cannot json-serialize
-    c = filter_dict(c, exclude_prefix)
-    if isinstance(c, (Container, dict)):
-        r = {k: normalize_construct(v) for (k, v) in c.items()}
-    elif isinstance(c, ListContainer):
-        r = [normalize_construct(x) for x in c]
-    elif isinstance(c, list):
-        r = [normalize_construct(x) for x in c]
-    elif isinstance(c, EnumIntegerString):
-        r = str(c)
-    else:
-        r = c
-    return r
-
-
-def parse_construct(c, raw_bin_data: bytes, length: typing.Optional[int] = None, exclude_prefix: str = '_', context: dict = {}):
-    """Helper function to wrap around normalize_construct() and filter_dict()."""
-    if not length:
-        length = len(raw_bin_data)
-    try:
-        parsed = c.parse(raw_bin_data, total_len=length, **context)
-    except StreamError as e:
-        # if the input is all-ff, this means the content is undefined.  Let's avoid passing StreamError
-        # exceptions in those situations (which might occur if a length field 0xff is 255 but then there's
-        # actually less bytes in the remainder of the file.
-        if all(v == 0xff for v in raw_bin_data):
-            return None
-        else:
-            raise e
-    return normalize_construct(parsed, exclude_prefix)
-
-def build_construct(c, decoded_data, context: dict = {}):
-    """Helper function to handle total_len."""
-    return c.build(decoded_data, total_len=None, **context)
-
-# here we collect some shared / common definitions of data types
-LV = Prefixed(Int8ub, HexAdapter(GreedyBytes))
-
-# Default value for Reserved for Future Use (RFU) bits/bytes
-# See TS 31.101 Sec. "3.4 Coding Conventions"
-__RFU_VALUE = 0
-
-# Field that packs Reserved for Future Use (RFU) bit
-FlagRFU = Default(Flag, __RFU_VALUE)
-
-# Field that packs Reserved for Future Use (RFU) byte
-ByteRFU = Default(Byte, __RFU_VALUE)
-
-# Field that packs all remaining Reserved for Future Use (RFU) bytes
-GreedyBytesRFU = Default(GreedyBytes, b'')
-
-
-def BitsRFU(n=1):
-    '''
-    Field that packs Reserved for Future Use (RFU) bit(s)
-    as defined in TS 31.101 Sec. "3.4 Coding Conventions"
-
-    Use this for (currently) unused/reserved bits whose contents
-    should be initialized automatically but should not be cleared
-    in the future or when restoring read data (unlike padding).
-
-    Parameters:
-        n (Integer): Number of bits (default: 1)
-    '''
-    return Default(BitsInteger(n), __RFU_VALUE)
-
-
-def BytesRFU(n=1):
-    '''
-    Field that packs Reserved for Future Use (RFU) byte(s)
-    as defined in TS 31.101 Sec. "3.4 Coding Conventions"
-
-    Use this for (currently) unused/reserved bytes whose contents
-    should be initialized automatically but should not be cleared
-    in the future or when restoring read data (unlike padding).
-
-    Parameters:
-        n (Integer): Number of bytes (default: 1)
-    '''
-    return Default(Bytes(n), __RFU_VALUE)
-
-
-def GsmString(n):
-    '''
-    GSM 03.38 encoded byte string of fixed length n.
-    Encoder appends padding bytes (b'\\xff') to maintain
-    length. Decoder removes those trailing bytes.
-
-    Exceptions are raised for invalid characters
-    and length excess.
-
-    Parameters:
-        n (Integer): Fixed length of the encoded byte string
-    '''
-    return GsmStringAdapter(Rpad(Bytes(n), pattern=b'\xff'), codec='gsm03.38')
-
-def GsmOrUcs2String(n):
-    '''
-    GSM 03.38 or UCS-2 (TS 102 221 Annex A) encoded byte string of fixed length n.
-    Encoder appends padding bytes (b'\\xff') to maintain
-    length. Decoder removes those trailing bytes.
-
-    Exceptions are raised for invalid characters
-    and length excess.
-
-    Parameters:
-        n (Integer): Fixed length of the encoded byte string
-    '''
-    return GsmOrUcs2Adapter(Rpad(Bytes(n), pattern=b'\xff'))
-
-class GreedyInteger(Construct):
-    """A variable-length integer implementation, think of combining GrredyBytes with BytesInteger."""
-    def __init__(self, signed=False, swapped=False, minlen=0):
-        super().__init__()
-        self.signed = signed
-        self.swapped = swapped
-        self.minlen = minlen
-
-    def _parse(self, stream, context, path):
-        data = stream_read_entire(stream, path)
-        if evaluate(self.swapped, context):
-            data = swapbytes(data)
-        try:
-            return int.from_bytes(data, byteorder='big', signed=self.signed)
-        except ValueError as e:
-            raise IntegerError(str(e), path=path)
-
-    def __bytes_required(self, i, minlen=0):
-        if self.signed:
-            raise NotImplementedError("FIXME: Implement support for encoding signed integer")
-
-        # compute how many bytes we need
-        nbytes = 1
-        while True:
-            i = i >> 8
-            if i == 0:
-                break
-            else:
-                nbytes = nbytes + 1
-
-        # round up to the minimum number
-        # of bytes we anticipate
-        nbytes = max(nbytes, minlen)
-
-        return nbytes
-
-    def _build(self, obj, stream, context, path):
-        if not isinstance(obj, integertypes):
-            raise IntegerError(f"value {obj} is not an integer", path=path)
-        length = self.__bytes_required(obj, self.minlen)
-        try:
-            data = obj.to_bytes(length, byteorder='big', signed=self.signed)
-        except ValueError as e:
-            raise IntegerError(str(e), path=path) from e
-        if evaluate(self.swapped, context):
-            data = swapbytes(data)
-        stream_write(stream, data, length, path)
-        return obj
-
-# merged definitions of 24.008 + 23.040
-TypeOfNumber = Enum(BitsInteger(3), unknown=0, international=1, national=2, network_specific=3,
-                    short_code=4, alphanumeric=5, abbreviated=6, reserved_for_extension=7)
-NumberingPlan = Enum(BitsInteger(4), unknown=0, isdn_e164=1, data_x121=3, telex_f69=4,
-                     sc_specific_5=5, sc_specific_6=6, national=8, private=9,
-                     ermes=10, reserved_cts=11, reserved_for_extension=15)
-TonNpi = BitStruct('ext'/Flag, 'type_of_number'/TypeOfNumber, 'numbering_plan_id'/NumberingPlan)

pySim/esim/__init__.py

@@ -1,9 +1,53 @@
 import sys
-from typing import Optional
+from typing import Optional, Tuple
 from importlib import resources
 
+class PMO:
+    """Convenience conversion class for ProfileManagementOperation as used in ES9+ notifications."""
+    pmo4operation = {
+        'install': 0x80,
+        'enable': 0x40,
+        'disable': 0x20,
+        'delete': 0x10,
+    }
 
-def compile_asn1_subdir(subdir_name:str):
+    def __init__(self, op: str):
+        if not op in self.pmo4operation:
+            raise ValueError('Unknown operation "%s"' % op)
+        self.op = op
+
+    def to_int(self):
+        return self.pmo4operation[self.op]
+
+    @staticmethod
+    def _num_bits(data: int)-> int:
+        for i in range(0, 8):
+            if data & (1 << i):
+                return 8-i
+        return 0
+
+    def to_bitstring(self) -> Tuple[bytes, int]:
+        """return value in a format as used by asn1tools for BITSTRING."""
+        val = self.to_int()
+        return (bytes([val]), self._num_bits(val))
+
+    @classmethod
+    def from_int(cls, i: int) -> 'PMO':
+        """Parse an integer representation."""
+        for k, v in cls.pmo4operation.items():
+            if v == i:
+                return cls(k)
+        raise ValueError('Unknown PMO 0x%02x' % i)
+
+    @classmethod
+    def from_bitstring(cls, bstr: Tuple[bytes, int]) -> 'PMO':
+        """Parse a asn1tools BITSTRING representation."""
+        return cls.from_int(bstr[0][0])
+
+    def __str__(self):
+        return self.op
+
+def compile_asn1_subdir(subdir_name:str, codec='der'):
     """Helper function that compiles ASN.1 syntax from all files within given subdir"""
     import asn1tools
     asn_txt = ''
@@ -14,7 +58,7 @@ def compile_asn1_subdir(subdir_name:str):
             asn_txt += "\n"
     #else:
         #print(resources.read_text(__name__, 'asn1/rsp.asn'))
-    return asn1tools.compile_string(asn_txt, codec='der')
+    return asn1tools.compile_string(asn_txt, codec=codec)
 
 
 # SGP.22 section 4.1 Activation Code

pySim/esim/bsp.py

@@ -35,7 +35,8 @@ from cryptography.hazmat.primitives.kdf.x963kdf import X963KDF
 from Cryptodome.Cipher import AES
 from Cryptodome.Hash import CMAC
 
-from pySim.utils import bertlv_encode_len, bertlv_parse_one, b2h
+from osmocom.utils import b2h
+from osmocom.tlv import bertlv_encode_len, bertlv_parse_one
 
 # don't log by default
 logger = logging.getLogger(__name__)
@@ -287,6 +288,8 @@ class BspInstance:
     def demac_only_one(self, ciphertext: bytes) -> bytes:
         payload = self.m_algo.verify(ciphertext)
         _tdict, _l, val, _remain = bertlv_parse_one(payload)
+        # The data block counter for ICV caluclation is incremented also for each segment with C-MAC only.
+        self.c_algo.block_nr += 1
         return val
 
     def demac_only(self, ciphertext_list: List[bytes]) -> bytes:

pySim/esim/es2p.py

@@ -15,96 +15,16 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import abc
 import requests
 import logging
-import json
 from datetime import datetime
 import time
-import base64
+
+from pySim.esim.http_json_api import *
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
-class ApiParam(abc.ABC):
-    """A class reprsenting a single parameter in the ES2+ API."""
-    @classmethod
-    def verify_decoded(cls, data):
-        """Verify the decoded reprsentation of a value. Should raise an exception if somthing is odd."""
-        pass
-
-    @classmethod
-    def verify_encoded(cls, data):
-        """Verify the encoded reprsentation of a value. Should raise an exception if somthing is odd."""
-        pass
-
-    @classmethod
-    def encode(cls, data):
-        """[Validate and] Encode the given value."""
-        cls.verify_decoded(data)
-        encoded = cls._encode(data)
-        cls.verify_decoded(encoded)
-        return encoded
-
-    @classmethod
-    def _encode(cls, data):
-        """encoder function, typically [but not always] overridden by derived class."""
-        return data
-
-    @classmethod
-    def decode(cls, data):
-        """[Validate and] Decode the given value."""
-        cls.verify_encoded(data)
-        decoded = cls._decode(data)
-        cls.verify_decoded(decoded)
-        return decoded
-
-    @classmethod
-    def _decode(cls, data):
-        """decoder function, typically [but not always] overridden by derived class."""
-        return data
-
-class ApiParamString(ApiParam):
-    """Base class representing an API parameter of 'string' type."""
-    pass
-
-
-class ApiParamInteger(ApiParam):
-    """Base class representing an API parameter of 'integer' type."""
-    @classmethod
-    def _decode(cls, data):
-        return int(data)
-
-    @classmethod
-    def _encode(cls, data):
-        return str(data)
-
-    @classmethod
-    def verify_decoded(cls, data):
-        if not isinstance(data, int):
-            raise TypeError('Expected an integer input data type')
-
-    @classmethod
-    def verify_encoded(cls, data):
-        if not data.isdecimal():
-            raise ValueError('integer (%s) contains non-decimal characters' % data)
-        assert str(int(data)) == data
-
-class ApiParamBoolean(ApiParam):
-    """Base class representing an API parameter of 'boolean' type."""
-    @classmethod
-    def _encode(cls, data):
-        return bool(data)
-
-class ApiParamFqdn(ApiParam):
-    """String, as a list of domain labels concatenated using the full stop (dot, period) character as
-    separator between labels. Labels are restricted to the Alphanumeric mode character set defined in table 5
-    of ISO/IEC 18004"""
-    @classmethod
-    def verify_encoded(cls, data):
-        # FIXME
-        pass
-
 class param:
     class Iccid(ApiParamString):
         """String representation of 19 or 20 digits, where the 20th digit MAY optionally be the padding
@@ -170,9 +90,6 @@ class param:
     class SmdsAddress(ApiParamFqdn):
         pass
 
-    class SmdpAddress(ApiParamFqdn):
-        pass
-
     class ReleaseFlag(ApiParamBoolean):
         pass
 
@@ -187,7 +104,7 @@ class param:
 
         @classmethod
         def _encode(cls, data):
-            return datetime.toisoformat(data)
+            return datetime.isoformat(data)
 
     class NotificationPointId(ApiParamInteger):
         pass
@@ -195,148 +112,12 @@ class param:
     class NotificationPointStatus(ApiParam):
         pass
 
-    class ResultData(ApiParam):
-        @classmethod
-        def _decode(cls, data):
-            return base64.b64decode(data)
+    class ResultData(ApiParamBase64):
+        pass
 
-        @classmethod
-        def _encode(cls, data):
-            return base64.b64encode(data)
-
-    class JsonResponseHeader(ApiParam):
-        """SGP.22 section 6.5.1.4."""
-        @classmethod
-        def verify_decoded(cls, data):
-            fe_status = data.get('functionExecutionStatus')
-            if not fe_status:
-                raise ValueError('Missing mandatory functionExecutionStatus in header')
-            status = fe_status.get('status')
-            if not status:
-                raise ValueError('Missing mandatory status in header functionExecutionStatus')
-            if status not in ['Executed-Success', 'Executed-WithWarning', 'Failed', 'Expired']:
-                raise ValueError('Unknown/unspecified status "%s"' % status)
-
-
-class HttpStatusError(Exception):
-    pass
-
-class HttpHeaderError(Exception):
-    pass
-
-class Es2PlusApiError(Exception):
-    """Exception representing an error at the ES2+ API level (status != Executed)."""
-    def __init__(self, func_ex_status: dict):
-        self.status = func_ex_status['status']
-        sec = {
-            'subjectCode': None,
-            'reasonCode': None,
-            'subjectIdentifier': None,
-            'message': None,
-        }
-        actual_sec = func_ex_status.get('statusCodeData', None)
-        sec.update(actual_sec)
-        self.subject_code = sec['subjectCode']
-        self.reason_code = sec['reasonCode']
-        self.subject_id = sec['subjectIdentifier']
-        self.message = sec['message']
-
-    def __str__(self):
-        return f'{self.status}("{self.subject_code}","{self.reason_code}","{self.subject_id}","{self.message}")'
-
-class Es2PlusApiFunction(abc.ABC):
+class Es2PlusApiFunction(JsonHttpApiFunction):
     """Base classs for representing an ES2+ API Function."""
-    # the below class variables are expected to be overridden in derived classes
-
-    path = None
-    # dictionary of input parameters. key is parameter name, value is ApiParam class
-    input_params = {}
-    # list of mandatory input parameters
-    input_mandatory = []
-    # dictionary of output parameters. key is parameter name, value is ApiParam class
-    output_params = {}
-    # list of mandatory output parameters (for successful response)
-    output_mandatory = []
-    # expected HTTP status code of the response
-    expected_http_status = 200
-
-    def __init__(self, url_prefix: str, func_req_id: str, session):
-        self.url_prefix = url_prefix
-        self.func_req_id = func_req_id
-        self.session = session
-
-    def encode(self, data: dict, func_call_id: str) -> dict:
-        """Validate an encode input dict into JSON-serializable dict for request body."""
-        output = {
-            'header': {
-                'functionRequesterIdentifier': self.func_req_id,
-                'functionCallIdentifier': func_call_id
-            }
-        }
-        for p in self.input_mandatory:
-            if not p in data:
-                raise ValueError('Mandatory input parameter %s missing' % p)
-        for p, v in data.items():
-            p_class = self.input_params.get(p)
-            if not p_class:
-                logger.warning('Unexpected/unsupported input parameter %s=%s', p, v)
-                output[p] = v
-            else:
-                output[p] = p_class.encode(v)
-        return output
-
-
-    def decode(self, data: dict) -> dict:
-        """[further] Decode and validate the JSON-Dict of the respnse body."""
-        output = {}
-        # let's first do the header, it's special
-        if not 'header' in data:
-            raise ValueError('Mandatory output parameter "header" missing')
-        hdr_class = self.output_params.get('header')
-        output['header'] = hdr_class.decode(data['header'])
-
-        if output['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
-            raise Es2PlusApiError(output['header']['functionExecutionStatus'])
-        # we can only expect mandatory parameters to be present in case of successful execution
-        for p in self.output_mandatory:
-            if p == 'header':
-                continue
-            if not p in data:
-                raise ValueError('Mandatory output parameter "%s" missing' % p)
-        for p, v in data.items():
-            p_class = self.output_params.get(p)
-            if not p_class:
-                logger.warning('Unexpected/unsupported output parameter "%s"="%s"', p, v)
-                output[p] = v
-            else:
-                output[p] = p_class.decode(v)
-        return output
-
-    def call(self, data: dict, func_call_id:str, timeout=10) -> dict:
-        """Make an API call to the ES2+ API endpoint represented by this object.
-        Input data is passed in `data` as json-serializable dict.  Output data
-        is returned as json-deserialized dict."""
-        url = self.url_prefix + self.path
-        encoded = json.dumps(self.encode(data, func_call_id))
-        headers = {
-            'Content-Type': 'application/json',
-            'X-Admin-Protocol': 'gsma/rsp/v2.5.0',
-        }
-
-        logger.debug("HTTP REQ %s - '%s'" % (url, encoded))
-        response = self.session.post(url, data=encoded, headers=headers, timeout=timeout)
-        logger.debug("HTTP RSP-STS: [%u] hdr: %s" % (response.status_code, response.headers))
-        logger.debug("HTTP RSP: %s" % (response.content))
-
-        if response.status_code != self.expected_http_status:
-            raise HttpStatusError(response)
-        if not response.headers.get('Content-Type').startswith(headers['Content-Type']):
-            raise HttpHeaderError(response)
-        if not response.headers.get('X-Admin-Protocol', 'gsma/rsp/v2.unknown').startswith('gsma/rsp/v2.'):
-            raise HttpHeaderError(response)
-
-        return self.decode(response.json())
-
+    pass
 
 # ES2+ DownloadOrder function (SGP.22 section 5.3.1)
 class DownloadOrder(Es2PlusApiFunction):
@@ -347,7 +128,7 @@ class DownloadOrder(Es2PlusApiFunction):
         'profileType': param.ProfileType
       }
     output_params = {
-        'header': param.JsonResponseHeader,
+        'header': JsonResponseHeader,
         'iccid': param.Iccid,
       }
     output_mandatory = ['header', 'iccid']
@@ -365,10 +146,10 @@ class ConfirmOrder(Es2PlusApiFunction):
       }
     input_mandatory = ['iccid', 'releaseFlag']
     output_params = {
-        'header': param.JsonResponseHeader,
+        'header': JsonResponseHeader,
         'eid': param.Eid,
         'matchingId': param.MatchingId,
-        'smdpAddress': param.SmdpAddress,
+        'smdpAddress': SmdpAddress,
       }
     output_mandatory = ['header', 'matchingId']
 
@@ -383,7 +164,7 @@ class CancelOrder(Es2PlusApiFunction):
       }
     input_mandatory = ['finalProfileStatusIndicator', 'iccid']
     output_params = {
-        'header': param.JsonResponseHeader,
+        'header': JsonResponseHeader,
       }
     output_mandatory = ['header']
 
@@ -395,7 +176,7 @@ class ReleaseProfile(Es2PlusApiFunction):
       }
     input_mandatory = ['iccid']
     output_params = {
-        'header': param.JsonResponseHeader,
+        'header': JsonResponseHeader,
       }
     output_mandatory = ['header']
 

pySim/esim/es8p.py

@@ -17,10 +17,14 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from typing import Dict, List, Optional
-from pySim.utils import b2h, h2b, bertlv_encode_tag, bertlv_encode_len
+from cryptography.hazmat.primitives.asymmetric import ec
+from osmocom.utils import b2h, h2b
+from osmocom.tlv import bertlv_encode_tag, bertlv_encode_len, bertlv_parse_one_rawtag
+from osmocom.tlv import bertlv_return_one_rawtlv
 
 import pySim.esim.rsp as rsp
 from pySim.esim.bsp import BspInstance
+from pySim.esim import PMO
 
 # Given that GSMA RSP uses ASN.1 in a very weird way, we actually cannot encode the full data type before
 # signing, but we have to build parts of it separately first, then sign that, so we can put the signature
@@ -74,6 +78,23 @@ class ProfileMetadata:
         self.iccid_bin = iccid_bin
         self.spn = spn
         self.profile_name = profile_name
+        self.icon = None
+        self.icon_type = None
+        self.notifications = []
+
+    def set_icon(self, is_png: bool, icon_data: bytes):
+        """Set the icon that is part of the metadata."""
+        if len(icon_data) > 1024:
+            raise ValueError('Icon data must not exceed 1024 bytes')
+        self.icon = icon_data
+        if is_png:
+            self.icon_type = 1
+        else:
+            self.icon_type = 0
+
+    def add_notification(self, event: str, address: str):
+        """Add an 'other' notification to the notification configuration of the metadata"""
+        self.notifications.append((event, address))
 
     def gen_store_metadata_request(self) -> bytes:
         """Generate encoded (but unsigned) StoreMetadataReqest DO (SGP.22 5.5.3)"""
@@ -82,6 +103,15 @@ class ProfileMetadata:
             'serviceProviderName': self.spn,
             'profileName': self.profile_name,
         }
+        if self.icon:
+            smr['icon'] = self.icon
+            smr['iconType'] = self.icon_type
+        nci = []
+        for n in self.notifications:
+            pmo = PMO(n[0])
+            nci.append({'profileManagementOperation': pmo.to_bitstring(), 'notificationAddress': n[1]})
+        if len(nci):
+            smr['notificationConfigurationInfo'] = nci
         return rsp.asn1.encode('StoreMetadataRequest', smr)
 
 
@@ -183,3 +213,79 @@ class BoundProfilePackage(ProfilePackage):
 
         # manual DER encode: wrap in outer SEQUENCE
         return bertlv_encode_tag(0xbf36) + bertlv_encode_len(len(bpp_seq)) + bpp_seq
+
+    def decode(self, euicc_ot, eid: str, bpp_bin: bytes):
+        """Decode a BPP into the PPP and subsequently UPP. This is what happens inside an eUICC."""
+
+        def split_bertlv_sequence(sequence: bytes) -> List[bytes]:
+            remainder = sequence
+            ret = []
+            while remainder:
+                _tag, _l, tlv, remainder = bertlv_return_one_rawtlv(remainder)
+                ret.append(tlv)
+            return ret
+
+        # we don't use rsp.asn1.decode('boundProfilePackage') here, as the BSP needs
+        # fully encoded + MACed TLVs including their tag + length values.
+        #bpp = rsp.asn1.decode('BoundProfilePackage', bpp_bin)
+
+        tag, _l, v, _remainder = bertlv_parse_one_rawtag(bpp_bin)
+        if len(_remainder):
+            raise ValueError('Excess data at end of TLV')
+        if tag != 0xbf36:
+            raise ValueError('Unexpected outer tag: %s' % tag)
+
+        # InitialiseSecureChannelRequest
+        tag, _l, iscr_bin, remainder = bertlv_return_one_rawtlv(v)
+        iscr = rsp.asn1.decode('InitialiseSecureChannelRequest', iscr_bin)
+
+        # configureIsdpRequest
+        tag, _l, firstSeqOf87, remainder = bertlv_parse_one_rawtag(remainder)
+        if tag != 0xa0:
+            raise ValueError("Unexpected 'firstSequenceOf87' tag: %s" % tag)
+        firstSeqOf87 = split_bertlv_sequence(firstSeqOf87)
+
+        # storeMetadataRequest
+        tag, _l, seqOf88, remainder = bertlv_parse_one_rawtag(remainder)
+        if tag != 0xa1:
+            raise ValueError("Unexpected 'sequenceOf88' tag: %s" % tag)
+        seqOf88 = split_bertlv_sequence(seqOf88)
+
+        tag, _l, tlv, remainder = bertlv_parse_one_rawtag(remainder)
+        if tag == 0xa2:
+            secondSeqOf87 = split_bertlv_sequence(tlv)
+            tag2, _l, seqOf86, remainder = bertlv_parse_one_rawtag(remainder)
+            if tag2 != 0xa3:
+                raise ValueError("Unexpected 'sequenceOf86' tag: %s" % tag)
+            seqOf86 = split_bertlv_sequence(seqOf86)
+        elif tag == 0xa3:
+            secondSeqOf87 = None
+            seqOf86 = split_bertlv_sequence(tlv)
+        else:
+            raise ValueError("Unexpected 'secondSequenceOf87' tag: %s" % tag)
+
+        # extract smdoOtpk from initialiseSecureChannel
+        smdp_otpk = iscr['smdpOtpk']
+
+        # Generate Session Keys using the CRT, opPK.DP.ECKA and otSK.EUICC.ECKA according to annex G
+        smdp_public_key = ec.EllipticCurvePublicKey.from_encoded_point(euicc_ot.curve, smdp_otpk)
+        self.shared_secret = euicc_ot.exchange(ec.ECDH(), smdp_public_key)
+
+        crt = iscr['controlRefTemplate']
+        bsp = BspInstance.from_kdf(self.shared_secret, int.from_bytes(crt['keyType'], 'big'), int.from_bytes(crt['keyLen'], 'big'), crt['hostId'], h2b(eid))
+
+        self.encoded_configureISDPRequest = bsp.demac_and_decrypt(firstSeqOf87)
+        self.configureISDPRequest = rsp.asn1.decode('ConfigureISDPRequest', self.encoded_configureISDPRequest)
+
+        self.encoded_storeMetadataRequest = bsp.demac_only(seqOf88)
+        self.storeMetadataRequest = rsp.asn1.decode('StoreMetadataRequest', self.encoded_storeMetadataRequest)
+
+        if secondSeqOf87 != None:
+            rsk_bin = bsp.demac_and_decrypt(secondSeqOf87)
+            rsk = rsp.asn1.decode('ReplaceSessionKeysRequest', rsk_bin)
+            # process replace_session_keys!
+            bsp = BspInstance(rsk['ppkEnc'], rsk['ppkCmac'], rsk['initialMacChainingValue'])
+            self.replaceSessionKeysRequest = rsk
+
+        self.upp = bsp.demac_and_decrypt(seqOf86)
+        return self.upp

pySim/esim/es9p.py

@@ -0,0 +1,177 @@
+"""GSMA eSIM RSP ES9+ interface according ot SGP.22 v2.5"""
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import requests
+import logging
+import time
+
+import pySim.esim.rsp as rsp
+from pySim.esim.http_json_api import *
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+class param:
+    class RspAsn1Par(ApiParamBase64):
+        """Generalized RSP ASN.1 parameter: base64-wrapped ASN.1 DER. Derived classes must provide
+        the asn1_type class variable to indicate the name of the ASN.1 type to use for encode/decode."""
+        asn1_type = None # must be overridden by derived class
+
+        @classmethod
+        def _decode(cls, data):
+            data = ApiParamBase64.decode(data)
+            return rsp.asn1.decode(cls.asn1_type, data)
+
+        @classmethod
+        def _encode(cls, data):
+            data = rsp.asn1.encode(cls.asn1_type, data)
+            return ApiParamBase64.encode(data)
+
+    class EuiccInfo1(RspAsn1Par):
+        asn1_type = 'EUICCInfo1'
+
+    class ServerSigned1(RspAsn1Par):
+        asn1_type = 'ServerSigned1'
+
+    class PrepareDownloadResponse(RspAsn1Par):
+        asn1_type = 'PrepareDownloadResponse'
+
+    class AuthenticateServerResponse(RspAsn1Par):
+        asn1_type = 'AuthenticateServerResponse'
+
+    class SmdpSigned2(RspAsn1Par):
+        asn1_type = 'SmdpSigned2'
+
+    class StoreMetadataRequest(RspAsn1Par):
+        asn1_type = 'StoreMetadataRequest'
+
+    class PendingNotification(RspAsn1Par):
+        asn1_type = 'PendingNotification'
+
+    class CancelSessionResponse(RspAsn1Par):
+        asn1_type = 'CancelSessionResponse'
+
+    class TransactionId(ApiParamString):
+        pass
+
+class Es9PlusApiFunction(JsonHttpApiFunction):
+    pass
+
+# ES9+ InitiateAuthentication function (SGP.22 section 6.5.2.6)
+class InitiateAuthentication(Es9PlusApiFunction):
+    path = '/gsma/rsp2/es9plus/initiateAuthentication'
+    extra_http_req_headers = { 'User-Agent': 'gsma-rsp-lpad' }
+    input_params = {
+        'euiccChallenge': ApiParamBase64,
+        'euiccInfo1': param.EuiccInfo1,
+        'smdpAddress': SmdpAddress,
+      }
+    input_mandatory = ['euiccChallenge', 'euiccInfo1', 'smdpAddress']
+    output_params = {
+        'header': JsonResponseHeader,
+        'transactionId': param.TransactionId,
+        'serverSigned1': param.ServerSigned1,
+        'serverSignature1': ApiParamBase64,
+        'euiccCiPKIdToBeUsed': ApiParamBase64,
+        'serverCertificate': ApiParamBase64,
+      }
+    output_mandatory = ['header', 'transactionId', 'serverSigned1', 'serverSignature1',
+                        'euiccCiPKIdToBeUsed', 'serverCertificate']
+
+# ES9+ GetBoundProfilePackage function (SGP.22 section 6.5.2.7)
+class GetBoundProfilePackage(Es9PlusApiFunction):
+    path = '/gsma/rsp2/es9plus/getBoundProfilePackage'
+    extra_http_req_headers = { 'User-Agent': 'gsma-rsp-lpad' }
+    input_params = {
+        'transactionId': param.TransactionId,
+        'prepareDownloadResponse': param.PrepareDownloadResponse,
+      }
+    input_mandatory = ['transactionId', 'prepareDownloadResponse']
+    output_params = {
+        'header': JsonResponseHeader,
+        'transactionId': param.TransactionId,
+        'boundProfilePackage': ApiParamBase64,
+      }
+    output_mandatory = ['header', 'transactionId', 'boundProfilePackage']
+
+# ES9+ AuthenticateClient function (SGP.22 section 6.5.2.8)
+class AuthenticateClient(Es9PlusApiFunction):
+    path= '/gsma/rsp2/es9plus/authenticateClient'
+    extra_http_req_headers = { 'User-Agent': 'gsma-rsp-lpad' }
+    input_params = {
+        'transactionId': param.TransactionId,
+        'authenticateServerResponse': param.AuthenticateServerResponse,
+      }
+    input_mandatory = ['transactionId', 'authenticateServerResponse']
+    output_params = {
+        'header': JsonResponseHeader,
+        'transactionId': param.TransactionId,
+        'profileMetadata': param.StoreMetadataRequest,
+        'smdpSigned2': param.SmdpSigned2,
+        'smdpSignature2': ApiParamBase64,
+        'smdpCertificate': ApiParamBase64,
+      }
+    output_mandatory = ['header', 'transactionId', 'profileMetadata', 'smdpSigned2',
+                        'smdpSignature2', 'smdpCertificate']
+
+# ES9+ HandleNotification function (SGP.22 section 6.5.2.9)
+class HandleNotification(Es9PlusApiFunction):
+    path = '/gsma/rsp2/es9plus/handleNotification'
+    extra_http_req_headers = { 'User-Agent': 'gsma-rsp-lpad' }
+    input_params = {
+        'pendingNotification': param.PendingNotification,
+      }
+    input_mandatory = ['pendingNotification']
+    expected_http_status = 204
+
+# ES9+ CancelSession function (SGP.22 section 6.5.2.10)
+class CancelSession(Es9PlusApiFunction):
+    path = '/gsma/rsp2/es9plus/cancelSession'
+    extra_http_req_headers = { 'User-Agent': 'gsma-rsp-lpad' }
+    input_params = {
+        'transactionId': param.TransactionId,
+        'cancelSessionResponse': param.CancelSessionResponse,
+      }
+    input_mandatory = ['transactionId', 'cancelSessionResponse']
+
+class Es9pApiClient:
+    def __init__(self, url_prefix:str, server_cert_verify: str = None):
+        self.session = requests.Session()
+        self.session.verify = False # FIXME HACK
+        if server_cert_verify:
+            self.session.verify = server_cert_verify
+
+        self.initiateAuthentication = InitiateAuthentication(url_prefix, '', self.session)
+        self.authenticateClient = AuthenticateClient(url_prefix, '', self.session)
+        self.getBoundProfilePackage = GetBoundProfilePackage(url_prefix, '', self.session)
+        self.handleNotification = HandleNotification(url_prefix, '', self.session)
+        self.cancelSession = CancelSession(url_prefix, '', self.session)
+
+    def call_initiateAuthentication(self, data: dict) -> dict:
+        return self.initiateAuthentication.call(data)
+
+    def call_authenticateClient(self, data: dict) -> dict:
+        return self.authenticateClient.call(data)
+
+    def call_getBoundProfilePackage(self, data: dict) -> dict:
+        return self.getBoundProfilePackage.call(data)
+
+    def call_handleNotification(self, data: dict) -> dict:
+        return self.handleNotification.call(data)
+
+    def call_cancelSession(self, data: dict) -> dict:
+        return self.cancelSession.call(data)

pySim/esim/http_json_api.py

@@ -0,0 +1,259 @@
+"""GSMA eSIM RSP HTTP/REST/JSON interface according to SGP.22 v2.5"""
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import abc
+import requests
+import logging
+import json
+from typing import Optional
+import base64
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+class ApiParam(abc.ABC):
+    """A class reprsenting a single parameter in the API."""
+    @classmethod
+    def verify_decoded(cls, data):
+        """Verify the decoded reprsentation of a value. Should raise an exception if somthing is odd."""
+        pass
+
+    @classmethod
+    def verify_encoded(cls, data):
+        """Verify the encoded reprsentation of a value. Should raise an exception if somthing is odd."""
+        pass
+
+    @classmethod
+    def encode(cls, data):
+        """[Validate and] Encode the given value."""
+        cls.verify_decoded(data)
+        encoded = cls._encode(data)
+        cls.verify_decoded(encoded)
+        return encoded
+
+    @classmethod
+    def _encode(cls, data):
+        """encoder function, typically [but not always] overridden by derived class."""
+        return data
+
+    @classmethod
+    def decode(cls, data):
+        """[Validate and] Decode the given value."""
+        cls.verify_encoded(data)
+        decoded = cls._decode(data)
+        cls.verify_decoded(decoded)
+        return decoded
+
+    @classmethod
+    def _decode(cls, data):
+        """decoder function, typically [but not always] overridden by derived class."""
+        return data
+
+class ApiParamString(ApiParam):
+    """Base class representing an API parameter of 'string' type."""
+    pass
+
+
+class ApiParamInteger(ApiParam):
+    """Base class representing an API parameter of 'integer' type."""
+    @classmethod
+    def _decode(cls, data):
+        return int(data)
+
+    @classmethod
+    def _encode(cls, data):
+        return str(data)
+
+    @classmethod
+    def verify_decoded(cls, data):
+        if not isinstance(data, int):
+            raise TypeError('Expected an integer input data type')
+
+    @classmethod
+    def verify_encoded(cls, data):
+        if isinstance(data, int):
+            return
+        if not data.isdecimal():
+            raise ValueError('integer (%s) contains non-decimal characters' % data)
+        assert str(int(data)) == data
+
+class ApiParamBoolean(ApiParam):
+    """Base class representing an API parameter of 'boolean' type."""
+    @classmethod
+    def _encode(cls, data):
+        return bool(data)
+
+class ApiParamFqdn(ApiParam):
+    """String, as a list of domain labels concatenated using the full stop (dot, period) character as
+    separator between labels. Labels are restricted to the Alphanumeric mode character set defined in table 5
+    of ISO/IEC 18004"""
+    @classmethod
+    def verify_encoded(cls, data):
+        # FIXME
+        pass
+
+class ApiParamBase64(ApiParam):
+    @classmethod
+    def _decode(cls, data):
+        return base64.b64decode(data)
+
+    @classmethod
+    def _encode(cls, data):
+        return base64.b64encode(data).decode('ascii')
+
+class SmdpAddress(ApiParamFqdn):
+    pass
+
+class JsonResponseHeader(ApiParam):
+    """SGP.22 section 6.5.1.4."""
+    @classmethod
+    def verify_decoded(cls, data):
+        fe_status = data.get('functionExecutionStatus')
+        if not fe_status:
+            raise ValueError('Missing mandatory functionExecutionStatus in header')
+        status = fe_status.get('status')
+        if not status:
+            raise ValueError('Missing mandatory status in header functionExecutionStatus')
+        if status not in ['Executed-Success', 'Executed-WithWarning', 'Failed', 'Expired']:
+            raise ValueError('Unknown/unspecified status "%s"' % status)
+
+
+class HttpStatusError(Exception):
+    pass
+
+class HttpHeaderError(Exception):
+    pass
+
+class ApiError(Exception):
+    """Exception representing an error at the API level (status != Executed)."""
+    def __init__(self, func_ex_status: dict):
+        self.status = func_ex_status['status']
+        sec = {
+            'subjectCode': None,
+            'reasonCode': None,
+            'subjectIdentifier': None,
+            'message': None,
+        }
+        actual_sec = func_ex_status.get('statusCodeData', None)
+        sec.update(actual_sec)
+        self.subject_code = sec['subjectCode']
+        self.reason_code = sec['reasonCode']
+        self.subject_id = sec['subjectIdentifier']
+        self.message = sec['message']
+
+    def __str__(self):
+        return f'{self.status}("{self.subject_code}","{self.reason_code}","{self.subject_id}","{self.message}")'
+
+class JsonHttpApiFunction(abc.ABC):
+    """Base classs for representing an HTTP[s] API Function."""
+     # the below class variables are expected to be overridden in derived classes
+
+    path = None
+    # dictionary of input parameters. key is parameter name, value is ApiParam class
+    input_params = {}
+    # list of mandatory input parameters
+    input_mandatory = []
+    # dictionary of output parameters. key is parameter name, value is ApiParam class
+    output_params = {}
+    # list of mandatory output parameters (for successful response)
+    output_mandatory = []
+    # expected HTTP status code of the response
+    expected_http_status = 200
+    # the HTTP method used (GET, OPTIONS, HEAD, POST, PUT, PATCH or DELETE)
+    http_method = 'POST'
+    extra_http_req_headers = {}
+
+    def __init__(self, url_prefix: str, func_req_id: Optional[str], session: requests.Session):
+        self.url_prefix = url_prefix
+        self.func_req_id = func_req_id
+        self.session = session
+
+    def encode(self, data: dict, func_call_id: Optional[str] = None) -> dict:
+        """Validate an encode input dict into JSON-serializable dict for request body."""
+        output = {}
+        if func_call_id:
+            output['header'] = {
+                    'functionRequesterIdentifier': self.func_req_id,
+                    'functionCallIdentifier': func_call_id
+                }
+
+        for p in self.input_mandatory:
+            if not p in data:
+                raise ValueError('Mandatory input parameter %s missing' % p)
+        for p, v in data.items():
+            p_class = self.input_params.get(p)
+            if not p_class:
+                logger.warning('Unexpected/unsupported input parameter %s=%s', p, v)
+                output[p] = v
+            else:
+                output[p] = p_class.encode(v)
+        return output
+
+    def decode(self, data: dict) -> dict:
+        """[further] Decode and validate the JSON-Dict of the response body."""
+        output = {}
+        if 'header' in self.output_params:
+            # let's first do the header, it's special
+            if not 'header' in data:
+                raise ValueError('Mandatory output parameter "header" missing')
+            hdr_class = self.output_params.get('header')
+            output['header'] = hdr_class.decode(data['header'])
+
+            if output['header']['functionExecutionStatus']['status'] not in ['Executed-Success','Executed-WithWarning']:
+                raise ApiError(output['header']['functionExecutionStatus'])
+        # we can only expect mandatory parameters to be present in case of successful execution
+        for p in self.output_mandatory:
+            if p == 'header':
+                continue
+            if not p in data:
+                raise ValueError('Mandatory output parameter "%s" missing' % p)
+        for p, v in data.items():
+            p_class = self.output_params.get(p)
+            if not p_class:
+                logger.warning('Unexpected/unsupported output parameter "%s"="%s"', p, v)
+                output[p] = v
+            else:
+                output[p] = p_class.decode(v)
+        return output
+
+    def call(self, data: dict, func_call_id: Optional[str] = None, timeout=10) -> Optional[dict]:
+        """Make an API call to the HTTP API endpoint represented by this object.
+        Input data is passed in `data` as json-serializable dict.  Output data
+        is returned as json-deserialized dict."""
+        url = self.url_prefix + self.path
+        encoded = json.dumps(self.encode(data, func_call_id))
+        req_headers = {
+            'Content-Type': 'application/json',
+            'X-Admin-Protocol': 'gsma/rsp/v2.5.0',
+        }
+        req_headers.update(self.extra_http_req_headers)
+
+        logger.debug("HTTP REQ %s - hdr: %s '%s'" % (url, req_headers, encoded))
+        response = self.session.request(self.http_method, url, data=encoded, headers=req_headers, timeout=timeout)
+        logger.debug("HTTP RSP-STS: [%u] hdr: %s" % (response.status_code, response.headers))
+        logger.debug("HTTP RSP: %s" % (response.content))
+
+        if response.status_code != self.expected_http_status:
+            raise HttpStatusError(response)
+        if not response.headers.get('Content-Type').startswith(req_headers['Content-Type']):
+            raise HttpHeaderError(response)
+        if not response.headers.get('X-Admin-Protocol', 'gsma/rsp/v2.unknown').startswith('gsma/rsp/v2.'):
+            raise HttpHeaderError(response)
+
+        if response.content:
+            return self.decode(response.json())
+        return None

pySim/esim/rsp.py

@@ -23,6 +23,8 @@ import shelve
 from cryptography.hazmat.primitives.asymmetric import ec
 from cryptography.hazmat.primitives.serialization import Encoding
 from cryptography import x509
+from osmocom.utils import b2h
+from osmocom.tlv import bertlv_parse_one_rawtag, bertlv_return_one_rawtlv
 
 from pySim.esim import compile_asn1_subdir
 
@@ -36,7 +38,7 @@ class RspSessionState:
     def __init__(self, transactionId: str, serverChallenge: bytes, ci_cert_id: bytes):
         self.transactionId = transactionId
         self.serverChallenge = serverChallenge
-        #  used at a later point between API calsl
+        #  used at a later point between API calls
         self.ci_cert_id = ci_cert_id
         self.euicc_cert: Optional[x509.Certificate] = None
         self.eum_cert: Optional[x509.Certificate] = None
@@ -96,3 +98,35 @@ class RspSessionState:
 class RspSessionStore(shelve.DbfilenameShelf):
     """A derived class as wrapper around the database-backed non-volatile storage 'shelve', in case we might
     need to extend it in the future. We use it to store RspSessionState objects indexed by transactionId."""
+
+def extract_euiccSigned1(authenticateServerResponse: bytes) -> bytes:
+    """Extract the raw, DER-encoded binary euiccSigned1 field from the given AuthenticateServerResponse. This
+    is needed due to the very peculiar SGP.22 notion of signing sections of DER-encoded ASN.1 objects."""
+    rawtag, l, v, remainder = bertlv_parse_one_rawtag(authenticateServerResponse)
+    if len(remainder):
+        raise ValueError('Excess data at end of TLV')
+    if rawtag != 0xbf38:
+        raise ValueError('Unexpected outer tag: %s' % b2h(rawtag))
+    rawtag, l, v1, remainder = bertlv_parse_one_rawtag(v)
+    if rawtag != 0xa0:
+        raise ValueError('Unexpected tag where CHOICE was expected')
+    rawtag, l, tlv2, remainder = bertlv_return_one_rawtlv(v1)
+    if rawtag != 0x30:
+        raise ValueError('Unexpected tag where SEQUENCE was expected')
+    return tlv2
+
+def extract_euiccSigned2(prepareDownloadResponse: bytes) -> bytes:
+    """Extract the raw, DER-encoded binary euiccSigned2 field from the given prepareDownloadrResponse. This is
+    needed due to the very peculiar SGP.22 notion of signing sections of DER-encoded ASN.1 objects."""
+    rawtag, l, v, remainder = bertlv_parse_one_rawtag(prepareDownloadResponse)
+    if len(remainder):
+        raise ValueError('Excess data at end of TLV')
+    if rawtag != 0xbf21:
+        raise ValueError('Unexpected outer tag: %s' % b2h(rawtag))
+    rawtag, l, v1, remainder = bertlv_parse_one_rawtag(v)
+    if rawtag != 0xa0:
+        raise ValueError('Unexpected tag where CHOICE was expected')
+    rawtag, l, tlv2, remainder = bertlv_return_one_rawtlv(v1)
+    if rawtag != 0x30:
+        raise ValueError('Unexpected tag where SEQUENCE was expected')
+    return tlv2

pySim/esim/saip/oid.py

@@ -15,8 +15,10 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from functools import total_ordering
 from typing import List, Union
 
+@total_ordering
 class OID:
     @staticmethod
     def intlist_from_str(instr: str) -> List[int]:
@@ -26,6 +28,10 @@ class OID:
     def str_from_intlist(intlist: List[int]) -> str:
         return '.'.join([str(x) for x in intlist])
 
+    @staticmethod
+    def highest_oid(oids: List['OID']) -> 'OID':
+        return sorted(oids)[-1]
+
     def __init__(self, initializer: Union[List[int], str]):
         if isinstance(initializer, str):
             self.intlist = self.intlist_from_str(initializer)
@@ -38,6 +44,43 @@ class OID:
     def __repr__(self) -> str:
         return 'OID(%s)' % (str(self))
 
+    def __eq__(self, other: 'OID'):
+        return (self.intlist == other.intlist)
+
+    def __ne__(self, other: 'OID'):
+        # implement based on __eq__
+        return not (self == other)
+
+    def cmp(self, other: 'OID'):
+        self_len = len(self.intlist)
+        other_len = len(other.intlist)
+        common_len = min(self_len, other_len)
+        max_len = max(self_len, other_len)
+
+        for i in range(0, max_len+1):
+            if i >= self_len:
+                # other list is longer
+                return -1
+            if i >= other_len:
+                # our list is longer
+                return 1
+            if self.intlist[i] > other.intlist[i]:
+                # our version is higher
+                return 1
+            if self.intlist[i] < other.intlist[i]:
+                # other version is higher
+                return -1
+            # continue to next digit
+        return 0
+
+    def __gt__(self, other: 'OID'):
+        if self.cmp(other) > 0:
+            return True
+
+    def prefix_match(self, oid_str: Union[str, 'OID']):
+        """determine if oid_str is equal or below our OID."""
+        return str(oid_str).startswith(str(self))
+
 
 class eOID(OID):
     """OID helper for TCA eUICC prefix"""
@@ -53,18 +96,18 @@ DF_TELECOM = eOID("2.3")
 DF_TELECOM_v2 = eOID("2.3.2")
 ADF_USIM_by_default = eOID("2.4")
 ADF_USIM_by_default_v2 = eOID("2.4.2")
-ADF_USIM_not_by_default = eOID("2.5")
-ADF_USIM_not_by_default_v2 = eOID("2.5.2")
-ADF_USIM_not_by_default_v3 = eOID("2.5.3")
+ADF_USIMopt_not_by_default = eOID("2.5")
+ADF_USIMopt_not_by_default_v2 = eOID("2.5.2")
+ADF_USIMopt_not_by_default_v3 = eOID("2.5.3")
 DF_PHONEBOOK_ADF_USIM = eOID("2.6")
 DF_GSM_ACCESS_ADF_USIM = eOID("2.7")
 ADF_ISIM_by_default = eOID("2.8")
-ADF_ISIM_not_by_default = eOID("2.9")
-ADF_ISIM_not_by_default_v2 = eOID("2.9.2")
+ADF_ISIMopt_not_by_default = eOID("2.9")
+ADF_ISIMopt_not_by_default_v2 = eOID("2.9.2")
 ADF_CSIM_by_default = eOID("2.10")
 ADF_CSIM_by_default_v2 = eOID("2.10.2")
-ADF_CSIM_not_by_default = eOID("2.11")
-ADF_CSIM_not_by_default_v2 = eOID("2.11.2")
+ADF_CSIMopt_not_by_default = eOID("2.11")
+ADF_CSIMopt_not_by_default_v2 = eOID("2.11.2")
 DF_EAP = eOID("2.12")
 DF_5GS = eOID("2.13")
 DF_5GS_v2 = eOID("2.13.2")
@@ -73,5 +116,5 @@ DF_5GS_v4 = eOID("2.13.4")
 DF_SAIP = eOID("2.14")
 DF_SNPN = eOID("2.15")
 DF_5GProSe = eOID("2.16")
-IoT_default = eOID("2.17")
-IoT_default = eOID("2.18")
+IoT_by_default = eOID("2.17")
+IoTopt_not_by_default = eOID("2.18")

pySim/esim/saip/personalization.py

@@ -19,7 +19,7 @@ import abc
 import io
 from typing import List, Tuple
 
-from pySim.tlv import camel_to_snake
+from osmocom.tlv import camel_to_snake
 from pySim.utils import enc_iccid, enc_imsi, h2b, rpad, sanitize_iccid
 from pySim.esim.saip import ProfileElement, ProfileElementSequence
 

pySim/esim/saip/templates.py

@@ -17,14 +17,36 @@
 
 from typing import *
 from copy import deepcopy
+from pySim.utils import all_subclasses, h2b
+from pySim.filesystem import Path
 import pySim.esim.saip.oid as OID
 
 class FileTemplate:
-    """Representation of a single file in a SimAlliance/TCA Profile Template."""
+    """Representation of a single file in a SimAlliance/TCA Profile Template. The argument order
+    is done to match that of the tables in Section 9 of the SAIP specification."""
     def __init__(self, fid:int, name:str, ftype, nb_rec: Optional[int], size:Optional[int], arr:int,
                  sfi:Optional[int] = None, default_val:Optional[str] = None, content_rqd:bool = True,
                  params:Optional[List] = None, ass_serv:Optional[List[int]]=None, high_update:bool = False,
-                 pe_name:Optional[str] = None):
+                 pe_name:Optional[str] = None, repeat:bool = False, ppath: List[int] = []):
+        """
+        Args:
+            fid: The 16bit file-identifier of the file
+            name: The name of the file in human-readable "EF.FOO", "DF.BAR" notation
+            ftype: The type of the file; can be 'MF', 'ADF', 'DF', 'TR', 'LF', 'CY', 'BT'
+            nb_rec: Then number of records (only valid for 'LF' and 'CY')
+            size: The size of the file ('TR', 'BT'); size of each record ('LF, 'CY')
+            arr: The record number of EF.ARR for referenced access rules
+            sfi: The short file identifier, if any
+            default_val: The default value [pattern] of the file
+            content_rqd: Whether an instance of template *must* specify file contents
+            params: A list of parameters that an instance of the template *must* specify
+            ass_serv: The associated service[s] of the service table
+            high_update: Is this file of "high update frequency" type?
+            pe_name: The name of this file in the ASN.1 type of the PE. Auto-generated for most.
+            repeat: Whether the default_val pattern is a repeating pattern.
+            ppath: The intermediate path between the base_df of the ProfileTemplate and this file.  If not
+                   specified, the file will be created immediately underneath the base_df.
+        """
         # initialize from arguments
         self.fid = fid
         self.name = name
@@ -36,18 +58,24 @@ class FileTemplate:
         if ftype in ['LF', 'CY']:
             self.nb_rec = nb_rec
             self.rec_len = size
-        elif ftype in ['TR']:
+        elif ftype in ['TR', 'BT']:
             self.file_size = size
         self.arr = arr
         self.sfi = sfi
         self.default_val = default_val
+        self.default_val_repeat = repeat
         self.content_rqd = content_rqd
         self.params = params
         self.ass_serv = ass_serv
         self.high_update = high_update
+        self.ppath = ppath # parent path, if this FileTemplate is not immediately below the base_df
         # initialize empty
         self.parent = None
         self.children = []
+        if self.default_val:
+            length = self._default_value_len() or 100
+            # run the method once to verify the pattern can be processed
+            self.expand_default_value_pattern(length)
 
     def __str__(self) -> str:
         return "FileTemplate(%s)" % (self.name)
@@ -56,26 +84,122 @@ class FileTemplate:
         s_fid = "%04x" % self.fid if self.fid is not None else 'None'
         s_arr = self.arr if self.arr is not None else 'None'
         s_sfi = "%02x" % self.sfi if self.sfi is not None else 'None'
-        return "FileTemplate(%s/%s, %s, %s, arr=%s, sfi=%s)" % (self.name, self.pe_name, s_fid,
-                                                                self.file_type, s_arr, s_sfi)
+        return "FileTemplate(%s/%s, %s, %s, arr=%s, sfi=%s, ppath=%s)" % (self.name, self.pe_name, s_fid, self.file_type, s_arr, s_sfi, self.ppath)
+
+    def print_tree(self, indent:str = ""):
+        """recursive printing of FileTemplate tree structure."""
+        print("%s%s (%s)" % (indent, repr(self), self.path))
+        indent += " "
+        for c in self.children:
+            c.print_tree(indent)
+
+    @property
+    def path(self):
+        """Return the path of the given File within the hierarchy."""
+        if self.parent:
+            return self.parent.path + self.name
+        else:
+            return Path(self.name)
+
+    def get_file_by_path(self, path: List[str]) -> Optional['FileTemplate']:
+        """Return a FileTemplate matching the given path within this ProfileTemplate."""
+        if path[0].lower() != self.name.lower():
+            return None
+        for c in self.children:
+            if path[1].lower() == c.name.lower():
+                return c.get_file_by_path(path[1:])
+
+    def _default_value_len(self):
+        if self.file_type in ['TR']:
+            return self.file_size
+        elif self.file_type in ['LF', 'CY']:
+            return self.rec_len
+
+    def expand_default_value_pattern(self, length: Optional[int] = None) -> Optional[bytes]:
+        """Expand the default value pattern to the specified length."""
+        if length is None:
+            length = self._default_value_len()
+        if length is None:
+            raise ValueError("%s does not have a default length" % self)
+        if not self.default_val:
+            return None
+        if not '...' in self.default_val:
+            return h2b(self.default_val)
+        l = self.default_val.split('...')
+        if len(l) != 2:
+            raise ValueError("Pattern '%s' contains more than one ..." % self.default_val)
+        prefix = h2b(l[0])
+        suffix = h2b(l[1])
+        pad_len = length - len(prefix) - len(suffix)
+        if pad_len <= 0:
+            ret = prefix + suffix
+            return ret[:length]
+        return prefix + prefix[-1:] * pad_len + suffix
+
 
 class ProfileTemplate:
     """Representation of a SimAlliance/TCA Profile Template.  Each Template is identified by its OID and
     consists of a number of file definitions.  We implement each profile template as a class derived from this
     base class.  Each such derived class is a singleton and has no instances."""
     created_by_default: bool = False
+    optional: bool = False
     oid: Optional[OID.eOID] = None
     files: List[FileTemplate] = []
-    files_by_pename: dict[str,FileTemplate] = {}
+
+    # indicates that a given template does not have its own 'base DF', but that its contents merely
+    # extends that of the 'base DF' of another template
+    extends: Optional['ProfileTemplate'] = None
+
+    # indicates a parent ProfileTemplate below whose 'base DF' our files should be placed.
+    parent: Optional['ProfileTemplate'] = None
 
     def __init_subclass__(cls, **kwargs):
         """This classmethod is called automatically after executing the subclass body. We use it to
         initialize the cls.files_by_pename from the cls.files"""
         super().__init_subclass__(**kwargs)
+        cur_df = None
+
+        cls.files_by_pename: dict[str,FileTemplate] = {}
+        cls.tree: List[FileTemplate] = []
+
+        if not cls.optional and not cls.files[0].file_type in ['MF', 'DF', 'ADF']:
+            raise ValueError('First file in non-optional template must be MF, DF or ADF (is: %s)' % cls.files[0])
         for f in cls.files:
+            if f.file_type in ['MF', 'DF', 'ADF']:
+                if cur_df == None:
+                    cls.tree.append(f)
+                    f.parent = None
+                    cur_df = f
+                else:
+                    # "cd .."
+                    if cur_df.parent:
+                        cur_df = cur_df.parent
+                    f.parent = cur_df
+                    cur_df.children.append(f)
+                    cur_df = f
+            else:
+                if cur_df == None:
+                    cls.tree.append(f)
+                    f.parent = None
+                else:
+                    cur_df.children.append(f)
+                    f.parent = cur_df
             cls.files_by_pename[f.pe_name] = f
         ProfileTemplateRegistry.add(cls)
 
+    @classmethod
+    def print_tree(cls):
+        for c in cls.tree:
+            c.print_tree()
+
+    @classmethod
+    def base_df(cls) -> FileTemplate:
+        """Return the FileTemplate for the base DF of the given template.  This may be a DF or ADF
+        within this template, or refer to another template (e.g. mandatory USIM if we are optional USIM."""
+        if cls.extends:
+            return cls.extends.base_df
+        return cls.files[0]
+
 class ProfileTemplateRegistry:
     """A registry of profile templates.  Exists as a singleton class with no instances and only
     classmethods."""
@@ -98,7 +222,7 @@ class ProfileTemplateRegistry:
         return cls.by_oid.get(oid, None)
 
 # below are transcribed template definitions from "ANNEX A (Normative): File Structure Templates Definition"
-# of "Profile interoperability specification V3.1 Final" (unless other version explicitly specified).
+# of "Profile interoperability specification V3.3.1 Final" (unless other version explicitly specified).
 
 # Section 9.2
 class FilesAtMF(ProfileTemplate):
@@ -129,46 +253,47 @@ class FilesCD(ProfileTemplate):
 # Section 9.4: Do this separately, so we can use them also from 9.5.3
 df_pb_files = [
     FileTemplate(0x5f3a, 'DF.PHONEBOOK', 'DF', None, None,  14, None, None, True, ['pinStatusTemplateDO']),
-    FileTemplate(0x4f30, 'EF.PBR',       'LF', None, None,   2, None, None, True, ['nb_rec', 'size']),
+    FileTemplate(0x4f30, 'EF.PBR',       'LF', None, None,   2, None, None, True, ['nb_rec', 'size'], ppath=[0x5f3a]),
 ]
 for i in range(0x38, 0x40):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.EXT1', 'LF', None,   13,  5, None, '00FF...FF', False, ['size','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.EXT1', 'LF', None,   13,  5, None, '00FF...FF', False, ['size','sfi'], ppath=[0x5f3a]))
 for i in range(0x40, 0x48):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.AAS', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.AAS', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size'], ppath=[0x5f3a]))
 for i in range(0x48, 0x50):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.GAS', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.GAS', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size'], ppath=[0x5f3a]))
 df_pb_files += [
-    FileTemplate(0x4f22, 'EF.PSC',       'TR', None,    4,   5, None, '00000000', False, ['sfi']),
-    FileTemplate(0x4f23, 'EF.CC',        'TR', None,    2,   5, None, '0000', False, ['sfi'], high_update=True),
-    FileTemplate(0x4f24, 'EF.PUID',      'TR', None,    2,   5, None, '0000', False, ['sfi'], high_update=True),
+    FileTemplate(0x4f22, 'EF.PSC',       'TR', None,    4,   5, None, '00000000', False, ['sfi'], ppath=[0x5f3a]),
+    FileTemplate(0x4f23, 'EF.CC',        'TR', None,    2,   5, None, '0000', False, ['sfi'], high_update=True, ppath=[0x5f3a]),
+    FileTemplate(0x4f24, 'EF.PUID',      'TR', None,    2,   5, None, '0000', False, ['sfi'], high_update=True, ppath=[0x5f3a]),
 ]
 for i in range(0x50, 0x58):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.IAP', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.IAP', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 for i in range(0x58, 0x60):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.ADN', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.ADN', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 for i in range(0x60, 0x68):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.ADN', 'LF', None,    2,  5, None, '00...00', False, ['nb_rec','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.ADN', 'LF', None,    2,  5, None, '00...00', False, ['nb_rec','sfi'], ppath=[0x5f3a]))
 for i in range(0x68, 0x70):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.ANR', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.ANR', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 for i in range(0x70, 0x78):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.PURI', 'LF', None, None,  5, None, None, True, ['nb_rec','size','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.PURI', 'LF', None, None,  5, None, None, True, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 for i in range(0x78, 0x80):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.EMAIL', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.EMAIL', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 for i in range(0x80, 0x88):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.SNE', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.SNE', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 for i in range(0x88, 0x90):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.UID', 'LF', None,    2,  5, None, '0000', False, ['nb_rec','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.UID', 'LF', None,    2,  5, None, '0000', False, ['nb_rec','sfi'], ppath=[0x5f3a]))
 for i in range(0x90, 0x98):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.GRP', 'LF', None, None,  5, None, '00...00', False, ['nb_rec','size','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.GRP', 'LF', None, None,  5, None, '00...00', False, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 for i in range(0x98, 0xa0):
-    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.CCP1', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi']))
+    df_pb_files.append(FileTemplate(0x4f00+i, 'EF.CCP1', 'LF', None, None,  5, None, 'FF...FF', False, ['nb_rec','size','sfi'], ppath=[0x5f3a]))
 
 # Section 9.4 v2.3.1
 class FilesTelecom(ProfileTemplate):
     created_by_default = False
     oid = OID.DF_TELECOM
+    base_path = Path('MF')
     files = [
-        FileTemplate(0x7f11, 'DF.TELECOM',   'DF', None, None,  14, None, None, False, params=['pinStatusTemplateDO']),
+        FileTemplate(0x7f10, 'DF.TELECOM',   'DF', None, None,  14, None, None, False, params=['pinStatusTemplateDO']),
         FileTemplate(0x6f06, 'EF.ARR',       'LF', None, None,  10, None, None, True, ['nb_rec', 'size']),
         FileTemplate(0x6f53, 'EF.RMA',       'LF', None, None,   3, None, None, True, ['nb_rec', 'size']),
         FileTemplate(0x6f54, 'EF.SUME',      'TR', None,   22,   3, None, None, True),
@@ -176,29 +301,30 @@ class FilesTelecom(ProfileTemplate):
         FileTemplate(0x6fe1, 'EF.ICE_FF',    'LF', None, None,   9, None, 'FF...FF', False, ['nb_rec', 'size']),
         FileTemplate(0x6fe5, 'EF.PSISMSC',   'LF', None, None,   5, None, None, True, ['nb_rec', 'size'], ass_serv=[12,91]),
         FileTemplate(0x5f50, 'DF.GRAPHICS',  'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO']),
-        FileTemplate(0x4f20, 'EF.IMG',       'LF', None, None,   2, None, '00FF...FF', False, ['nb_rec', 'size']),
+        FileTemplate(0x4f20, 'EF.IMG',       'LF', None, None,   2, None, '00FF...FF', False, ['nb_rec', 'size'], ppath=[0x5f50]),
         # EF.IIDF below
-        FileTemplate(0x4f21, 'EF.ICE_GRAPHICS','BT',None,None,   9, None, None, False, ['size']),
-        FileTemplate(0x4f01, 'EF.LAUNCH_SCWS','TR',None, None,  10, None, None, True, ['size']),
+        FileTemplate(0x4f21, 'EF.ICE_GRAPHICS','BT',None,None,   9, None, None, False, ['size'], ppath=[0x5f50]),
+        FileTemplate(0x4f01, 'EF.LAUNCH_SCWS','TR',None, None,  10, None, None, True, ['size'], ppath=[0x5f50]),
         # EF.ICON below
     ]
     for i in range(0x40, 0x80):
-        files.append(FileTemplate(0x4f00+i, 'EF.IIDF', 'TR', None, None, 2, None, 'FF...FF', False, ['size']))
+        files.append(FileTemplate(0x4f00+i, 'EF.IIDF', 'TR', None, None, 2, None, 'FF...FF', False, ['size'], ppath=[0x5f50]))
     for i in range(0x80, 0xC0):
-        files.append(FileTemplate(0x4f00+i, 'EF.ICON', 'TR', None, None, 10, None, None, True, ['size']))
+        files.append(FileTemplate(0x4f00+i, 'EF.ICON', 'TR', None, None, 10, None, None, True, ['size'], ppath=[0x5f50]))
 
     # we copy the objects (instances) here as we also use them below from FilesUsimDfPhonebook
-    files += [deepcopy(x) for x in df_pb_files]
+    df_pb = deepcopy(df_pb_files)
+    files += df_pb
 
     files += [
         FileTemplate(0x5f3b, 'DF.MULTIMEDIA','DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[67]),
-        FileTemplate(0x4f47, 'EF.MML',       'BT', None, None,   5, None, None, False, ['size'], ass_serv=[67]),
-        FileTemplate(0x4f48, 'EF.MMDF',      'BT', None, None,   5, None, None, False, ['size'], ass_serv=[67]),
+        FileTemplate(0x4f47, 'EF.MML',       'BT', None, None,   5, None, None, False, ['size'], ass_serv=[67], ppath=[0x5f3b]),
+        FileTemplate(0x4f48, 'EF.MMDF',      'BT', None, None,   5, None, None, False, ['size'], ass_serv=[67], ppath=[0x5f3b]),
 
         FileTemplate(0x5f3c, 'DF.MMSS',      'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO']),
-        FileTemplate(0x4f20, 'EF.MLPL',      'TR', None, None,   2, 0x01, None, True, ['size']),
-        FileTemplate(0x4f21, 'EF.MSPL',      'TR', None, None,   2, 0x02, None, True, ['size']),
-        FileTemplate(0x4f21, 'EF.MMSSMODE',  'TR', None,    1,   2, 0x03, None, True),
+        FileTemplate(0x4f20, 'EF.MLPL',      'TR', None, None,   2, 0x01, None, True, ['size'], ppath=[0x5f3c]),
+        FileTemplate(0x4f21, 'EF.MSPL',      'TR', None, None,   2, 0x02, None, True, ['size'], ppath=[0x5f3c]),
+        FileTemplate(0x4f21, 'EF.MMSSMODE',  'TR', None,    1,   2, 0x03, None, True, ppath=[0x5f3c]),
     ]
 
 
@@ -206,8 +332,9 @@ class FilesTelecom(ProfileTemplate):
 class FilesTelecomV2(ProfileTemplate):
     created_by_default = False
     oid = OID.DF_TELECOM_v2
+    base_path = Path('MF')
     files = [
-        FileTemplate(0x7f11, 'DF.TELECOM',   'DF', None, None,  14, None, None, False, params=['pinStatusTemplateDO']),
+        FileTemplate(0x7f10, 'DF.TELECOM',   'DF', None, None,  14, None, None, False, params=['pinStatusTemplateDO']),
         FileTemplate(0x6f06, 'EF.ARR',       'LF', None, None,  10, None, None, True, ['nb_rec', 'size']),
         FileTemplate(0x6f53, 'EF.RMA',       'LF', None, None,   3, None, None, True, ['nb_rec', 'size']),
         FileTemplate(0x6f54, 'EF.SUME',      'TR', None,   22,   3, None, None, True),
@@ -215,40 +342,40 @@ class FilesTelecomV2(ProfileTemplate):
         FileTemplate(0x6fe1, 'EF.ICE_FF',    'LF', None, None,   9, None, 'FF...FF', False, ['nb_rec', 'size']),
         FileTemplate(0x6fe5, 'EF.PSISMSC',   'LF', None, None,   5, None, None, True, ['nb_rec', 'size'], ass_serv=[12,91]),
         FileTemplate(0x5f50, 'DF.GRAPHICS',  'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO']),
-        FileTemplate(0x4f20, 'EF.IMG',       'LF', None, None,   2, None, '00FF...FF', False, ['nb_rec', 'size']),
+        FileTemplate(0x4f20, 'EF.IMG',       'LF', None, None,   2, None, '00FF...FF', False, ['nb_rec', 'size'], ppath=[0x5f50]),
         # EF.IIDF below
-        FileTemplate(0x4f21, 'EF.ICE_GRRAPHICS','BT',None,None,   9, None, None, False, ['size']),
-        FileTemplate(0x4f01, 'EF.LAUNCH_SCWS','TR',None, None,  10, None, None, True, ['size']),
+        FileTemplate(0x4f21, 'EF.ICE_GRRAPHICS','BT',None,None,   9, None, None, False, ['size'], ppath=[0x5f50]),
+        FileTemplate(0x4f01, 'EF.LAUNCH_SCWS','TR',None, None,  10, None, None, True, ['size'], ppath=[0x5f50]),
         # EF.ICON below
     ]
     for i in range(0x40, 0x80):
-        files.append(FileTemplate(0x4f00+i, 'EF.IIDF', 'TR', None, None, 2, None, 'FF...FF', False, ['size']))
+        files.append(FileTemplate(0x4f00+i, 'EF.IIDF', 'TR', None, None, 2, None, 'FF...FF', False, ['size'], ppath=[0x5f50]))
     for i in range(0x80, 0xC0):
-        files.append(FileTemplate(0x4f00+i, 'EF.ICON', 'TR', None, None, 10, None, None, True, ['size']))
+        files.append(FileTemplate(0x4f00+i, 'EF.ICON', 'TR', None, None, 10, None, None, True, ['size'],ppath=[0x5f50]))
 
     # we copy the objects (instances) here as we also use them below from FilesUsimDfPhonebook
-    files += [deepcopy(x) for x in df_pb_files]
+    df_pb = deepcopy(df_pb_files)
+    files += df_pb
 
     files += [
         FileTemplate(0x5f3b, 'DF.MULTIMEDIA','DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[67]),
-        FileTemplate(0x4f47, 'EF.MML',       'BT', None, None,   5, None, None, False, ['size'], ass_serv=[67]),
-        FileTemplate(0x4f48, 'EF.MMDF',      'BT', None, None,   5, None, None, False, ['size'], ass_serv=[67]),
+        FileTemplate(0x4f47, 'EF.MML',       'BT', None, None,   5, None, None, False, ['size'], ass_serv=[67], ppath=[0x5f3b]),
+        FileTemplate(0x4f48, 'EF.MMDF',      'BT', None, None,   5, None, None, False, ['size'], ass_serv=[67], ppath=[0x5f3b]),
 
         FileTemplate(0x5f3c, 'DF.MMSS',      'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO']),
-        FileTemplate(0x4f20, 'EF.MLPL',      'TR', None, None,   2, 0x01, None, True, ['size']),
-        FileTemplate(0x4f21, 'EF.MSPL',      'TR', None, None,   2, 0x02, None, True, ['size']),
-        FileTemplate(0x4f21, 'EF.MMSSMODE',  'TR', None,    1,   2, 0x03, None, True),
-
+        FileTemplate(0x4f20, 'EF.MLPL',      'TR', None, None,   2, 0x01, None, True, ['size'], ppath=[0x5f3c]),
+        FileTemplate(0x4f21, 'EF.MSPL',      'TR', None, None,   2, 0x02, None, True, ['size'], ppath=[0x5f3c]),
+        FileTemplate(0x4f21, 'EF.MMSSMODE',  'TR', None,    1,   2, 0x03, None, True, ppath=[0x5f3c]),
 
         FileTemplate(0x5f3d, 'DF.MCS',       'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv={'usim':109, 'isim': 15}),
-        FileTemplate(0x4f01, 'EF.MST',       'TR', None, None,   2, 0x01, None, True, ['size'], ass_serv={'usim':109, 'isim': 15}),
-        FileTemplate(0x4f02, 'EF.MCSCONFIG', 'BT', None, None,   2, 0x02, None, True, ['size'], ass_serv={'usim':109, 'isim': 15}),
+        FileTemplate(0x4f01, 'EF.MST',       'TR', None, None,   2, 0x01, None, True, ['size'], ass_serv={'usim':109, 'isim': 15}, ppath=[0x5f3d]),
+        FileTemplate(0x4f02, 'EF.MCSCONFIG', 'BT', None, None,   2, 0x02, None, True, ['size'], ass_serv={'usim':109, 'isim': 15}, ppath=[0x5f3d]),
 
         FileTemplate(0x5f3e, 'DF.V2X',       'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[119]),
-        FileTemplate(0x4f01, 'EF.VST',       'TR', None, None,   2, 0x01, None, True, ['size'], ass_serv=[119]),
-        FileTemplate(0x4f02, 'EF.V2X_CONFIG','BT', None, None,   2, 0x02, None, True, ['size'], ass_serv=[119]),
-        FileTemplate(0x4f03, 'EF.V2XP_PC5',  'TR', None, None,   2, None, None, True, ['size'], ass_serv=[119]), # VST: 2
-        FileTemplate(0x4f04, 'EF.V2XP_Uu',   'TR', None, None,   2, None, None, True, ['size'], ass_serv=[119]), # VST: 3
+        FileTemplate(0x4f01, 'EF.VST',       'TR', None, None,   2, 0x01, None, True, ['size'], ass_serv=[119], ppath=[0x5f3e]),
+        FileTemplate(0x4f02, 'EF.V2X_CONFIG','BT', None, None,   2, 0x02, None, True, ['size'], ass_serv=[119], ppath=[0x5f3e]),
+        FileTemplate(0x4f03, 'EF.V2XP_PC5',  'TR', None, None,   2, None, None, True, ['size'], ass_serv=[119], ppath=[0x5f3e]), # VST: 2
+        FileTemplate(0x4f04, 'EF.V2XP_Uu',   'TR', None, None,   2, None, None, True, ['size'], ass_serv=[119], ppath=[0x5f3e]), # VST: 3
     ]
 
 
@@ -318,7 +445,10 @@ class FilesUsimMandatoryV2(ProfileTemplate):
 # Section 9.5.2 v2.3.1
 class FilesUsimOptional(ProfileTemplate):
     created_by_default = False
-    oid = OID.ADF_USIM_not_by_default
+    optional = True
+    oid = OID.ADF_USIMopt_not_by_default
+    base_path = Path('ADF.USIM')
+    extends = FilesUsimMandatory
     files = [
         FileTemplate(0x6f05, 'EF.LI',        'TR', None,    6,   1, 0x02, 'FF...FF', False),
         FileTemplate(0x6f37, 'EF.ACMmax',    'TR', None,    3,   5, None, '000000', False, ass_serv=[13], pe_name='ef-acmax'),
@@ -333,9 +463,9 @@ class FilesUsimOptional(ProfileTemplate):
         FileTemplate(0x6f4b, 'EF.EXT2',      'LF',   10,   13,   8, None, '00FF...FF', False, ass_serv=[3]),
         FileTemplate(0x6f4c, 'EF.EXT3',      'LF',   10,   13,   2, None, '00FF...FF', False, ass_serv=[5]),
         FileTemplate(0x6f50, 'EF.CBMIR',     'TR', None,   20,   5, None, 'FF...FF', False, ass_serv=[16]),
-        FileTemplate(0x6f60, 'EF.PLMNwAcT',  'TR', None,   40,   5, 0x0a, 'FFFFFF0000'*8, False, ass_serv=[20]),
-        FileTemplate(0x6f61, 'EF.OPLMNwAcT', 'TR', None,   40,   2, 0x11, 'FFFFFF0000'*8, False, ass_serv=[42]),
-        FileTemplate(0x6f62, 'EF.HPLMNwAcT', 'TR', None,    5,   2, 0x13, 'FFFFFF0000', False, ass_serv=[43]),
+        FileTemplate(0x6f60, 'EF.PLMNwAcT',  'TR', None,   40,   5, 0x0a, 'FFFFFF0000', False, ass_serv=[20], repeat=True),
+        FileTemplate(0x6f61, 'EF.OPLMNwAcT', 'TR', None,   40,   2, 0x11, 'FFFFFF0000', False, ass_serv=[42], repeat=True),
+        FileTemplate(0x6f62, 'EF.HPLMNwAcT', 'TR', None,    5,   2, 0x13, 'FFFFFF0000', False, ass_serv=[43], repeat=True),
         FileTemplate(0x6f2c, 'EF.DCK',       'TR', None,   16,   5, None, 'FF...FF', False, ass_serv=[36]),
         FileTemplate(0x6f32, 'EF.CNL',       'TR', None,   30,   2, None, 'FF...FF', False, ass_serv=[37]),
         FileTemplate(0x6f47, 'EF.SMSR',      'LF',   10,   30,   5, None, '00FF...FF', False, ass_serv=[11]),
@@ -400,7 +530,10 @@ class FilesUsimOptional(ProfileTemplate):
 # Section 9.5.2
 class FilesUsimOptionalV2(ProfileTemplate):
     created_by_default = False
-    oid = OID.ADF_USIM_not_by_default_v2
+    optional = True
+    oid = OID.ADF_USIMopt_not_by_default_v2
+    base_path = Path('ADF.USIM')
+    extends = FilesUsimMandatoryV2
     files = [
         FileTemplate(0x6f05, 'EF.LI',        'TR', None,    6,   1, 0x02, 'FF...FF', False),
         FileTemplate(0x6f37, 'EF.ACMmax',    'TR', None,    3,   5, None, '000000', False, ass_serv=[13]),
@@ -489,11 +622,22 @@ class FilesUsimOptionalV2(ProfileTemplate):
         FileTemplate(0x6ffd, 'EF.MudMidCfgdata','BT', None, None,2, None, None, True, ['size'], ass_serv=[134]),
     ]
 
+# Section 9.5.2.3 v3.3.1
+class FilesUsimOptionalV3(ProfileTemplate):
+    created_by_default = False
+    optional = True
+    oid = OID.ADF_USIMopt_not_by_default_v3
+    base_path = Path('ADF.USIM')
+    extends = FilesUsimMandatoryV2
+    files = FilesUsimOptionalV2.files + [
+        FileTemplate(0x6f01, 'EF.eAKA', 'TR', None, 1, 3, None, None, True, ['size'], ass_serv=[134]),
+    ]
 
 # Section 9.5.3
 class FilesUsimDfPhonebook(ProfileTemplate):
     created_by_default = False
     oid = OID.DF_PHONEBOOK_ADF_USIM
+    base_path = Path('ADF.USIM')
     files = df_pb_files
 
 
@@ -501,6 +645,8 @@ class FilesUsimDfPhonebook(ProfileTemplate):
 class FilesUsimDfGsmAccess(ProfileTemplate):
     created_by_default = False
     oid = OID.DF_GSM_ACCESS_ADF_USIM
+    base_path = Path('ADF.USIM')
+    parent = FilesUsimMandatory
     files = [
         FileTemplate(0x5f3b, 'DF.GSM-ACCESS','DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[27]),
         FileTemplate(0x4f20, 'EF.Kc',        'TR', None,    9,   5, 0x01, 'FF...FF07', False, ass_serv=[27], high_update=True),
@@ -514,6 +660,8 @@ class FilesUsimDfGsmAccess(ProfileTemplate):
 class FilesUsimDf5GS(ProfileTemplate):
     created_by_default = False
     oid = OID.DF_5GS
+    base_path = Path('ADF.USIM')
+    parent = FilesUsimMandatory
     files = [
         FileTemplate(0x6fc0, 'DF.5GS',               'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[122,126,127,128,129,130], pe_name='df-df-5gs'),
         FileTemplate(0x4f01, 'EF.5GS3GPPLOCI',       'TR', None,   20,   5, 0x01, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00000001', False, ass_serv=[122], high_update=True),
@@ -533,6 +681,8 @@ class FilesUsimDf5GS(ProfileTemplate):
 class FilesUsimDf5GSv2(ProfileTemplate):
     created_by_default = False
     oid = OID.DF_5GS_v2
+    base_path = Path('ADF.USIM')
+    parent = FilesUsimMandatory
     files = [
         FileTemplate(0x6fc0, 'DF.5GS',               'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[122,126,127,128,129,130], pe_name='df-df-5gs'),
         FileTemplate(0x4f01, 'EF.5GS3GPPLOCI',       'TR', None,   20,   5, 0x01, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00000001', False, ass_serv=[122], high_update=True),
@@ -554,6 +704,8 @@ class FilesUsimDf5GSv2(ProfileTemplate):
 class FilesUsimDf5GSv3(ProfileTemplate):
     created_by_default = False
     oid = OID.DF_5GS_v3
+    base_path = Path('ADF.USIM')
+    parent = FilesUsimMandatory
     files = [
         FileTemplate(0x6fc0, 'DF.5GS',               'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[122,126,127,128,129,130], pe_name='df-df-5gs'),
         FileTemplate(0x4f01, 'EF.5GS3GPPLOCI',       'TR', None,   20,   5, 0x01, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00000001', False, ass_serv=[122], high_update=True),
@@ -572,16 +724,75 @@ class FilesUsimDf5GSv3(ProfileTemplate):
         FileTemplate(0x4f0c, 'EF.TN3GPPSNN',         'TR', None,    1,   2, 0x0c, '00', False, ass_serv=[135]),
     ]
 
+# Section 9.5.11.4
+class FilesUsimDf5GSv4(ProfileTemplate):
+    created_by_default = False
+    oid = OID.DF_5GS_v4
+    base_path = Path('ADF.USIM')
+    parent = FilesUsimMandatory
+    files = [
+        FileTemplate(0x6fc0, 'DF.5GS',               'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[122,126,127,128,129,130], pe_name='df-df-5gs'),
+        FileTemplate(0x4f01, 'EF.5GS3GPPLOCI',       'TR', None,   20,   5, 0x01, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00000001', False, ass_serv=[122], high_update=True),
+        FileTemplate(0x4f02, 'EF.5GSN3GPPLOCI',      'TR', None,   20,   5, 0x02, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00000001', False, ass_serv=[122], high_update=True),
+        FileTemplate(0x4f03, 'EF.5GS3GPPNSC',        'LF',    2,   62,   5, 0x03, 'FF...FF', False, ass_serv=[122,136], high_update=True),
+        # ^ If Service n°136 is not "available" in EF UST, the Profile Creator shall ensure that these files shall contain one record; otherwise, they shall contain 2 records.
+        FileTemplate(0x4f04, 'EF.5GSN3GPPNSC',       'LF',    2,   62,   5, 0x04, 'FF...FF', False, ass_serv=[122,136], high_update=True),
+        # ^ If Service n°136 is not "available" in EF UST, the Profile Creator shall ensure that these files shall contain one record; otherwise, they shall contain 2 records.
+        FileTemplate(0x4f05, 'EF.5GAUTHKEYS',        'TR', None,  110,   5, 0x05, None, True, ass_serv=[123], high_update=True),
+        FileTemplate(0x4f06, 'EF.UAC_AIC',           'TR', None,    4,   2, 0x06, None, True, ass_serv=[126]),
+        FileTemplate(0x4f07, 'EF.SUCI_Calc_Info',    'TR', None, None,   2, 0x07, 'FF...FF', False, ass_serv=[124]),
+        FileTemplate(0x4f08, 'EF.OPL5G',             'LF', None,   10,  10, 0x08, 'FF...FF', False, ['nb_rec'], ass_serv=[129]),
+        FileTemplate(0x4f09, 'EF.SUPI_NAI',          'TR', None, None,   2, 0x09, None, True, ['size'], ass_serv=[130], pe_name='ef-supinai'),
+        FileTemplate(0x4f0a, 'EF.Routing_Indicator', 'TR', None,    4,   2, 0x0a, 'F0FF0000', False, ass_serv=[124]),
+        FileTemplate(0x4f0b, 'EF.URSP',              'BT', None, None,   2, None, None, False, ass_serv=[132]),
+        FileTemplate(0x4f0c, 'EF.TN3GPPSNN',         'TR', None,    1,   2, 0x0c, '00', False, ass_serv=[135]),
+        FileTemplate(0x4f0d, 'EF.CAG',               'TR', None,    2,   2, 0x0d, None, True, ass_serv=[137]),
+        FileTemplate(0x4f0e, 'EF.SOR_CMCI',          'TR', None, None,   2, 0x0e, None, True, ass_serv=[138]),
+        FileTemplate(0x4f0f, 'EF.DRI',               'TR', None,    7,   2, 0x0f, None, True, ass_serv=[150]),
+        FileTemplate(0x4f10, 'EF.5GSEDRX',           'TR', None,    2,   2, 0x10, None, True, ass_serv=[141]),
+        FileTemplate(0x4f11, 'EF.5GNSWO_CONF',       'TR', None,    1,   2, 0x11, None, True, ass_serv=[142]),
+        FileTemplate(0x4f15, 'EF.MCHPPLMN',          'TR', None,    1,   2, 0x15, None, True, ass_serv=[144]),
+        FileTemplate(0x4f16, 'EF.KAUSF_DERIVATION',  'TR', None,    1,   2, 0x16, None, True, ass_serv=[145]),
+    ]
+
 
 # Section 9.5.12
 class FilesUsimDfSaip(ProfileTemplate):
     created_by_default = False
     oid = OID.DF_SAIP
+    base_path = Path('ADF.USIM')
+    parent = FilesUsimMandatory
     files = [
         FileTemplate(0x6fd0, 'DF.SAIP',        'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[(124, 125)], pe_name='df-df-saip'),
-        FileTemplate(0x4f01, 'EF.SUCICalcInfo','TR', None, None, 3, None, 'FF..FF', False, ['size'], ass_serv=[125], pe_name='ef-suci-calc-info-usim'),
+        FileTemplate(0x4f01, 'EF.SUCICalcInfo','TR', None, None, 3, None, 'FF...FF', False, ['size'], ass_serv=[125], pe_name='ef-suci-calc-info-usim'),
     ]
 
+# Section 9.5.13
+class FilesDfSnpn(ProfileTemplate):
+    created_by_default = False
+    oid = OID.DF_SNPN
+    base_path = Path('ADF.USIM')
+    parent = FilesUsimMandatory
+    files = [
+        FileTemplate(0x5fe0, 'DF.SNPN',         'DF', None, None,  14, None, None, False, ['pinStatusTemplateDO'], ass_serv=[143], pe_name='df-df-snpn'),
+        FileTemplate(0x4f01, 'EF.PWS_SNPN',     'TR', None,    1,  10, None, None, True, ass_serv=[143]),
+    ]
+
+# Section 9.5.14
+class FilesDf5GProSe(ProfileTemplate):
+    created_by_default = False
+    oid = OID.DF_5GProSe
+    base_path = Path('ADF.USIM')
+    parent = FilesUsimMandatory
+    files = [
+        FileTemplate(0x5ff0, 'DF.5G_ProSe',       'DF', None, None,  14, None, None, False, ['pinStatusTeimplateDO'], ass_serv=[139], pe_name='df-df-5g-prose'),
+        FileTemplate(0x4f01, 'EF.5G_PROSE_ST',    'TR', None,    1,   2, 0x01, None,  True, ass_serv=[139]),
+        FileTemplate(0x4f02, 'EF.5G_PROSE_DD',    'TR', None,   26,   2, 0x02, None,  True, ass_serv=[139,1001]),
+        FileTemplate(0x4f03, 'EF.5G_PROSE_DC',    'TR', None,   12,   2, 0x03, None,  True, ass_serv=[139,1002]),
+        FileTemplate(0x4f04, 'EF.5G_PROSE_U2NRU', 'TR', None,   32,   2, 0x04, None,  True, ass_serv=[139,1003]),
+        FileTemplate(0x4f05, 'EF.5G_PROSE_RU',    'TR', None,   29,   2, 0x05, None,  True, ass_serv=[139,1004]),
+        FileTemplate(0x4f06, 'EF.5G_PROSE_UIR',   'TR', None,   32,   2, 0x06, None,  True, ass_serv=[139,1005]),
+    ]
 
 # Section 9.6.1
 class FilesIsimMandatory(ProfileTemplate):
@@ -601,7 +812,10 @@ class FilesIsimMandatory(ProfileTemplate):
 # Section 9.6.2 v2.3.1
 class FilesIsimOptional(ProfileTemplate):
     created_by_default = False
-    oid = OID.ADF_ISIM_not_by_default
+    optional = True
+    oid = OID.ADF_ISIMopt_not_by_default
+    base_path = Path('ADF.ISIM')
+    extends = FilesIsimMandatory
     files = [
         FileTemplate(0x6f09, 'EF.P-CSCF',      'LF',    1, None,   2, None, None, True, ['size'], ass_serv=[1,5]),
         FileTemplate(0x6f3c, 'EF.SMS',         'LF',   10,  176,   5, None, '00FF...FF', False, ass_serv=[6,8]),
@@ -618,7 +832,10 @@ class FilesIsimOptional(ProfileTemplate):
 # Section 9.6.2
 class FilesIsimOptionalv2(ProfileTemplate):
     created_by_default = False
-    oid = OID.ADF_ISIM_not_by_default_v2
+    optional = True
+    oid = OID.ADF_ISIMopt_not_by_default_v2
+    base_path = Path('ADF.ISIM')
+    extends = FilesIsimMandatory
     files = [
         FileTemplate(0x6f09, 'EF.PCSCF',       'LF',    1, None,   2, None, None, True, ['size'], ass_serv=[1,5]),
         FileTemplate(0x6f3c, 'EF.SMS',         'LF',   10,  176,   5, None, '00FF...FF', False, ass_serv=[6,8]),
@@ -649,8 +866,8 @@ class FilesEap(ProfileTemplate):
         FileTemplate(0x4f01, 'EF.EAPKEYS',     'TR', None, None,   2, None, None, True, ['size'], high_update=True),
         FileTemplate(0x4f02, 'EF.EAPSTATUS',   'TR', None,    1,   2, None, '00', False, high_update=True),
         FileTemplate(0x4f03, 'EF.PUId',        'TR', None, None,   2, None, None, True, ['size']),
-        FileTemplate(0x4f04, 'EF.Ps',          'TR', None, None,   5, None, 'FF..FF', False, ['size'], high_update=True),
-        FileTemplate(0x4f20, 'EF.CurID',       'TR', None, None,   5, None, 'FF..FF', False, ['size'], high_update=True),
+        FileTemplate(0x4f04, 'EF.Ps',          'TR', None, None,   5, None, 'FF...FF', False, ['size'], high_update=True),
+        FileTemplate(0x4f20, 'EF.CurID',       'TR', None, None,   5, None, 'FF...FF', False, ['size'], high_update=True),
         FileTemplate(0x4f21, 'EF.RelID',       'TR', None, None,   5, None, None, True, ['size']),
         FileTemplate(0x4f22, 'EF.Realm',       'TR', None, None,   5, None, None, True, ['size']),
     ]
@@ -673,3 +890,85 @@ ARR_DEFINITION = {
     13: ['800113A406830101950108', '800148A40683010A950108'],
     14: ['80015EA40683010A950108'],
 }
+
+class SaipSpecVersionMeta(type):
+    def __getitem__(self, ver: str):
+        """Syntactic sugar so that SaipSpecVersion['2.3.0'] will work."""
+        return SaipSpecVersion.for_version(ver)
+
+class SaipSpecVersion(object, metaclass=SaipSpecVersionMeta):
+    """Represents a specific version of the SIMalliance / TCA eUICC Profile Package:
+       Interoperable Format Technical Specification."""
+    version = None
+    oids = []
+
+    @classmethod
+    def suports_template_OID(cls, OID: OID.OID) -> bool:
+        """Return if a given spec version supports a template of given OID."""
+        return OID in cls.oids
+
+    @classmethod
+    def version_match(cls, ver: str) -> bool:
+        """Check if the given version-string matches the classes version.  trailing zeroes are ignored,
+        so that for example 2.2.0 will be considered equal to 2.2"""
+        def strip_trailing_zeroes(l: List):
+            while l[-1] == '0':
+                l.pop()
+        cls_ver_l = cls.version.split('.')
+        strip_trailing_zeroes(cls_ver_l)
+        ver_l = ver.split('.')
+        strip_trailing_zeroes(ver_l)
+        return cls_ver_l == ver_l
+
+    @staticmethod
+    def for_version(req_version: str) -> Optional['SaipSpecVersion']:
+        """Return the subclass for the requested version number string."""
+        for cls in all_subclasses(SaipSpecVersion):
+            if cls.version_match(req_version):
+                return cls
+
+
+class SaipSpecVersion101(SaipSpecVersion):
+    version = '1.0.1'
+    oids = [OID.MF, OID.DF_CD, OID.DF_TELECOM, OID.ADF_USIM_by_default, OID.ADF_USIMopt_not_by_default,
+            OID.DF_PHONEBOOK_ADF_USIM, OID.DF_GSM_ACCESS_ADF_USIM, OID.ADF_ISIM_by_default,
+            OID.ADF_ISIMopt_not_by_default, OID.ADF_CSIM_by_default, OID.ADF_CSIMopt_not_by_default]
+
+class SaipSpecVersion20(SaipSpecVersion):
+    version = '2.0'
+    # no changes in filesystem teplates to previous 1.0.1
+    oids = SaipSpecVersion101.oids
+
+class SaipSpecVersion21(SaipSpecVersion):
+    version = '2.1'
+    # no changes in filesystem teplates to previous 2.0
+    oids = SaipSpecVersion20.oids
+
+class SaipSpecVersion22(SaipSpecVersion):
+    version = '2.2'
+    oids = SaipSpecVersion21.oids + [OID.DF_EAP]
+
+class SaipSpecVersion23(SaipSpecVersion):
+    version = '2.3'
+    oids = SaipSpecVersion22.oids + [OID.DF_5GS, OID.DF_SAIP]
+
+class SaipSpecVersion231(SaipSpecVersion):
+    version = '2.3.1'
+    # no changes in filesystem teplates to previous 2.3
+    oids = SaipSpecVersion23.oids
+
+class SaipSpecVersion31(SaipSpecVersion):
+    version = '3.1'
+    oids = [OID.MF, OID.DF_CD, OID.DF_TELECOM_v2, OID.ADF_USIM_by_default_v2, OID.ADF_USIMopt_not_by_default_v2,
+            OID.DF_PHONEBOOK_ADF_USIM, OID.DF_GSM_ACCESS_ADF_USIM, OID.DF_5GS_v2, OID.DF_5GS_v3, OID.DF_SAIP,
+            OID.ADF_ISIM_by_default, OID.ADF_ISIMopt_not_by_default_v2, OID.ADF_CSIM_by_default_v2,
+            OID.ADF_CSIMopt_not_by_default_v2, OID.DF_EAP]
+
+class SaipSpecVersion32(SaipSpecVersion):
+    version = '3.2'
+    # no changes in filesystem teplates to previous 3.1
+    oids = SaipSpecVersion31.oids
+
+class SaipSpecVersion331(SaipSpecVersion):
+    version = '3.3.1'
+    oids = SaipSpecVersion32.oids + [OID.ADF_USIMopt_not_by_default_v3, OID.DF_5GS_v4, OID.DF_SAIP, OID.DF_SNPN, OID.DF_5GProSe, OID.IoT_by_default, OID.IoTopt_not_by_default]

pySim/esim/saip/validation.py

@@ -95,6 +95,12 @@ class CheckBasicStructure(ProfileConstraintChecker):
         if 'profile-a-p256' in m_svcs and not ('usim' in m_svcs or 'isim' in m_svcs):
             raise ProfileError('profile-a-p256 mandatory, but no usim or isim')
 
+    def check_identification_unique(self, pes: ProfileElementSequence):
+        """Ensure that each PE has a unique identification value."""
+        id_list = [pe.header['identification'] for pe in pes.pe_list if pe.header]
+        if len(id_list) != len(set(id_list)):
+            raise ProfileError('PE identification values are not unique')
+
 FileChoiceList = List[Tuple]
 
 class FileError(ProfileError):

pySim/euicc.py

@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 
 """
-Various definitions related to GSMA eSIM / eUICC
+Various definitions related to GSMA consumer + IoT eSIM / eUICC
 
-Related Specs: GSMA SGP.22, GSMA SGP.02, etc.
+Does *not* implement anything related to M2M eUICC
+
+Related Specs: GSMA SGP.21, SGP.22, SGP.31, SGP32
 """
 
 # Copyright (C) 2023 Harald Welte <laforge@osmocom.org>
@@ -25,13 +27,22 @@ import argparse
 
 from construct import Array, Struct, FlagsEnum, GreedyRange
 from cmd2 import cmd2, CommandSet, with_default_category
+from osmocom.utils import Hexstr
+from osmocom.tlv import *
+from osmocom.construct import *
 
-from pySim.tlv import *
-from pySim.construct import *
-from pySim.commands import SimCardCommands
+from pySim.exceptions import SwMatchError
 from pySim.utils import Hexstr, SwHexstr, SwMatchstr
+from pySim.commands import SimCardCommands
 import pySim.global_platform
 
+# SGP.02 Section 2.2.2
+class Sgp02Eid(BER_TLV_IE, tag=0x5a):
+    _construct = BcdAdapter(GreedyBytes)
+
+# patch this into global_platform, to allow 'get_data sgp02_eid' in EF.ECASD
+pySim.global_platform.DataCollection.possible_nested.append(Sgp02Eid)
+
 def compute_eid_checksum(eid) -> str:
     """Compute and add/replace check digits of an EID value according to GSMA SGP.29 Section 10."""
     if isinstance(eid, str):
@@ -165,7 +176,7 @@ class ProfileMgmtOperation(BER_TLV_IE, tag=0x81):
 class ListNotificationReq(BER_TLV_IE, tag=0xbf28, nested=[ProfileMgmtOperation]):
     pass
 class SeqNumber(BER_TLV_IE, tag=0x80):
-    _construct = GreedyInteger()
+    _construct = Asn1DerInteger()
 class NotificationAddress(BER_TLV_IE, tag=0x0c):
     _construct = Utf8Adapter(GreedyBytes)
 class Iccid(BER_TLV_IE, tag=0x5a):
@@ -279,7 +290,7 @@ class EumCertificate(BER_TLV_IE, tag=0xa5):
     _construct = GreedyBytes
 class EuiccCertificate(BER_TLV_IE, tag=0xa6):
     _construct = GreedyBytes
-class GetCertsError(BER_TLV_IE, tag=0x80):
+class GetCertsError(BER_TLV_IE, tag=0x81):
     _construct = Enum(Int8ub, invalidCiPKId=1, undefinedError=127)
 class GetCertsResp(BER_TLV_IE, tag=0xbf56, nested=[EumCertificate, EuiccCertificate, GetCertsError]):
     pass
@@ -292,9 +303,9 @@ class EimFqdn(BER_TLV_IE, tag=0x81):
 class EimIdType(BER_TLV_IE, tag=0x82):
     _construct = Enum(Int8ub, eimIdTypeOid=1, eimIdTypeFqdn=2, eimIdTypeProprietary=3)
 class CounterValue(BER_TLV_IE, tag=0x83):
-    _construct = GreedyInteger
+    _construct = Asn1DerInteger()
 class AssociationToken(BER_TLV_IE, tag=0x84):
-    _construct = GreedyInteger
+    _construct = Asn1DerInteger()
 class EimSupportedProtocol(BER_TLV_IE, tag=0x87):
     _construct = Enum(Int8ub, eimRetrieveHttps=0, eimRetrieveCoaps=1, eimInjectHttps=2, eimInjectCoaps=3,
                       eimProprietary=4)
@@ -313,12 +324,14 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
                          desc='ISD-R (Issuer Security Domain Root) Application')
         self.adf.decode_select_response = self.decode_select_response
         self.adf.shell_commands += [self.AddlShellCommands()]
+        # we attempt to retrieve ISD-R key material from CardKeyProvider identified by EID
+        self.adf.scp_key_identity = 'EID'
 
     @staticmethod
     def store_data(scc: SimCardCommands, tx_do: Hexstr, exp_sw: SwMatchstr ="9000") -> Tuple[Hexstr, SwHexstr]:
         """Perform STORE DATA according to Table 47+48 in Section 5.7.2 of SGP.22.
         Only single-block store supported for now."""
-        capdu = '%sE29100%02x%s' % (scc.cla4lchan('80'), len(tx_do)//2, tx_do)
+        capdu = '80E29100%02x%s00' % (len(tx_do)//2, tx_do)
         return scc.send_apdu_checksw(capdu, exp_sw)
 
     @staticmethod
@@ -343,6 +356,13 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
         else:
             return None
 
+    @staticmethod
+    def get_eid(scc: SimCardCommands) -> str:
+        ged_cmd = GetEuiccData(children=[TagList(decoded=[0x5A])])
+        ged = CardApplicationISDR.store_data_tlv(scc, ged_cmd, GetEuiccData)
+        d = ged.to_dict()
+        return flatten_dict_lists(d['get_euicc_data'])['eid_value']
+
     def decode_select_response(self, data_hex: Hexstr) -> object:
         t = FciTemplate()
         t.from_tlv(h2b(data_hex))
@@ -429,8 +449,11 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             """Perform an ES10c EnableProfile function."""
             if opts.isdp_aid:
                 p_id = ProfileIdentifier(children=[IsdpAid(decoded=opts.isdp_aid)])
-            if opts.iccid:
+            elif opts.iccid:
                 p_id = ProfileIdentifier(children=[Iccid(decoded=opts.iccid)])
+            else:
+                # this is guaranteed by argparse; but we need this to make pylint happy
+                raise ValueError('Either ISD-P AID or ICCID must be given')
             ep_cmd_contents = [p_id, RefreshFlag(decoded=opts.refresh_required)]
             ep_cmd = EnableProfileReq(children=ep_cmd_contents)
             ep = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, ep_cmd, EnableProfileResp)
@@ -448,8 +471,11 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             """Perform an ES10c DisableProfile function."""
             if opts.isdp_aid:
                 p_id = ProfileIdentifier(children=[IsdpAid(decoded=opts.isdp_aid)])
-            if opts.iccid:
+            elif opts.iccid:
                 p_id = ProfileIdentifier(children=[Iccid(decoded=opts.iccid)])
+            else:
+                # this is guaranteed by argparse; but we need this to make pylint happy
+                raise ValueError('Either ISD-P AID or ICCID must be given')
             dp_cmd_contents = [p_id, RefreshFlag(decoded=opts.refresh_required)]
             dp_cmd = DisableProfileReq(children=dp_cmd_contents)
             dp = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, dp_cmd, DisableProfileResp)
@@ -466,8 +492,11 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             """Perform an ES10c DeleteProfile function."""
             if opts.isdp_aid:
                 p_id = IsdpAid(decoded=opts.isdp_aid)
-            if opts.iccid:
+            elif opts.iccid:
                 p_id = Iccid(decoded=opts.iccid)
+            else:
+                # this is guaranteed by argparse; but we need this to make pylint happy
+                raise ValueError('Either ISD-P AID or ICCID must be given')
             dp_cmd_contents = [p_id]
             dp_cmd = DeleteProfileReq(children=dp_cmd_contents)
             dp = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, dp_cmd, DeleteProfileResp)
@@ -477,15 +506,14 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
 
         def do_get_eid(self, _opts):
             """Perform an ES10c GetEID function."""
-            (_data, _sw) = CardApplicationISDR.store_data(self._cmd.lchan.scc, 'BF3E035C015A')
             ged_cmd = GetEuiccData(children=[TagList(decoded=[0x5A])])
             ged = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, ged_cmd, GetEuiccData)
             d = ged.to_dict()
             self._cmd.poutput_json(flatten_dict_lists(d['get_euicc_data']))
 
         set_nickname_parser = argparse.ArgumentParser()
-        set_nickname_parser.add_argument('ICCID', help='ICCID of the profile whose nickname to set')
         set_nickname_parser.add_argument('--profile-nickname', help='Nickname of the profile')
+        set_nickname_parser.add_argument('ICCID', help='ICCID of the profile whose nickname to set')
 
         @cmd2.with_argparser(set_nickname_parser)
         def do_set_nickname(self, opts):
@@ -501,7 +529,7 @@ class CardApplicationISDR(pySim.global_platform.CardApplicationSD):
             """Perform an ES10c GetCerts() function on an IoT eUICC."""
             gc = CardApplicationISDR.store_data_tlv(self._cmd.lchan.scc, GetCertsReq(), GetCertsResp)
             d = gc.to_dict()
-            self._cmd.poutput_json(flatten_dict_lists(d['get_certficiates_resp']))
+            self._cmd.poutput_json(flatten_dict_lists(d['get_certs_resp']))
 
         def do_get_eim_configuration_data(self, _opts):
             """Perform an ES10b GetEimConfigurationData function on an Iot eUICC."""
@@ -522,6 +550,8 @@ class CardApplicationECASD(pySim.global_platform.CardApplicationSD):
                          desc='ECASD (eUICC Controlling Authority Security Domain) Application')
         self.adf.decode_select_response = self.decode_select_response
         self.adf.shell_commands += [self.AddlShellCommands()]
+        # we attempt to retrieve ECASD key material from CardKeyProvider identified by EID
+        self.adf.scp_key_identity = 'EID'
 
     @with_default_category('Application-Specific Commands')
     class AddlShellCommands(CommandSet):

pySim/exceptions.py

@@ -49,9 +49,18 @@ class SwMatchError(Exception):
         self.sw_expected = sw_expected
         self.rs = rs
 
-    def __str__(self):
+    @property
+    def description(self):
         if self.rs and self.rs.lchan[0]:
             r = self.rs.lchan[0].interpret_sw(self.sw_actual)
             if r:
-                return "SW match failed! Expected %s and got %s: %s - %s" % (self.sw_expected, self.sw_actual, r[0], r[1])
-        return "SW match failed! Expected %s and got %s." % (self.sw_expected, self.sw_actual)
+                return "%s - %s" % (r[0], r[1])
+        return ''
+
+    def __str__(self):
+        description = self.description
+        if description:
+            description = ": " + description
+        else:
+            description = "."
+        return "SW match failed! Expected %s and got %s%s" % (self.sw_expected, self.sw_actual, description)

pySim/filesystem.py

@@ -9,7 +9,7 @@ The classes are intended to represent the *specification* of the filesystem,
 not the actual contents / runtime state of interacting with a given smart card.
 """
 
-# (C) 2021 by Harald Welte <laforge@osmocom.org>
+# (C) 2021-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -35,10 +35,14 @@ import cmd2
 from cmd2 import CommandSet, with_default_category
 from smartcard.util import toBytes
 
-from pySim.utils import sw_match, h2b, b2h, is_hex, auto_int, auto_uint8, auto_uint16, is_hexstr
-from pySim.construct import filter_dict, parse_construct, build_construct
+from osmocom.utils import h2b, b2h, is_hex, auto_int, auto_uint8, auto_uint16, is_hexstr, JsonEncoder
+from osmocom.tlv import bertlv_parse_one
+from osmocom.construct import filter_dict, parse_construct, build_construct
+
+from pySim.utils import sw_match
 from pySim.jsonpath import js_path_modify
 from pySim.commands import SimCardCommands
+from pySim.exceptions import SwMatchError
 
 # int: a single service is associated with this file
 # list: any of the listed services requires this file
@@ -61,7 +65,7 @@ class CardFile:
         Args:
             fid : File Identifier (4 hex digits)
             sfid : Short File Identifier (2 hex digits, optional)
-            name : Brief name of the file, lik EF_ICCID
+            name : Brief name of the file, like EF_ICCID
             desc : Description of the file
             parent : Parent CardFile object within filesystem hierarchy
             profile : Card profile that this file should be part of
@@ -175,7 +179,7 @@ class CardFile:
         """Return a dict of {'identifier': self} tuples.
 
         Args:
-            alias : Add an alias with given name to 'self' 
+            alias : Add an alias with given name to 'self'
             flags : Specify which selectables to return 'FIDS' and/or 'NAMES';
                     If not specified, all selectables will be returned.
         Returns:
@@ -295,6 +299,17 @@ class CardFile:
             return True
         raise ValueError("self.service must be either int or list or tuple")
 
+    @staticmethod
+    def export(as_json: bool, lchan):
+        """
+        Export file contents in the form of commandline script. This method is meant to be overloaded by a subclass in
+        case any exportable contents are present. The generated script may contain multiple command lines separated by
+        line breaks ("\n"), where the last commandline shall have no line break at the end
+        (e.g. "update_record 1 112233\nupdate_record 1 445566"). Naturally this export method will always refer to the
+        currently selected file of the presented lchan.
+        """
+        return "# %s has no exportable contents" % str(lchan.selected_file)
+
 
 class CardDF(CardFile):
     """DF (Dedicated File) in the smart card filesystem.  Those are basically sub-directories."""
@@ -512,7 +527,7 @@ class CardADF(CardDF):
         super().__init__(**kwargs)
         # reference to CardApplication may be set from CardApplication constructor
         self.application = None  # type: Optional[CardApplication]
-        self.aid = aid           # Application Identifier
+        self.aid = aid.lower()   # Application Identifier
         self.has_fs = has_fs     # Flag to tell whether the ADF supports a filesystem or not
         mf = self.get_mf()
         if mf:
@@ -527,6 +542,15 @@ class CardADF(CardDF):
         else:
             return self.aid
 
+    @staticmethod
+    def export(as_json: bool, lchan):
+        """
+        Export application specific parameters that are not part of the UICC filesystem.
+        """
+        if not isinstance(lchan.selected_file, CardADF):
+            raise TypeError('currently selected file is not of type CardADF')
+        return lchan.selected_file.application.export(as_json, lchan)
+
 
 class CardEF(CardFile):
     """EF (Entry File) in the smart card filesystem"""
@@ -603,19 +627,19 @@ class TransparentEF(CardEF):
         upd_bin_parser = argparse.ArgumentParser()
         upd_bin_parser.add_argument(
             '--offset', type=auto_uint16, default=0, help='Byte offset for start of read')
-        upd_bin_parser.add_argument('data', type=is_hexstr, help='Data bytes (hex format) to write')
+        upd_bin_parser.add_argument('DATA', type=is_hexstr, help='Data bytes (hex format) to write')
 
         @cmd2.with_argparser(upd_bin_parser)
         def do_update_binary(self, opts):
             """Update (Write) data of a transparent EF"""
-            (data, _sw) = self._cmd.lchan.update_binary(opts.data, opts.offset)
+            (data, _sw) = self._cmd.lchan.update_binary(opts.DATA, opts.offset)
             if data:
                 self._cmd.poutput(data)
 
         upd_bin_dec_parser = argparse.ArgumentParser()
-        upd_bin_dec_parser.add_argument('data', help='Abstract data (JSON format) to write')
         upd_bin_dec_parser.add_argument('--json-path', type=str,
                                         help='JSON path to modify specific element of file only')
+        upd_bin_dec_parser.add_argument('DATA', help='Abstract data (JSON format) to write')
 
         @cmd2.with_argparser(upd_bin_dec_parser)
         def do_update_binary_decoded(self, opts):
@@ -623,9 +647,9 @@ class TransparentEF(CardEF):
             if opts.json_path:
                 (data_json, _sw) = self._cmd.lchan.read_binary_dec()
                 js_path_modify(data_json, opts.json_path,
-                               json.loads(opts.data))
+                               json.loads(opts.DATA))
             else:
-                data_json = json.loads(opts.data)
+                data_json = json.loads(opts.DATA)
             (data, _sw) = self._cmd.lchan.update_binary_dec(data_json)
             if data:
                 self._cmd.poutput_json(data)
@@ -719,7 +743,26 @@ class TransparentEF(CardEF):
             return t.to_dict()
         return {'raw': raw_bin_data.hex()}
 
-    def encode_bin(self, abstract_data: dict) -> bytearray:
+    def __get_size(self, total_len: Optional[int] = None) -> Optional[int]:
+        """Get the size (total length) of the file"""
+
+        # Caller has provided the actual total length of the file, this should be the default case
+        if total_len is not None:
+            return total_len
+
+        if self.size is None:
+            return None
+
+        # Alternatively use the recommended size from the specification
+        if self.size[1] is not None:
+            return self.size[1]
+        # In case no recommended size is specified, use the minimum size
+        if self.size[0] is not None:
+            return self.size[0]
+
+        return None
+
+    def encode_bin(self, abstract_data: dict, total_len: Optional[int] = None) -> bytearray:
         """Encode abstract representation into raw (binary) data.
 
         A derived class would typically provide an _encode_bin() or _encode_hex() method
@@ -728,17 +771,18 @@ class TransparentEF(CardEF):
 
         Args:
             abstract_data : dict representing the decoded data
+            total_len : expected total length of the encoded data (file size)
         Returns:
             binary encoded data
         """
         method = getattr(self, '_encode_bin', None)
         if callable(method):
-            return method(abstract_data)
+            return method(abstract_data, total_len = self.__get_size(total_len))
         method = getattr(self, '_encode_hex', None)
         if callable(method):
-            return h2b(method(abstract_data))
+            return h2b(method(abstract_data, total_len = self.__get_size(total_len)))
         if self._construct:
-            return build_construct(self._construct, abstract_data)
+            return build_construct(self._construct, abstract_data, {'total_len' : self.__get_size(total_len)})
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -746,7 +790,7 @@ class TransparentEF(CardEF):
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
-    def encode_hex(self, abstract_data: dict) -> str:
+    def encode_hex(self, abstract_data: dict, total_len: Optional[int] = None) -> str:
         """Encode abstract representation into raw (hex string) data.
 
         A derived class would typically provide an _encode_bin() or _encode_hex() method
@@ -755,18 +799,19 @@ class TransparentEF(CardEF):
 
         Args:
             abstract_data : dict representing the decoded data
+            total_len : expected total length of the encoded data (file size)
         Returns:
             hex string encoded data
         """
         method = getattr(self, '_encode_hex', None)
         if callable(method):
-            return method(abstract_data)
+            return method(abstract_data, total_len = self.__get_size(total_len))
         method = getattr(self, '_encode_bin', None)
         if callable(method):
-            raw_bin_data = method(abstract_data)
+            raw_bin_data = method(abstract_data, total_len = self.__get_size(total_len))
             return b2h(raw_bin_data)
         if self._construct:
-            return b2h(build_construct(self._construct, abstract_data))
+            return b2h(build_construct(self._construct, abstract_data, {'total_len':self.__get_size(total_len)}))
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -774,6 +819,25 @@ class TransparentEF(CardEF):
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
+    @staticmethod
+    def export(as_json: bool, lchan):
+        """
+        Export the file contents of a TransparentEF. This method returns a shell command string (See also ShellCommand
+        definition in this class) that can be used to write the file contents back.
+        """
+
+        if lchan.selected_file_structure() != 'transparent':
+            raise ValueError("selected file has structure type '%s', expecting a file with structure 'transparent'" %
+                             lchan.selected_file_structure())
+        export_str = ""
+        if as_json:
+            result = lchan.read_binary_dec()
+            export_str += ("update_binary_decoded '%s'\n" % json.dumps(result[0], cls=JsonEncoder))
+        else:
+            result = lchan.read_binary()
+            export_str += ("update_binary %s\n" % str(result[0]))
+        return export_str.strip()
+
 
 class LinFixedEF(CardEF):
     """Linear Fixed EF (Entry File) in the smart card filesystem.
@@ -796,16 +860,16 @@ class LinFixedEF(CardEF):
             self._cmd.poutput_json(data, opts.oneline)
 
         read_rec_parser = argparse.ArgumentParser()
-        read_rec_parser.add_argument(
-            'record_nr', type=auto_uint8, help='Number of record to be read')
         read_rec_parser.add_argument(
             '--count', type=auto_uint8, default=1, help='Number of records to be read, beginning at record_nr')
+        read_rec_parser.add_argument(
+            'RECORD_NR', type=auto_uint8, help='Number of record to be read')
 
         @cmd2.with_argparser(read_rec_parser)
         def do_read_record(self, opts):
             """Read one or multiple records from a record-oriented EF"""
             for r in range(opts.count):
-                recnr = opts.record_nr + r
+                recnr = opts.RECORD_NR + r
                 (data, _sw) = self._cmd.lchan.read_record(recnr)
                 if len(data) > 0:
                     recstr = str(data)
@@ -814,15 +878,15 @@ class LinFixedEF(CardEF):
                 self._cmd.poutput("%03d %s" % (recnr, recstr))
 
         read_rec_dec_parser = argparse.ArgumentParser()
-        read_rec_dec_parser.add_argument(
-            'record_nr', type=auto_uint8, help='Number of record to be read')
         read_rec_dec_parser.add_argument('--oneline', action='store_true',
                                          help='No JSON pretty-printing, dump as a single line')
+        read_rec_dec_parser.add_argument(
+            'RECORD_NR', type=auto_uint8, help='Number of record to be read')
 
         @cmd2.with_argparser(read_rec_dec_parser)
         def do_read_record_decoded(self, opts):
             """Read + decode a record from a record-oriented EF"""
-            (data, _sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
+            (data, _sw) = self._cmd.lchan.read_record_dec(opts.RECORD_NR)
             self._cmd.poutput_json(data, opts.oneline)
 
         read_recs_parser = argparse.ArgumentParser()
@@ -856,45 +920,45 @@ class LinFixedEF(CardEF):
 
         upd_rec_parser = argparse.ArgumentParser()
         upd_rec_parser.add_argument(
-            'record_nr', type=auto_uint8, help='Number of record to be read')
-        upd_rec_parser.add_argument('data', type=is_hexstr, help='Data bytes (hex format) to write')
+            'RECORD_NR', type=auto_uint8, help='Number of record to be read')
+        upd_rec_parser.add_argument('DATA', type=is_hexstr, help='Data bytes (hex format) to write')
 
         @cmd2.with_argparser(upd_rec_parser)
         def do_update_record(self, opts):
             """Update (write) data to a record-oriented EF"""
-            (data, _sw) = self._cmd.lchan.update_record(opts.record_nr, opts.data)
+            (data, _sw) = self._cmd.lchan.update_record(opts.RECORD_NR, opts.DATA)
             if data:
                 self._cmd.poutput(data)
 
         upd_rec_dec_parser = argparse.ArgumentParser()
-        upd_rec_dec_parser.add_argument(
-            'record_nr', type=auto_uint8, help='Number of record to be read')
-        upd_rec_dec_parser.add_argument('data', help='Abstract data (JSON format) to write')
         upd_rec_dec_parser.add_argument('--json-path', type=str,
                                         help='JSON path to modify specific element of record only')
+        upd_rec_dec_parser.add_argument(
+            'RECORD_NR', type=auto_uint8, help='Number of record to be read')
+        upd_rec_dec_parser.add_argument('data', help='Abstract data (JSON format) to write')
 
         @cmd2.with_argparser(upd_rec_dec_parser)
         def do_update_record_decoded(self, opts):
             """Encode + Update (write) data to a record-oriented EF"""
             if opts.json_path:
-                (data_json, _sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
+                (data_json, _sw) = self._cmd.lchan.read_record_dec(opts.RECORD_NR)
                 js_path_modify(data_json, opts.json_path,
                                json.loads(opts.data))
             else:
                 data_json = json.loads(opts.data)
             (data, _sw) = self._cmd.lchan.update_record_dec(
-                opts.record_nr, data_json)
+                opts.RECORD_NR, data_json)
             if data:
                 self._cmd.poutput(data)
 
         edit_rec_dec_parser = argparse.ArgumentParser()
         edit_rec_dec_parser.add_argument(
-            'record_nr', type=auto_uint8, help='Number of record to be edited')
+            'RECORD_NR', type=auto_uint8, help='Number of record to be edited')
 
         @cmd2.with_argparser(edit_rec_dec_parser)
         def do_edit_record_decoded(self, opts):
             """Edit the JSON representation of one record in an editor."""
-            (orig_json, _sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
+            (orig_json, _sw) = self._cmd.lchan.read_record_dec(opts.RECORD_NR)
             with tempfile.TemporaryDirectory(prefix='pysim_') as dirname:
                 filename = '%s/file' % dirname
                 # write existing data as JSON to file
@@ -908,7 +972,7 @@ class LinFixedEF(CardEF):
                     self._cmd.poutput("Data not modified, skipping write")
                 else:
                     (data, _sw) = self._cmd.lchan.update_record_dec(
-                        opts.record_nr, edited_json)
+                        opts.RECORD_NR, edited_json)
                     if data:
                         self._cmd.poutput_json(data)
 
@@ -987,7 +1051,26 @@ class LinFixedEF(CardEF):
             return t.to_dict()
         return {'raw': raw_hex_data}
 
-    def encode_record_hex(self, abstract_data: dict, record_nr: int) -> str:
+    def __get_rec_len(self, total_len: Optional[int] = None) -> Optional[int]:
+        """Get the length (total length) of the file record"""
+
+        # Caller has provided the actual total length of the record, this should be the default case
+        if total_len is not None:
+            return total_len
+
+        if self.rec_len is None:
+            return None
+
+        # Alternatively use the recommended length from the specification
+        if self.rec_len[1] is not None:
+            return self.rec_len[1]
+        # In case no recommended length is specified, use the minimum length
+        if self.rec_len[0] is not None:
+            return self.rec_len[0]
+
+        return None
+
+    def encode_record_hex(self, abstract_data: dict, record_nr: int, total_len: Optional[int] = None) -> str:
         """Encode abstract representation into raw (hex string) data.
 
         A derived class would typically provide an _encode_record_bin() or _encode_record_hex()
@@ -997,18 +1080,19 @@ class LinFixedEF(CardEF):
         Args:
             abstract_data : dict representing the decoded data
             record_nr : record number (1 for first record, ...)
+            total_len : expected total length of the encoded data (record length)
         Returns:
             hex string encoded data
         """
         method = getattr(self, '_encode_record_hex', None)
         if callable(method):
-            return method(abstract_data, record_nr=record_nr)
+            return method(abstract_data, record_nr=record_nr, total_len = self.__get_rec_len(total_len))
         method = getattr(self, '_encode_record_bin', None)
         if callable(method):
-            raw_bin_data = method(abstract_data, record_nr=record_nr)
+            raw_bin_data = method(abstract_data, record_nr=record_nr, total_len = self.__get_rec_len(total_len))
             return b2h(raw_bin_data)
         if self._construct:
-            return b2h(build_construct(self._construct, abstract_data))
+            return b2h(build_construct(self._construct, abstract_data, {'total_len':self.__get_rec_len(total_len)}))
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -1016,7 +1100,7 @@ class LinFixedEF(CardEF):
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
-    def encode_record_bin(self, abstract_data: dict, record_nr : int) -> bytearray:
+    def encode_record_bin(self, abstract_data: dict, record_nr : int, total_len: Optional[int] = None) -> bytearray:
         """Encode abstract representation into raw (binary) data.
 
         A derived class would typically provide an _encode_record_bin() or _encode_record_hex()
@@ -1026,17 +1110,18 @@ class LinFixedEF(CardEF):
         Args:
             abstract_data : dict representing the decoded data
             record_nr : record number (1 for first record, ...)
+            total_len : expected total length of the encoded data (record length)
         Returns:
             binary encoded data
         """
         method = getattr(self, '_encode_record_bin', None)
         if callable(method):
-            return method(abstract_data, record_nr=record_nr)
+            return method(abstract_data, record_nr=record_nr, total_len = self.__get_rec_len(total_len))
         method = getattr(self, '_encode_record_hex', None)
         if callable(method):
-            return h2b(method(abstract_data, record_nr=record_nr))
+            return h2b(method(abstract_data, record_nr=record_nr, total_len = self.__get_rec_len(total_len)))
         if self._construct:
-            return build_construct(self._construct, abstract_data)
+            return build_construct(self._construct, abstract_data, {'total_len':self.__get_rec_len(total_len)})
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -1044,6 +1129,54 @@ class LinFixedEF(CardEF):
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
+    @staticmethod
+    def export(as_json: bool, lchan):
+        """
+        Export the file contents of a LinFixedEF (or a CyclicEF). This method returns a shell command string (See also
+        ShellCommand definition in this class) that can be used to write the file contents back.
+        """
+
+        # A CyclicEF is a subclass of LinFixedEF.
+        if lchan.selected_file_structure() != 'linear_fixed' and lchan.selected_file_structure() != 'cyclic':
+            raise ValueError("selected file has structure type '%s', expecting a file with structure 'linear_fixed' or 'cyclic'" %
+                             lchan.selected_file_structure())
+
+        export_str = ""
+
+        # Use number of records specified in select response
+        num_of_rec = lchan.selected_file_num_of_rec()
+        if num_of_rec:
+            for r in range(1, num_of_rec + 1):
+                if as_json:
+                    result = lchan.read_record_dec(r)
+                    export_str += ("update_record_decoded %d '%s'\n" % (r, json.dumps(result[0], cls=JsonEncoder)))
+                else:
+                    result = lchan.read_record(r)
+                    export_str += ("update_record %d %s\n" % (r, str(result[0])))
+
+        # In case the select response does not return the number of records, read until we hit the first record that
+        # cannot be read.
+        else:
+            r = 1
+            while True:
+                try:
+                    if as_json:
+                        result = lchan.read_record_dec(r)
+                        export_str += ("update_record_decoded %d '%s'\n" % (r, json.dumps(result[0], cls=JsonEncoder)))
+                    else:
+                        result = lchan.read_record(r)
+                        export_str += ("update_record %d %s\n" % (r, str(result[0])))
+                except SwMatchError as e:
+                    # We are past the last valid record - stop
+                    if e.sw_actual == "9402":
+                        break
+                # Some other problem occurred
+                else:
+                    raise e
+                r = r + 1
+
+        return export_str.strip()
+
 
 class CyclicEF(LinFixedEF):
     """Cyclic EF (Entry File) in the smart card filesystem"""
@@ -1133,7 +1266,20 @@ class TransRecEF(TransparentEF):
             return t.to_dict()
         return {'raw': raw_hex_data}
 
-    def encode_record_hex(self, abstract_data: dict) -> str:
+    def __get_rec_len(self, total_len: Optional[int] = None) -> Optional[int]:
+        """Get the length (total length) of the file record"""
+
+        # Caller has provided the actual total length of the record, this should be the default case
+        if total_len is not None:
+            return total_len
+
+        # Alternatively use the record length from the specification
+        if self.rec_len:
+            return self.rec_len
+
+        return None
+
+    def encode_record_hex(self, abstract_data: dict, total_len: Optional[int] = None) -> str:
         """Encode abstract representation into raw (hex string) data.
 
         A derived class would typically provide an _encode_record_bin() or _encode_record_hex()
@@ -1142,17 +1288,19 @@ class TransRecEF(TransparentEF):
 
         Args:
             abstract_data : dict representing the decoded data
+            total_len : expected total length of the encoded data (record length)
         Returns:
             hex string encoded data
         """
         method = getattr(self, '_encode_record_hex', None)
         if callable(method):
-            return method(abstract_data)
+            return method(abstract_data, total_len = self.__get_rec_len(total_len))
         method = getattr(self, '_encode_record_bin', None)
         if callable(method):
-            return b2h(method(abstract_data))
+            return b2h(method(abstract_data, total_len = self.__get_rec_len(total_len)))
         if self._construct:
-            return b2h(filter_dict(build_construct(self._construct, abstract_data)))
+            return b2h(filter_dict(build_construct(self._construct, abstract_data,
+                                                   {'total_len':self.__get_rec_len(total_len)})))
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -1160,7 +1308,7 @@ class TransRecEF(TransparentEF):
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
-    def encode_record_bin(self, abstract_data: dict) -> bytearray:
+    def encode_record_bin(self, abstract_data: dict, total_len: Optional[int] = None) -> bytearray:
         """Encode abstract representation into raw (binary) data.
 
         A derived class would typically provide an _encode_record_bin() or _encode_record_hex()
@@ -1169,17 +1317,19 @@ class TransRecEF(TransparentEF):
 
         Args:
             abstract_data : dict representing the decoded data
+            total_len : expected total length of the encoded data (record length)
         Returns:
             binary encoded data
         """
         method = getattr(self, '_encode_record_bin', None)
         if callable(method):
-            return method(abstract_data)
+            return method(abstract_data, total_len = self.__get_rec_len(total_len))
         method = getattr(self, '_encode_record_hex', None)
         if callable(method):
-            return h2b(method(abstract_data))
+            return h2b(method(abstract_data, total_len = self.__get_rec_len(total_len)))
         if self._construct:
-            return filter_dict(build_construct(self._construct, abstract_data))
+            return filter_dict(build_construct(self._construct, abstract_data,
+                                               {'total_len':self.__get_rec_len(total_len)}))
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -1192,8 +1342,8 @@ class TransRecEF(TransparentEF):
                   for i in range(0, len(raw_bin_data), self.rec_len)]
         return [self.decode_record_bin(x) for x in chunks]
 
-    def _encode_bin(self, abstract_data) -> bytes:
-        chunks = [self.encode_record_bin(x) for x in abstract_data]
+    def _encode_bin(self, abstract_data, **kwargs) -> bytes:
+        chunks = [self.encode_record_bin(x, total_len = kwargs.get('total_len', None)) for x in abstract_data]
         # FIXME: pad to file size
         return b''.join(chunks)
 
@@ -1212,12 +1362,12 @@ class BerTlvEF(CardEF):
 
         retrieve_data_parser = argparse.ArgumentParser()
         retrieve_data_parser.add_argument(
-            'tag', type=auto_int, help='BER-TLV Tag of value to retrieve')
+            'TAG', type=auto_int, help='BER-TLV Tag of value to retrieve')
 
         @cmd2.with_argparser(retrieve_data_parser)
         def do_retrieve_data(self, opts):
             """Retrieve (Read) data from a BER-TLV EF"""
-            (data, _sw) = self._cmd.lchan.retrieve_data(opts.tag)
+            (data, _sw) = self._cmd.lchan.retrieve_data(opts.TAG)
             self._cmd.poutput(data)
 
         def do_retrieve_tags(self, _opts):
@@ -1227,27 +1377,33 @@ class BerTlvEF(CardEF):
 
         set_data_parser = argparse.ArgumentParser()
         set_data_parser.add_argument(
-            'tag', type=auto_int, help='BER-TLV Tag of value to set')
+            'TAG', type=auto_int, help='BER-TLV Tag of value to set')
         set_data_parser.add_argument('data', type=is_hexstr, help='Data bytes (hex format) to write')
 
         @cmd2.with_argparser(set_data_parser)
         def do_set_data(self, opts):
             """Set (Write) data for a given tag in a BER-TLV EF"""
-            (data, _sw) = self._cmd.lchan.set_data(opts.tag, opts.data)
+            (data, _sw) = self._cmd.lchan.set_data(opts.TAG, opts.data)
             if data:
                 self._cmd.poutput(data)
 
         del_data_parser = argparse.ArgumentParser()
         del_data_parser.add_argument(
-            'tag', type=auto_int, help='BER-TLV Tag of value to set')
+            'TAG', type=auto_int, help='BER-TLV Tag of value to set')
 
         @cmd2.with_argparser(del_data_parser)
         def do_delete_data(self, opts):
-            """Delete  data for a given tag in a BER-TLV EF"""
-            (data, _sw) = self._cmd.lchan.set_data(opts.tag, None)
+            """Delete data for a given tag in a BER-TLV EF"""
+            (data, _sw) = self._cmd.lchan.set_data(opts.TAG, None)
             if data:
                 self._cmd.poutput(data)
 
+        def do_delete_all(self, opts):
+            """Delete all data from a BER-TLV EF"""
+            tags = self._cmd.lchan.retrieve_tags()
+            for tag in tags:
+                self._cmd.lchan.set_data(tag, None)
+
     def __init__(self, fid: str, sfid: str = None, name: str = None, desc: str = None, parent: CardDF = None,
                  size: Size = (1, None), **kwargs):
         """
@@ -1264,6 +1420,34 @@ class BerTlvEF(CardEF):
         self.size = size
         self.shell_commands = [self.ShellCommands()]
 
+    @staticmethod
+    def export(as_json: bool, lchan):
+        """
+        Export the file contents of a BerTlvEF. This method returns a shell command string (See also ShellCommand
+        definition in this class) that can be used to write the file contents back.
+        """
+
+        if lchan.selected_file_structure() != 'ber_tlv':
+            raise ValueError("selected file has structure type '%s', expecting a file with structure 'ber_tlv'" %
+                             lchan.selected_file_structure())
+
+        # TODO: Add JSON output as soon as we have a set_data_decoded command and a retrieve_data_dec method.
+        if as_json:
+            raise NotImplementedError("BerTlvEF encoder not yet implemented. Patches welcome.")
+
+        export_str = ""
+        tags = lchan.retrieve_tags()
+        if tags == []:
+            export_str += "# empty file, no tags"
+        else:
+            export_str += "delete_all\n"
+            for t in tags:
+                result = lchan.retrieve_data(t)
+                (tag, l, val, remainer) = bertlv_parse_one(h2b(result[0]))
+                export_str += ("set_data 0x%02x %s\n" % (t, b2h(val)))
+        return export_str.strip()
+
+
 def interpret_sw(sw_data: dict, sw: str):
     """Interpret a given status word.
 
@@ -1294,6 +1478,8 @@ class CardApplication:
             adf : ADF name
             sw : Dict of status word conversions
         """
+        if aid:
+            aid = aid.lower()
         self.name = name
         self.adf = adf
         self.sw = sw or {}
@@ -1318,6 +1504,15 @@ class CardApplication:
         """
         return interpret_sw(self.sw, sw)
 
+    @staticmethod
+    def export(as_json: bool, lchan):
+        """
+        Export application specific parameters, in the form of commandline script. (see also comment in the export
+        method of class "CardFile")
+        """
+        return "# %s has no exportable features" % str(lchan.selected_file)
+
+
 
 class CardModel(abc.ABC):
     """A specific card model, typically having some additional vendor-specific files. All
@@ -1349,3 +1544,53 @@ class CardModel(abc.ABC):
         for m in CardModel.__subclasses__():
             if m.match(scc):
                 m.add_files(rs)
+
+
+class Path:
+    """Representation of a file-system path."""
+    def __init__(self, p: Union[str, List[str], List[int]]):
+        # split if given as single string with slahes
+        if isinstance(p, str):
+            p = p.split('/')
+        elif len(p) and isinstance(p[0], int):
+            p = ['%04x' % x for x in p]
+        # make sure internal representation alwas is uppercase only
+        self.list = [x.upper() for x in p]
+
+    def __str__(self) -> str:
+        return '/'.join(self.list)
+
+    def __repr__(self) -> str:
+        return 'Path(%s)' % (str(self))
+
+    def __eq__(self, other: 'Path') -> bool:
+        return self.list == other.list
+
+    def __getitem__(self, i):
+        return self.list[i]
+
+    def __len__(self):
+        return len(self.list)
+
+    def __add__(self, a):
+        if isinstance(a, list):
+            l = self.list + a
+        elif isinstance(a, Path):
+            l = self.list + a.list
+        else:
+            l = self.list + [a]
+        return Path(l)
+
+    def relative_to_mf(self) -> 'Path':
+        """Return a path relative to MF, i.e. without initial explicit MF."""
+        if len(self.list) and self.list[0] in ['MF', '3F00']:
+            return Path(self.list[1:])
+        return self
+
+    def is_parent(self, other: 'Path') -> bool:
+        """Is this instance a parent of the given other instance?"""
+        if len(self.list) >= len(other.list):
+            return False
+        if other.list[:len(self.list)] == self.list:
+            return True
+        return False

pySim/global_platform/__init__.py

@@ -23,12 +23,88 @@ from construct import Optional as COptional
 from construct import Struct, GreedyRange, FlagsEnum, Int16ub, Int24ub, Padding, Bit, Const
 from Cryptodome.Random import get_random_bytes
 from Cryptodome.Cipher import DES, DES3, AES
+from osmocom.utils import *
+from osmocom.tlv import *
+from osmocom.construct import *
+from pySim.utils import ResTuple
+from pySim.card_key_provider import card_key_provider_get_field
 from pySim.global_platform.scp import SCP02, SCP03
-from pySim.construct import *
-from pySim.utils import *
 from pySim.filesystem import *
-from pySim.tlv import *
 from pySim.profile import CardProfile
+from pySim.ota import SimFileAccessAndToolkitAppSpecParams
+
+# GPCS Table 11-48 Load Parameter Tags
+class NonVolatileCodeMinMemoryReq(BER_TLV_IE, tag=0xC6):
+    _construct = GreedyInteger()
+
+# GPCS Table 11-48 Load Parameter Tags
+class VolatileMinMemoryReq(BER_TLV_IE, tag=0xC7):
+    _construct = GreedyInteger()
+
+# GPCS Table 11-48 Load Parameter Tags
+class NonVolatileDataMinMemoryReq(BER_TLV_IE, tag=0xC8):
+    _construct = GreedyInteger()
+
+# GPCS Table 11-49: Install Parameter Tags
+class GlobalServiceParams(BER_TLV_IE, tag=0xCB):
+    pass
+
+# GPCS Table 11-49: Install Parameter Tags
+class VolatileReservedMemory(BER_TLV_IE, tag=0xD7):
+    _construct = GreedyInteger()
+
+# GPCS Table 11-49: Install Parameter Tags
+class NonVolatileReservedMemory(BER_TLV_IE, tag=0xD8):
+    _construct = GreedyInteger()
+
+# GPCS Table 11-49: Install Parameter Tags
+class Ts102226SpecificParameter(BER_TLV_IE, tag=0xCA):
+    _construct = SimFileAccessAndToolkitAppSpecParams
+
+# GPCS Table 11-48 Load Parameter Tags
+class LoadFileDataBlockFormatId(BER_TLV_IE, tag=0xCD):
+    pass
+
+# GPCS Table 11-50: Make Selectable Parameter Tags
+class ImplicitSelectionParam(BER_TLV_IE, tag=0xCF):
+    pass
+
+# GPCS Table 11-48 Load Parameter Tags
+class LoadFileDtaBlockParameters(BER_TLV_IE, tag=0xDD):
+    pass
+
+# GPCS Table 11-48 Load Parameter Tags / 11-49: Install Parameter Tags
+class SystemSpecificParams(BER_TLV_IE, tag=0xEF,
+                           nested=[NonVolatileCodeMinMemoryReq,
+                                   VolatileMinMemoryReq,
+                                   NonVolatileDataMinMemoryReq,
+                                   GlobalServiceParams,
+                                   VolatileReservedMemory,
+                                   NonVolatileReservedMemory,
+                                   Ts102226SpecificParameter,
+                                   LoadFileDataBlockFormatId,
+                                   ImplicitSelectionParam,
+                                   LoadFileDtaBlockParameters]):
+    pass
+
+# GPCS Table 11-49: Install Parameter Tags
+class ApplicationSpecificParams(BER_TLV_IE, tag=0xC9):
+    _construct = GreedyBytes
+
+class Ts102226SpecificTemplate(BER_TLV_IE, tag=0xEA):
+    pass
+
+class CrtForDigitalSignature(BER_TLV_IE, tag=0xB6):
+    # FIXME: nested
+    pass
+
+
+class InstallParameters(TLV_IE_Collection, nested=[ApplicationSpecificParams,
+                                                   SystemSpecificParams,
+                                                   Ts102226SpecificTemplate,
+                                                   CrtForDigitalSignature]):
+    pass
+
 
 sw_table = {
     'Warnings': {
@@ -504,7 +580,7 @@ class ADF_SD(CardADF):
                                       {'last_block': len(remainder) == 0, 'encryption': encryption,
                                        'structure': structure, 'response': response_permitted})
                 hdr = "80E2%02x%02x%02x" % (p1b[0], block_nr, len(chunk))
-                data, _sw = self._cmd.lchan.scc.send_apdu_checksw(hdr + b2h(chunk))
+                data, _sw = self._cmd.lchan.scc.send_apdu_checksw(hdr + b2h(chunk) + "00")
                 block_nr += 1
                 response += data
             return data
@@ -513,7 +589,7 @@ class ADF_SD(CardADF):
         put_key_parser.add_argument('--old-key-version-nr', type=auto_uint8, default=0, help='Old Key Version Number')
         put_key_parser.add_argument('--key-version-nr', type=auto_uint8, required=True, help='Key Version Number')
         put_key_parser.add_argument('--key-id', type=auto_uint7, required=True, help='Key Identifier (base)')
-        put_key_parser.add_argument('--key-type', choices=KeyType.ksymapping.values(), action='append', required=True, help='Key Type')
+        put_key_parser.add_argument('--key-type', choices=list(KeyType.ksymapping.values()), action='append', required=True, help='Key Type')
         put_key_parser.add_argument('--key-data', type=is_hexstr, action='append', required=True, help='Key Data Block')
         put_key_parser.add_argument('--key-check', type=is_hexstr, action='append', help='Key Check Value')
         put_key_parser.add_argument('--suppress-key-check', action='store_true', help='Suppress generation of Key Check Values')
@@ -550,7 +626,7 @@ class ADF_SD(CardADF):
                     kcv = b2h(kcv_bin)
                 if self._cmd.lchan.scc.scp:
                     # encrypte key data with DEK of current SCP
-                    kcb = b2h(self._cmd.lchan.scc.scp.card_keys.encrypt_key(h2b(opts.key_data[i])))
+                    kcb = b2h(self._cmd.lchan.scc.scp.encrypt_key(h2b(opts.key_data[i])))
                 else:
                     # (for example) during personalization, DEK might not be required)
                     kcb = opts.key_data[i]
@@ -570,11 +646,11 @@ class ADF_SD(CardADF):
             See GlobalPlatform CardSpecification v2.3 Section 11.8 for details."""
             key_data = kvn.to_bytes(1, 'big') + build_construct(ADF_SD.AddlShellCommands.KeyDataBasic, key_dict)
             hdr = "80D8%02x%02x%02x" % (old_kvn, kid, len(key_data))
-            data, _sw = self._cmd.lchan.scc.send_apdu_checksw(hdr + b2h(key_data))
+            data, _sw = self._cmd.lchan.scc.send_apdu_checksw(hdr + b2h(key_data) + "00")
             return data
 
         get_status_parser = argparse.ArgumentParser()
-        get_status_parser.add_argument('subset', choices=StatusSubset.ksymapping.values(),
+        get_status_parser.add_argument('subset', choices=list(StatusSubset.ksymapping.values()),
                                        help='Subset of statuses to be included in the response')
         get_status_parser.add_argument('--aid', type=is_hexstr, default='',
                                        help='AID Search Qualifier (search only for given AID)')
@@ -595,7 +671,7 @@ class ADF_SD(CardADF):
             grd_list = []
             while True:
                 hdr = "80F2%s%02x%02x" % (subset_hex, p2, len(cmd_data))
-                data, sw = self._cmd.lchan.scc.send_apdu(hdr + b2h(cmd_data))
+                data, sw = self._cmd.lchan.scc.send_apdu(hdr + b2h(cmd_data) + "00")
                 remainder = h2b(data)
                 while len(remainder):
                     # tlv sequence, each element is one GpRegistryRelatedData()
@@ -609,9 +685,9 @@ class ADF_SD(CardADF):
             return grd_list
 
         set_status_parser = argparse.ArgumentParser()
-        set_status_parser.add_argument('scope', choices=SetStatusScope.ksymapping.values(),
+        set_status_parser.add_argument('scope', choices=list(SetStatusScope.ksymapping.values()),
                                        help='Defines the scope of the requested status change')
-        set_status_parser.add_argument('status', choices=CLifeCycleState.ksymapping.values(),
+        set_status_parser.add_argument('status', choices=list(CLifeCycleState.ksymapping.values()),
                                        help='Specify the new intended status')
         set_status_parser.add_argument('--aid', type=is_hexstr,
                                        help='AID of the target Application or Security Domain')
@@ -631,7 +707,7 @@ class ADF_SD(CardADF):
             _data, _sw = self._cmd.lchan.scc.send_apdu_checksw(b2h(apdu))
 
         inst_perso_parser = argparse.ArgumentParser()
-        inst_perso_parser.add_argument('application-aid', type=is_hexstr, help='Application AID')
+        inst_perso_parser.add_argument('application_aid', type=is_hexstr, help='Application AID')
 
         @cmd2.with_argparser(inst_perso_parser)
         def do_install_for_personalization(self, opts):
@@ -650,7 +726,7 @@ class ADF_SD(CardADF):
         inst_inst_parser.add_argument('--install-parameters', type=is_hexstr, default='',
                                       help='Install Parameters')
         inst_inst_parser.add_argument('--privilege', action='append', dest='privileges', default=[],
-                                      choices=Privileges._construct.flags.keys(),
+                                      choices=list(Privileges._construct.flags.keys()),
                                       help='Privilege granted to newly installed Application')
         inst_inst_parser.add_argument('--install-token', type=is_hexstr, default='',
                                       help='Install Token (Section GPCS C.4.2/C.4.7)')
@@ -676,7 +752,7 @@ class ADF_SD(CardADF):
             self.install(p1, 0x00, b2h(ifi_bytes))
 
         def install(self, p1:int, p2:int, data:Hexstr) -> ResTuple:
-            cmd_hex = "80E6%02x%02x%02x%s" % (p1, p2, len(data)//2, data)
+            cmd_hex = "80E6%02x%02x%02x%s00" % (p1, p2, len(data)//2, data)
             return self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)
 
         del_cc_parser = argparse.ArgumentParser()
@@ -719,23 +795,33 @@ class ADF_SD(CardADF):
             return self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)
 
         est_scp02_parser = argparse.ArgumentParser()
-        est_scp02_parser.add_argument('--key-ver', type=auto_uint8, required=True,
-                                      help='Key Version Number (KVN)')
-        est_scp02_parser.add_argument('--key-enc', type=is_hexstr, required=True,
-                                      help='Secure Channel Encryption Key')
-        est_scp02_parser.add_argument('--key-mac', type=is_hexstr, required=True,
-                                      help='Secure Channel MAC Key')
-        est_scp02_parser.add_argument('--key-dek', type=is_hexstr, required=True,
-                                      help='Data Encryption Key')
+        est_scp02_parser.add_argument('--key-ver', type=auto_uint8, required=True, help='Key Version Number (KVN)')
         est_scp02_parser.add_argument('--host-challenge', type=is_hexstr,
                                       help='Hard-code the host challenge; default: random')
         est_scp02_parser.add_argument('--security-level', type=auto_uint8, default=0x01,
                                       help='Security Level. Default: 0x01 (C-MAC only)')
+        est_scp02_p_k = est_scp02_parser.add_argument_group('Manual key specification')
+        est_scp02_p_k.add_argument('--key-enc', type=is_hexstr, help='Secure Channel Encryption Key')
+        est_scp02_p_k.add_argument('--key-mac', type=is_hexstr, help='Secure Channel MAC Key')
+        est_scp02_p_k.add_argument('--key-dek', type=is_hexstr, help='Data Encryption Key')
+        est_scp02_p_csv = est_scp02_parser.add_argument_group('Obtain keys from CardKeyProvider (e.g. CSV')
+        est_scp02_p_csv.add_argument('--key-provider-suffix', help='Suffix for key names in CardKeyProvider')
 
         @cmd2.with_argparser(est_scp02_parser)
         def do_establish_scp02(self, opts):
             """Establish a secure channel using the GlobalPlatform SCP02 protocol.  It can be released
             again by using `release_scp`."""
+            if opts.key_provider_suffix:
+                suffix = opts.key_provider_suffix
+                id_field_name = self._cmd.lchan.selected_adf.scp_key_identity
+                identity = self._cmd.rs.identity.get(id_field_name)
+                opts.key_enc = card_key_provider_get_field('SCP02_ENC_' + suffix, key=id_field_name, value=identity)
+                opts.key_mac = card_key_provider_get_field('SCP02_MAC_' + suffix, key=id_field_name, value=identity)
+                opts.key_dek = card_key_provider_get_field('SCP02_DEK_' + suffix, key=id_field_name, value=identity)
+            else:
+                if not opts.key_enc or not opts.key_mac:
+                    self._cmd.poutput("Cannot establish SCP02 without at least ENC and MAC keys given!")
+                    return
             if self._cmd.lchan.scc.scp:
                 self._cmd.poutput("Cannot establish SCP02 as this lchan already has a SCP instance!")
                 return
@@ -745,12 +831,24 @@ class ADF_SD(CardADF):
             self._establish_scp(scp02, host_challenge, opts.security_level)
 
         est_scp03_parser = deepcopy(est_scp02_parser)
+        est_scp03_parser.description = None
         est_scp03_parser.add_argument('--s16-mode', action='store_true', help='S16 mode (S8 is default)')
 
         @cmd2.with_argparser(est_scp03_parser)
         def do_establish_scp03(self, opts):
             """Establish a secure channel using the GlobalPlatform SCP03 protocol.  It can be released
             again by using `release_scp`."""
+            if opts.key_provider_suffix:
+                suffix = opts.key_provider_suffix
+                id_field_name = self._cmd.lchan.selected_adf.scp_key_identity
+                identity = self._cmd.rs.identity.get(id_field_name)
+                opts.key_enc = card_key_provider_get_field('SCP03_ENC_' + suffix, key=id_field_name, value=identity)
+                opts.key_mac = card_key_provider_get_field('SCP03_MAC_' + suffix, key=id_field_name, value=identity)
+                opts.key_dek = card_key_provider_get_field('SCP03_DEK_' + suffix, key=id_field_name, value=identity)
+            else:
+                if not opts.key_enc or not opts.key_mac:
+                    self._cmd.poutput("Cannot establish SCP03 without at least ENC and MAC keys given!")
+                    return
             if self._cmd.lchan.scc.scp:
                 self._cmd.poutput("Cannot establish SCP03 as this lchan already has a SCP instance!")
                 return
@@ -787,6 +885,9 @@ class CardApplicationSD(CardApplication):
     __intermediate = True
     def __init__(self, aid: str, name: str, desc: str):
         super().__init__(name, adf=ADF_SD(aid, name, desc), sw=sw_table)
+        # the identity (e.g. 'ICCID', 'EID') that should be used as a look-up key to attempt to retrieve
+        # the key material for the security domain from the CardKeyProvider
+        self.adf.scp_key_identity = None
 
 # Card Application of Issuer Security Domain
 class CardApplicationISD(CardApplicationSD):
@@ -794,6 +895,7 @@ class CardApplicationISD(CardApplicationSD):
     # application using '00a4040000' and then parse the response FCI to get the ISD AID
     def __init__(self, aid='a000000003000000'):
         super().__init__(aid=aid, name='ADF.ISD', desc='Issuer Security Domain')
+        self.adf.scp_key_identity = 'ICCID'
 
 #class CardProfileGlobalPlatform(CardProfile):
 #    ORDER = 23

pySim/global_platform/http.py

@@ -0,0 +1,93 @@
+"""GlobalPlatform Remote Application Management over HTTP Card Specification v2.3 - Amendment B.
+Also known as SCP81 for SIM/USIM/UICC/eUICC/eSIM OTA.
+"""
+
+# (C) 2024 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from construct import Struct, Int8ub, Int16ub, Bytes, GreedyBytes, GreedyString, BytesInteger
+from construct import this, len_, Rebuild, Const
+from construct import Optional as COptional
+from osmocom.tlv import BER_TLV_IE
+
+from pySim import cat
+
+
+# Table 3-3 + Section 3.8.1
+class RasConnectionParams(BER_TLV_IE, tag=0x84, nested=cat.OpenChannel.nested_collection_cls.possible_nested):
+    pass
+
+# Table 3-3 + Section 3.8.2
+class SecurityParams(BER_TLV_IE, tag=0x85):
+    _test_de_encode = [
+        ( '850804deadbeef020040', {'kid': 64,'kvn': 0, 'psk_id': b'\xde\xad\xbe\xef', 'sha_type': None} )
+    ]
+    _construct = Struct('_psk_id_len'/Rebuild(Int8ub, len_(this.psk_id)), 'psk_id'/Bytes(this._psk_id_len),
+                        '_kid_kvn_len'/Const(2, Int8ub), 'kvn'/Int8ub, 'kid'/Int8ub,
+                        'sha_type'/COptional(Int8ub))
+
+# Table 3-3 + ?
+class ExtendedSecurityParams(BER_TLV_IE, tag=0xA5):
+    _construct = GreedyBytes
+
+# Table 3-3 + Section 3.8.3
+class SessionRetryPolicyParams(BER_TLV_IE, tag=0x86):
+    _construct = Struct('retry_counter'/Int16ub,
+                        'retry_waiting_delay'/BytesInteger(5),
+                        'retry_report_failure'/COptional(GreedyBytes))
+
+# Table 3-3 + Section 3.8.4
+class AdminHostParam(BER_TLV_IE, tag=0x8A):
+    _test_de_encode = [
+        ( '8a0a61646d696e2e686f7374', 'admin.host' ),
+    ]
+    _construct = GreedyString('utf-8')
+
+# Table 3-3 + Section 3.8.5
+class AgentIdParam(BER_TLV_IE, tag=0x8B):
+    _construct = GreedyString('utf-8')
+
+# Table 3-3 + Section 3.8.6
+class AdminUriParam(BER_TLV_IE, tag=0x8C):
+    _test_de_encode = [
+        ( '8c1668747470733a2f2f61646d696e2e686f73742f757269', 'https://admin.host/uri' ),
+    ]
+    _construct = GreedyString('utf-8')
+
+# Table 3-3
+class HttpPostParams(BER_TLV_IE, tag=0x89, nested=[AdminHostParam, AgentIdParam, AdminUriParam]):
+    pass
+
+# Table 3-3
+class AdmSessionParams(BER_TLV_IE, tag=0x83, nested=[RasConnectionParams, SecurityParams,
+                                                     ExtendedSecurityParams, SessionRetryPolicyParams,
+                                                     HttpPostParams]):
+    pass
+
+# Table 3-3 + Section 3.11.4
+class RasFqdn(BER_TLV_IE, tag=0xD6):
+    _construct = GreedyBytes # FIXME: DNS String
+
+# Table 3-3 + Section 3.11.7
+class DnsConnectionParams(BER_TLV_IE, tag=0xFA, nested=cat.OpenChannel.nested_collection_cls.possible_nested):
+    pass
+
+# Table 3-3
+class DnsResolutionParams(BER_TLV_IE, tag=0xB3, nested=[RasFqdn, DnsConnectionParams]):
+    pass
+
+# Table 3-3
+class AdmSessTriggerParams(BER_TLV_IE, tag=0x81, nested=[AdmSessionParams, DnsResolutionParams]):
+    pass

pySim/global_platform/scp.py

@@ -22,7 +22,9 @@ from Cryptodome.Cipher import DES3, DES
 from Cryptodome.Util.strxor import strxor
 from construct import Struct, Bytes, Int8ub, Int16ub, Const
 from construct import Optional as COptional
-from pySim.utils import b2h, bertlv_parse_len, bertlv_encode_len
+from osmocom.utils import b2h
+from osmocom.tlv import bertlv_parse_len, bertlv_encode_len
+from pySim.utils import parse_command_apdu
 from pySim.secure_channel import SecureChannel
 
 logger = logging.getLogger(__name__)
@@ -115,6 +117,12 @@ class SCP(SecureChannel, abc.ABC):
             if not card_keys.kvn in range(self.kvn_range[0], self.kvn_range[1]+1):
                 raise ValueError('%s cannot be used with KVN outside range 0x%02x..0x%02x' %
                                  (self.__class__.__name__, self.kvn_range[0], self.kvn_range[1]))
+        elif hasattr(self, 'kvn_ranges'):
+            # pylint: disable=no-member
+            if all([not card_keys.kvn in range(x[0], x[1]+1) for x in self.kvn_ranges]):
+                raise ValueError('%s cannot be used with KVN outside permitted ranges %s' %
+                                 (self.__class__.__name__, self.kvn_ranges))
+
         self.lchan_nr = lchan_nr
         self.card_keys = card_keys
         self.sk = None
@@ -216,18 +224,19 @@ class SCP02(SCP):
 
     constr_iur = Struct('key_div_data'/Bytes(10), 'key_ver'/Int8ub, Const(b'\x02'),
                         'seq_counter'/Int16ub, 'card_challenge'/Bytes(6), 'card_cryptogram'/Bytes(8))
-    kvn_range = [0x20, 0x2f]
+    # The 0x70 is a non-spec special-case of sysmoISIM-SJA2/SJA5 and possibly more sysmocom products
+    kvn_ranges = [[0x20, 0x2f], [0x70, 0x70]]
 
     def __init__(self, *args, **kwargs):
         self.overhead = 8
         super().__init__(*args, **kwargs)
 
     def dek_encrypt(self, plaintext:bytes) -> bytes:
-        cipher = DES.new(self.card_keys.dek, DES.MODE_ECB)
+        cipher = DES.new(self.card_keys.dek[:8], DES.MODE_ECB)
         return cipher.encrypt(plaintext)
 
     def dek_decrypt(self, ciphertext:bytes) -> bytes:
-        cipher = DES.new(self.card_keys.dek, DES.MODE_ECB)
+        cipher = DES.new(self.card_keys.dek[:8], DES.MODE_ECB)
         return cipher.decrypt(ciphertext)
 
     def _compute_cryptograms(self, card_challenge: bytes, host_challenge: bytes):
@@ -239,7 +248,7 @@ class SCP02(SCP):
     def gen_init_update_apdu(self, host_challenge: bytes = b'\x00'*8) -> bytes:
         """Generate INITIALIZE UPDATE APDU."""
         self.host_challenge = host_challenge
-        return bytes([self._cla(), INS_INIT_UPDATE, self.card_keys.kvn, 0, 8]) + self.host_challenge
+        return bytes([self._cla(), INS_INIT_UPDATE, self.card_keys.kvn, 0, 8]) + self.host_challenge + b'\x00'
 
     def parse_init_update_resp(self, resp_bin: bytes):
         """Parse response to INITIALZIE UPDATE."""
@@ -266,34 +275,52 @@ class SCP02(SCP):
 
     def _wrap_cmd_apdu(self, apdu: bytes, *args, **kwargs) -> bytes:
         """Wrap Command APDU for SCP02: calculate MAC and encrypt."""
-        lc = len(apdu) - 5
-        assert len(apdu) >= 5, "Wrong APDU length: %d" % len(apdu)
-        assert len(apdu) == 5 or apdu[4] == lc, "Lc differs from length of data: %d vs %d" % (apdu[4], lc)
-
         logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
 
+        if not self.do_cmac:
+            return apdu
+
+        (case, lc, le, data) = parse_command_apdu(apdu)
+
+        # TODO: add support for extended length fields.
+        assert lc <= 256
+        assert le <= 256
+        lc &= 0xFF
+        le &= 0xFF
+
+        # CLA without log. channel can be 80 or 00 only
         cla = apdu[0]
         b8 = cla & 0x80
         if cla & 0x03 or cla & CLA_SM:
             # nonzero logical channel in APDU, check that are the same
             assert cla == self._cla(False, b8), "CLA mismatch"
-        # CLA without log. channel can be 80 or 00 only
-        if self.do_cmac:
-            if self.mac_on_unmodified:
-                mlc = lc
-                clac = cla
-            else:                      # CMAC on modified APDU
-                mlc = lc + 8
-                clac = cla | CLA_SM
-            mac = self.sk.calc_mac_1des(bytes([clac]) + apdu[1:4] + bytes([mlc]) + apdu[5:])
-            if self.do_cenc:
-                k = DES3.new(self.sk.enc, DES.MODE_CBC, b'\x00'*8)
-                data = k.encrypt(pad80(apdu[5:], 8))
-                lc = len(data)
-            else:
-                data = apdu[5:]
-            lc += 8
-            apdu = bytes([self._cla(True, b8)]) + apdu[1:4] + bytes([lc]) + data + mac
+
+        if self.mac_on_unmodified:
+            mlc = lc
+            clac = cla
+        else:
+            # CMAC on modified APDU
+            mlc = lc + 8
+            clac = cla | CLA_SM
+        mac = self.sk.calc_mac_1des(bytes([clac]) + apdu[1:4] + bytes([mlc]) + data)
+        if self.do_cenc:
+            k = DES3.new(self.sk.enc, DES.MODE_CBC, b'\x00'*8)
+            data = k.encrypt(pad80(data, 8))
+            lc = len(data)
+
+        lc += 8
+        apdu = bytes([self._cla(True, b8)]) + apdu[1:4] + bytes([lc]) + data + mac
+
+        # Since we attach a signature, we will always send some data. This means that if the APDU is of case #4
+        # or case #2, we must attach an additional Le byte to signal that we expect a response. It is technically
+        # legal to use 0x00 (=256) as Le byte, even when the caller has specified a different value in the original
+        # APDU. This is due to the fact that Le always describes the maximum expected length of the response
+        # (see also ISO/IEC 7816-4, section 5.1). In addition to that, it should also important that depending on
+        # the configuration of the SCP, the response may also contain a signature that makes the response larger
+        # than specified in the Le field of the original APDU.
+        if case == 4 or case == 2:
+            apdu += b'\x00'
+
         return apdu
 
     def unwrap_rsp_apdu(self, sw: bytes, rsp_apdu: bytes) -> bytes:
@@ -440,7 +467,7 @@ class SCP03(SCP):
         if len(host_challenge) != self.s_mode:
             raise ValueError('Host Challenge must be %u bytes long' % self.s_mode)
         self.host_challenge = host_challenge
-        return bytes([self._cla(), INS_INIT_UPDATE, self.card_keys.kvn, 0, len(host_challenge)]) + host_challenge
+        return bytes([self._cla(), INS_INIT_UPDATE, self.card_keys.kvn, 0, len(host_challenge)]) + host_challenge + b'\x00'
 
     def parse_init_update_resp(self, resp_bin: bytes):
         """Parse response to INITIALIZE UPDATE."""
@@ -465,18 +492,26 @@ class SCP03(SCP):
         return self.wrap_cmd_apdu(header + self.host_cryptogram, skip_cenc=True)
 
     def _wrap_cmd_apdu(self, apdu: bytes, skip_cenc: bool = False) -> bytes:
-        """Wrap Command APDU for SCP02: calculate MAC and encrypt."""
+        """Wrap Command APDU for SCP03: calculate MAC and encrypt."""
+        logger.debug("wrap_cmd_apdu(%s)", b2h(apdu))
+
+        if not self.do_cmac:
+            return apdu
+
         cla = apdu[0]
         ins = apdu[1]
         p1 = apdu[2]
         p2 = apdu[3]
-        lc = apdu[4]
-        assert lc == len(apdu) - 5
-        cmd_data = apdu[5:]
+        (case, lc, le, cmd_data) = parse_command_apdu(apdu)
+
+        # TODO: add support for extended length fields.
+        assert lc <= 256
+        assert le <= 256
+        lc &= 0xFF
+        le &= 0xFF
 
         if self.do_cenc and not skip_cenc:
-            assert self.do_cmac
-            if lc == 0:
+            if case <= 2:
                 # No encryption shall be applied to a command where there is no command data field. In this
                 # case, the encryption counter shall still be incremented
                 self.sk.block_nr += 1
@@ -489,21 +524,24 @@ class SCP03(SCP):
                 # perform AES-CBC with ICV + S_ENC
                 cmd_data = self.sk._encrypt(padded_data)
 
-        if self.do_cmac:
-            # The length of the command message (Lc) shall be incremented by 8 (in S8 mode) or 16 (in S16
-            # mode) to indicate the inclusion of the C-MAC in the data field of the command message.
-            mlc = lc + self.s_mode
-            if mlc >= 256:
-                raise ValueError('Modified Lc (%u) would exceed maximum when appending %u bytes of mac' % (mlc, self.s_mode))
-            # The class byte shall be modified for the generation or verification of the C-MAC: The logical
-            # channel number shall be set to zero, bit 4 shall be set to 0 and bit 3 shall be set to 1 to indicate
-            # GlobalPlatform proprietary secure messaging.
-            mcla = (cla & 0xF0) | CLA_SM
-            mapdu = bytes([mcla, ins, p1, p2, mlc]) + cmd_data
-            cmac = self.sk.calc_cmac(mapdu)
-            mapdu += cmac[:self.s_mode]
+        # The length of the command message (Lc) shall be incremented by 8 (in S8 mode) or 16 (in S16
+        # mode) to indicate the inclusion of the C-MAC in the data field of the command message.
+        mlc = lc + self.s_mode
+        if mlc >= 256:
+            raise ValueError('Modified Lc (%u) would exceed maximum when appending %u bytes of mac' % (mlc, self.s_mode))
+        # The class byte shall be modified for the generation or verification of the C-MAC: The logical
+        # channel number shall be set to zero, bit 4 shall be set to 0 and bit 3 shall be set to 1 to indicate
+        # GlobalPlatform proprietary secure messaging.
+        mcla = (cla & 0xF0) | CLA_SM
+        apdu = bytes([mcla, ins, p1, p2, mlc]) + cmd_data
+        cmac = self.sk.calc_cmac(apdu)
+        apdu += cmac[:self.s_mode]
 
-        return mapdu
+        # See comment in SCP03._wrap_cmd_apdu()
+        if case == 4 or case == 2:
+            apdu += b'\x00'
+
+        return apdu
 
     def unwrap_rsp_apdu(self, sw: bytes, rsp_apdu: bytes) -> bytes:
         # No R-MAC shall be generated and no protection shall be applied to a response that includes an error

pySim/global_platform/uicc.py

@@ -0,0 +1,107 @@
+# coding=utf-8
+"""GlobalPLatform UICC Configuration 1.0 parameters
+
+(C) 2024 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from construct import Optional as COptional
+from construct import Struct, GreedyRange, FlagsEnum, Int16ub, Int24ub, Padding, Bit, Const
+from osmocom.construct import *
+from osmocom.utils import *
+from osmocom.tlv import *
+
+# Section 11.6.2.3 / Table 11-58
+class SecurityDomainAid(BER_TLV_IE, tag=0x4f):
+    _construct = GreedyBytes
+class LoadFileDataBlockSignature(BER_TLV_IE, tag=0xc3):
+    _construct = GreedyBytes
+class DapBlock(BER_TLV_IE, tag=0xe2, nested=[SecurityDomainAid, LoadFileDataBlockSignature]):
+    pass
+class LoadFileDataBlock(BER_TLV_IE, tag=0xc4):
+    _construct = GreedyBytes
+class Icv(BER_TLV_IE, tag=0xd3):
+    _construct = GreedyBytes
+class CipheredLoadFileDataBlock(BER_TLV_IE, tag=0xd4):
+    _construct = GreedyBytes
+class LoadFile(TLV_IE_Collection, nested=[DapBlock, LoadFileDataBlock, Icv, CipheredLoadFileDataBlock]):
+    pass
+
+# UICC Configuration v1.0.1 / Section 4.3.2
+class UiccScp(BER_TLV_IE, tag=0x81):
+    _construct = Struct('scp'/Int8ub, 'i'/Int8ub)
+
+class AcceptExtradAppsAndElfToSd(BER_TLV_IE, tag=0x82):
+    _construct = GreedyBytes
+
+class AcceptDelOfAssocSd(BER_TLV_IE, tag=0x83):
+    _construct = GreedyBytes
+
+class LifeCycleTransitionToPersonalized(BER_TLV_IE, tag=0x84):
+    _construct = GreedyBytes
+
+class CasdCapabilityInformation(BER_TLV_IE, tag=0x86):
+    _construct = GreedyBytes
+
+class AcceptExtradAssocAppsAndElf(BER_TLV_IE, tag=0x87):
+    _construct = GreedyBytes
+
+# Security Domain Install Parameters (inside C9 during INSTALL [for install])
+class UiccSdInstallParams(TLV_IE_Collection, nested=[UiccScp, AcceptExtradAppsAndElfToSd, AcceptDelOfAssocSd,
+                                                     LifeCycleTransitionToPersonalized,
+                                                     CasdCapabilityInformation, AcceptExtradAssocAppsAndElf]):
+    def has_scp(self, scp: int) -> bool:
+        """Determine if SD Installation parameters already specify given SCP."""
+        for c in self.children:
+            if not isinstance(c, UiccScp):
+                continue
+            if c.decoded['scp'] == scp:
+                return True
+        return False
+
+    def add_scp(self, scp: int, i: int):
+        """Add given SCP (and i parameter) to list of SCP of the Security Domain Install Params.
+        Example: add_scp(0x03, 0x70) for SCP03, or add_scp(0x02, 0x55) for SCP02."""
+        if self.has_scp(scp):
+            raise ValueError('SCP%02x already present' % scp)
+        self.children.append(UiccScp(decoded={'scp': scp, 'i': i}))
+
+    def remove_scp(self, scp: int):
+        """Remove given SCP from list of SCP of the Security Domain Install Params."""
+        for c in self.children:
+            if not isinstance(c, UiccScp):
+                continue
+            if c.decoded['scp'] == scp:
+                self.children.remove(c)
+                return
+        raise ValueError("SCP%02x not present" % scp)
+
+
+# Key Usage:
+# KVN 0x01 .. 0x0F reserved for SCP80
+# KVN 0x11 reserved for DAP specified in ETSI TS 102 226
+# KVN 0x20 .. 0x2F reserved for SCP02
+#   KID 0x01 = ENC; 0x02 = MAC; 0x03 = DEK
+# KVN 0x30 .. 0x3F reserved for SCP03
+#   KID 0x01 = ENC; 0x02 = MAC; 0x03 = DEK
+# KVN 0x70 KID 0x01: Token key (RSA public or DES)
+# KVN 0x71 KID 0x01: Receipt key (DES)
+# KVN 0x73 KID 0x01: DAP verifiation key (RS public or DES)
+# KVN 0x74 reserved for CASD
+#   KID 0x01: PK.CA.AUT
+#   KID 0x02: SK.CASD.AUT (PK) and KS.CASD.AUT (Non-PK)
+#   KID 0x03: SK.CASD.CT (P) and KS.CASD.CT (Non-PK)
+# KVN 0x75 KID 0x01: 16-byte DES key for  Ciphered Load File Data Block
+# KVN 0xFF reserved for ISD with SCP02 without SCP80 s upport

pySim/gsmtap.py

@@ -1,214 +0,0 @@
-# -*- coding: utf-8 -*-
-
-""" Osmocom GSMTAP python implementation.
-GSMTAP is a packet format used for conveying a number of different
-telecom-related protocol traces over UDP.
-"""
-
-#
-# Copyright (C) 2022  Harald Welte <laforge@gnumonks.org>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-import socket
-from construct import Optional as COptional
-from construct import Int8ub, Int8sb, Int32ub, BitStruct, Enum, GreedyBytes, Struct, Switch
-from construct import this, PaddedString
-from pySim.construct import *
-
-# The root definition of GSMTAP can be found at
-# https://cgit.osmocom.org/cgit/libosmocore/tree/include/osmocom/core/gsmtap.h
-
-GSMTAP_UDP_PORT = 4729
-
-# GSMTAP_TYPE_*
-gsmtap_type_construct = Enum(Int8ub,
-                             gsm_um = 0x01,
-                             gsm_abis = 0x02,
-                             gsm_um_burst = 0x03,
-                             sim = 0x04,
-                             tetra_i1 = 0x05,
-                             tetra_i1_burst = 0x06,
-                             wimax_burst = 0x07,
-                             gprs_gb_llc = 0x08,
-                             gprs_gb_sndcp = 0x09,
-                             gmr1_um = 0x0a,
-                             umts_rlc_mac = 0x0b,
-                             umts_rrc = 0x0c,
-                             lte_rrc = 0x0d,
-                             lte_mac = 0x0e,
-                             lte_mac_framed = 0x0f,
-                             osmocore_log = 0x10,
-                             qc_diag = 0x11,
-                             lte_nas = 0x12,
-                             e1_t1 = 0x13)
-
-
-# TYPE_UM_BURST
-gsmtap_subtype_burst_construct = Enum(Int8ub,
-                                      unknown = 0x00,
-                                      fcch = 0x01,
-                                      partial_sch = 0x02,
-                                      sch = 0x03,
-                                      cts_sch = 0x04,
-                                      compact_sch = 0x05,
-                                      normal = 0x06,
-                                      dummy = 0x07,
-                                      access = 0x08,
-                                      none = 0x09)
-
-gsmtap_subtype_wimax_burst_construct = Enum(Int8ub,
-                                            cdma_code = 0x10,
-                                            fch = 0x11,
-                                            ffb = 0x12,
-                                            pdu = 0x13,
-                                            hack = 0x14,
-                                            phy_attributes = 0x15)
-
-# GSMTAP_CHANNEL_*
-gsmtap_subtype_um_construct = Enum(Int8ub,
-                                   unknown = 0x00,
-                                   bcch = 0x01,
-                                   ccch = 0x02,
-                                   rach = 0x03,
-                                   agch = 0x04,
-                                   pch = 0x05,
-                                   sdcch = 0x06,
-                                   sdcch4 = 0x07,
-                                   sdcch8 = 0x08,
-                                   facch_f = 0x09,
-                                   facch_h = 0x0a,
-                                   pacch = 0x0b,
-                                   cbch52 = 0x0c,
-                                   pdtch = 0x0d,
-                                   ptcch = 0x0e,
-                                   cbch51 = 0x0f,
-                                   voice_f = 0x10,
-                                   voice_h = 0x11)
-
-
-# GSMTAP_SIM_*
-gsmtap_subtype_sim_construct = Enum(Int8ub,
-                                    apdu = 0x00,
-                                    atr = 0x01,
-                                    pps_req = 0x02,
-                                    pps_rsp = 0x03,
-                                    tpdu_hdr = 0x04,
-                                    tpdu_cmd = 0x05,
-                                    tpdu_rsp = 0x06,
-                                    tpdu_sw = 0x07)
-
-gsmtap_subtype_tetra_construct = Enum(Int8ub,
-                                      bsch = 0x01,
-                                      aach = 0x02,
-                                      sch_hu = 0x03,
-                                      sch_hd = 0x04,
-                                      sch_f = 0x05,
-                                      bnch = 0x06,
-                                      stch = 0x07,
-                                      tch_f = 0x08,
-                                      dmo_sch_s = 0x09,
-                                      dmo_sch_h = 0x0a,
-                                      dmo_sch_f = 0x0b,
-                                      dmo_stch = 0x0c,
-                                      dmo_tch = 0x0d)
-
-gsmtap_subtype_gmr1_construct = Enum(Int8ub,
-                                     unknown = 0x00,
-                                     bcch = 0x01,
-                                     ccch = 0x02,
-                                     pch = 0x03,
-                                     agch = 0x04,
-                                     bach = 0x05,
-                                     rach = 0x06,
-                                     cbch = 0x07,
-                                     sdcch = 0x08,
-                                     tachh = 0x09,
-                                     gbch = 0x0a,
-                                     tch3 = 0x10,
-                                     tch6 = 0x14,
-                                     tch9 = 0x18)
-
-gsmtap_subtype_e1t1_construct = Enum(Int8ub,
-                                     lapd = 0x01,
-                                     fr = 0x02,
-                                     raw = 0x03,
-                                     trau16 = 0x04,
-                                     trau8 = 0x05)
-
-gsmtap_arfcn_construct = BitStruct('pcs'/Flag, 'uplink'/Flag, 'arfcn'/BitsInteger(14))
-
-gsmtap_hdr_construct = Struct('version'/Int8ub,
-                              'hdr_len'/Int8ub,
-                              'type'/gsmtap_type_construct,
-                              'timeslot'/Int8ub,
-                              'arfcn'/gsmtap_arfcn_construct,
-                              'signal_dbm'/Int8sb,
-                              'snr_db'/Int8sb,
-                              'frame_nr'/Int32ub,
-                              'sub_type'/Switch(this.type, {
-                                                'gsm_um': gsmtap_subtype_um_construct,
-                                                'gsm_um_burst': gsmtap_subtype_burst_construct,
-                                                'sim': gsmtap_subtype_sim_construct,
-                                                'tetra_i1': gsmtap_subtype_tetra_construct,
-                                                'tetra_i1_burst': gsmtap_subtype_tetra_construct,
-                                                'wimax_burst': gsmtap_subtype_wimax_burst_construct,
-                                                'gmr1_um': gsmtap_subtype_gmr1_construct,
-                                                'e1_t1': gsmtap_subtype_e1t1_construct,
-                                                }),
-                              'antenna_nr'/Int8ub,
-                              'sub_slot'/Int8ub,
-                              'res'/Int8ub,
-                              'body'/GreedyBytes)
-
-osmocore_log_ts_construct = Struct('sec'/Int32ub, 'usec'/Int32ub)
-osmocore_log_level_construct = Enum(Int8ub, debug=1, info=3, notice=5, error=7, fatal=8)
-gsmtap_osmocore_log_hdr_construct = Struct('ts'/osmocore_log_ts_construct,
-                                           'proc_name'/PaddedString(16, 'ascii'),
-                                           'pid'/Int32ub,
-                                           'level'/osmocore_log_level_construct,
-                                           Bytes(3),
-                                           'subsys'/PaddedString(16, 'ascii'),
-                                           'src_file'/Struct('name'/PaddedString(32, 'ascii'), 'line_nr'/Int32ub))
-
-
-class GsmtapMessage:
-    """Class whose objects represent a single GSMTAP message. Can encode and decode messages."""
-    def __init__(self, encoded = None):
-        self.encoded = encoded
-        self.decoded = None
-
-    def decode(self):
-        self.decoded = parse_construct(gsmtap_hdr_construct, self.encoded)
-        return self.decoded
-
-    def encode(self, decoded):
-        self.encoded = gsmtap_hdr_construct.build(decoded)
-        return self.encoded
-
-class GsmtapSource:
-    def __init__(self, bind_ip:str='127.0.0.1', bind_port:int=4729):
-        self.bind_ip = bind_ip
-        self.bind_port = bind_port
-        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        self.sock.bind((self.bind_ip, self.bind_port))
-
-    def read_packet(self) -> GsmtapMessage:
-        data, addr = self.sock.recvfrom(1024)
-        gsmtap_msg = GsmtapMessage(data)
-        gsmtap_msg.decode()
-        if gsmtap_msg.decoded['version'] != 0x02:
-            raise ValueError('Unknown GSMTAP version 0x%02x' % gsmtap_msg.decoded['version'])
-        return gsmtap_msg.decoded, addr

pySim/iso7816_4.py

@@ -18,10 +18,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
 from construct import GreedyBytes, GreedyString
-from pySim.construct import *
-from pySim.utils import *
-from pySim.filesystem import *
-from pySim.tlv import *
+from osmocom.tlv import *
+from osmocom.construct import *
 
 # Table 91 + Section 8.2.1.2
 class ApplicationId(BER_TLV_IE, tag=0x4f):

pySim/javacard.py

@@ -0,0 +1,19 @@
+# JavaCard related utilities
+
+import zipfile
+import struct
+import sys
+import io
+
+def ijc_to_cap(in_file: io.IOBase, out_zip: zipfile.ZipFile, p : str = "foo"):
+    """Convert an ICJ (Interoperable Java Card) file [back] to a CAP file."""
+    TAGS = ["Header", "Directory", "Applet", "Import", "ConstantPool", "Class", "Method", "StaticField", "RefLocation", "Export", "Descriptor", "Debug"]
+    b = in_file.read()
+    while len(b):
+        tag, size = struct.unpack('!BH', b[0:3])
+        out_zip.writestr(p+"/javacard/"+TAGS[tag-1]+".cap", b[0:3+size])
+        b = b[3+size:]
+
+# example usage:
+# with io.open(sys.argv[1],"rb") as f, zipfile.ZipFile(sys.argv[2], "wb") as z:
+#     ijc_to_cap(f, z)

pySim/legacy/cards.py

@@ -7,16 +7,16 @@ from smartcard.util import toBytes
 from pytlv.TLV import *
 
 from pySim.cards import SimCardBase, UiccCardBase
-from pySim.utils import dec_iccid, enc_iccid, dec_imsi, enc_imsi, dec_msisdn, enc_msisdn
+from pySim.utils import dec_iccid, enc_iccid, dec_imsi, enc_imsi
 from pySim.utils import enc_plmn, get_addr_type
 from pySim.utils import is_hex, h2b, b2h, h2s, s2h, lpad, rpad
 from pySim.legacy.utils import enc_ePDGSelection, format_xplmn_w_act, format_xplmn, dec_st, enc_st
-from pySim.legacy.utils import format_ePDGSelection, dec_addr_tlv, enc_addr_tlv
+from pySim.legacy.utils import format_ePDGSelection, dec_addr_tlv, enc_addr_tlv, dec_msisdn, enc_msisdn
 from pySim.legacy.ts_51_011 import EF, DF
 from pySim.legacy.ts_31_102 import EF_USIM_ADF_map
 from pySim.legacy.ts_31_103 import EF_ISIM_ADF_map
 
-from pySim.ts_51_011 import EF_AD, EF_SPN
+from pySim.profile.ts_51_011 import EF_AD, EF_SPN
 
 def format_addr(addr: str, addr_type: str) -> str:
     """

pySim/legacy/utils.py

@@ -20,8 +20,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from typing import Optional, Tuple
 from pySim.utils import Hexstr, rpad, enc_plmn, h2i, i2s, s2h
 from pySim.utils import dec_xplmn_w_act, dec_xplmn, dec_mcc_from_plmn, dec_mnc_from_plmn
+from osmocom.utils import swap_nibbles, h2b, b2h
 
 def hexstr_to_Nbytearr(s, nbytes):
     return [s[i:i+(nbytes*2)] for i in range(0, len(s), (nbytes*2))]
@@ -146,7 +148,7 @@ def dec_st(st, table="sim") -> str:
         from pySim.ts_31_102 import EF_UST_map
         lookup_map = EF_UST_map
     else:
-        from pySim.ts_51_011 import EF_SST_map
+        from pySim.profile.ts_51_011 import EF_SST_map
         lookup_map = EF_SST_map
 
     st_bytes = [st[i:i+2] for i in range(0, len(st), 2)]
@@ -330,3 +332,82 @@ def enc_addr_tlv(addr, addr_type='00'):
         s += '80' + ('%02x' % ((len(ipv4_str)//2)+2)) + '01' + 'ff' + ipv4_str
 
     return s
+
+
+def dec_msisdn(ef_msisdn: Hexstr) -> Optional[Tuple[int, int, Optional[str]]]:
+    """
+    Decode MSISDN from EF.MSISDN or EF.ADN (same structure).
+    See 3GPP TS 31.102, section 4.2.26 and 4.4.2.3.
+    """
+
+    # Convert from str to (kind of) 'bytes'
+    ef_msisdn = h2b(ef_msisdn)
+
+    # Make sure mandatory fields are present
+    if len(ef_msisdn) < 14:
+        raise ValueError("EF.MSISDN is too short")
+
+    # Skip optional Alpha Identifier
+    xlen = len(ef_msisdn) - 14
+    msisdn_lhv = ef_msisdn[xlen:]
+
+    # Parse the length (in bytes) of the BCD encoded number
+    bcd_len = msisdn_lhv[0]
+    # BCD length = length of dial num (max. 10 bytes) + 1 byte ToN and NPI
+    if bcd_len == 0xff:
+        return None
+    elif bcd_len > 11 or bcd_len < 1:
+        raise ValueError(
+            "Length of MSISDN (%d bytes) is out of range" % bcd_len)
+
+    # Parse ToN / NPI
+    ton = (msisdn_lhv[1] >> 4) & 0x07
+    npi = msisdn_lhv[1] & 0x0f
+    bcd_len -= 1
+
+    # No MSISDN?
+    if not bcd_len:
+        return (npi, ton, None)
+
+    msisdn = swap_nibbles(b2h(msisdn_lhv[2:][:bcd_len])).rstrip('f')
+    # International number 10.5.118/3GPP TS 24.008
+    if ton == 0x01:
+        msisdn = '+' + msisdn
+
+    return (npi, ton, msisdn)
+
+
+def enc_msisdn(msisdn: str, npi: int = 0x01, ton: int = 0x03) -> Hexstr:
+    """
+    Encode MSISDN as LHV so it can be stored to EF.MSISDN.
+    See 3GPP TS 31.102, section 4.2.26 and 4.4.2.3. (The result
+    will not contain the optional Alpha Identifier at the beginning.)
+
+    Default NPI / ToN values:
+      - NPI: ISDN / telephony numbering plan (E.164 / E.163),
+      - ToN: network specific or international number (if starts with '+').
+    """
+
+    # If no MSISDN is supplied then encode the file contents as all "ff"
+    if msisdn in ["", "+"]:
+        return "ff" * 14
+
+    # Leading '+' indicates International Number
+    if msisdn[0] == '+':
+        msisdn = msisdn[1:]
+        ton = 0x01
+
+    # An MSISDN must not exceed 20 digits
+    if len(msisdn) > 20:
+        raise ValueError("msisdn must not be longer than 20 digits")
+
+    # Append 'f' padding if number of digits is odd
+    if len(msisdn) % 2 > 0:
+        msisdn += 'f'
+
+    # BCD length also includes NPI/ToN header
+    bcd_len = len(msisdn) // 2 + 1
+    npi_ton = (npi & 0x0f) | ((ton & 0x07) << 4) | 0x80
+    bcd = rpad(swap_nibbles(msisdn), 10 * 2)  # pad to 10 octets
+
+    return ('%02x' % bcd_len) + ('%02x' % npi_ton) + bcd + ("ff" * 2)

pySim/ota.py

@@ -1,6 +1,6 @@
 """Code related to SIM/UICC OTA according to TS 102 225 + TS 31.115."""
 
-# (C) 2021-2023 by Harald Welte <laforge@osmocom.org>
+# (C) 2021-2024 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -18,12 +18,12 @@
 import zlib
 import abc
 import struct
-from typing import Optional
+from typing import Optional, Tuple
 from construct import Enum, Int8ub, Int16ub, Struct, Bytes, GreedyBytes, BitsInteger, BitStruct
-from construct import Flag, Padding, Switch, this
+from construct import Flag, Padding, Switch, this, PrefixedArray, GreedyRange
+from osmocom.construct import *
+from osmocom.utils import b2h
 
-from pySim.construct import *
-from pySim.utils import b2h
 from pySim.sms import UserDataHeader
 
 # ETS TS 102 225 gives the general command structure and the dialects for CAT_TP, TCP/IP and HTTPS
@@ -99,6 +99,17 @@ SmsCommandPacket = Struct('cmd_pkt_len'/Int16ub,
                           'tar'/Bytes(3),
                           'secured_data'/GreedyBytes)
 
+# TS 102 226 Section 8.2.1.3.2.1
+SimFileAccessAndToolkitAppSpecParams = Struct('access_domain'/Prefixed(Int8ub, GreedyBytes),
+                                              'prio_level_of_tk_app_inst'/Int8ub,
+                                              'max_num_of_timers'/Int8ub,
+                                              'max_text_length_for_menu_entry'/Int8ub,
+                                              'menu_entries'/PrefixedArray(Int8ub, Struct('id'/Int8ub,
+                                                                                          'pos'/Int8ub)),
+                                              'max_num_of_channels'/Int8ub,
+                                              'msl'/Prefixed(Int8ub, GreedyBytes),
+                                              'tar_values'/Prefixed(Int8ub, GreedyRange(Bytes(3))))
+
 class OtaKeyset:
     """The OTA related data (key material, counter) to be used in encrypt/decrypt."""
     def __init__(self, algo_crypt: str, kic_idx: int, kic: bytes,
@@ -322,6 +333,7 @@ class OtaDialectSms(OtaDialect):
                                'response_status'/ResponseStatus,
                                'cc_rc'/Bytes(this.rhl-10),
                                'secured_data'/GreedyBytes)
+    hdr_construct = Struct('chl'/Int8ub, 'spi'/SPI, 'kic'/KIC, 'kid'/KID_CC, 'tar'/Bytes(3))
 
     def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict, apdu: bytes) -> bytes:
         # length of signature in octets
@@ -332,6 +344,7 @@ class OtaDialectSms(OtaDialect):
             len_cipher = 6 + len_sig + len(apdu)
             padding = otak.crypt._get_padding(len_cipher, otak.crypt.blocksize)
             pad_cnt = len(padding)
+            apdu = bytes(apdu) # make a copy so we don't modify the input data
             apdu += padding
 
         kic = {'key': otak.kic_idx, 'algo': otak.algo_crypt}
@@ -342,8 +355,7 @@ class OtaDialectSms(OtaDialect):
         chl = 13 + len_sig
 
         # CHL + SPI (+ KIC + KID)
-        c = Struct('chl'/Int8ub, 'spi'/SPI, 'kic'/KIC, 'kid'/KID_CC, 'tar'/Bytes(3))
-        part_head = c.build({'chl': chl, 'spi':spi, 'kic':kic, 'kid':kid, 'tar':tar})
+        part_head = self.hdr_construct.build({'chl': chl, 'spi':spi, 'kic':kic, 'kid':kid, 'tar':tar})
         #print("part_head: %s" % b2h(part_head))
 
         # CNTR + PCNTR (CNTR not used)
@@ -385,8 +397,54 @@ class OtaDialectSms(OtaDialect):
 
         #print("envelope_data: %s" % b2h(envelope_data))
 
+        if len(envelope_data) > 140:
+            raise ValueError('Cannot encode command in a single SMS; Fragmentation not implemented')
+
         return envelope_data
 
+    def decode_cmd(self, otak: OtaKeyset, encoded: bytes) -> Tuple[bytes, dict, bytes]:
+        """Decode an encoded (encrypted, signed) OTA SMS Command-APDU."""
+        if True: # TODO: how to decide?
+            cpl = int.from_bytes(encoded[:2], 'big')
+            part_head = encoded[2:2+8]
+            ciph = encoded[2+8:]
+            envelope_data = otak.crypt.decrypt(ciph)
+        else:
+            part_head = encoded[:8]
+            envelope_data = encoded[8:]
+
+        hdr_dec = self.hdr_construct.parse(part_head)
+
+        # strip counter part from front of envelope_data
+        part_cnt = envelope_data[:6]
+        cntr = int.from_bytes(part_cnt[:5], 'big')
+        pad_cnt = int.from_bytes(part_cnt[5:], 'big')
+        envelope_data = envelope_data[6:]
+
+        spi = hdr_dec['spi']
+        if spi['rc_cc_ds'] == 'cc':
+            # split cc from front of APDU
+            cc = envelope_data[:8]
+            apdu = envelope_data[8:]
+            # verify CC
+            temp_data = cpl.to_bytes(2, 'big') + part_head + part_cnt + apdu
+            otak.auth.check_sig(temp_data, cc)
+        elif spi['rc_cc_ds'] == 'rc':
+            # CRC32
+            crc32_rx = int.from_bytes(envelope_data[:4], 'big')
+            # FIXME: crc32_computed = zlip.crc32(
+            # FIXME: verify RC
+            raise NotImplementedError
+            apdu = envelope_data[4:]
+        elif spi['rc_cc_ds'] == 'no_rc_cc_ds':
+            apdu = envelope_data
+        else:
+            raise ValueError("Invalid rc_cc_ds: %s" % spi['rc_cc_ds'])
+
+        apdu = apdu[:len(apdu)-pad_cnt]
+        return hdr_dec['tar'], spi, apdu
+
+
     def decode_resp(self, otak: OtaKeyset, spi: dict, data: bytes) -> ("OtaDialectSms.SmsResponsePacket", Optional["CompactRemoteResp"]):
         if isinstance(data, str):
             data = h2b(data)
@@ -434,7 +492,7 @@ class OtaDialectSms(OtaDialect):
             raise OtaCheckError('Unknown por_rc_cc_ds: %s' % spi['por_rc_cc_ds'])
 
         # TODO: ExpandedRemoteResponse according to TS 102 226 5.2.2
-        if res.response_status == 'por_ok':
+        if res.response_status == 'por_ok' and len(res['secured_data']):
             dec = CompactRemoteResp.parse(res['secured_data'])
         else:
             dec = None

pySim/pprint.py

@@ -0,0 +1,77 @@
+import pprint
+from pprint import PrettyPrinter
+from functools import singledispatch, wraps
+from typing import get_type_hints
+
+from pySim.utils import b2h
+
+def common_container_checks(f):
+    type_ = get_type_hints(f)['object']
+    base_impl = type_.__repr__
+    empty_repr = repr(type_())   # {}, [], ()
+    too_deep_repr = f'{empty_repr[0]}...{empty_repr[-1]}'  # {...}, [...], (...)
+    @wraps(f)
+    def wrapper(object, context, maxlevels, level):
+        if type(object).__repr__ is not base_impl:  # subclassed repr
+            return repr(object)
+        if not object:                              # empty, short-circuit
+            return empty_repr
+        if maxlevels and level >= maxlevels:        # exceeding the max depth
+            return too_deep_repr
+        oid = id(object)
+        if oid in context:                          # self-reference
+            return pprint._recursion(object)
+        context[oid] = 1
+        result = f(object, context, maxlevels, level)
+        del context[oid]
+        return result
+    return wrapper
+
+@singledispatch
+def saferepr(object, context, maxlevels, level):
+    return repr(object)
+
+@saferepr.register
+def _handle_bytes(object: bytes, *args):
+    if len(object) <= 40:
+        return '"%s"' % b2h(object)
+    else:
+        return '"%s...%s"' % (b2h(object[:20]), b2h(object[-20:]))
+
+@saferepr.register
+@common_container_checks
+def _handle_dict(object: dict, context, maxlevels, level):
+    level += 1
+    contents = [
+        f'{saferepr(k, context, maxlevels, level)}: '
+        f'{saferepr(v, context, maxlevels, level)}'
+        for k, v in sorted(object.items(), key=pprint._safe_tuple)
+    ]
+    return f'{{{", ".join(contents)}}}'
+
+@saferepr.register
+@common_container_checks
+def _handle_list(object: list, context, maxlevels, level):
+    level += 1
+    contents = [
+        f'{saferepr(v, context, maxlevels, level)}'
+        for v in object
+    ]
+    return f'[{", ".join(contents)}]'
+
+@saferepr.register
+@common_container_checks
+def _handle_tuple(object: tuple, context, maxlevels, level):
+    level += 1
+    if len(object) == 1:
+        return f'({saferepr(object[0], context, maxlevels, level)},)'
+    contents = [
+        f'{saferepr(v, context, maxlevels, level)}'
+        for v in object
+    ]
+    return f'({", ".join(contents)})'
+
+class HexBytesPrettyPrinter(PrettyPrinter):
+    def format(self, *args):
+        # it doesn't matter what the boolean values are here
+        return saferepr(*args), True, False

pySim/profile/__init__.py

@@ -25,54 +25,11 @@ import abc
 import operator
 from typing import List
 
+from pySim.exceptions import SwMatchError
 from pySim.commands import SimCardCommands
 from pySim.filesystem import CardApplication, interpret_sw
 from pySim.utils import all_subclasses
 
-def _mf_select_test(scc: SimCardCommands,
-                    cla_byte: str, sel_ctrl: str,
-                    fids: List[str]) -> bool:
-    cla_byte_bak = scc.cla_byte
-    sel_ctrl_bak = scc.sel_ctrl
-    scc.reset_card()
-
-    scc.cla_byte = cla_byte
-    scc.sel_ctrl = sel_ctrl
-    rc = True
-    try:
-        for fid in fids:
-            scc.select_file(fid)
-    except:
-        rc = False
-
-    scc.reset_card()
-    scc.cla_byte = cla_byte_bak
-    scc.sel_ctrl = sel_ctrl_bak
-    return rc
-
-
-def match_uicc(scc: SimCardCommands) -> bool:
-    """ Try to access MF via UICC APDUs (3GPP TS 102.221), if this works, the
-    card is considered a UICC card.
-    """
-    return _mf_select_test(scc, "00", "0004", ["3f00"])
-
-
-def match_sim(scc: SimCardCommands) -> bool:
-    """ Try to access MF via 2G APDUs (3GPP TS 11.11), if this works, the card
-    is also a simcard. This will be the case for most UICC cards, but there may
-    also be plain UICC cards without 2G support as well.
-    """
-    return _mf_select_test(scc, "a0", "0000", ["3f00"])
-
-
-def match_ruim(scc: SimCardCommands) -> bool:
-    """ Try to access MF/DF.CDMA via 2G APDUs (3GPP TS 11.11), if this works,
-    the card is considered an R-UIM card for CDMA.
-    """
-    return _mf_select_test(scc, "a0", "0000", ["3f00", "7f25"])
-
-
 class CardProfile:
     """A Card Profile describes a card, it's filesystem hierarchy, an [initial] list of
     applications as well as profile-specific SW and shell commands.  Every card has
@@ -138,13 +95,39 @@ class CardProfile:
         return data_hex
 
     @staticmethod
+    def _mf_select_test(scc: SimCardCommands,
+                        cla_byte: str, sel_ctrl: str,
+                        fids: List[str]) -> bool:
+        """Helper function used by some derived _try_match_card() methods."""
+        scc.reset_card()
+
+        scc.cla_byte = cla_byte
+        scc.sel_ctrl = sel_ctrl
+        for fid in fids:
+            scc.select_file(fid)
+
+    @classmethod
     @abc.abstractmethod
-    def match_with_card(scc: SimCardCommands) -> bool:
-        """Check if the specific profile matches the card. This method is a
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        """Try to see if the specific profile matches the card. This method is a
         placeholder that is overloaded by specific dirived classes. The method
         actively probes the card to make sure the profile class matches the
         physical card. This usually also means that the card is reset during
         the process, so this method must not be called at random times. It may
+        only be called on startup.  If there is no exception raised, we assume
+        the card matches the profile.
+
+        Args:
+                scc: SimCardCommands class
+        """
+        pass
+
+    @classmethod
+    def match_with_card(cls, scc: SimCardCommands) -> bool:
+        """Check if the specific profile matches the card. The method
+        actively probes the card to make sure the profile class matches the
+        physical card. This usually also means that the card is reset during
+        the process, so this method must not be called at random times. It may
         only be called on startup.
 
         Args:
@@ -152,7 +135,17 @@ class CardProfile:
         Returns:
                 match = True, no match = False
         """
-        return False
+        sel_backup = scc.sel_ctrl
+        cla_backup = scc.cla_byte
+        try:
+            cls._try_match_card(scc)
+            return True
+        except SwMatchError:
+            return False
+        finally:
+            scc.sel_ctrl = sel_backup
+            scc.cla_byte = cla_backup
+            scc.reset_card()
 
     @staticmethod
     def pick(scc: SimCardCommands):

pySim/profile/cdma_ruim.py

@@ -20,15 +20,14 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import enum
 
 from construct import Bytewise, BitStruct, BitsInteger, Struct, FlagsEnum
+from osmocom.utils import *
+from osmocom.construct import *
 
-from pySim.utils import *
 from pySim.filesystem import *
-from pySim.profile import match_ruim
 from pySim.profile import CardProfile, CardProfileAddon
-from pySim.ts_51_011 import CardProfileSIM
-from pySim.ts_51_011 import DF_TELECOM, DF_GSM
-from pySim.ts_51_011 import EF_ServiceTable
-from pySim.construct import *
+from pySim.profile.ts_51_011 import CardProfileSIM
+from pySim.profile.ts_51_011 import DF_TELECOM, DF_GSM
+from pySim.profile.ts_51_011 import EF_ServiceTable
 
 
 # Mapping between CDMA Service Number and its description
@@ -179,7 +178,7 @@ class DF_CDMA(CardDF):
 class CardProfileRUIM(CardProfile):
     '''R-UIM card profile as per 3GPP2 C.S0023-D'''
 
-    ORDER = 2
+    ORDER = 20
 
     def __init__(self):
         super().__init__('R-UIM', desc='CDMA R-UIM Card', cla="a0",
@@ -190,9 +189,12 @@ class CardProfileRUIM(CardProfile):
         # TODO: Response parameters/data in case of DF_CDMA (section 2.6)
         return CardProfileSIM.decode_select_response(data_hex)
 
-    @staticmethod
-    def match_with_card(scc: SimCardCommands) -> bool:
-        return match_ruim(scc)
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        """ Try to access MF/DF.CDMA via 2G APDUs (3GPP TS 11.11), if this works,
+        the card is considered an R-UIM card for CDMA."""
+        cls._mf_select_test(scc, "a0", "0000", ["3f00", "7f25"])
+
 
 class AddonRUIM(CardProfileAddon):
     """An Addon that can be found on on a combined SIM + RUIM or UICC + RUIM to support CDMA."""

pySim/profile/euicc.py

@@ -0,0 +1,58 @@
+# Copyright (C) 2023 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from pySim.profile.ts_102_221 import CardProfileUICC
+from pySim.commands import SimCardCommands
+from pySim.euicc import CardApplicationISDR, AID_ISD_R, AID_ECASD, GetCertsReq, GetCertsResp
+
+class CardProfileEuiccSGP32(CardProfileUICC):
+    ORDER = 5
+
+    def __init__(self):
+        super().__init__(name='IoT eUICC (SGP.32)')
+
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        # try a command only supported by SGP.32
+        scc.cla_byte = "00"
+        scc.select_adf(AID_ISD_R)
+        CardApplicationISDR.store_data_tlv(scc, GetCertsReq(), GetCertsResp)
+
+class CardProfileEuiccSGP22(CardProfileUICC):
+    ORDER = 6
+
+    def __init__(self):
+        super().__init__(name='Consumer eUICC (SGP.22)')
+
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        # try to read EID from ISD-R
+        scc.cla_byte = "00"
+        scc.select_adf(AID_ISD_R)
+        eid = CardApplicationISDR.get_eid(scc)
+        # TODO: Store EID identity?
+
+class CardProfileEuiccSGP02(CardProfileUICC):
+    ORDER = 7
+
+    def __init__(self):
+        super().__init__(name='M2M eUICC (SGP.02)')
+
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        scc.cla_byte = "00"
+        scc.select_adf(AID_ECASD)
+        scc.get_data(0x5a)
+        # TODO: Store EID identity?

pySim/profile/gsm_r.py

@@ -28,8 +28,8 @@ from pySim.utils import *
 from struct import pack, unpack
 from construct import Struct, Bytes, Int8ub, Int16ub, Int24ub, Int32ub, FlagsEnum
 from construct import Optional as COptional
+from osmocom.construct import *
 
-from pySim.construct import *
 from pySim.profile import CardProfileAddon
 from pySim.filesystem import *
 
@@ -290,7 +290,7 @@ class EF_Predefined(LinFixedEF):
         else:
             return parse_construct(self.construct_others, raw_bin_data)
 
-    def _encode_record_bin(self, abstract_data : dict, record_nr : int) -> bytearray:
+    def _encode_record_bin(self, abstract_data : dict, record_nr : int, **kwargs) -> bytearray:
         r = None
         if record_nr == 1:
             r = self.construct_first.build(abstract_data)

pySim/profile/ts_102_221.py

@@ -0,0 +1,394 @@
+# coding=utf-8
+"""Card Profile of ETSI TS 102 221, the core UICC spec.
+
+(C) 2021-2024 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from construct import Struct, FlagsEnum, GreedyString
+
+from osmocom.construct import *
+from osmocom.utils import *
+from osmocom.tlv import BER_TLV_IE
+
+from pySim.utils import *
+from pySim.filesystem import *
+from pySim.profile import CardProfile
+from pySim import iso7816_4
+from pySim.ts_102_221 import decode_select_response, ts_102_22x_cmdset
+from pySim.ts_102_221 import AM_DO_EF, SC_DO, AdditionalInterfacesSupport, AdditionalTermCapEuicc
+from pySim.ts_102_221 import TerminalPowerSupply, ExtendedLchanTerminalSupport, TerminalCapability
+
+# A UICC will usually also support 2G functionality. If this is the case, we
+# need to add DF_GSM and DF_TELECOM along with the UICC related files
+from pySim.profile.ts_51_011 import AddonSIM, EF_ICCID, EF_PL
+from pySim.profile.gsm_r import AddonGSMR
+from pySim.profile.cdma_ruim import AddonRUIM
+
+
+# TS 102 221 Section 13.1
+class EF_DIR(LinFixedEF):
+    _test_de_encode = [
+        ( '61294f10a0000000871002ffffffff890709000050055553696d31730ea00c80011781025f608203454150',
+          { "application_template": [ { "application_id": h2b("a0000000871002ffffffff8907090000") },
+                                      { "application_label": "USim1" },
+                                      { "discretionary_template": h2b("a00c80011781025f608203454150") } ] }
+        ),
+        ( '61194f10a0000000871004ffffffff890709000050054953696d31',
+          { "application_template": [ { "application_id": h2b("a0000000871004ffffffff8907090000") },
+                                      { "application_label": "ISim1" } ] }
+        ),
+    ]
+    class ApplicationLabel(BER_TLV_IE, tag=0x50):
+        # TODO: UCS-2 coding option as per Annex A of TS 102 221
+        _construct = GreedyString('ascii')
+
+    # see https://github.com/PyCQA/pylint/issues/5794
+    #pylint: disable=undefined-variable
+    class ApplicationTemplate(BER_TLV_IE, tag=0x61,
+                              nested=[iso7816_4.ApplicationId, ApplicationLabel, iso7816_4.FileReference,
+                                      iso7816_4.CommandApdu, iso7816_4.DiscretionaryData,
+                                      iso7816_4.DiscretionaryTemplate, iso7816_4.URL,
+                                      iso7816_4.ApplicationRelatedDOSet]):
+        pass
+
+    def __init__(self, fid='2f00', sfid=0x1e, name='EF.DIR', desc='Application Directory'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(5, 54))
+        self._tlv = EF_DIR.ApplicationTemplate
+
+
+# TS 102 221 Section 13.4
+class EF_ARR(LinFixedEF):
+    _test_de_encode = [
+        ( '800101a40683010a950108800106900080016097008401d4a40683010a950108',
+         [ [ { "access_mode": [ "read_search_compare" ] },
+             { "control_reference_template": "ADM1" } ],
+           [ { "access_mode": [ "write_append", "update_erase" ] },
+             { "always": None } ],
+           [ { "access_mode": [ "delete_file", "terminate_ef" ] },
+             { "never": None } ],
+           [ { "command_header": { "INS": 212 } },
+             { "control_reference_template": "ADM1" } ]
+         ] ),
+        ( '80010190008001029700800118a40683010a9501088401d4a40683010a950108',
+         [ [ { "access_mode": [ "read_search_compare" ] },
+             { "always": None } ],
+           [ { "access_mode": [ "update_erase" ] },
+             { "never": None } ],
+           [ { "access_mode": [ "activate_file_or_record", "deactivate_file_or_record" ] },
+             { "control_reference_template": "ADM1" } ],
+           [ { "command_header": { "INS": 212 } },
+             { "control_reference_template": "ADM1" } ]
+         ] ),
+    ]
+    def __init__(self, fid='2f06', sfid=0x06, name='EF.ARR', desc='Access Rule Reference'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc)
+        # add those commands to the general commands of a TransparentEF
+        self.shell_commands += [self.AddlShellCommands()]
+
+    @staticmethod
+    def flatten(inp: list):
+        """Flatten the somewhat deep/complex/nested data returned from decoder."""
+        def sc_abbreviate(sc):
+            if 'always' in sc:
+                return 'always'
+            elif 'never' in sc:
+                return 'never'
+            elif 'control_reference_template' in sc:
+                return sc['control_reference_template']
+            else:
+                return sc
+
+        by_mode = {}
+        for t in inp:
+            am = t[0]
+            sc = t[1]
+            sc_abbr = sc_abbreviate(sc)
+            if 'access_mode' in am:
+                for m in am['access_mode']:
+                    by_mode[m] = sc_abbr
+            elif 'command_header' in am:
+                ins = am['command_header']['INS']
+                if 'CLA' in am['command_header']:
+                    cla = am['command_header']['CLA']
+                else:
+                    cla = None
+                cmd = ts_102_22x_cmdset.lookup(ins, cla)
+                if cmd:
+                    name = cmd.name.lower().replace(' ', '_')
+                    by_mode[name] = sc_abbr
+                else:
+                    raise ValueError
+            else:
+                raise ValueError
+        return by_mode
+
+    def _decode_record_bin(self, raw_bin_data, **kwargs):
+        # we can only guess if we should decode for EF or DF here :(
+        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        dec = arr_seq.decode_multi(raw_bin_data)
+        # we cannot pass the result through flatten() here, as we don't have a related
+        # 'un-flattening' decoder, and hence would be unable to encode :(
+        return dec[0]
+
+    def _encode_record_bin(self, in_json, **kwargs):
+        # we can only guess if we should decode for EF or DF here :(
+        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        return arr_seq.encode_multi(in_json)
+
+    @with_default_category('File-Specific Commands')
+    class AddlShellCommands(CommandSet):
+        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
+        def do_read_arr_record(self, opts):
+            """Read one EF.ARR record in flattened, human-friendly form."""
+            (data, _sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
+            data = self._cmd.lchan.selected_file.flatten(data)
+            self._cmd.poutput_json(data, opts.oneline)
+
+        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
+        def do_read_arr_records(self, opts):
+            """Read + decode all EF.ARR records in flattened, human-friendly form."""
+            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
+            # collect all results in list so they are rendered as JSON list when printing
+            data_list = []
+            for recnr in range(1, 1 + num_of_rec):
+                (data, _sw) = self._cmd.lchan.read_record_dec(recnr)
+                data = self._cmd.lchan.selected_file.flatten(data)
+                data_list.append(data)
+            self._cmd.poutput_json(data_list, opts.oneline)
+
+
+# TS 102 221 Section 13.6
+class EF_UMPC(TransparentEF):
+    _test_de_encode = [
+        ( '3cff02', { "max_current_mA": 60, "t_op_s": 255,
+                      "addl_info": { "req_inc_idle_current": False, "support_uicc_suspend": True } } ),
+        ( '320500', { "max_current_mA": 50, "t_op_s": 5, "addl_info": {"req_inc_idle_current": False,
+                                                                       "support_uicc_suspend": False } } ),
+    ]
+    def __init__(self, fid='2f08', sfid=0x08, name='EF.UMPC', desc='UICC Maximum Power Consumption'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(5, 5))
+        addl_info = FlagsEnum(Byte, req_inc_idle_current=1,
+                              support_uicc_suspend=2)
+        self._construct = Struct(
+            'max_current_mA'/Int8ub, 't_op_s'/Int8ub, 'addl_info'/addl_info)
+
+
+class CardProfileUICC(CardProfile):
+
+    ORDER = 10
+
+    def __init__(self, name='UICC'):
+        files = [
+            EF_DIR(),
+            EF_ICCID(),
+            EF_PL(),
+            EF_ARR(),
+            # FIXME: DF.CD
+            EF_UMPC(),
+        ]
+        addons = [
+            AddonSIM,
+            AddonGSMR,
+            AddonRUIM,
+        ]
+        sw = {
+            'Normal': {
+                '9000': 'Normal ending of the command',
+                '91xx': 'Normal ending of the command, with extra information from the proactive UICC containing a command for the terminal',
+                '92xx': 'Normal ending of the command, with extra information concerning an ongoing data transfer session',
+            },
+            'Postponed processing': {
+                '9300': 'SIM Application Toolkit is busy. Command cannot be executed at present, further normal commands are allowed',
+            },
+            'Warnings': {
+                '6200': 'No information given, state of non-volatile memory unchanged',
+                '6281': 'Part of returned data may be corrupted',
+                '6282': 'End of file/record reached before reading Le bytes or unsuccessful search',
+                '6283': 'Selected file invalidated/disabled; needs to be activated before use',
+                '6284': 'Selected file in termination state',
+                '62f1': 'More data available',
+                '62f2': 'More data available and proactive command pending',
+                '62f3': 'Response data available',
+                '63f1': 'More data expected',
+                '63f2': 'More data expected and proactive command pending',
+                '63cx': 'Command successful but after using an internal update retry routine X times',
+            },
+            'Execution errors': {
+                '6400': 'No information given, state of non-volatile memory unchanged',
+                '6500': 'No information given, state of non-volatile memory changed',
+                '6581': 'Memory problem',
+            },
+            'Checking errors': {
+                '6700': 'Wrong length',
+                '67xx': 'The interpretation of this status word is command dependent',
+                '6b00': 'Wrong parameter(s) P1-P2',
+                '6d00': 'Instruction code not supported or invalid',
+                '6e00': 'Class not supported',
+                '6f00': 'Technical problem, no precise diagnosis',
+                '6fxx': 'The interpretation of this status word is command dependent',
+            },
+            'Functions in CLA not supported': {
+                '6800': 'No information given',
+                '6881': 'Logical channel not supported',
+                '6882': 'Secure messaging not supported',
+            },
+            'Command not allowed': {
+                '6900': 'No information given',
+                '6981': 'Command incompatible with file structure',
+                '6982': 'Security status not satisfied',
+                '6983': 'Authentication/PIN method blocked',
+                '6984': 'Referenced data invalidated',
+                '6985': 'Conditions of use not satisfied',
+                '6986': 'Command not allowed (no EF selected)',
+                '6989': 'Command not allowed - secure channel - security not satisfied',
+            },
+            'Wrong parameters': {
+                '6a80': 'Incorrect parameters in the data field',
+                '6a81': 'Function not supported',
+                '6a82': 'File not found',
+                '6a83': 'Record not found',
+                '6a84': 'Not enough memory space',
+                '6a86': 'Incorrect parameters P1 to P2',
+                '6a87': 'Lc inconsistent with P1 to P2',
+                '6a88': 'Referenced data not found',
+            },
+            'Application errors': {
+                '9850': 'INCREASE cannot be performed, max value reached',
+                '9862': 'Authentication error, application specific',
+                '9863': 'Security session or association expired',
+                '9864': 'Minimum UICC suspension time is too long',
+            },
+        }
+
+        super().__init__(name, desc='ETSI TS 102 221', cla="00",
+                         sel_ctrl="0004", files_in_mf=files, sw=sw,
+                         shell_cmdsets = [self.AddlShellCommands()], addons = addons)
+
+    @staticmethod
+    def decode_select_response(data_hex: str) -> object:
+        """ETSI TS 102 221 Section 11.1.1.3"""
+        return decode_select_response(data_hex)
+
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        """ Try to access MF via UICC APDUs (3GPP TS 102.221), if this works, the
+        card is considered a UICC card."""
+        cls._mf_select_test(scc, "00", "0004", ["3f00"])
+
+    @with_default_category('TS 102 221 Specific Commands')
+    class AddlShellCommands(CommandSet):
+        suspend_uicc_parser = argparse.ArgumentParser()
+        suspend_uicc_parser.add_argument('--min-duration-secs', type=int, default=60,
+                                         help='Proposed minimum duration of suspension')
+        suspend_uicc_parser.add_argument('--max-duration-secs', type=int, default=24*60*60,
+                                         help='Proposed maximum duration of suspension')
+
+        # not ISO7816-4 but TS 102 221
+        @cmd2.with_argparser(suspend_uicc_parser)
+        def do_suspend_uicc(self, opts):
+            """Perform the SUSPEND UICC command. Only supported on some UICC (check EF.UMPC)."""
+            (duration, token, sw) = self._cmd.card._scc.suspend_uicc(min_len_secs=opts.min_duration_secs,
+                                                                     max_len_secs=opts.max_duration_secs)
+            self._cmd.poutput(
+                'Negotiated Duration: %u secs, Token: %s, SW: %s' % (duration, token, sw))
+
+        resume_uicc_parser = argparse.ArgumentParser()
+        resume_uicc_parser.add_argument('TOKEN', type=str, help='Token provided during SUSPEND')
+
+        @cmd2.with_argparser(resume_uicc_parser)
+        def do_resume_uicc(self, opts):
+            """Perform the REUSME UICC operation. Only supported on some UICC. Also: A power-cycle
+            of the card is required between SUSPEND and RESUME, and only very few non-RESUME
+            commands are permitted between SUSPEND and RESUME.  See TS 102 221 Section 11.1.22."""
+            self._cmd.card._scc.resume_uicc(opts.TOKEN)
+
+        term_cap_parser = argparse.ArgumentParser()
+        # power group
+        tc_power_grp = term_cap_parser.add_argument_group('Terminal Power Supply')
+        tc_power_grp.add_argument('--used-supply-voltage-class', type=str, choices=['a','b','c','d','e'],
+                                  help='Actual used Supply voltage class')
+        tc_power_grp.add_argument('--maximum-available-power-supply', type=auto_uint8,
+                                  help='Maximum available power supply of the terminal')
+        tc_power_grp.add_argument('--actual-used-freq-100k', type=auto_uint8,
+                                  help='Actual used clock frequency (in units of 100kHz)')
+        # no separate groups for those two
+        tc_elc_grp = term_cap_parser.add_argument_group('Extended logical channels terminal support')
+        tc_elc_grp.add_argument('--extended-logical-channel', action='store_true',
+                                help='Extended Logical Channel supported')
+        tc_aif_grp = term_cap_parser.add_argument_group('Additional interfaces support')
+        tc_aif_grp.add_argument('--uicc-clf', action='store_true',
+                                help='Local User Interface in the Device (LUId) supported')
+        # eUICC group
+        tc_euicc_grp = term_cap_parser.add_argument_group('Additional Terminal capability indications related to eUICC')
+        tc_euicc_grp.add_argument('--lui-d', action='store_true',
+                                  help='Local User Interface in the Device (LUId) supported')
+        tc_euicc_grp.add_argument('--lpd-d', action='store_true',
+                                  help='Local Profile Download in the Device (LPDd) supported')
+        tc_euicc_grp.add_argument('--lds-d', action='store_true',
+                                  help='Local Discovery Service in the Device (LPDd) supported')
+        tc_euicc_grp.add_argument('--lui-e-scws', action='store_true',
+                                  help='LUIe based on SCWS supported')
+        tc_euicc_grp.add_argument('--metadata-update-alerting', action='store_true',
+                                  help='Metadata update alerting supported')
+        tc_euicc_grp.add_argument('--enterprise-capable-device', action='store_true',
+                                  help='Enterprise Capable Device')
+        tc_euicc_grp.add_argument('--lui-e-e4e', action='store_true',
+                                  help='LUIe using E4E (ENVELOPE tag E4) supported')
+        tc_euicc_grp.add_argument('--lpr', action='store_true',
+                                  help='LPR (LPA Proxy) supported')
+
+        @cmd2.with_argparser(term_cap_parser)
+        def do_terminal_capability(self, opts):
+            """Perform the TERMINAL CAPABILITY function. Used to inform the UICC about terminal capability."""
+            ps_flags = {}
+            addl_if_flags = {}
+            euicc_flags = {}
+
+            opts_dict = vars(opts)
+
+            power_items = ['used_supply_voltage_class', 'maximum_available_power_supply', 'actual_used_freq_100k']
+            if any(opts_dict[x] for x in power_items):
+                if not all(opts_dict[x] for x in power_items):
+                    raise argparse.ArgumentTypeError('If any of the Terminal Power Supply group options are used, all must be specified')
+
+            for k, v in opts_dict.items():
+                if k in AdditionalInterfacesSupport._construct.flags.keys():
+                    addl_if_flags[k] = v
+                elif k in AdditionalTermCapEuicc._construct.flags.keys():
+                    euicc_flags[k] = v
+                elif k in [f.name for f in TerminalPowerSupply._construct.subcons]:
+                    if k == 'used_supply_voltage_class' and v:
+                        v = {v: True}
+                    ps_flags[k] = v
+
+            child_list = []
+            if any(x for x in ps_flags.values()):
+                child_list.append(TerminalPowerSupply(decoded=ps_flags))
+
+            if opts.extended_logical_channel:
+                child_list.append(ExtendedLchanTerminalSupport())
+            if any(x for x in addl_if_flags.values()):
+                child_list.append(AdditionalInterfacesSupport(decoded=addl_if_flags))
+            if any(x for x in euicc_flags.values()):
+                child_list.append(AdditionalTermCapEuicc(decoded=euicc_flags))
+
+            print(child_list)
+            tc = TerminalCapability(children=child_list)
+            self.terminal_capability(b2h(tc.to_tlv()))
+
+        def terminal_capability(self, data:Hexstr):
+            cmd_hex = "80AA0000%02x%s" % (len(data)//2, data)
+            _rsp_hex, _sw = self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)

pySim/profile/ts_51_011.py

@@ -29,19 +29,21 @@ order to describe the files specified in the relevant ETSI + 3GPP specifications
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from pySim.profile import match_sim
+import enum
+from struct import pack, unpack
+from typing import Tuple
+
+from construct import Optional as COptional
+from construct import *
+from osmocom.tlv import *
+from osmocom.utils import *
+from osmocom.construct import *
+
+from pySim.utils import dec_iccid, enc_iccid, dec_imsi, enc_imsi, dec_plmn, enc_plmn, dec_xplmn_w_act
 from pySim.profile import CardProfile, CardProfileAddon
 from pySim.filesystem import *
 from pySim.ts_31_102_telecom import DF_PHONEBOOK, DF_MULTIMEDIA, DF_MCS, DF_V2X
-from pySim.gsm_r import AddonGSMR
-import enum
-from pySim.construct import *
-from construct import Optional as COptional
-from construct import *
-from struct import pack, unpack
-from typing import Tuple
-from pySim.tlv import *
-from pySim.utils import *
+from pySim.profile.gsm_r import AddonGSMR
 
 # Mapping between SIM Service Number and its description
 EF_SST_map = {
@@ -129,13 +131,13 @@ class ExtendedBcdAdapter(Adapter):
 
 # TS 51.011 Section 10.5.1
 class EF_ADN(LinFixedEF):
-    _test_decode = [
-            ( '42204841203120536963ffffffff06810628560810ffffffffffffff',
+    _test_de_encode = [
+            ( '42204841203120536963FFFFFFFFFFFF06810628560810FFFFFFFFFFFFFF',
               { "alpha_id": "B HA 1 Sic", "len_of_bcd": 6, "ton_npi": { "ext": True, "type_of_number":
                                                                        "unknown", "numbering_plan_id":
                                                                        "isdn_e164" }, "dialing_nr":
                "6082658001", "cap_conf_id": 255, "ext1_record_id": 255 }),
-            ( '4B756E64656E626574726575756E67FFFFFF0791947112122721ffffffffffff',
+            ( '4B756E64656E626574726575756E67FF0791947112122721ffffffffffff',
               {"alpha_id": "Kundenbetreuung", "len_of_bcd": 7, "ton_npi": {"ext": True, "type_of_number":
                                                                            "international",
                                                                            "numbering_plan_id": "isdn_e164"},
@@ -186,20 +188,54 @@ class EF_SMS(LinFixedEF):
 
 # TS 51.011 Section 10.5.5
 class EF_MSISDN(LinFixedEF):
+    _test_de_encode = [
+        ( 'ffffffffffffffffffffffffffffffffffffffff04b12143f5ffffffffffffffffff',
+            {"alpha_id": "", "len_of_bcd": 4, "ton_npi": {"ext": True, "type_of_number": "network_specific",
+                                                          "numbering_plan_id": "isdn_e164"},
+             "dialing_nr": "12345f"}),
+        ( '456967656e65205275666e756d6d6572ffffffff0891947172199181f3ffffffffff',
+            {"alpha_id": "Eigene Rufnummer", "len_of_bcd": 8, "ton_npi": {"ext": True, "type_of_number": "international",
+                                                                          "numbering_plan_id": "isdn_e164"},
+             "dialing_nr": "4917279119183f"}),
+    ]
+
+    # Ensure deprecated representations still work
+    _test_encode = [
+        ( 'ffffffffffffffffffffffffffffffffffffffff05b1716662f6ffffffffffffffff',
+            {"msisdn": [ 1, 3, "1766266"]}),
+        ( 'ffffffffffffffffffffffffffffffffffffffff06b121436587f9ffffffffffffff',
+            {"msisdn": "123456789"}),
+    ]
+
+    _test_no_pad = True
+
     def __init__(self, fid='6f40', sfid=None, name='EF.MSISDN', desc='MSISDN', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(15, 34), leftpad=True, **kwargs)
+        self._construct = Struct('alpha_id'/COptional(GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-14)))),
+                                 'len_of_bcd'/Int8ub,
+                                 'ton_npi'/TonNpi,
+                                 'dialing_nr'/ExtendedBcdAdapter(BcdAdapter(Rpad(Bytes(10)))),
+                                  Padding(2, pattern=b'\xff'))
 
-    def _decode_record_hex(self, raw_hex_data, **kwargs):
-        return {'msisdn': dec_msisdn(raw_hex_data)}
-
-    def _encode_record_hex(self, abstract, **kwargs):
-        msisdn = abstract['msisdn']
-        if type(msisdn) == str:
-            encoded_msisdn = enc_msisdn(msisdn)
-        else:
-            encoded_msisdn = enc_msisdn(msisdn[2], msisdn[0], msisdn[1])
-        alpha_identifier = (list(self.rec_len)[0] - len(encoded_msisdn) // 2) * "ff"
-        return alpha_identifier + encoded_msisdn
+    # Maintain compatibility with deprecated representations
+    def encode_record_hex(self, abstract_data: dict, record_nr: int, total_len: int = None) -> str:
+        if 'msisdn' in abstract_data:
+            msisdn = abstract_data['msisdn']
+            if type(msisdn) == str:
+                npi = 'isdn_e164'
+                ton = 'network_specific'
+                dialing_nr = msisdn + len(msisdn) % 2 * "f"
+            else:
+                npi = msisdn[0]
+                ton = msisdn[1]
+                dialing_nr = msisdn[2] + len(msisdn[2]) % 2 * "f"
+            abstract_data = {'alpha_id' : "",
+                             'len_of_bcd' : len(dialing_nr) // 2 + 1,
+                             'ton_npi' : {'ext' : True,
+                                          'type_of_number' : ton,
+                                          'numbering_plan_id' : npi},
+                             'dialing_nr' : dialing_nr}
+        return super().encode_record_hex(abstract_data, record_nr, total_len)
 
 # TS 51.011 Section 10.5.6
 class EF_SMSP(LinFixedEF):
@@ -355,7 +391,7 @@ class EF_IMSI(TransparentEF):
     def _decode_hex(self, raw_hex):
         return {'imsi': dec_imsi(raw_hex)}
 
-    def _encode_hex(self, abstract):
+    def _encode_hex(self, abstract, **kwargs):
         return enc_imsi(abstract['imsi'])
 
     @with_default_category('File-Specific Commands')
@@ -443,7 +479,7 @@ class EF_ServiceTable(TransparentEF):
             }
         return ret
 
-    def _encode_bin(self, in_json):
+    def _encode_bin(self, in_json, **kwargs):
         # compute the required binary size
         bin_len = 0
         for srv in in_json.keys():
@@ -805,7 +841,7 @@ class EF_InvScan(TransparentEF):
 
 # TS 51.011 Section 10.3.46
 class EF_CFIS(LinFixedEF):
-    _test_decode = [
+    _test_de_encode = [
         ( '0100ffffffffffffffffffffffffffff',
           {"msp_number": 1, "cfu_indicator_status": { "voice": False, "fax": False, "data": False, "rfu": 0 },
                                                       "len_of_bcd": 255, "ton_npi": {"ext": True,
@@ -955,6 +991,44 @@ class EF_MMSUCP(TransparentEF):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
 
 
+# TS 102 221 Section 13.2 / TS 31.101 Section 13 / TS 51.011 Section 10.1.1
+class EF_ICCID(TransparentEF):
+    _test_de_encode = [
+        ( '988812010000400310f0', { "iccid": "8988211000000430010" } ),
+    ]
+    def __init__(self, fid='2fe2', sfid=0x02, name='EF.ICCID', desc='ICC Identification'):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(10, 10))
+
+    def _decode_hex(self, raw_hex):
+        return {'iccid': dec_iccid(raw_hex)}
+
+    def _encode_hex(self, abstract, **kwargs):
+        return enc_iccid(abstract['iccid'])
+
+# TS 102 221 Section 13.3 / TS 31.101 Secction 13 / TS 51.011 Section 10.1.2
+class EF_PL(TransRecEF):
+    _test_de_encode = [
+        ( '6465', "de" ),
+        ( '656e', "en" ),
+        ( 'ffff', None ),
+    ]
+
+    def __init__(self, fid='2f05', sfid=0x05, name='EF.PL', desc='Preferred Languages'):
+        super().__init__(fid, sfid=sfid, name=name,
+                         desc=desc, rec_len=2, size=(2, None))
+
+    def _decode_record_bin(self, bin_data, **kwargs):
+        if bin_data == b'\xff\xff':
+            return None
+        else:
+            return bin_data.decode('ascii')
+
+    def _encode_record_bin(self, in_json, **kwargs):
+        if in_json is None:
+            return b'\xff\xff'
+        else:
+            return in_json.encode('ascii')
+
 class DF_GSM(CardDF):
     def __init__(self, fid='7f20', name='DF.GSM', desc='GSM Network related files'):
         super().__init__(fid=fid, name=name, desc=desc)
@@ -1034,18 +1108,18 @@ class DF_GSM(CardDF):
             super().__init__()
 
         authenticate_parser = argparse.ArgumentParser()
-        authenticate_parser.add_argument('rand', type=is_hexstr, help='Random challenge')
+        authenticate_parser.add_argument('RAND', type=is_hexstr, help='Random challenge')
 
         @cmd2.with_argparser(authenticate_parser)
         def do_authenticate(self, opts):
             """Perform GSM Authentication."""
-            (data, sw) = self._cmd.lchan.scc.run_gsm(opts.rand)
+            (data, sw) = self._cmd.lchan.scc.run_gsm(opts.RAND)
             self._cmd.poutput_json(data)
 
 
 class CardProfileSIM(CardProfile):
 
-    ORDER = 3
+    ORDER = 30
 
     def __init__(self):
         sw = {
@@ -1085,12 +1159,19 @@ class CardProfileSIM(CardProfile):
             },
         }
 
+        files = [
+            EF_ICCID(),
+            EF_PL(),
+            DF_TELECOM(),
+            DF_GSM(),
+        ]
+
         addons = [
             AddonGSMR,
         ]
 
         super().__init__('SIM', desc='GSM SIM Card', cla="a0",
-                         sel_ctrl="0000", files_in_mf=[DF_TELECOM(), DF_GSM()], sw=sw, addons = addons)
+                         sel_ctrl="0000", files_in_mf=files, sw=sw, addons = addons)
 
     @staticmethod
     def decode_select_response(resp_hex: str) -> object:
@@ -1143,9 +1224,9 @@ class CardProfileSIM(CardProfile):
                 ret['life_cycle_status_int'] = 'terminated'
         return ret
 
-    @staticmethod
-    def match_with_card(scc: SimCardCommands) -> bool:
-        return match_sim(scc)
+    @classmethod
+    def _try_match_card(cls, scc: SimCardCommands) -> None:
+        cls._mf_select_test(scc, "a0", "0000", ["3f00"])
 
 
 class AddonSIM(CardProfileAddon):

pySim/runtime.py

@@ -18,8 +18,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from typing import Optional, Tuple
+from osmocom.utils import h2b, i2h, is_hex, Hexstr
+from osmocom.tlv import bertlv_parse_one
 
-from pySim.utils import h2b, i2h, is_hex, bertlv_parse_one, Hexstr
 from pySim.exceptions import *
 from pySim.filesystem import *
 
@@ -49,6 +50,9 @@ class RuntimeState:
         self.lchan = {}
         # the basic logical channel always exists
         self.lchan[0] = RuntimeLchan(0, self)
+        # this is a dict of card identities which different parts of the code might populate,
+        # typically with something like ICCID, EID, ATR, ...
+        self.identity = {}
 
         # make sure the class and selection control bytes, which are specified
         # by the card profile are used
@@ -131,13 +135,18 @@ class RuntimeState:
         """
         # delete all lchan != 0 (basic lchan)
         for lchan_nr in list(self.lchan.keys()):
+            self.lchan[lchan_nr].scc.scp = None
             if lchan_nr == 0:
                 continue
             del self.lchan[lchan_nr]
         atr = i2h(self.card.reset())
+        if cmd_app:
+            cmd_app.lchan = self.lchan[0]
         # select MF to reset internal state and to verify card really works
         self.lchan[0].select('MF', cmd_app)
         self.lchan[0].selected_adf = None
+        # store ATR as part of our card identies dict
+        self.identity['ATR'] = atr
         return atr
 
     def add_lchan(self, lchan_nr: int) -> 'RuntimeLchan':
@@ -202,6 +211,18 @@ class RuntimeLchan:
     def selected_file_num_of_rec(self) -> Optional[int]:
         return self.selected_file_fcp['file_descriptor'].get('num_of_rec')
 
+    def selected_file_record_len(self) -> Optional[int]:
+        return self.selected_file_fcp['file_descriptor'].get('record_len')
+
+    def selected_file_size(self) -> Optional[int]:
+        return self.selected_file_fcp.get('file_size')
+
+    def selected_file_reserved_file_size(self) -> Optional[int]:
+        return self.selected_file_fcp['proprietary_information'].get('reserved_file_size')
+
+    def selected_file_maximum_file_size(self) -> Optional[int]:
+        return self.selected_file_fcp['proprietary_information'].get('maximum_file_size')
+
     def get_cwd(self) -> CardDF:
         """Obtain the current working directory.
 
@@ -226,6 +247,42 @@ class RuntimeLchan:
             node = node.parent
         return None
 
+    def get_file_by_name(self, name: str) -> CardFile:
+        """Obtain the file object from the file system tree by its name without actually selecting the file.
+
+        Returns:
+            CardFile() instance or None"""
+
+        # handling of entire paths with multiple directories/elements
+        if '/' in name:
+            pathlist = name.split('/')
+            # treat /DF.GSM/foo like MF/DF.GSM/foo
+            if pathlist[0] == '':
+                pathlist[0] = 'MF'
+        else:
+            pathlist = [name]
+
+        # start in the current working directory (we can still
+        # select any ADF and the MF from here, so those will be
+        # among the selectables).
+        file = self.get_cwd()
+
+        for p in pathlist:
+            # Look for the next file in the path list
+            selectables = file.get_selectables()
+            file = None
+            for selectable in selectables:
+                if selectable == p:
+                    file = selectables[selectable]
+                    break
+
+            # When we hit none, then the given path must be invalid
+            if file is None:
+                return None
+
+        # Return the file object found at the tip of the path
+        return file
+
     def interpret_sw(self, sw: str):
         """Interpret a given status word relative to the currently selected application
         or the underlying card profile.
@@ -254,7 +311,8 @@ class RuntimeLchan:
             raise ValueError(
                 "Cannot select unknown file by name %s, only hexadecimal 4 digit FID is allowed" % fid)
 
-        self._select_pre(cmd_app)
+        # unregister commands of old file
+        self.unregister_cmds(cmd_app)
 
         try:
             # We access the card through the select_file method of the scc object.
@@ -287,12 +345,6 @@ class RuntimeLchan:
 
         self._select_post(cmd_app, f, data)
 
-    def _select_pre(self, cmd_app):
-        # unregister commands of old file
-        if cmd_app and self.selected_file.shell_commands:
-            for c in self.selected_file.shell_commands:
-                cmd_app.unregister_command_set(c)
-
     def _select_post(self, cmd_app, file:Optional[CardFile] = None, select_resp_data = None):
         # we store some reference data (see above) about the currently selected file.
         # This data must be updated after every select.
@@ -308,9 +360,7 @@ class RuntimeLchan:
                 self.selected_file_fcp = None
 
         # register commands of new file
-        if cmd_app and self.selected_file.shell_commands:
-            for c in self.selected_file.shell_commands:
-                cmd_app.register_command_set(c)
+        self.register_cmds(cmd_app)
 
     def select_file(self, file: CardFile, cmd_app=None):
         """Select a file (EF, DF, ADF, MF, ...).
@@ -319,11 +369,31 @@ class RuntimeLchan:
             file : CardFile [or derived class] instance
             cmd_app : Command Application State (for unregistering old file commands)
         """
+
+        if not isinstance(file, CardADF) and self.selected_adf and self.selected_adf.has_fs == False:
+            # Not every application that may be present on a GlobalPlatform card will support the SELECT
+            # command as we know it from ETSI TS 102 221, section 11.1.1. In fact the only subset of
+            # SELECT we may rely on is the OPEN SELECT command as specified in GlobalPlatform Card
+            # Specification, section 11.9. Unfortunately the OPEN SELECT command only supports the
+            # "select by name" method, which means we can only select an application and not a file.
+            # The consequence of this is that we may get trapped in an application that does not have
+            # ISIM/USIM like file system support and the only way to leave that application is to select
+            # an ISIM/USIM application in order to get the file system access back.
+            #
+            # To automate this escape-route we will first select an arbitrary ADF that has file system support first
+            # and then continue normally.
+            for selectable in self.rs.mf.get_selectables().items():
+                if isinstance(selectable[1], CardADF) and selectable[1].has_fs == True:
+                    self.select(selectable[1].name, cmd_app)
+                    break
+
         # we need to find a path from our self.selected_file to the destination
         inter_path = self.selected_file.build_select_path_to(file)
         if not inter_path:
             raise RuntimeError('Cannot determine path from %s to %s' % (self.selected_file, file))
-        self._select_pre(cmd_app)
+
+        # unregister commands of old file
+        self.unregister_cmds(cmd_app)
 
         # be sure the variables that we pass to _select_post contain valid values.
         selected_file = self.selected_file
@@ -396,7 +466,8 @@ class RuntimeLchan:
         (data, _sw) = self.scc.status()
         return self.selected_file.decode_select_response(data)
 
-    def get_file_for_selectable(self, name: str):
+    def get_file_for_filename(self, name: str):
+        """Get the related CardFile object for a specified filename."""
         sels = self.selected_file.get_selectables()
         return sels[name]
 
@@ -417,7 +488,8 @@ class RuntimeLchan:
             binary data read from the file
         """
         if not isinstance(self.selected_file, TransparentEF):
-            raise TypeError("Only works with TransparentEF")
+            raise TypeError("Only works with TransparentEF, but %s is %s" % (self.selected_file,
+                                                                             self.selected_file.__class__.__mro__))
         return self.scc.read_binary(self.selected_file.fid, length, offset)
 
     def read_binary_dec(self) -> Tuple[dict, str]:
@@ -441,7 +513,8 @@ class RuntimeLchan:
             offset : Offset into the file from which to write 'data_hex'
         """
         if not isinstance(self.selected_file, TransparentEF):
-            raise TypeError("Only works with TransparentEF")
+            raise TypeError("Only works with TransparentEF, but %s is %s" % (self.selected_file,
+                                                                             self.selected_file.__class__.__mro__))
         return self.scc.update_binary(self.selected_file.fid, data_hex, offset, conserve=self.rs.conserve_write)
 
     def update_binary_dec(self, data: dict):
@@ -451,7 +524,7 @@ class RuntimeLchan:
         Args:
             data : abstract data which is to be encoded and written
         """
-        data_hex = self.selected_file.encode_hex(data)
+        data_hex = self.selected_file.encode_hex(data, self.selected_file_size())
         return self.update_binary(data_hex)
 
     def read_record(self, rec_nr: int = 0):
@@ -463,7 +536,8 @@ class RuntimeLchan:
             hex string of binary data contained in record
         """
         if not isinstance(self.selected_file, LinFixedEF):
-            raise TypeError("Only works with Linear Fixed EF")
+            raise TypeError("Only works with Linear Fixed EF, but %s is %s" % (self.selected_file,
+                                                                               self.selected_file.__class__.__mro__))
         # returns a string of hex nibbles
         return self.scc.read_record(self.selected_file.fid, rec_nr)
 
@@ -486,7 +560,8 @@ class RuntimeLchan:
             data_hex : Hex string binary data to be written
         """
         if not isinstance(self.selected_file, LinFixedEF):
-            raise TypeError("Only works with Linear Fixed EF")
+            raise TypeError("Only works with Linear Fixed EF, but %s is %s" % (self.selected_file,
+                                                                               self.selected_file.__class__.__mro__))
         return self.scc.update_record(self.selected_file.fid, rec_nr, data_hex,
 					       conserve=self.rs.conserve_write,
 					       leftpad=self.selected_file.leftpad)
@@ -499,7 +574,7 @@ class RuntimeLchan:
             rec_nr : Record number to read
             data_hex : Abstract data to be written
         """
-        data_hex = self.selected_file.encode_record_hex(data, rec_nr)
+        data_hex = self.selected_file.encode_record_hex(data, rec_nr, self.selected_file_record_len())
         return self.update_record(rec_nr, data_hex)
 
     def retrieve_data(self, tag: int = 0):
@@ -522,7 +597,8 @@ class RuntimeLchan:
             list of integer tags contained in EF
         """
         if not isinstance(self.selected_file, BerTlvEF):
-            raise TypeError("Only works with BER-TLV EF")
+            raise TypeError("Only works with BER-TLV EF, but %s is %s" % (self.selected_file,
+                                                                          self.selected_file.__class__.__mro__))
         data, _sw = self.scc.retrieve_data(self.selected_file.fid, 0x5c)
         _tag, _length, value, _remainder = bertlv_parse_one(h2b(data))
         return list(value)
@@ -535,11 +611,18 @@ class RuntimeLchan:
             data_hex : Hex string binary data to be written (value portion)
         """
         if not isinstance(self.selected_file, BerTlvEF):
-            raise TypeError("Only works with BER-TLV EF")
+            raise TypeError("Only works with BER-TLV EF, but %s is %s" % (self.selected_file,
+                                                                          self.selected_file.__class__.__mro__))
         return self.scc.set_data(self.selected_file.fid, tag, data_hex, conserve=self.rs.conserve_write)
 
+    def register_cmds(self, cmd_app=None):
+        """Register command set that is associated with the currently selected file"""
+        if cmd_app and self.selected_file.shell_commands:
+            for c in self.selected_file.shell_commands:
+                cmd_app.register_command_set(c)
+
     def unregister_cmds(self, cmd_app=None):
-        """Unregister all file specific commands."""
+        """Unregister command set that is associated with the currently selected file"""
         if cmd_app and self.selected_file.shell_commands:
             for c in self.selected_file.shell_commands:
                 cmd_app.unregister_command_set(c)

pySim/secure_channel.py

@@ -16,7 +16,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import abc
-from pySim.utils import b2h, h2b, ResTuple, Hexstr
+from osmocom.utils import b2h, h2b, Hexstr
+
+from pySim.utils import ResTuple
 
 class SecureChannel(abc.ABC):
     @abc.abstractmethod

pySim/sms.py

@@ -23,9 +23,8 @@ from bidict import bidict
 from construct import Int8ub, Byte, Bytes, Bit, Flag, BitsInteger
 from construct import Struct, Enum, Tell, BitStruct, this, Padding
 from construct import Prefixed, GreedyRange, GreedyBytes
-
-from pySim.construct import HexAdapter, BcdAdapter, TonNpi
-from pySim.utils import Hexstr, h2b, b2h
+from osmocom.construct import HexAdapter, BcdAdapter, TonNpi
+from osmocom.utils import Hexstr, h2b, b2h
 
 from smpp.pdu import pdu_types, operations
 

pySim/sysmocom_sja2.py

@@ -19,13 +19,14 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from struct import unpack
 from construct import FlagsEnum, Byte, Struct, Int8ub, Bytes, Mapping, Enum, Padding, BitsInteger
-from construct import Bit, this, Int32ub, Int16ub, Nibble, BytesInteger, GreedyRange
+from construct import Bit, this, Int32ub, Int16ub, Nibble, BytesInteger, GreedyRange, Const
 from construct import Optional as COptional
+from osmocom.utils import *
+from osmocom.construct import *
 
-from pySim.utils import *
 from pySim.filesystem import *
+from pySim.profile.ts_102_221 import CardProfileUICC
 from pySim.runtime import RuntimeState
-from pySim.construct import *
 import pySim
 
 key_type2str = {
@@ -180,7 +181,7 @@ class DF_SYSTEM(CardDF):
         self.add_files(files)
 
     def decode_select_response(self, resp_hex):
-        return pySim.ts_102_221.CardProfileUICC.decode_select_response(resp_hex)
+        return CardProfileUICC.decode_select_response(resp_hex)
 
 
 class EF_USIM_SQN(TransparentEF):
@@ -206,6 +207,18 @@ class EF_USIM_SQN(TransparentEF):
 
 
 class EF_USIM_AUTH_KEY(TransparentEF):
+    _test_de_encode = [
+        ( '141898d827f70120d33b3e7462ee5fd6fe6ca53d7a0a804561646816d7b0c702fb',
+          { "cfg": { "only_4bytes_res_in_3g": False, "sres_deriv_func_in_2g": 1, "use_opc_instead_of_op": True, "algorithm": "milenage" },
+            "key": "1898d827f70120d33b3e7462ee5fd6fe", "op_opc": "6ca53d7a0a804561646816d7b0c702fb" } ),
+        ( '160a04101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f000102030405060708090a0b0c0d0e0f',
+         { "cfg" : { "algorithm" : "tuak", "key_length" : 128, "sres_deriv_func_in_2g" : 1, "use_opc_instead_of_op" : True },
+           "tuak_cfg" : { "ck_and_ik_size" : 128, "mac_size" : 128, "res_size" : 128 },
+           "num_of_keccak_iterations" : 4,
+           "k" : "000102030405060708090a0b0c0d0e0f",
+           "op_opc" : "101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f"
+         } ),
+      ]
     def __init__(self, fid='af20', name='EF.USIM_AUTH_KEY'):
         super().__init__(fid, name=name, desc='USIM authentication key')
         Algorithm = Enum(Nibble, milenage=4, sha1_aka=5, tuak=6, xor=15)
@@ -240,7 +253,7 @@ class EF_USIM_AUTH_KEY(TransparentEF):
         else:
             return parse_construct(self._construct, raw_bin_data)
 
-    def _encode_bin(self, abstract_data: dict) -> bytearray:
+    def _encode_bin(self, abstract_data: dict, **kwargs) -> bytearray:
         if abstract_data['cfg']['algorithm'] == 'tuak':
             return build_construct(self._constr_tuak, abstract_data)
         else:

pySim/tlv.py

@@ -1,464 +0,0 @@
-"""object-oriented TLV parser/encoder library."""
-
-# (C) 2021 by Harald Welte <laforge@osmocom.org>
-# All Rights Reserved
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import inspect
-import abc
-import re
-from typing import List, Tuple
-
-from pySim.utils import bertlv_encode_len, bertlv_parse_len, bertlv_encode_tag, bertlv_parse_tag
-from pySim.utils import comprehensiontlv_encode_tag, comprehensiontlv_parse_tag
-from pySim.utils import bertlv_parse_tag_raw, comprehensiontlv_parse_tag_raw
-from pySim.utils import dgi_parse_tag_raw, dgi_parse_len, dgi_encode_tag, dgi_encode_len
-
-from pySim.construct import build_construct, parse_construct
-
-
-def camel_to_snake(name):
-    name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
-    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower()
-
-class TlvMeta(abc.ABCMeta):
-    """Metaclass which we use to set some class variables at the time of defining a subclass.
-    This allows us to create subclasses for each TLV/IE type, where the class represents fixed
-    parameters like the tag/type and instances of it represent the actual TLV data."""
-    def __new__(mcs, name, bases, namespace, **kwargs):
-        #print("TlvMeta_new_(mcs=%s, name=%s, bases=%s, namespace=%s, kwargs=%s)" % (mcs, name, bases, namespace, kwargs))
-        x = super().__new__(mcs, name, bases, namespace)
-        # this becomes a _class_ variable, not an instance variable
-        x.tag = namespace.get('tag', kwargs.get('tag', None))
-        x.desc = namespace.get('desc', kwargs.get('desc', None))
-        nested = namespace.get('nested', kwargs.get('nested', None))
-        if nested is None or inspect.isclass(nested) and issubclass(nested, TLV_IE_Collection):
-            # caller has specified TLV_IE_Collection sub-class, we can directly reference it
-            x.nested_collection_cls = nested
-        else:
-            # caller passed list of other TLV classes that might possibly appear within us,
-            # build a dynamically-created TLV_IE_Collection sub-class and reference it
-            name = 'auto_collection_%s' % (name)
-            cls = type(name, (TLV_IE_Collection,), {'nested': nested})
-            x.nested_collection_cls = cls
-        return x
-
-
-class TlvCollectionMeta(abc.ABCMeta):
-    """Metaclass which we use to set some class variables at the time of defining a subclass.
-    This allows us to create subclasses for each Collection type, where the class represents fixed
-    parameters like the nested IE classes and instances of it represent the actual TLV data."""
-    def __new__(mcs, name, bases, namespace, **kwargs):
-        #print("TlvCollectionMeta_new_(mcs=%s, name=%s, bases=%s, namespace=%s, kwargs=%s)" % (mcs, name, bases, namespace, kwargs))
-        x = super().__new__(mcs, name, bases, namespace)
-        # this becomes a _class_ variable, not an instance variable
-        x.possible_nested = namespace.get('nested', kwargs.get('nested', None))
-        return x
-
-
-class Transcodable(abc.ABC):
-    _construct = None
-    """Base class for something that can be encoded + encoded.  Decoding and Encoding happens either
-     * via a 'construct' object stored in a derived class' _construct variable, or
-     * via a 'construct' object stored in an instance _construct variable, or
-     * via a derived class' _{to,from}_bytes() methods."""
-
-    def __init__(self):
-        self.encoded = None
-        self.decoded = None
-        self._construct = None
-
-    def to_bytes(self, context: dict = {}) -> bytes:
-        """Convert from internal representation to binary bytes.  Store the binary result
-        in the internal state and return it."""
-        if self.decoded is None:
-            do = b''
-        elif self._construct:
-            do = build_construct(self._construct, self.decoded, context)
-        elif self.__class__._construct:
-            do = build_construct(self.__class__._construct, self.decoded, context)
-        else:
-            do = self._to_bytes()
-        self.encoded = do
-        return do
-
-    # not an abstractmethod, as it is only required if no _construct exists
-    def _to_bytes(self):
-        raise NotImplementedError('%s._to_bytes' % type(self).__name__)
-
-    def from_bytes(self, do: bytes, context: dict = {}):
-        """Convert from binary bytes to internal representation. Store the decoded result
-        in the internal state and return it."""
-        self.encoded = do
-        if self.encoded == b'':
-            self.decoded = None
-        elif self._construct:
-            self.decoded = parse_construct(self._construct, do, context=context)
-        elif self.__class__._construct:
-            self.decoded = parse_construct(self.__class__._construct, do, context=context)
-        else:
-            self.decoded = self._from_bytes(do)
-        return self.decoded
-
-    # not an abstractmethod, as it is only required if no _construct exists
-    def _from_bytes(self, do: bytes):
-        raise NotImplementedError('%s._from_bytes' % type(self).__name__)
-
-
-class IE(Transcodable, metaclass=TlvMeta):
-    # we specify the metaclass so any downstream subclasses will automatically use it
-    """Base class for various Information Elements. We understand the notion of a hierarchy
-    of IEs on top of the Transcodable class."""
-    # this is overridden by the TlvMeta metaclass, if it is used to create subclasses
-    nested_collection_cls = None
-    tag = None
-
-    def __init__(self, **kwargs):
-        super().__init__()
-        self.nested_collection = None
-        if self.nested_collection_cls:
-            self.nested_collection = self.nested_collection_cls()
-        # if we are a constructed IE, [ordered] list of actual child-IE instances
-        self.children = kwargs.get('children', [])
-        self.decoded = kwargs.get('decoded', None)
-
-    def __repr__(self):
-        """Return a string representing the [nested] IE data (for print)."""
-        if len(self.children):
-            member_strs = [repr(x) for x in self.children]
-            return '%s(%s)' % (type(self).__name__, ','.join(member_strs))
-        else:
-            return '%s(%s)' % (type(self).__name__, self.decoded)
-
-    def to_dict(self):
-        """Return a JSON-serializable dict representing the [nested] IE data."""
-        if len(self.children):
-            v = [x.to_dict() for x in self.children]
-        else:
-            v = self.decoded
-        return {camel_to_snake(type(self).__name__): v}
-
-    def from_dict(self, decoded: dict):
-        """Set the IE internal decoded representation to data from the argument.
-        If this is a nested IE, the child IE instance list is re-created."""
-        expected_key_name = camel_to_snake(type(self).__name__)
-        if not expected_key_name in decoded:
-            raise ValueError("Dict %s doesn't contain expected key %s" % (decoded, expected_key_name))
-        if self.nested_collection:
-            self.children = self.nested_collection.from_dict(decoded[expected_key_name])
-        else:
-            self.children = []
-            self.decoded = decoded[expected_key_name]
-
-    def is_constructed(self):
-        """Is this IE constructed by further nested IEs?"""
-        return bool(len(self.children) > 0)
-
-    @abc.abstractmethod
-    def to_ie(self, context: dict = {}) -> bytes:
-        """Convert the internal representation to entire IE including IE header."""
-
-    def to_bytes(self, context: dict = {}) -> bytes:
-        """Convert the internal representation *of the value part* to binary bytes."""
-        if self.is_constructed():
-            # concatenate the encoded IE of all children to form the value part
-            out = b''
-            for c in self.children:
-                out += c.to_ie(context=context)
-            return out
-        else:
-            return super().to_bytes(context=context)
-
-    def from_bytes(self, do: bytes, context: dict = {}):
-        """Parse *the value part* from binary bytes to internal representation."""
-        if self.nested_collection:
-            self.children = self.nested_collection.from_bytes(do, context=context)
-        else:
-            self.children = []
-            return super().from_bytes(do, context=context)
-
-
-class TLV_IE(IE):
-    """Abstract base class for various TLV type Information Elements."""
-
-    def _compute_tag(self) -> int:
-        """Compute the tag (sometimes the tag encodes part of the value)."""
-        return self.tag
-
-    @classmethod
-    @abc.abstractmethod
-    def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]:
-        """Obtain the raw TAG at the start of the bytes provided by the user."""
-
-    @classmethod
-    @abc.abstractmethod
-    def _parse_len(cls, do: bytes) -> Tuple[int, bytes]:
-        """Obtain the length encoded at the start of the bytes provided by the user."""
-
-    @abc.abstractmethod
-    def _encode_tag(self) -> bytes:
-        """Encode the tag part. Must be provided by derived (TLV format specific) class."""
-
-    @abc.abstractmethod
-    def _encode_len(self, val: bytes) -> bytes:
-        """Encode the length part assuming a certain binary value. Must be provided by
-        derived (TLV format specific) class."""
-
-    def to_ie(self, context: dict = {}):
-        return self.to_tlv(context=context)
-
-    def to_tlv(self, context: dict = {}):
-        """Convert the internal representation to binary TLV bytes."""
-        val = self.to_bytes(context=context)
-        return self._encode_tag() + self._encode_len(val) + val
-
-    def from_tlv(self, do: bytes, context: dict = {}):
-        if len(do) == 0:
-            return {}, b''
-        (rawtag, remainder) = self.__class__._parse_tag_raw(do)
-        if rawtag:
-            if rawtag != self._compute_tag():
-                raise ValueError("%s: Encountered tag %s doesn't match our supported tag %s" %
-                                 (self, rawtag, self.tag))
-            (length, remainder) = self.__class__._parse_len(remainder)
-            value = remainder[:length]
-            remainder = remainder[length:]
-        else:
-            value = do
-            remainder = b''
-        dec = self.from_bytes(value, context=context)
-        return dec, remainder
-
-
-class BER_TLV_IE(TLV_IE):
-    """TLV_IE formatted as ASN.1 BER described in ITU-T X.690 8.1.2."""
-
-    @classmethod
-    def _decode_tag(cls, do: bytes) -> Tuple[dict, bytes]:
-        return bertlv_parse_tag(do)
-
-    @classmethod
-    def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]:
-        return bertlv_parse_tag_raw(do)
-
-    @classmethod
-    def _parse_len(cls, do: bytes) -> Tuple[int, bytes]:
-        return bertlv_parse_len(do)
-
-    def _encode_tag(self) -> bytes:
-        return bertlv_encode_tag(self._compute_tag())
-
-    def _encode_len(self, val: bytes) -> bytes:
-        return bertlv_encode_len(len(val))
-
-
-class COMPR_TLV_IE(TLV_IE):
-    """TLV_IE formated as COMPREHENSION-TLV as described in ETSI TS 101 220."""
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.comprehension = False
-
-    @classmethod
-    def _decode_tag(cls, do: bytes) -> Tuple[dict, bytes]:
-        return comprehensiontlv_parse_tag(do)
-
-    @classmethod
-    def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]:
-        return comprehensiontlv_parse_tag_raw(do)
-
-    @classmethod
-    def _parse_len(cls, do: bytes) -> Tuple[int, bytes]:
-        return bertlv_parse_len(do)
-
-    def _encode_tag(self) -> bytes:
-        return comprehensiontlv_encode_tag(self._compute_tag())
-
-    def _encode_len(self, val: bytes) -> bytes:
-        return bertlv_encode_len(len(val))
-
-
-class DGI_TLV_IE(TLV_IE):
-    """TLV_IE formated as  GlobalPlatform Systems Scripting Language Specification v1.1.0 Annex B."""
-
-    @classmethod
-    def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]:
-        return dgi_parse_tag_raw(do)
-
-    @classmethod
-    def _parse_len(cls, do: bytes) -> Tuple[int, bytes]:
-        return dgi_parse_len(do)
-
-    def _encode_tag(self) -> bytes:
-        return dgi_encode_tag(self._compute_tag())
-
-    def _encode_len(self, val: bytes) -> bytes:
-        return dgi_encode_len(len(val))
-
-
-class TLV_IE_Collection(metaclass=TlvCollectionMeta):
-    # we specify the metaclass so any downstream subclasses will automatically use it
-    """A TLV_IE_Collection consists of multiple TLV_IE classes identified by their tags.
-    A given encoded DO may contain any of them in any order, and may contain multiple instances
-    of each DO."""
-    # this is overridden by the TlvCollectionMeta metaclass, if it is used to create subclasses
-    possible_nested = []
-
-    def __init__(self, desc=None, **kwargs):
-        self.desc = desc
-        #print("possible_nested: ", self.possible_nested)
-        self.members = kwargs.get('nested', self.possible_nested)
-        self.members_by_tag = {}
-        self.members_by_name = {}
-        self.members_by_tag = {m.tag: m for m in self.members}
-        self.members_by_name = {camel_to_snake(m.__name__): m for m in self.members}
-        # if we are a constructed IE, [ordered] list of actual child-IE instances
-        self.children = kwargs.get('children', [])
-        self.encoded = None
-
-    def __str__(self):
-        member_strs = [str(x) for x in self.members]
-        return '%s(%s)' % (type(self).__name__, ','.join(member_strs))
-
-    def __repr__(self):
-        member_strs = [repr(x) for x in self.members]
-        return '%s(%s)' % (self.__class__, ','.join(member_strs))
-
-    def __add__(self, other):
-        """Extending TLV_IE_Collections with other TLV_IE_Collections or TLV_IEs."""
-        if isinstance(other, TLV_IE_Collection):
-            # adding one collection to another
-            members = self.members + other.members
-            return TLV_IE_Collection(self.desc, nested=members)
-        elif inspect.isclass(other) and issubclass(other, TLV_IE):
-            # adding a member to a collection
-            return TLV_IE_Collection(self.desc, nested=self.members + [other])
-        else:
-            raise TypeError
-
-    def from_bytes(self, binary: bytes, context: dict = {}) -> List[TLV_IE]:
-        """Create a list of TLV_IEs from the collection based on binary input data.
-        Args:
-            binary : binary bytes of encoded data
-        Returns:
-            list of instances of TLV_IE sub-classes containing parsed data
-        """
-        self.encoded = binary
-        # list of instances of TLV_IE collection member classes appearing in the data
-        res = []
-        remainder = binary
-        first = next(iter(self.members_by_tag.values()))
-        # iterate until no binary trailer is left
-        while len(remainder):
-            context['siblings'] = res
-            # obtain the tag at the start of the remainder
-            tag, _r = first._parse_tag_raw(remainder)
-            if tag is None:
-                break
-            if tag in self.members_by_tag:
-                cls = self.members_by_tag[tag]
-                # create an instance and parse accordingly
-                inst = cls()
-                _dec, remainder = inst.from_tlv(remainder, context=context)
-                res.append(inst)
-            else:
-                # unknown tag; create the related class on-the-fly using the same base class
-                name = 'unknown_%s_%X' % (first.__base__.__name__, tag)
-                cls = type(name, (first.__base__,), {'tag': tag, 'possible_nested': [],
-                                                     'nested_collection_cls': None})
-                cls._from_bytes = lambda s, a: {'raw': a.hex()}
-                cls._to_bytes = lambda s: bytes.fromhex(s.decoded['raw'])
-                # create an instance and parse accordingly
-                inst = cls()
-                _dec, remainder = inst.from_tlv(remainder, context=context)
-                res.append(inst)
-        self.children = res
-        return res
-
-    def from_dict(self, decoded: List[dict]) -> List[TLV_IE]:
-        """Create a list of TLV_IE instances from the collection based on an array
-        of dicts, where they key indicates the name of the TLV_IE subclass to use."""
-        # list of instances of TLV_IE collection member classes appearing in the data
-        res = []
-        # iterate over members of the list passed into "decoded"
-        for i in decoded:
-            # iterate over all the keys (typically one!) within the current list item dict
-            for k in i.keys():
-                # check if we have a member identified by the dict key
-                if k in self.members_by_name:
-                    # resolve the class for that name; create an instance of it
-                    cls = self.members_by_name[k]
-                    inst = cls()
-                    if cls.nested_collection_cls:
-                        # in case of collections, we want to pass the raw "value" portion to from_dict,
-                        # as to_dict() below intentionally omits the collection-class-name as key
-                        inst.from_dict(i[k])
-                    else:
-                        inst.from_dict({k: i[k]})
-                    res.append(inst)
-                else:
-                    raise ValueError('%s: Unknown TLV Class %s in %s; expected %s' %
-                                     (self, k, decoded, self.members_by_name.keys()))
-        self.children = res
-        return res
-
-    def to_dict(self):
-        # we intentionally return not a dict, but a list of dicts.  We could prefix by
-        # self.__class__.__name__, but that is usually some meaningless auto-generated  collection name.
-        return [x.to_dict() for x in self.children]
-
-    def to_bytes(self, context: dict = {}):
-        out = b''
-        context['siblings'] = self.children
-        for c in self.children:
-            out += c.to_tlv(context=context)
-        return out
-
-    def from_tlv(self, do, context: dict = {}):
-        return self.from_bytes(do, context=context)
-
-    def to_tlv(self, context: dict = {}):
-        return self.to_bytes(context=context)
-
-
-def flatten_dict_lists(inp):
-    """hierarchically flatten each list-of-dicts into a single dict. This is useful to
-       make the output of hierarchical TLV decoder structures flatter and more easy to read."""
-    def are_all_elements_dict(l):
-        for e in l:
-            if not isinstance(e, dict):
-                return False
-        return True
-
-    def are_elements_unique(lod):
-        set_of_keys = {list(x.keys())[0] for x in lod}
-        return len(lod) == len(set_of_keys)
-
-    if isinstance(inp, list):
-        if are_all_elements_dict(inp) and are_elements_unique(inp):
-            # flatten into one shared dict
-            newdict = {}
-            for e in inp:
-                key = list(e.keys())[0]
-                newdict[key] = e[key]
-            inp = newdict
-            # process result as any native dict
-            return {k:flatten_dict_lists(v) for k,v in inp.items()}
-        else:
-            return [flatten_dict_lists(x) for x in inp]
-    elif isinstance(inp, dict):
-        return {k:flatten_dict_lists(v) for k,v in inp.items()}
-    else:
-        return inp

pySim/transport/__init__.py

@@ -8,9 +8,10 @@ import abc
 import argparse
 from typing import Optional, Tuple
 from construct import Construct
+from osmocom.utils import b2h, h2b, i2h, Hexstr
 
 from pySim.exceptions import *
-from pySim.utils import sw_match, b2h, h2b, i2h, Hexstr, SwHexstr, SwMatchstr, ResTuple
+from pySim.utils import SwHexstr, SwMatchstr, ResTuple, sw_match, parse_command_apdu
 from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
 
 #
@@ -39,6 +40,18 @@ class ApduTracer:
     def trace_response(self, cmd, sw, resp):
         pass
 
+    def trace_reset(self):
+        pass
+
+class StdoutApduTracer(ApduTracer):
+    """Minimalistic APDU tracer, printing commands to stdout."""
+    def trace_response(self, cmd, sw, resp):
+        print("-> %s %s" % (cmd[:10], cmd[10:]))
+        print("<- %s: %s" % (sw, resp))
+
+    def trace_reset(self):
+        print("-- RESET")
+
 class ProactiveHandler(abc.ABC):
     """Abstract base class representing the interface of some code that handles
     the proactive commands, as returned by the card in responses to the FETCH
@@ -56,7 +69,18 @@ class ProactiveHandler(abc.ABC):
         """Default handler for not otherwise handled proactive commands."""
         raise NotImplementedError('No handler method for %s' % pcmd.decoded)
 
-
+    def prepare_response(self, pcmd: ProactiveCommand, general_result: str = 'performed_successfully'):
+        # The Command Details are echoed from the command that has been processed.
+        (command_details,) = [c for c in pcmd.children if isinstance(c, CommandDetails)]
+        # invert the device identities
+        (command_dev_ids,) = [c for c in pcmd.children if isinstance(c, DeviceIdentities)]
+        rsp_dev_ids = DeviceIdentities()
+        rsp_dev_ids.from_dict({'device_identities': {
+                                    'dest_dev_id': command_dev_ids.decoded['source_dev_id'],
+                                    'source_dev_id': command_dev_ids.decoded['dest_dev_id']}})
+        result = Result()
+        result.from_dict({'result': {'general_result': general_result, 'additional_information': ''}})
+        return [command_details, rsp_dev_ids, result]
 
 class LinkBase(abc.ABC):
     """Base class for link/transport to card."""
@@ -66,14 +90,16 @@ class LinkBase(abc.ABC):
         self.sw_interpreter = sw_interpreter
         self.apdu_tracer = apdu_tracer
         self.proactive_handler = proactive_handler
+        self.apdu_strict = False
 
     @abc.abstractmethod
     def __str__(self) -> str:
         """Implementation specific method for printing an information to identify the device."""
 
     @abc.abstractmethod
-    def _send_apdu_raw(self, pdu: Hexstr) -> ResTuple:
-        """Implementation specific method for sending the PDU."""
+    def _send_apdu(self, apdu: Hexstr) -> ResTuple:
+        """Implementation specific method for sending the APDU. This method must accept APDUs as defined in
+        ISO/IEC 7816-3, section 12.1 """
 
     def set_sw_interpreter(self, interp):
         """Set an (optional) status word interpreter."""
@@ -99,62 +125,62 @@ class LinkBase(abc.ABC):
         """
 
     @abc.abstractmethod
-    def reset_card(self):
+    def _reset_card(self):
         """Resets the card (power down/up)
         """
 
-    def send_apdu_raw(self, pdu: Hexstr) -> ResTuple:
+    def reset_card(self):
+        """Resets the card (power down/up)
+        """
+        if self.apdu_tracer:
+            self.apdu_tracer.trace_reset()
+        return self._reset_card()
+
+    def send_apdu(self, apdu: Hexstr) -> ResTuple:
         """Sends an APDU with minimal processing
 
         Args:
-           pdu : string of hexadecimal characters (ex. "A0A40000023F00")
+           apdu : string of hexadecimal characters (ex. "A0A40000023F00", must comply to ISO/IEC 7816-3, section 12.1)
         Returns:
            tuple(data, sw), where
                         data : string (in hex) of returned data (ex. "074F4EFFFF")
                         sw   : string (in hex) of status word (ex. "9000")
         """
+
+        # To make sure that no invalid APDUs can be passed further down into the transport layer, we parse the APDU.
+        (case, _lc, _le, _data) = parse_command_apdu(h2b(apdu))
+
         if self.apdu_tracer:
-            self.apdu_tracer.trace_command(pdu)
-        (data, sw) = self._send_apdu_raw(pdu)
+            self.apdu_tracer.trace_command(apdu)
+
+        # Handover APDU to concrete transport layer implementation
+        (data, sw) = self._send_apdu(apdu)
+
         if self.apdu_tracer:
-            self.apdu_tracer.trace_response(pdu, sw, data)
+            self.apdu_tracer.trace_response(apdu, sw, data)
+
+        # The APDU case (See aso ISO/IEC 7816-3, table 12) dictates if we should receive a response or not. If we
+        # receive a response in an APDU case that does not allow the reception of a respnse we print a warning to
+        # make the user/caller aware of the problem. Since the transaction is over at this point and data was received
+        # we count it as a successful transaction anyway, even though the spec was violated. The problem is most likely
+        # caused by a missing Le field in the APDU. This is an error that the caller/user should correct to avoid
+        # problems at some later point when a different transport protocol or transport layer implementation is used.
+        # All APDUs passed to this function must comply to ISO/IEC 7816-3, section 12.
+        if len(data) > 0 and (case == 3 or case == 1):
+            exeption_str = 'received unexpected response data, incorrect APDU-case ' + \
+            '(%d, should be %d, missing Le field?)!' % (case, case + 1)
+            if self.apdu_strict:
+                raise ValueError(exeption_str)
+            else:
+                print('Warning: %s' % exeption_str)
+
         return (data, sw)
 
-    def send_apdu(self, pdu: Hexstr) -> ResTuple:
-        """Sends an APDU and auto fetch response data
-
-        Args:
-           pdu : string of hexadecimal characters (ex. "A0A40000023F00")
-        Returns:
-           tuple(data, sw), where
-                        data : string (in hex) of returned data (ex. "074F4EFFFF")
-                        sw   : string (in hex) of status word (ex. "9000")
-        """
-        data, sw = self.send_apdu_raw(pdu)
-
-        # When we have sent the first APDU, the SW may indicate that there are response bytes
-        # available. There are two SWs commonly used for this 9fxx (sim) and 61xx (usim), where
-        # xx is the number of response bytes available.
-        # See also:
-        if sw is not None:
-            while ((sw[0:2] == '9f') or (sw[0:2] == '61')):
-                # SW1=9F: 3GPP TS 51.011 9.4.1, Responses to commands which are correctly executed
-                # SW1=61: ISO/IEC 7816-4, Table 5 — General meaning of the interindustry values of SW1-SW2
-                pdu_gr = pdu[0:2] + 'c00000' + sw[2:4]
-                d, sw = self.send_apdu_raw(pdu_gr)
-                data += d
-            if sw[0:2] == '6c':
-                # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
-                pdu_gr = pdu[0:8] + sw[2:4]
-                data, sw = self.send_apdu_raw(pdu_gr)
-
-        return data, sw
-
-    def send_apdu_checksw(self, pdu: Hexstr, sw: SwMatchstr = "9000") -> ResTuple:
+    def send_apdu_checksw(self, apdu: Hexstr, sw: SwMatchstr = "9000") -> ResTuple:
         """Sends an APDU and check returned SW
 
         Args:
-           pdu : string of hexadecimal characters (ex. "A0A40000023F00")
+           apdu : string of hexadecimal characters (ex. "A0A40000023F00", must comply to ISO/IEC 7816-3, section 12.1)
            sw : string of 4 hexadecimal characters (ex. "9000"). The user may mask out certain
                         digits using a '?' to add some ambiguity if needed.
         Returns:
@@ -162,7 +188,7 @@ class LinkBase(abc.ABC):
                         data : string (in hex) of returned data (ex. "074F4EFFFF")
                         sw   : string (in hex) of status word (ex. "9000")
         """
-        rv = self.send_apdu(pdu)
+        rv = self.send_apdu(apdu)
         last_sw = rv[1]
 
         while sw == '9000' and sw_match(last_sw, '91xx'):
@@ -181,31 +207,26 @@ class LinkBase(abc.ABC):
             pcmd = ProactiveCommand()
             parsed = pcmd.from_tlv(h2b(fetch_rv[0]))
             print("FETCH: %s (%s)" % (fetch_rv[0], type(parsed).__name__))
-            result = Result()
             if self.proactive_handler:
                 # Extension point: If this does return a list of TLV objects,
                 # they could be appended after the Result; if the first is a
                 # Result, that cuold replace the one built here.
-                self.proactive_handler.receive_fetch_raw(pcmd, parsed)
-                result.from_dict({'general_result': 'performed_successfully', 'additional_information': ''})
+                ti_list = self.proactive_handler.receive_fetch_raw(pcmd, parsed)
+                if not ti_list:
+                    ti_list = self.proactive_handler.prepare_response(pcmd, 'FIXME')
             else:
-                result.from_dict({'general_result': 'command_beyond_terminal_capability', 'additional_information': ''})
+                ti_list = self.proactive_handler.prepare_response(pcmd, 'command_beyond_terminal_capability')
 
             # Send response immediately, thus also flushing out any further
             # proactive commands that the card already wants to send
             #
             # Structure as per TS 102 223 V4.4.0 Section 6.8
 
-            # The Command Details are echoed from the command that has been processed.
-            (command_details,) = [c for c in pcmd.decoded.children if isinstance(c, CommandDetails)]
-            # The Device Identities are fixed. (TS 102 223 V4.0.0 Section 6.8.2)
-            device_identities = DeviceIdentities()
-            device_identities.from_dict({'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'})
-
             # Testing hint: The value of tail does not influence the behavior
             # of an SJA2 that sent ans SMS, so this is implemented only
             # following TS 102 223, and not fully tested.
-            tail = command_details.to_tlv() + device_identities.to_tlv() + result.to_tlv()
+            ti_list_bin = [x.to_tlv() for x in ti_list]
+            tail = b''.join(ti_list_bin)
             # Testing hint: In contrast to the above, this part is positively
             # essential to get the SJA2 to provide the later parts of a
             # multipart SMS in response to an OTA RFM command.
@@ -218,17 +239,104 @@ class LinkBase(abc.ABC):
             raise SwMatchError(rv[1], sw.lower(), self.sw_interpreter)
         return rv
 
+
+class LinkBaseTpdu(LinkBase):
+
+    # Use the T=0 TPDU format by default as this is the most commonly used transport protocol.
+    protocol = 0
+
+    def set_tpdu_format(self, protocol: int):
+        """Set TPDU format. Each transport protocol has its specific TPDU format. This method allows the
+        concrete transport layer implementation to set the TPDU format it expects. (This method must not be
+        called by higher layers. Switching the TPDU format does not switch the transport protocol that the
+        reader uses on the wire)
+
+        Args:
+           protocol : number of the transport protocol used. (0 => T=0, 1 => T=1)
+        """
+        self.protocol = protocol
+
+    @abc.abstractmethod
+    def send_tpdu(self, tpdu: Hexstr) -> ResTuple:
+        """Implementation specific method for sending the resulting TPDU. This method must accept TPDUs as defined in
+        ETSI TS 102 221, section 7.3.1 and 7.3.2, depending on the protocol selected. """
+
+    def _send_apdu(self, apdu: Hexstr) -> ResTuple:
+        """Transforms APDU into a TPDU and sends it. The response TPDU is returned as APDU back to the caller.
+
+        Args:
+           apdu : string of hexadecimal characters (eg. "A0A40000023F00", must comply to ISO/IEC 7816-3, section 12)
+        Returns:
+           tuple(data, sw), where
+                        data : string (in hex) of returned data (ex. "074F4EFFFF")
+                        sw   : string (in hex) of status word (ex. "9000")
+        """
+
+        if self.protocol == 0:
+            return self.__send_apdu_T0(apdu)
+        elif self.protocol == 1:
+            return self.__send_apdu_transparent(apdu)
+        raise ValueError('unspported protocol selected (T=%d)' % self.protocol)
+
+    def __send_apdu_T0(self, apdu: Hexstr) -> ResTuple:
+        # Transform the given APDU to the T=0 TPDU format and send it. Automatically fetch the response (case #4 APDUs)
+        # (see also ETSI TS 102 221, section 7.3.1.1)
+
+        # Transform APDU to T=0 TPDU (see also ETSI TS 102 221, section 7.3.1)
+        (case, _lc, _le, _data) = parse_command_apdu(h2b(apdu))
+
+        if case == 1:
+            # Attach an Le field to all case #1 APDUs (see also ETSI TS 102 221, section 7.3.1.1.1)
+            tpdu = apdu + '00'
+        elif case == 4:
+            # Remove the Le field from all case #4 APDUs (see also ETSI TS 102 221, section 7.3.1.1.4)
+            tpdu = apdu[:-2]
+        else:
+            tpdu = apdu
+
+        prev_tpdu = tpdu
+        data, sw = self.send_tpdu(tpdu)
+
+        # When we have sent the first APDU, the SW may indicate that there are response bytes
+        # available. There are two SWs commonly used for this 9fxx (sim) and 61xx (usim), where
+        # xx is the number of response bytes available.
+        # See also:
+        if sw is not None:
+            while (sw[0:2] in ['9f', '61', '62', '63']):
+                # SW1=9F: 3GPP TS 51.011 9.4.1, Responses to commands which are correctly executed
+                # SW1=61: ISO/IEC 7816-4, Table 5 — General meaning of the interindustry values of SW1-SW2
+                # SW1=62: ETSI TS 102 221 7.3.1.1.4 Clause 4b): 62xx, 63xx, 9xxx != 9000
+                tpdu_gr = tpdu[0:2] + 'c00000' + sw[2:4]
+                prev_tpdu = tpdu_gr
+                d, sw = self.send_tpdu(tpdu_gr)
+                data += d
+            if sw[0:2] == '6c':
+                # SW1=6C: ETSI TS 102 221 Table 7.1: Procedure byte coding
+                tpdu_gr = prev_tpdu[0:8] + sw[2:4]
+                data, sw = self.send_tpdu(tpdu_gr)
+
+        return data, sw
+
+    def __send_apdu_transparent(self, apdu: Hexstr) -> ResTuple:
+        # In cases where the TPDU format is the same as the APDU format, we may pass the given APDU through without modification
+        # (This is the case for T=1, see also  ETSI TS 102 221, section 7.3.2.0.)
+        return self.send_tpdu(apdu)
+
 def argparse_add_reader_args(arg_parser: argparse.ArgumentParser):
     """Add all reader related arguments to the given argparse.Argumentparser instance."""
     from pySim.transport.serial import SerialSimLink
     from pySim.transport.pcsc import PcscSimLink
     from pySim.transport.modem_atcmd import ModemATCommandLink
     from pySim.transport.calypso import CalypsoSimLink
+    from pySim.transport.wsrc import WsrcSimLink
 
     SerialSimLink.argparse_add_reader_args(arg_parser)
     PcscSimLink.argparse_add_reader_args(arg_parser)
     ModemATCommandLink.argparse_add_reader_args(arg_parser)
     CalypsoSimLink.argparse_add_reader_args(arg_parser)
+    WsrcSimLink.argparse_add_reader_args(arg_parser)
+    arg_parser.add_argument('--apdu-trace', action='store_true',
+                            help='Trace the command/response APDUs exchanged with the card')
 
     return arg_parser
 
@@ -237,6 +345,9 @@ def init_reader(opts, **kwargs) -> LinkBase:
     """
     Init card reader driver
     """
+    if opts.apdu_trace and not 'apdu_tracer' in kwargs:
+        kwargs['apdu_tracer'] = StdoutApduTracer()
+
     if opts.pcsc_dev is not None or opts.pcsc_regex is not None:
         from pySim.transport.pcsc import PcscSimLink
         sl = PcscSimLink(opts, **kwargs)
@@ -246,6 +357,9 @@ def init_reader(opts, **kwargs) -> LinkBase:
     elif opts.modem_dev is not None:
         from pySim.transport.modem_atcmd import ModemATCommandLink
         sl = ModemATCommandLink(opts, **kwargs)
+    elif opts.wsrc_server_url is not None:
+        from pySim.transport.wsrc import WsrcSimLink
+        sl = WsrcSimLink(opts, **kwargs)
     else:  # Serial reader is default
         print("No reader/driver specified; falling back to default (Serial reader)")
         from pySim.transport.serial import SerialSimLink

pySim/transport/calypso.py

@@ -22,10 +22,11 @@ import socket
 import os
 import argparse
 from typing import Optional
+from osmocom.utils import h2b, b2h, Hexstr
 
-from pySim.transport import LinkBase
+from pySim.transport import LinkBaseTpdu
 from pySim.exceptions import ReaderError, ProtocolError
-from pySim.utils import h2b, b2h, Hexstr, ResTuple
+from pySim.utils import ResTuple
 
 
 class L1CTLMessage:
@@ -69,12 +70,12 @@ class L1CTLMessageSIM(L1CTLMessage):
     L1CTL_SIM_REQ = 0x16
     L1CTL_SIM_CONF = 0x17
 
-    def __init__(self, pdu):
+    def __init__(self, tpdu):
         super().__init__(self.L1CTL_SIM_REQ)
-        self.data += pdu
+        self.data += tpdu
 
 
-class CalypsoSimLink(LinkBase):
+class CalypsoSimLink(LinkBaseTpdu):
     """Transport Link for Calypso based phones."""
     name = 'Calypso-based (OsmocomBB) reader'
 
@@ -108,7 +109,7 @@ class CalypsoSimLink(LinkBase):
         rsp = self.sock.recv(exp_len)
         return rsp
 
-    def reset_card(self):
+    def _reset_card(self):
         # Request FULL reset
         req_msg = L1CTLMessageReset()
         self.sock.send(req_msg.gen_msg())
@@ -128,10 +129,10 @@ class CalypsoSimLink(LinkBase):
     def wait_for_card(self, timeout: Optional[int] = None, newcardonly: bool = False):
         pass  # Nothing to do really ...
 
-    def _send_apdu_raw(self, pdu: Hexstr) -> ResTuple:
+    def send_tpdu(self, tpdu: Hexstr) -> ResTuple:
 
-        # Request FULL reset
-        req_msg = L1CTLMessageSIM(h2b(pdu))
+        # Request sending of TPDU
+        req_msg = L1CTLMessageSIM(h2b(tpdu))
         self.sock.send(req_msg.gen_msg())
 
         # Read message length first

pySim/transport/modem_atcmd.py

@@ -22,16 +22,17 @@ import re
 import argparse
 from typing import Optional
 import serial
+from osmocom.utils import Hexstr
 
-from pySim.utils import Hexstr, ResTuple
-from pySim.transport import LinkBase
+from pySim.utils import ResTuple
+from pySim.transport import LinkBaseTpdu
 from pySim.exceptions import ReaderError, ProtocolError
 
 # HACK: if somebody needs to debug this thing
 # log.root.setLevel(log.DEBUG)
 
 
-class ModemATCommandLink(LinkBase):
+class ModemATCommandLink(LinkBaseTpdu):
     """Transport Link for 3GPP TS 27.007 compliant modems."""
     name = "modem for Generic SIM Access (3GPP TS 27.007)"
 
@@ -124,7 +125,7 @@ class ModemATCommandLink(LinkBase):
                 return
         raise ReaderError('Interface \'%s\' does not respond to \'AT\' command' % self._device)
 
-    def reset_card(self):
+    def _reset_card(self):
         # Reset the modem, just to be sure
         if self.send_at_cmd('ATZ') != [b'OK']:
             raise ReaderError('Failed to reset the modem')
@@ -144,12 +145,12 @@ class ModemATCommandLink(LinkBase):
     def wait_for_card(self, timeout: Optional[int] = None, newcardonly: bool = False):
         pass  # Nothing to do really ...
 
-    def _send_apdu_raw(self, pdu: Hexstr) -> ResTuple:
+    def send_tpdu(self, tpdu: Hexstr) -> ResTuple:
         # Make sure pdu has upper case hex digits [A-F]
-        pdu = pdu.upper()
+        tpdu = tpdu.upper()
 
         # Prepare the command as described in 8.17
-        cmd = 'AT+CSIM=%d,\"%s\"' % (len(pdu), pdu)
+        cmd = 'AT+CSIM=%d,\"%s\"' % (len(tpdu), tpdu)
         log.debug('Sending command: %s',  cmd)
 
         # Send AT+CSIM command to the modem
@@ -163,14 +164,14 @@ class ModemATCommandLink(LinkBase):
         # Make sure that the response has format: b'+CSIM: %d,\"%s\"'
         try:
             result = re.match(b'\+CSIM: (\d+),\"([0-9A-F]+)\"', rsp)
-            (_rsp_pdu_len, rsp_pdu) = result.groups()
+            (_rsp_tpdu_len, rsp_tpdu) = result.groups()
         except Exception as exc:
             raise ReaderError('Failed to parse response from modem: %s' % rsp) from exc
 
         # TODO: make sure we have at least SW
-        data = rsp_pdu[:-4].decode().lower()
-        sw = rsp_pdu[-4:].decode().lower()
-        log.debug('Command response: %s, %s',  data, sw)
+        data = rsp_tpdu[:-4].decode().lower()
+        sw = rsp_tpdu[-4:].decode().lower()
+        log.debug('Command response: %s, %s', data, sw)
         return data, sw
 
     def __str__(self) -> str:

pySim/transport/pcsc.py

@@ -25,13 +25,16 @@ from smartcard.CardConnection import CardConnection
 from smartcard.CardRequest import CardRequest
 from smartcard.Exceptions import NoCardException, CardRequestTimeoutException, CardConnectionException
 from smartcard.System import readers
+from smartcard.ExclusiveConnectCardConnection import ExclusiveConnectCardConnection
+
+from osmocom.utils import h2i, i2h, Hexstr
 
 from pySim.exceptions import NoCardError, ProtocolError, ReaderError
-from pySim.transport import LinkBase
-from pySim.utils import h2i, i2h, Hexstr, ResTuple
+from pySim.transport import LinkBaseTpdu
+from pySim.utils import ResTuple
 
 
-class PcscSimLink(LinkBase):
+class PcscSimLink(LinkBaseTpdu):
     """ pySim: PCSC reader transport link."""
     name = 'PC/SC'
 
@@ -56,6 +59,8 @@ class PcscSimLink(LinkBase):
                 raise ReaderError('No matching reader found for regex %s' % opts.pcsc_regex)
 
         self._con = self._reader.createConnection()
+        if not getattr(opts, "pcsc_shared", False):
+            self._con = ExclusiveConnectCardConnection(self._con)
 
     def __del__(self):
         try:
@@ -79,8 +84,19 @@ class PcscSimLink(LinkBase):
             # is disconnected
             self.disconnect()
 
-            # Explicitly select T=0 communication protocol
-            self._con.connect(CardConnection.T0_protocol)
+            # Make card connection and select a suitable communication protocol
+            self._con.connect()
+            supported_protocols = self._con.getProtocol();
+            self.disconnect()
+            if (supported_protocols & CardConnection.T0_protocol):
+                protocol =  CardConnection.T0_protocol
+                self.set_tpdu_format(0)
+            elif (supported_protocols & CardConnection.T1_protocol):
+                protocol = CardConnection.T1_protocol
+                self.set_tpdu_format(1)
+            else:
+                raise ReaderError('Unsupported card protocol')
+            self._con.connect(protocol)
         except CardConnectionException as exc:
             raise ProtocolError() from exc
         except NoCardException as exc:
@@ -92,17 +108,13 @@ class PcscSimLink(LinkBase):
     def disconnect(self):
         self._con.disconnect()
 
-    def reset_card(self):
+    def _reset_card(self):
         self.disconnect()
         self.connect()
         return 1
 
-    def _send_apdu_raw(self, pdu: Hexstr) -> ResTuple:
-
-        apdu = h2i(pdu)
-
-        data, sw1, sw2 = self._con.transmit(apdu)
-
+    def send_tpdu(self, tpdu: Hexstr) -> ResTuple:
+        data, sw1, sw2 = self._con.transmit(h2i(tpdu))
         sw = [sw1, sw2]
 
         # Return value
@@ -119,6 +131,8 @@ access smart card readers, and is available on a variety of operating systems, s
 Windows, MacOS X and Linux.  Most vendors of smart card readers provide drivers that offer a PC/SC
 interface, if not even a generic USB CCID driver is used.  You can use a tool like ``pcsc_scan -r``
 to obtain a list of readers available on your system. """)
+        pcsc_group.add_argument('--pcsc-shared', action='store_true',
+                                help='Open PC/SC reaer in SHARED access (default: EXCLUSIVE)')
         dev_group = pcsc_group.add_mutually_exclusive_group()
         dev_group.add_argument('-p', '--pcsc-device', type=int, dest='pcsc_dev', metavar='PCSC', default=None,
                                help='Number of PC/SC reader to use for SIM access')

pySim/transport/serial.py

@@ -21,13 +21,14 @@ import os
 import argparse
 from typing import Optional
 import serial
+from osmocom.utils import h2b, b2h, Hexstr
 
 from pySim.exceptions import NoCardError, ProtocolError
-from pySim.transport import LinkBase
-from pySim.utils import h2b, b2h, Hexstr, ResTuple
+from pySim.transport import LinkBaseTpdu
+from pySim.utils import ResTuple
 
 
-class SerialSimLink(LinkBase):
+class SerialSimLink(LinkBaseTpdu):
     """ pySim: Transport Link for serial (RS232) based readers included with simcard"""
     name = 'Serial'
 
@@ -100,15 +101,15 @@ class SerialSimLink(LinkBase):
     def disconnect(self):
         pass  # Nothing to do really ...
 
-    def reset_card(self):
-        rv = self._reset_card()
+    def _reset_card(self):
+        rv = self.__reset_card()
         if rv == 0:
             raise NoCardError()
         if rv < 0:
             raise ProtocolError()
         return rv
 
-    def _reset_card(self):
+    def __reset_card(self):
         self._atr = None
         rst_meth_map = {
             'rts': self._sl.setRTS,
@@ -186,13 +187,13 @@ class SerialSimLink(LinkBase):
     def _rx_byte(self):
         return self._sl.read()
 
-    def _send_apdu_raw(self, pdu: Hexstr) -> ResTuple:
+    def send_tpdu(self, tpdu: Hexstr) -> ResTuple:
 
-        pdu = h2b(pdu)
-        data_len = pdu[4]  # P3
+        tpdu = h2b(tpdu)
+        data_len = tpdu[4]  # P3
 
         # Send first CLASS,INS,P1,P2,P3
-        self._tx_string(pdu[0:5])
+        self._tx_string(tpdu[0:5])
 
         # Wait ack which can be
         #  - INS: Command acked -> go ahead
@@ -200,7 +201,7 @@ class SerialSimLink(LinkBase):
         #  - SW1: The card can apparently proceed ...
         while True:
             b = self._rx_byte()
-            if ord(b) == pdu[1]:
+            if ord(b) == tpdu[1]:
                 break
             if b != '\x60':
                 # Ok, it 'could' be SW1
@@ -213,12 +214,12 @@ class SerialSimLink(LinkBase):
                 raise ProtocolError()
 
         # Send data (if any)
-        if len(pdu) > 5:
-            self._tx_string(pdu[5:])
+        if len(tpdu) > 5:
+            self._tx_string(tpdu[5:])
 
         # Receive data (including SW !)
-        #  length = [P3 - tx_data (=len(pdu)-len(hdr)) + 2 (SW1//2) ]
-        to_recv = data_len - len(pdu) + 5 + 2
+        #  length = [P3 - tx_data (=len(tpdu)-len(hdr)) + 2 (SW1//2) ]
+        to_recv = data_len - len(tpdu) + 5 + 2
 
         data = bytes(0)
         while len(data) < to_recv:

pySim/transport/wsrc.py

@@ -0,0 +1,99 @@
+# Copyright (C) 2024  Harald Welte <laforge@gnumonks.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import argparse
+from typing import Optional
+
+from osmocom.utils import h2i, i2h, Hexstr, is_hexstr
+
+from pySim.exceptions import NoCardError, ProtocolError, ReaderError
+from pySim.transport import LinkBase
+from pySim.utils import ResTuple
+from pySim.wsrc import WSRC_DEFAULT_PORT_USER
+from pySim.wsrc.client_blocking import WsClientBlocking
+
+class UserWsClientBlocking(WsClientBlocking):
+    def __init__(self, ws_uri: str, **kwargs):
+        super().__init__('user', ws_uri, **kwargs)
+
+    def select_card(self, id_type:str, id_str:str):
+        rx = self.transceive_json('select_card', {'id_type': id_type, 'id_str': id_str},
+                                  'select_card_ack')
+        return rx
+
+    def reset_card(self):
+        self.transceive_json('reset_req', {}, 'reset_resp')
+
+    def xceive_apdu_raw(self, cmd: Hexstr) -> ResTuple:
+        rx = self.transceive_json('c_apdu', {'command': cmd}, 'r_apdu')
+        return rx['response'], rx['sw']
+
+
+class WsrcSimLink(LinkBase):
+    """ pySim: WSRC (WebSocket Remote Card) reader transport link."""
+    name = 'WSRC'
+
+    def __init__(self, opts: argparse.Namespace, **kwargs):
+        super().__init__(**kwargs)
+        self.identities = {}
+        self.server_url = opts.wsrc_server_url
+        if opts.wsrc_eid:
+            self.id_type = 'eid'
+            self.id_str = opts.wsrc_eid
+        elif opts.wsrc_iccid:
+            self.id_type = 'iccid'
+            self.id_str = opts.wsrc_iccid
+        self.client = UserWsClientBlocking(self.server_url)
+        self.client.connect()
+
+    def __del__(self):
+       # FIXME: disconnect from server
+       pass
+
+    def wait_for_card(self, timeout: Optional[int] = None, newcardonly: bool = False):
+        self.connect()
+
+    def connect(self):
+        rx = self.client.select_card(self.id_type, self.id_str)
+        self.identities = rx['identities']
+
+    def get_atr(self) -> Hexstr:
+        return h2i(self.identities['ATR'])
+
+    def disconnect(self):
+        self.__delete__()
+
+    def _reset_card(self):
+        self.client.reset_card()
+        return 1
+
+    def _send_apdu_raw(self, pdu: Hexstr) -> ResTuple:
+        return self.client.xceive_apdu_raw(pdu)
+
+    def __str__(self) -> str:
+        return "WSRC[%s=%s]" % (self.id_type, self.id_str)
+
+    @staticmethod
+    def argparse_add_reader_args(arg_parser: argparse.ArgumentParser):
+        wsrc_group = arg_parser.add_argument_group('WebSocket Remote Card',
+            """WebSocket Remote Card (WSRC) is a protoocl by which remot cards / card readers
+            can be accessed via a network.""")
+        wsrc_group.add_argument('--wsrc-server-url', default='ws://localhost:%u' % WSRC_DEFAULT_PORT_USER,
+                                help='URI of the WSRC server to connect to')
+        wsrc_group.add_argument('--wsrc-iccid', type=is_hexstr,
+                                help='ICCID of the card to open via WSRC')
+        wsrc_group.add_argument('--wsrc-eid', type=is_hexstr,
+                                help='EID of the card to open via WSRC')

pySim/ts_102_221.py

@@ -18,22 +18,16 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 from bidict import bidict
 
-from construct import Select, Const, Bit, Struct, Int16ub, FlagsEnum, GreedyString, ValidationError
-from construct import Optional as COptional
+from construct import Select, Const, Bit, Struct, Int16ub, FlagsEnum, ValidationError
+from construct import Optional as COptional, Computed
 
-from pySim.construct import *
+from osmocom.construct import *
+from osmocom.utils import *
+from osmocom.tlv import BER_TLV_IE, flatten_dict_lists
 from pySim.utils import *
-from pySim.filesystem import *
-from pySim.tlv import *
-from pySim.profile import CardProfile
-from pySim.profile import match_uicc
-from pySim import iso7816_4
 
-# A UICC will usually also support 2G functionality. If this is the case, we
-# need to add DF_GSM and DF_TELECOM along with the UICC related files
-from pySim.ts_51_011 import AddonSIM
-from pySim.gsm_r import AddonGSMR
-from pySim.cdma_ruim import AddonRUIM
+#from pySim.filesystem import *
+#from pySim.profile import CardProfile
 
 ts_102_22x_cmdset = CardCommandSet('TS 102 22x', [
     # TS 102 221 Section 10.1.2 Table 10.5 "Coding of Instruction Byte"
@@ -94,27 +88,23 @@ class TotalFileSize(BER_TLV_IE, tag=0x81):
 
 # ETSI TS 102 221 11.1.1.4.3
 class FileDescriptor(BER_TLV_IE, tag=0x82):
-    _test_decode = [
-        # FIXME: this doesn't work as _encode test for some strange reason.
-        ( '82027921', { "file_descriptor_byte": { "shareable": True, "structure": "ber_tlv" }, "record_len": None, "num_of_rec": None } ),
-    ]
     _test_de_encode = [
+        ( '82027921', { "file_descriptor_byte": { "shareable": True, "file_type": "working_ef", "structure": "ber_tlv" }, "record_len": None, "num_of_rec": None } ),
         ( '82027821', { "file_descriptor_byte": { "shareable": True, "file_type": "df", "structure": "no_info_given" }, "record_len": None, "num_of_rec": None }),
         ( '82024121', { "file_descriptor_byte": { "shareable": True, "file_type": "working_ef", "structure": "transparent" }, "record_len": None, "num_of_rec": None } ),
         ( '82054221006e05', { "file_descriptor_byte": { "shareable": True, "file_type": "working_ef", "structure": "linear_fixed" }, "record_len": 110, "num_of_rec": 5 } ),
     ]
     class BerTlvAdapter(Adapter):
-        def _parse(self, obj, context, path):
-            data = obj.read()
-            if data == b'\x01\x01\x01\x00\x00\x01':
+        def _decode(self, obj, context, path):
+            if obj == 0x39:
                 return 'ber_tlv'
             raise ValidationError
-        def _build(self, obj, context, path):
+        def _encode(self, obj, context, path):
             if obj == 'ber_tlv':
-                return b'\x01\x01\x01\x00\x00\x01'
+                return 0x39
             raise ValidationError
 
-    FDB = Select(BitStruct(Const(0, Bit), 'shareable'/Flag, 'structure'/BerTlvAdapter(Const(0x39, BitsInteger(6)))),
+    FDB = Select(BitStruct(Const(0, Bit), 'shareable'/Flag, 'structure'/BerTlvAdapter(Const(0x39, BitsInteger(6))), 'file_type'/Computed('working_ef')),
                  BitStruct(Const(0, Bit), 'shareable'/Flag, 'file_type'/Enum(BitsInteger(3), working_ef=0, internal_ef=1, df=7),
                            'structure'/Enum(BitsInteger(3), no_info_given=0, transparent=1, linear_fixed=2, cyclic=6))
                 )
@@ -173,17 +163,40 @@ class SpecificUiccEnvironmentConditions(BER_TLV_IE, tag=0x88):
 class Platform2PlatformCatSecuredApdu(BER_TLV_IE, tag=0x89):
     _construct = GreedyBytes
 
+# TS 102 222 Table 4a + 5
+class SpecialFileInfo(BER_TLV_IE, tag=0xC0):
+    _construct = FlagsEnum(Byte, high_update_activity=0x80, readable_and_updatable_when_deactivated=0x40)
+
+# TS 102 222 Table 4a
+class FillingPattern(BER_TLV_IE, tag=0xC1):
+    # The first W-1 bytes of the transparent EF or the first W-1 bytes of each record of a record
+    # oriented EF shall be initialized with the first W-1 bytes of the Filling Pattern. All remaining
+    # bytes (if any) shall be initialized with the value of the last byte of the Filling Pattern. If
+    # the file or record length is shorter than the Filling Pattern, the Filling Pattern shall be
+    # truncated accordingly.
+    _construct = GreedyBytes
+
+# TS 102 222 Table 4a
+class RepeatPattern(BER_TLV_IE, tag=0xC2):
+    # The first X bytes of the transparent EF or the first X bytes of each record of a record oriented
+    # EF shall be initialized with the X bytes of the Repeat Pattern. This shall be repeated
+    # consecutively for all remaining blocks of X bytes of data in the file or in a record. If
+    # necessary, the Repeat Pattern shall be truncated at the end of the file or at the end of each
+    # record to initialize the remaining bytes.
+    _construct = GreedyBytes
+
 # sysmoISIM-SJA2 specific
 class ToolkitAccessConditions(BER_TLV_IE, tag=0xD2):
     _construct = FlagsEnum(Byte, rfm_create=1, rfm_delete_terminate=2, other_applet_create=4,
                            other_applet_delete_terminate=8)
 
-# ETSI TS 102 221 11.1.1.4.6.0
+# ETSI TS 102 221 11.1.1.4.6.0 + TS 102 222 Table 4A
 class ProprietaryInformation(BER_TLV_IE, tag=0xA5,
                              nested=[UiccCharacteristics, ApplicationPowerConsumption,
                                      MinApplicationClockFrequency, AvailableMemory,
                                      FileDetails, ReservedFileSize, MaximumFileSize,
                                      SupportedFilesystemCommands, SpecificUiccEnvironmentConditions,
+                                     SpecialFileInfo, FillingPattern, RepeatPattern,
                                      ToolkitAccessConditions]):
     pass
 
@@ -269,6 +282,12 @@ class FcpTemplate(BER_TLV_IE, tag=0x62, nested=[FileSize, TotalFileSize, FileDes
                                                 PinStatusTemplate_DO]):
     pass
 
+def decode_select_response(data_hex: str) -> object:
+    """ETSI TS 102 221 Section 11.1.1.3"""
+    t = FcpTemplate()
+    t.from_tlv(h2b(data_hex))
+    d = t.to_dict()
+    return flatten_dict_lists(d['fcp_template'])
 
 def tlv_key_replace(inmap, indata):
     def newkey(inmap, key):
@@ -615,398 +634,3 @@ NOT_DO = Nested_DO('not', 0xaf, NOT_Template)
 SC_DO = DataObjectChoice('security_condition', 'Security Condition',
                          members=[Always_DO, Never_DO, SecCondByte_DO(), SecCondByte_DO(0x9e), CRT_DO(),
                                   OR_DO, AND_DO, NOT_DO])
-
-# TS 102 221 Section 13.1
-class EF_DIR(LinFixedEF):
-    # FIXME: re-encode failure when changing to _test_de_encode
-    _test_decode = [
-        ( '61294f10a0000000871002ffffffff890709000050055553696d31730ea00c80011781025f608203454150',
-          { "application_template": [ { "application_id": h2b("a0000000871002ffffffff8907090000") },
-                                      { "application_label": "USim1" },
-                                      { "discretionary_template": h2b("a00c80011781025f608203454150") } ] }
-        ),
-        ( '61194f10a0000000871004ffffffff890709000050054953696d31',
-          { "application_template": [ { "application_id": h2b("a0000000871004ffffffff8907090000") },
-                                      { "application_label": "ISim1" } ] }
-        ),
-    ]
-    class ApplicationLabel(BER_TLV_IE, tag=0x50):
-        # TODO: UCS-2 coding option as per Annex A of TS 102 221
-        _construct = GreedyString('ascii')
-
-    # see https://github.com/PyCQA/pylint/issues/5794
-    #pylint: disable=undefined-variable
-    class ApplicationTemplate(BER_TLV_IE, tag=0x61,
-                              nested=[iso7816_4.ApplicationId, ApplicationLabel, iso7816_4.FileReference,
-                                      iso7816_4.CommandApdu, iso7816_4.DiscretionaryData,
-                                      iso7816_4.DiscretionaryTemplate, iso7816_4.URL,
-                                      iso7816_4.ApplicationRelatedDOSet]):
-        pass
-
-    def __init__(self, fid='2f00', sfid=0x1e, name='EF.DIR', desc='Application Directory'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(5, 54))
-        self._tlv = EF_DIR.ApplicationTemplate
-
-# TS 102 221 Section 13.2
-class EF_ICCID(TransparentEF):
-    _test_de_encode = [
-        ( '988812010000400310f0', { "iccid": "8988211000000430010" } ),
-    ]
-    def __init__(self, fid='2fe2', sfid=0x02, name='EF.ICCID', desc='ICC Identification'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(10, 10))
-
-    def _decode_hex(self, raw_hex):
-        return {'iccid': dec_iccid(raw_hex)}
-
-    def _encode_hex(self, abstract):
-        return enc_iccid(abstract['iccid'])
-
-# TS 102 221 Section 13.3
-class EF_PL(TransRecEF):
-    _test_de_encode = [
-        ( '6465', "de" ),
-        ( '656e', "en" ),
-        ( 'ffff', None ),
-    ]
-
-    def __init__(self, fid='2f05', sfid=0x05, name='EF.PL', desc='Preferred Languages'):
-        super().__init__(fid, sfid=sfid, name=name,
-                         desc=desc, rec_len=2, size=(2, None))
-
-    def _decode_record_bin(self, bin_data, **kwargs):
-        if bin_data == b'\xff\xff':
-            return None
-        else:
-            return bin_data.decode('ascii')
-
-    def _encode_record_bin(self, in_json, **kwargs):
-        if in_json is None:
-            return b'\xff\xff'
-        else:
-            return in_json.encode('ascii')
-
-
-# TS 102 221 Section 13.4
-class EF_ARR(LinFixedEF):
-    _test_de_encode = [
-        ( '800101a40683010a950108800106900080016097008401d4a40683010a950108',
-         [ [ { "access_mode": [ "read_search_compare" ] },
-             { "control_reference_template": "ADM1" } ],
-           [ { "access_mode": [ "write_append", "update_erase" ] },
-             { "always": None } ],
-           [ { "access_mode": [ "delete_file", "terminate_ef" ] },
-             { "never": None } ],
-           [ { "command_header": { "INS": 212 } },
-             { "control_reference_template": "ADM1" } ]
-         ] ),
-        ( '80010190008001029700800118a40683010a9501088401d4a40683010a950108',
-         [ [ { "access_mode": [ "read_search_compare" ] },
-             { "always": None } ],
-           [ { "access_mode": [ "update_erase" ] },
-             { "never": None } ],
-           [ { "access_mode": [ "activate_file_or_record", "deactivate_file_or_record" ] },
-             { "control_reference_template": "ADM1" } ],
-           [ { "command_header": { "INS": 212 } },
-             { "control_reference_template": "ADM1" } ]
-         ] ),
-    ]
-    def __init__(self, fid='2f06', sfid=0x06, name='EF.ARR', desc='Access Rule Reference'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc)
-        # add those commands to the general commands of a TransparentEF
-        self.shell_commands += [self.AddlShellCommands()]
-
-    @staticmethod
-    def flatten(inp: list):
-        """Flatten the somewhat deep/complex/nested data returned from decoder."""
-        def sc_abbreviate(sc):
-            if 'always' in sc:
-                return 'always'
-            elif 'never' in sc:
-                return 'never'
-            elif 'control_reference_template' in sc:
-                return sc['control_reference_template']
-            else:
-                return sc
-
-        by_mode = {}
-        for t in inp:
-            am = t[0]
-            sc = t[1]
-            sc_abbr = sc_abbreviate(sc)
-            if 'access_mode' in am:
-                for m in am['access_mode']:
-                    by_mode[m] = sc_abbr
-            elif 'command_header' in am:
-                ins = am['command_header']['INS']
-                if 'CLA' in am['command_header']:
-                    cla = am['command_header']['CLA']
-                else:
-                    cla = None
-                cmd = ts_102_22x_cmdset.lookup(ins, cla)
-                if cmd:
-                    name = cmd.name.lower().replace(' ', '_')
-                    by_mode[name] = sc_abbr
-                else:
-                    raise ValueError
-            else:
-                raise ValueError
-        return by_mode
-
-    def _decode_record_bin(self, raw_bin_data, **_kwargs):
-        # we can only guess if we should decode for EF or DF here :(
-        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
-        dec = arr_seq.decode_multi(raw_bin_data)
-        # we cannot pass the result through flatten() here, as we don't have a related
-        # 'un-flattening' decoder, and hence would be unable to encode :(
-        return dec[0]
-
-    def _encode_record_bin(self, in_json, **_kwargs):
-        # we can only guess if we should decode for EF or DF here :(
-        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
-        return arr_seq.encode_multi(in_json)
-
-    @with_default_category('File-Specific Commands')
-    class AddlShellCommands(CommandSet):
-        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
-        def do_read_arr_record(self, opts):
-            """Read one EF.ARR record in flattened, human-friendly form."""
-            (data, _sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
-            data = self._cmd.lchan.selected_file.flatten(data)
-            self._cmd.poutput_json(data, opts.oneline)
-
-        @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
-        def do_read_arr_records(self, opts):
-            """Read + decode all EF.ARR records in flattened, human-friendly form."""
-            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
-            # collect all results in list so they are rendered as JSON list when printing
-            data_list = []
-            for recnr in range(1, 1 + num_of_rec):
-                (data, _sw) = self._cmd.lchan.read_record_dec(recnr)
-                data = self._cmd.lchan.selected_file.flatten(data)
-                data_list.append(data)
-            self._cmd.poutput_json(data_list, opts.oneline)
-
-
-# TS 102 221 Section 13.6
-class EF_UMPC(TransparentEF):
-    _test_de_encode = [
-        ( '3cff02', { "max_current_mA": 60, "t_op_s": 255,
-                      "addl_info": { "req_inc_idle_current": False, "support_uicc_suspend": True } } ),
-        ( '320500', { "max_current_mA": 50, "t_op_s": 5, "addl_info": {"req_inc_idle_current": False,
-                                                                       "support_uicc_suspend": False } } ),
-    ]
-    def __init__(self, fid='2f08', sfid=0x08, name='EF.UMPC', desc='UICC Maximum Power Consumption'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(5, 5))
-        addl_info = FlagsEnum(Byte, req_inc_idle_current=1,
-                              support_uicc_suspend=2)
-        self._construct = Struct(
-            'max_current_mA'/Int8ub, 't_op_s'/Int8ub, 'addl_info'/addl_info)
-
-
-class CardProfileUICC(CardProfile):
-
-    ORDER = 1
-
-    def __init__(self, name='UICC'):
-        files = [
-            EF_DIR(),
-            EF_ICCID(),
-            EF_PL(),
-            EF_ARR(),
-            # FIXME: DF.CD
-            EF_UMPC(),
-        ]
-        addons = [
-            AddonSIM,
-            AddonGSMR,
-            AddonRUIM,
-        ]
-        sw = {
-            'Normal': {
-                '9000': 'Normal ending of the command',
-                '91xx': 'Normal ending of the command, with extra information from the proactive UICC containing a command for the terminal',
-                '92xx': 'Normal ending of the command, with extra information concerning an ongoing data transfer session',
-            },
-            'Postponed processing': {
-                '9300': 'SIM Application Toolkit is busy. Command cannot be executed at present, further normal commands are allowed',
-            },
-            'Warnings': {
-                '6200': 'No information given, state of non-volatile memory unchanged',
-                '6281': 'Part of returned data may be corrupted',
-                '6282': 'End of file/record reached before reading Le bytes or unsuccessful search',
-                '6283': 'Selected file invalidated/disabled; needs to be activated before use',
-                '6284': 'Selected file in termination state',
-                '62f1': 'More data available',
-                '62f2': 'More data available and proactive command pending',
-                '62f3': 'Response data available',
-                '63f1': 'More data expected',
-                '63f2': 'More data expected and proactive command pending',
-                '63cx': 'Command successful but after using an internal update retry routine X times',
-            },
-            'Execution errors': {
-                '6400': 'No information given, state of non-volatile memory unchanged',
-                '6500': 'No information given, state of non-volatile memory changed',
-                '6581': 'Memory problem',
-            },
-            'Checking errors': {
-                '6700': 'Wrong length',
-                '67xx': 'The interpretation of this status word is command dependent',
-                '6b00': 'Wrong parameter(s) P1-P2',
-                '6d00': 'Instruction code not supported or invalid',
-                '6e00': 'Class not supported',
-                '6f00': 'Technical problem, no precise diagnosis',
-                '6fxx': 'The interpretation of this status word is command dependent',
-            },
-            'Functions in CLA not supported': {
-                '6800': 'No information given',
-                '6881': 'Logical channel not supported',
-                '6882': 'Secure messaging not supported',
-            },
-            'Command not allowed': {
-                '6900': 'No information given',
-                '6981': 'Command incompatible with file structure',
-                '6982': 'Security status not satisfied',
-                '6983': 'Authentication/PIN method blocked',
-                '6984': 'Referenced data invalidated',
-                '6985': 'Conditions of use not satisfied',
-                '6986': 'Command not allowed (no EF selected)',
-                '6989': 'Command not allowed - secure channel - security not satisfied',
-            },
-            'Wrong parameters': {
-                '6a80': 'Incorrect parameters in the data field',
-                '6a81': 'Function not supported',
-                '6a82': 'File not found',
-                '6a83': 'Record not found',
-                '6a84': 'Not enough memory space',
-                '6a86': 'Incorrect parameters P1 to P2',
-                '6a87': 'Lc inconsistent with P1 to P2',
-                '6a88': 'Referenced data not found',
-            },
-            'Application errors': {
-                '9850': 'INCREASE cannot be performed, max value reached',
-                '9862': 'Authentication error, application specific',
-                '9863': 'Security session or association expired',
-                '9864': 'Minimum UICC suspension time is too long',
-            },
-        }
-
-        super().__init__(name, desc='ETSI TS 102 221', cla="00",
-                         sel_ctrl="0004", files_in_mf=files, sw=sw,
-                         shell_cmdsets = [self.AddlShellCommands()], addons = addons)
-
-    @staticmethod
-    def decode_select_response(data_hex: str) -> object:
-        """ETSI TS 102 221 Section 11.1.1.3"""
-        t = FcpTemplate()
-        t.from_tlv(h2b(data_hex))
-        d = t.to_dict()
-        return flatten_dict_lists(d['fcp_template'])
-
-    @staticmethod
-    def match_with_card(scc: SimCardCommands) -> bool:
-        return match_uicc(scc)
-
-    @with_default_category('TS 102 221 Specific Commands')
-    class AddlShellCommands(CommandSet):
-        suspend_uicc_parser = argparse.ArgumentParser()
-        suspend_uicc_parser.add_argument('--min-duration-secs', type=int, default=60,
-                                         help='Proposed minimum duration of suspension')
-        suspend_uicc_parser.add_argument('--max-duration-secs', type=int, default=24*60*60,
-                                         help='Proposed maximum duration of suspension')
-
-        # not ISO7816-4 but TS 102 221
-        @cmd2.with_argparser(suspend_uicc_parser)
-        def do_suspend_uicc(self, opts):
-            """Perform the SUSPEND UICC command. Only supported on some UICC (check EF.UMPC)."""
-            (duration, token, sw) = self._cmd.card._scc.suspend_uicc(min_len_secs=opts.min_duration_secs,
-                                                                     max_len_secs=opts.max_duration_secs)
-            self._cmd.poutput(
-                'Negotiated Duration: %u secs, Token: %s, SW: %s' % (duration, token, sw))
-
-        resume_uicc_parser = argparse.ArgumentParser()
-        resume_uicc_parser.add_argument('token', type=str, help='Token provided during SUSPEND')
-
-        @cmd2.with_argparser(resume_uicc_parser)
-        def do_resume_uicc(self, opts):
-            """Perform the REUSME UICC operation. Only supported on some UICC. Also: A power-cycle
-            of the card is required between SUSPEND and RESUME, and only very few non-RESUME
-            commands are permitted between SUSPEND and RESUME.  See TS 102 221 Section 11.1.22."""
-            self._cmd.card._scc.resume_uicc(opts.token)
-
-        term_cap_parser = argparse.ArgumentParser()
-        # power group
-        tc_power_grp = term_cap_parser.add_argument_group('Terminal Power Supply')
-        tc_power_grp.add_argument('--used-supply-voltage-class', type=str, choices=['a','b','c','d','e'],
-                                  help='Actual used Supply voltage class')
-        tc_power_grp.add_argument('--maximum-available-power-supply', type=auto_uint8,
-                                  help='Maximum available power supply of the terminal')
-        tc_power_grp.add_argument('--actual-used-freq-100k', type=auto_uint8,
-                                  help='Actual used clock frequency (in units of 100kHz)')
-        # no separate groups for those two
-        tc_elc_grp = term_cap_parser.add_argument_group('Extended logical channels terminal support')
-        tc_elc_grp.add_argument('--extended-logical-channel', action='store_true',
-                                help='Extended Logical Channel supported')
-        tc_aif_grp = term_cap_parser.add_argument_group('Additional interfaces support')
-        tc_aif_grp.add_argument('--uicc-clf', action='store_true',
-                                help='Local User Interface in the Device (LUId) supported')
-        # eUICC group
-        tc_euicc_grp = term_cap_parser.add_argument_group('Additional Terminal capability indications related to eUICC')
-        tc_euicc_grp.add_argument('--lui-d', action='store_true',
-                                  help='Local User Interface in the Device (LUId) supported')
-        tc_euicc_grp.add_argument('--lpd-d', action='store_true',
-                                  help='Local Profile Download in the Device (LPDd) supported')
-        tc_euicc_grp.add_argument('--lds-d', action='store_true',
-                                  help='Local Discovery Service in the Device (LPDd) supported')
-        tc_euicc_grp.add_argument('--lui-e-scws', action='store_true',
-                                  help='LUIe based on SCWS supported')
-        tc_euicc_grp.add_argument('--metadata-update-alerting', action='store_true',
-                                  help='Metadata update alerting supported')
-        tc_euicc_grp.add_argument('--enterprise-capable-device', action='store_true',
-                                  help='Enterprise Capable Device')
-        tc_euicc_grp.add_argument('--lui-e-e4e', action='store_true',
-                                  help='LUIe using E4E (ENVELOPE tag E4) supported')
-        tc_euicc_grp.add_argument('--lpr', action='store_true',
-                                  help='LPR (LPA Proxy) supported')
-
-        @cmd2.with_argparser(term_cap_parser)
-        def do_terminal_capability(self, opts):
-            """Perform the TERMINAL CAPABILITY function. Used to inform the UICC about terminal capability."""
-            ps_flags = {}
-            addl_if_flags = {}
-            euicc_flags = {}
-
-            opts_dict = vars(opts)
-
-            power_items = ['used_supply_voltage_class', 'maximum_available_power_supply', 'actual_used_freq_100k']
-            if any(opts_dict[x] for x in power_items):
-                if not all(opts_dict[x] for x in power_items):
-                    raise argparse.ArgumentTypeError('If any of the Terminal Power Supply group options are used, all must be specified')
-
-            for k, v in opts_dict.items():
-                if k in AdditionalInterfacesSupport._construct.flags.keys():
-                    addl_if_flags[k] = v
-                elif k in AdditionalTermCapEuicc._construct.flags.keys():
-                    euicc_flags[k] = v
-                elif k in [f.name for f in TerminalPowerSupply._construct.subcons]:
-                    if k == 'used_supply_voltage_class' and v:
-                        v = {v: True}
-                    ps_flags[k] = v
-
-            child_list = []
-            if any(x for x in ps_flags.values()):
-                child_list.append(TerminalPowerSupply(decoded=ps_flags))
-
-            if opts.extended_logical_channel:
-                child_list.append(ExtendedLchanTerminalSupport())
-            if any(x for x in addl_if_flags.values()):
-                child_list.append(AdditionalInterfacesSupport(decoded=addl_if_flags))
-            if any(x for x in euicc_flags.values()):
-                child_list.append(AdditionalTermCapEuicc(decoded=euicc_flags))
-
-            print(child_list)
-            tc = TerminalCapability(children=child_list)
-            self.terminal_capability(b2h(tc.to_tlv()))
-
-        def terminal_capability(self, data:Hexstr):
-            cmd_hex = "80AA0000%02x%s" % (len(data)//2, data)
-            _rsp_hex, _sw = self._cmd.lchan.scc.send_apdu_checksw(cmd_hex)

pySim/ts_102_222.py

@@ -19,11 +19,9 @@
 
 from typing import List
 import argparse
-
 import cmd2
 from cmd2 import CommandSet, with_default_category
-
-from pySim.utils import b2h, auto_uint8, auto_uint16, is_hexstr
+from osmocom.utils import b2h, auto_uint8, auto_uint16, is_hexstr
 
 from pySim.ts_102_221 import *
 
@@ -44,7 +42,7 @@ class Ts102222Commands(CommandSet):
         if not opts.force_delete:
             self._cmd.perror("Refusing to permanently delete the file, please read the help text.")
             return
-        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        f = self._cmd.lchan.get_file_for_filename(opts.NAME)
         (_data, _sw) = self._cmd.lchan.scc.delete_file(f.fid)
 
     def complete_delete_file(self, text, line, begidx, endidx) -> List[str]:
@@ -65,7 +63,7 @@ class Ts102222Commands(CommandSet):
         if not opts.force:
             self._cmd.perror("Refusing to terminate the file, please read the help text.")
             return
-        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        f = self._cmd.lchan.get_file_for_filename(opts.NAME)
         (_data, _sw) = self._cmd.lchan.scc.terminate_df(f.fid)
 
     def complete_terminate_df(self, text, line, begidx, endidx) -> List[str]:
@@ -81,7 +79,7 @@ class Ts102222Commands(CommandSet):
         if not opts.force:
             self._cmd.perror("Refusing to terminate the file, please read the help text.")
             return
-        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        f = self._cmd.lchan.get_file_for_filename(opts.NAME)
         (_data, _sw) = self._cmd.lchan.scc.terminate_ef(f.fid)
 
     def complete_terminate_ef(self, text, line, begidx, endidx) -> List[str]:
@@ -103,7 +101,6 @@ class Ts102222Commands(CommandSet):
         (_data, _sw) = self._cmd.lchan.scc.terminate_card_usage()
 
     create_parser = argparse.ArgumentParser()
-    create_parser.add_argument('FILE_ID', type=is_hexstr, help='File Identifier as 4-character hex string')
     create_parser._action_groups.pop()
     create_required = create_parser.add_argument_group('required arguments')
     create_optional = create_parser.add_argument_group('optional arguments')
@@ -115,6 +112,7 @@ class Ts102222Commands(CommandSet):
     create_optional.add_argument('--short-file-id', type=str, help='Short File Identifier as 2-digit hex string')
     create_optional.add_argument('--shareable', action='store_true', help='Should the file be shareable?')
     create_optional.add_argument('--record-length', type=auto_uint16, help='Length of each record in octets')
+    create_parser.add_argument('FILE_ID', type=is_hexstr, help='File Identifier as 4-character hex string')
 
     @cmd2.with_argparser(create_parser)
     def do_create_ef(self, opts):
@@ -150,7 +148,6 @@ class Ts102222Commands(CommandSet):
         self._cmd.lchan.select_file(self._cmd.lchan.selected_file)
 
     createdf_parser = argparse.ArgumentParser()
-    createdf_parser.add_argument('FILE_ID', type=is_hexstr, help='File Identifier as 4-character hex string')
     createdf_parser._action_groups.pop()
     createdf_required = createdf_parser.add_argument_group('required arguments')
     createdf_optional = createdf_parser.add_argument_group('optional arguments')
@@ -165,6 +162,7 @@ class Ts102222Commands(CommandSet):
     createdf_sja_optional.add_argument('--permit-rfm-delete-terminate', action='store_true')
     createdf_sja_optional.add_argument('--permit-other-applet-create', action='store_true')
     createdf_sja_optional.add_argument('--permit-other-applet-delete-terminate', action='store_true')
+    createdf_parser.add_argument('FILE_ID', type=is_hexstr, help='File Identifier as 4-character hex string')
 
     @cmd2.with_argparser(createdf_parser)
     def do_create_df(self, opts):
@@ -201,15 +199,15 @@ class Ts102222Commands(CommandSet):
         self._cmd.lchan.select_file(self._cmd.lchan.selected_file)
 
     resize_ef_parser = argparse.ArgumentParser()
-    resize_ef_parser.add_argument('NAME', type=str, help='Name or FID of file to be resized')
     resize_ef_parser._action_groups.pop()
     resize_ef_required = resize_ef_parser.add_argument_group('required arguments')
     resize_ef_required.add_argument('--file-size', required=True, type=auto_uint16, help='Size of file in octets')
+    resize_ef_parser.add_argument('NAME', type=str, help='Name or FID of file to be resized')
 
     @cmd2.with_argparser(resize_ef_parser)
     def do_resize_ef(self, opts):
         """Resize an existing EF below the currently selected DF.  Requires related privileges."""
-        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        f = self._cmd.lchan.get_file_for_filename(opts.NAME)
         ies = [FileIdentifier(decoded=f.fid),
                FileSize(decoded=opts.file_size)]
         fcp = FcpTemplate(children=ies)

pySim/ts_102_310.py

@@ -17,13 +17,12 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
-from pySim.construct import *
 from construct import *
 from construct import Optional as COptional
+from osmocom.construct import *
 
-#from pySim.utils import *
+from osmocom.tlv import BER_TLV_IE, TLV_IE_Collection
 from pySim.filesystem import CardDF, TransparentEF
-from pySim.tlv import BER_TLV_IE, TLV_IE_Collection
 
 # TS102 310 Section 7.1
 class EF_EAPKEYS(TransparentEF):

pySim/ts_31_102.py

@@ -30,21 +30,23 @@ import enum
 
 from construct import Optional as COptional
 from construct import Int32ub, Nibble, GreedyRange, Struct, FlagsEnum, Switch, this, Int16ub, Padding
-from construct import Bytewise, Int24ub, PaddedString
+from construct import Bytewise, Int24ub, Int24sb, PaddedString, PrefixedArray, If
 
-import pySim.ts_102_221
-from pySim.ts_51_011 import EF_ACMmax, EF_AAeM, EF_eMLPP, EF_CMI, EF_PNN
-from pySim.ts_51_011 import EF_MMSN, EF_MMSICP, EF_MMSUP, EF_MMSUCP, EF_VGCS, EF_VGCSS, EF_NIA
-from pySim.ts_51_011 import EF_SMSR, EF_DCK, EF_EXT, EF_CNL, EF_OPL, EF_MBI, EF_MWIS
-from pySim.ts_51_011 import EF_CBMID, EF_CBMIR, EF_ADN, EF_CFIS, EF_SMS, EF_MSISDN, EF_SMSP, EF_SMSS
-from pySim.ts_51_011 import EF_IMSI, EF_xPLMNwAcT, EF_SPN, EF_CBMI, EF_ACC, EF_PLMNsel
-from pySim.ts_51_011 import EF_Kc, EF_CPBCCH, EF_InvScan
-from pySim.ts_102_221 import EF_ARR
-from pySim.tlv import *
+from osmocom.utils import is_hexstr
+from osmocom.tlv import *
+from osmocom.construct import *
+
+from pySim.profile.ts_51_011 import EF_ACMmax, EF_AAeM, EF_eMLPP, EF_CMI, EF_PNN
+from pySim.profile.ts_51_011 import EF_MMSN, EF_MMSICP, EF_MMSUP, EF_MMSUCP, EF_VGCS, EF_VGCSS, EF_NIA
+from pySim.profile.ts_51_011 import EF_SMSR, EF_DCK, EF_EXT, EF_CNL, EF_OPL, EF_MBI, EF_MWIS
+from pySim.profile.ts_51_011 import EF_CBMID, EF_CBMIR, EF_ADN, EF_CFIS, EF_SMS, EF_MSISDN, EF_SMSP, EF_SMSS
+from pySim.profile.ts_51_011 import EF_IMSI, EF_xPLMNwAcT, EF_SPN, EF_CBMI, EF_ACC, EF_PLMNsel
+from pySim.profile.ts_51_011 import EF_Kc, EF_CPBCCH, EF_InvScan
+from pySim.profile.ts_102_221 import EF_ARR, CardProfileUICC
 from pySim.filesystem import *
 from pySim.ts_31_102_telecom import DF_PHONEBOOK, EF_UServiceTable
-from pySim.construct import *
-from pySim.utils import is_hexstr
+from pySim.ts_31_103_shared import EF_IMSConfigData, EF_XCAPConfigData, EF_MuDMiDConfigData
+from pySim.ts_31_103_shared import EF_AC_GBAUAPI, EF_IMSDCI
 from pySim.cat import SMS_TPDU, DeviceIdentities, SMSPPDownload
 
 # Mapping between USIM Service Number and its description
@@ -341,7 +343,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
                 out.append({k: v})
         return out
 
-    def _encode_hex(self, in_json):
+    def _encode_hex(self, in_json, **kwargs):
         out_bytes = self._encode_prot_scheme_id_list(
             in_json['prot_scheme_id_list'])
         d = self._expand_pubkey_list(in_json['hnet_pubkey_list'])
@@ -393,8 +395,8 @@ class EF_SUCI_Calc_Info(TransparentEF):
             'hnet_pubkey_list':    hnet_pubkey_list
         }
 
-    def _encode_bin(self, in_json):
-        return h2b(self._encode_hex(in_json))
+    def _encode_bin(self, in_json, **kwargs):
+        return h2b(self._encode_hex(in_json, **kwargs))
 
 
 class EF_LI(TransRecEF):
@@ -402,14 +404,14 @@ class EF_LI(TransRecEF):
                  desc='Language Indication'):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
 
-    def _decode_record_bin(self, in_bin, **_kwargs):
+    def _decode_record_bin(self, in_bin, **kwargs):
         if in_bin == b'\xff\xff':
             return None
         else:
             # officially this is 7-bit GSM alphabet with one padding bit in each byte
             return in_bin.decode('ascii')
 
-    def _encode_record_bin(self, in_json, **_kwargs):
+    def _encode_record_bin(self, in_json, **kwargs):
         if in_json is None:
             return b'\xff\xff'
         else:
@@ -502,7 +504,7 @@ class EF_ECC(LinFixedEF):
                  desc='Emergency Call Codes'):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(4, 20))
 
-    def _decode_record_bin(self, in_bin, **_kwargs):
+    def _decode_record_bin(self, in_bin, **kwargs):
         # mandatory parts
         code = in_bin[:3]
         if code == b'\xff\xff\xff':
@@ -516,7 +518,7 @@ class EF_ECC(LinFixedEF):
             ret['alpha_id'] = parse_construct(EF_ECC.alpha_construct, alpha_id)
         return ret
 
-    def _encode_record_bin(self, in_json, **_kwargs):
+    def _encode_record_bin(self, in_json, **kwargs):
         if in_json is None:
             return b'\xff\xff\xff\xff'
         code = EF_ECC.cc_construct.build(in_json['call_code'])
@@ -679,7 +681,7 @@ class EF_RPLMNAcT(TransRecEF):
     def __init__(self, fid='6f65', sfid=None, name='EF.RPLMNAcTD', size=(2, 4), rec_len=2,
                  desc='RPLMN Last used Access Technology', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
-    def _decode_record_hex(self, in_hex, **_kwargs):
+    def _decode_record_hex(self, in_hex, **kwargs):
         return dec_act(in_hex)
     # TODO: Encode
 
@@ -850,7 +852,7 @@ class EF_IPS(CyclicEF):
         self._construct = Struct('status'/PaddedString(2, 'ascii'),
                                  'link_to_ef_ipd'/Int8ub, 'rfu'/Byte)
 
-# TS 31.102 Section 4.2.103
+# TS 31.102 Section 4.2.103 (Rel 13)
 class EF_ePDGId(TransparentEF):
     _test_de_encode = [
         ( '801100657064672e6f736d6f636f6d2e6f7267', {'e_pdg_id': {"type_of_ePDG_address": "FQDN", "ePDG_address" : "epdg.osmocom.org" } } ),
@@ -868,7 +870,7 @@ class EF_ePDGId(TransparentEF):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_ePDGId.ePDGId
 
-# TS 31.102 Section 4.2.104
+# TS 31.102 Section 4.2.104 (Rel 13)
 class EF_ePDGSelection(TransparentEF):
     _test_de_encode = [
         ( '800600f110000100', {'e_pdg_selection': [{'plmn': '001-01', 'epdg_priority': 1, 'epdg_fqdn_format': 'operator_identified' }] }),
@@ -883,20 +885,76 @@ class EF_ePDGSelection(TransparentEF):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_ePDGSelection.ePDGSelection
 
-# TS 31.102 Section 4.2.106
+# TS 31.102 Section 4.2.106 (Rel 14)
 class EF_FromPreferred(TransparentEF):
     def __init__(self, fid='6ff7', sfid=None, name='EF.FromPreferred', size=(1, 1),
                  desc='From Preferred', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = BitStruct('rfu'/BitsRFU(7), 'from_preferred'/Flag)
 
-# TS 31.102 Section 4.2.114
+# TS 31.102 Section 4.2.112 + TS 23.032 Section 6.1
+GadPoint = Struct('latitude'/Int24sb, 'longitude'/Int24sb)
+
+# TS 31.102 Section 4.2.112 (Rel ??)
+class EF_EARFCNList(TransparentEF):
+    _test_de_encode = [
+        # single data object with one EARFCN + one area of 3 points
+        ('a01a8004000100008112000001100001000002100002000003100003',
+         [{'earfcn_list_tlv': [{'earfcn': 65536},
+                               {'geographical_area': [{'latitude': 1, 'longitude': 1048577},
+                                                      {'latitude': 2, 'longitude': 1048578},
+                                                      {'latitude': 3, 'longitude': 1048579}] }]}] ),
+        # single data object with one EARFCN + two areas of 3 + 4 points
+        ('a03480040001000081120000011000010000021000020000031000038118000001100001000002100002000003100003000004100004',
+         [{'earfcn_list_tlv': [{'earfcn': 65536},
+                               {'geographical_area': [{'latitude': 1, 'longitude': 1048577},
+                                                      {'latitude': 2, 'longitude': 1048578},
+                                                      {'latitude': 3, 'longitude': 1048579}] },
+                               {'geographical_area': [{'latitude': 1, 'longitude': 1048577},
+                                                      {'latitude': 2, 'longitude': 1048578},
+                                                      {'latitude': 3, 'longitude': 1048579},
+                                                      {'latitude': 4, 'longitude': 1048580}] }
+                               ] }] ),
+        # two concatenated data objects with 3 points each
+        ('a01a8004000100008112000001100001000002100002000003100003a01a8004000200008112000011100011000012100012000013100013',
+         [{'earfcn_list_tlv': [{'earfcn': 65536},
+                               {'geographical_area': [{'latitude': 1, 'longitude': 1048577},
+                                                      {'latitude': 2, 'longitude': 1048578},
+                                                      {'latitude': 3, 'longitude': 1048579}] }]},
+          {'earfcn_list_tlv': [{'earfcn': 131072},
+                               {'geographical_area': [{'latitude': 17, 'longitude': 1048593},
+                                                      {'latitude': 18, 'longitude': 1048594},
+                                                      {'latitude': 19, 'longitude': 1048595}] }]} ]),
+    ]
+    class Earfcn(BER_TLV_IE, tag=0x80):
+        _construct = Int32ub
+    class GeographicalArea(BER_TLV_IE, tag=0x81):
+        _construct = GreedyRange(GadPoint)
+    class EarfcnListTlv(BER_TLV_IE, tag=0xa0, nested=[Earfcn,GeographicalArea]):
+        pass
+    # we need a collection as there might be multiple concatenated instances
+    class EarfcnListTlvCollection(TLV_IE_Collection, nested=[EarfcnListTlv]):
+        pass
+    def __init__(self, fid='6ffd', sfid=None, name='EF.EARFCNList', size=(30, 100),
+                 desc='EARFCN list for MTC/NB-IOT UEs', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
+        self._tlv = self.EarfcnListTlvCollection
+
+# TS 31.102 Section 4.2.114 (Rel 18)
 class EF_eAKA(TransparentEF):
     def __init__(self, fid='6f01', sfid=None, name='EF.eAKA', size=(1, 1),
                  desc='enhanced AKA support', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = BitStruct('rfu'/BitsRFU(7), 'enhanced_sqn_calculation_supported'/Flag)
 
+# TS 31.102 Section 4.2.115 (Rel 18)
+class EF_OCST(TransparentEF):
+    def __init__(self, fid='6f02', sfid=None, name='EF.OCST', size=(2, 100),
+                 desc='Operator controlled signal threshold per access technology', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
+        self._construct = Struct('sense'/FlagsEnum(Byte, sense_enabled=1),
+                                 'ocst_tlv'/GreedyBytes)
+
 
 ######################################################################
 # DF.GSM-ACCESS
@@ -992,10 +1050,10 @@ class EF_OCSGL(LinFixedEF):
 
 
 ######################################################################
-# DF.5GS
+# DF.5GS (Rel 15)
 ######################################################################
 
-# TS 31.102 Section 4.4.11.2
+# TS 31.102 Section 4.4.11.2 (Rel 15)
 class EF_5GS3GPPLOCI(TransparentEF):
     def __init__(self, fid='4f01', sfid=0x01, name='EF.5GS3GPPLOCI', size=(20, 20),
                  desc='5S 3GP location information', **kwargs):
@@ -1006,7 +1064,7 @@ class EF_5GS3GPPLOCI(TransparentEF):
                                  'last_visited_registered_tai_in_5gs'/HexAdapter(Bytes(6)),
                                  '5gs_update_status'/upd_status_constr)
 
-# TS 31.102 Section 4.4.11.7
+# TS 31.102 Section 4.4.11.7 (Rel 15)
 class EF_UAC_AIC(TransparentEF):
     _test_de_encode = [
         ( '03', { "uac_access_id_config": { "multimedia_priority_service": True,
@@ -1019,7 +1077,7 @@ class EF_UAC_AIC(TransparentEF):
                                mission_critical_service=2)
         self._construct = Struct('uac_access_id_config'/cfg_constr)
 
-# TS 31.102 Section 4.4.11.9
+# TS 31.102 Section 4.4.11.9 (Rel 15)
 class EF_OPL5G(LinFixedEF):
     def __init__(self, fid='4f08', sfid=0x08, name='EF.OPL5G', desc='5GS Operator PLMN List', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=(10, None), **kwargs)
@@ -1027,7 +1085,7 @@ class EF_OPL5G(LinFixedEF):
                      'tac_max'/HexAdapter(Bytes(3)))
         self._construct = Struct('tai'/Tai, 'pnn_record_id'/Int8ub)
 
-# TS 31.102 Section 4.4.11.10
+# TS 31.102 Section 4.4.11.10 (Rel 15)
 class EF_SUPI_NAI(TransparentEF):
     class NetworkSpecificIdentifier(TLV_IE, tag=0x80):
         # RFC 7542 encoded as UTF-8 string
@@ -1049,7 +1107,7 @@ class EF_SUPI_NAI(TransparentEF):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_SUPI_NAI.NAI_TLV_Collection
 
-# TS 31.102 Section 4.4.11.11
+# TS 31.102 Section 4.4.11.11 (Rel 15)
 class EF_Routing_Indicator(TransparentEF):
     def __init__(self, fid='4f0a', sfid=0x0a, name='EF.Routing_Indicator', desc='Routing Indicator', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
@@ -1061,7 +1119,7 @@ class EF_Routing_Indicator(TransparentEF):
         self._construct = Struct('routing_indicator'/Rpad(BcdAdapter(Bytes(2)), 'f', 2),
                                  'rfu'/HexAdapter(Bytes(2)))
 
-# TS 31.102 Section 4.4.11.13
+# TS 31.102 Section 4.4.11.13 (Rel 16)
 class EF_TN3GPPSNN(TransparentEF):
     class ServingNetworkName(BER_TLV_IE, tag=0x80):
         _construct = Utf8Adapter(GreedyBytes)
@@ -1278,6 +1336,56 @@ class EF_5G_PROSE_UIR(TransparentEF):
         # contains TLV structure despite being TransparentEF, not BER-TLV ?!?
         self._tlv = EF_5G_PROSE_UIR.ProSeConfigDataForUeToNetworkRelayUE
 
+# TS 31.102 Section 4.4.13.8 (Rel 18)
+class EF_5G_PROSE_U2URU(TransparentEF):
+    class ValidityTimer(BER_TLV_IE, tag=0x85):
+        _construct = Bytes(5)
+    class ServedByNGRAN(BER_TLV_IE, tag=0x80):
+        _construct = GreedyBytes
+    class NotServedByNGRAN(BER_TLV_IE, tag=0x81):
+        _construct = GreedyBytes
+    class DefaultDstL2IdsForRxDisc(BER_TLV_IE, tag=0x99):
+        _construct = GreedyBytes
+    class UserInforIdForDiscovery(BER_TLV_IE, tag=0x8e):
+        _construct = GreedyBytes
+    class RSCInfoList(BER_TLV_IE, tag=0x8b):
+        _construct = GreedyBytes
+    class DefaultDstL2IdsForTxRxDirect(BER_TLV_IE, tag=0x9a):
+        _construct = GreedyBytes
+    class ProSeConfigDataForU2URelayUE(BER_TLV_IE, tag=0xa0,
+                                       nested=[ValidityTimer, ServedByNGRAN, NotServedByNGRAN,
+                                               DefaultDstL2IdsForRxDisc, UserInforIdForDiscovery,
+                                               RSCInfoList, DefaultDstL2IdsForTxRxDirect]):
+        pass
+    def __init__(self, fid='4f07', sfid=0x07, name='EF.5G_PROSE_U2URU',
+                 desc='5G ProSe configuration data for UE-to-UE relay UE', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_5G_PROSE_U2URU.ProSeConfigDataForU2URelayUE
+
+# TS 31.102 Section 4.4.13.9 (Rel 18)
+class EF_5G_PROSE_EU(TransparentEF):
+    class PKMFAddressInformation(BER_TLV_IE, tag=0x93):
+        Ipv4AddrList = PrefixedArray(Int8ub, Int32ub)
+        Ipv6AddrList = PrefixedArray(Int8ub, Bytes(16))
+        _construct = Struct('flags'/FlagsEnum(Byte, ipv4=1, ipv6=2, fqdn=4),
+                            'ipv4_addr_list'/If(this.flags.ipv4, Ipv4AddrList),
+                            'ipv6_addr_list'/If(this.flags.ipv6, Ipv6AddrList),
+                            'fqdn'/Prefixed(Int8ub, Utf8Adapter(GreedyBytes)))
+    class ProSeConfigDataForEndUE(BER_TLV_IE, tag=0xa0,
+                                  nested=[EF_5G_PROSE_U2URU.ValidityTimer,
+                                          EF_5G_PROSE_U2URU.ServedByNGRAN,
+                                          EF_5G_PROSE_U2URU.NotServedByNGRAN,
+                                          EF_5G_PROSE_U2URU.DefaultDstL2IdsForRxDisc,
+                                          EF_5G_PROSE_U2URU.UserInforIdForDiscovery,
+                                          EF_5G_PROSE_U2URU.RSCInfoList,
+                                          EF_5G_PROSE_U2URU.DefaultDstL2IdsForTxRxDirect,
+                                          PKMFAddressInformation]):
+        pass
+    def __init__(self, fid='4f08', sfid=0x08, name='EF.5G_PROSE_EU',
+                 desc='5G ProSe configuration data for end UE', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_5G_PROSE_EU.ProSeConfigDataForEndUE
+
 # TS 31.102 Section 4.4.13 (Rel 17)
 class DF_5G_ProSe(CardDF):
     def __init__(self, fid='5ff0', name='DF.5G_ProSe', desc='Files for 5G ProSe purpose', **kwargs):
@@ -1289,9 +1397,62 @@ class DF_5G_ProSe(CardDF):
             EF_5G_PROSE_U2NRU(service=3),
             EF_5G_PROSE_RU(service=4),
             EF_5G_PROSE_UIR(service=5),
+            # Rel 18 additions
+            EF_5G_PROSE_U2URU(service=6),
+            EF_5G_PROSE_EU(service=7),
         ]
         self.add_files(files)
 
+# TS 31.102 Section 4.4.14.2 (Rel 18)
+class EF_5MBSUECONFIG(TransparentEF):
+
+    class Plmn(BER_TLV_IE, tag=0x80):
+        _construct = Struct('plmn'/PlmnAdapter(Bytes(3)),
+                            'nid'/COptional(Bytes(6)))
+    class Tmgi(BER_TLV_IE, tag=0x81):
+        TmgiEntry = Struct('tmgi'/Bytes(6),
+                           'usd_fid'/HexAdapter(Bytes(2)),
+                           'service_type'/FlagsEnum(Byte, mbs_service_announcement=1, mbs_user_service=2))
+        _construct = GreedyRange(TmgiEntry)
+    class NrArfcnList(BER_TLV_IE, tag=0x82):
+        _construct = GreedyRange(Bytes(4))
+    class DNN(BER_TLV_IE, tag=0x83):
+        _construct = GreedyBytes
+    class SNSSAI(BER_TLV_IE, tag=0x84):
+        _construct = GreedyBytes
+    class PduInfoList(BER_TLV_IE, tag=0xa1, nested=[DNN, SNSSAI]):
+        pass
+    class Plmn5mbsPreconfiguration(BER_TLV_IE, tag=0xa0,
+                                   nested=[Plmn, Tmgi, NrArfcnList, PduInfoList]):
+        pass
+    def __init__(self, fid='4f01', sfid=None, name='EF.5MBSUECONFIG',
+                 desc='5MBS UE pre-configuration', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_5MBSUECONFIG.Plmn5mbsPreconfiguration
+
+# TS 31.102 Section 4.4.14.3 (Rel 18)
+class EF_5MBSUSD(TransparentEF):
+    """There can be any number of these files with undefined FID; the FIDs are contained
+    in EF.5BMSUECONFIG. FID range is 4f08...4fff"""
+    class USD(BER_TLV_IE, tag=0x80):
+        _construct = GreedyBytes
+    def __init__(self, fid, sfid=None, name='EF.5MBSUSD',
+                 desc='5MBS User Service Description', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = USD
+
+
+# TS 31.102 Section 4.4.14 (Rel 18)
+class DF_5MBSUECONFIG(CardDF):
+    def __init__(self, fid='5ff1', name='DF.5MBSUECONFIG', desc='', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_5MBSUECONFIG(),
+            # EF_5MBSUSD() wouild have to be dynamically registered based on EF_5MBSUECONFIG content
+        ]
+        self.add_files(files)
+
+
 # TS 31.102 Section 4.4.11.18 (Rel 17)
 class EF_5GSEDRX(TransparentEF):
     def __init__(self, fid='4f10', sfid=0x10, name='EF.5GSEDRX',
@@ -1526,7 +1687,7 @@ class ADF_USIM(CardADF):
             EF_VGCS('6fb3', None, 'EF.VBS', desc='Voice Broadcast Service', service=58),
             EF_VGCSS('6fb4', None, 'EF.VBSS', desc='Voice Broadcast Service Status', service=58),
             EF_VGCSCA(service=64),
-            EF_VGCSCA('6fd5', None, 'EF.VBCSCA', desc='Voice Broadcast Service Ciphering Algorithm', service=65),
+            EF_VGCSCA('6fd5', None, 'EF.VBSCA', desc='Voice Broadcast Service Ciphering Algorithm', service=65),
             EF_GBABP(service=68),
             EF_MSK(service=69),
             EF_MUK(service=69),
@@ -1556,7 +1717,17 @@ class ADF_USIM(CardADF):
             EF_ePDGSelection('6ff6', None, 'EF.ePDGSelectionEm',
                              desc='ePDG Selection Information for Emergency Services', service=(110, 111)),
             EF_FromPreferred(service=114),
+            EF_IMSConfigData(service=115),
+            # TODO: EF.TVCONFIG
+            # TODO: EF.3GPPPSDATAOFF
+            # TODO: EF.3GPPPSDATAOFFservicelist
+            EF_XCAPConfigData(service=120),
+            EF_EARFCNList(service=121),
+            EF_MuDMiDConfigData(service=134),
             EF_eAKA(),
+            EF_OCST(service=148),
+            EF_AC_GBAUAPI(service=68),
+            EF_IMSDCI(service=150),
             # FIXME: DF_SoLSA service=23
             DF_PHONEBOOK(),
             DF_GSM_ACCESS(),
@@ -1569,6 +1740,7 @@ class ADF_USIM(CardADF):
             DF_SNPN(service=[143,146]),
             DF_5G_ProSe(service=139),
             DF_SAIP(),
+            DF_5MBSUECONFIG(service=147),
         ]
 
         if has_imsi:
@@ -1577,19 +1749,19 @@ class ADF_USIM(CardADF):
         self.add_files(files)
 
     def decode_select_response(self, data_hex):
-        return pySim.ts_102_221.CardProfileUICC.decode_select_response(data_hex)
+        return CardProfileUICC.decode_select_response(data_hex)
 
     @with_default_category('Application-Specific Commands')
     class AddlShellCommands(CommandSet):
         authenticate_parser = argparse.ArgumentParser()
-        authenticate_parser.add_argument('rand', type=is_hexstr, help='Random challenge')
-        authenticate_parser.add_argument('autn', type=is_hexstr, help='Authentication Nonce')
+        authenticate_parser.add_argument('RAND', type=is_hexstr, help='Random challenge')
+        authenticate_parser.add_argument('AUTN', type=is_hexstr, help='Authentication Nonce')
         #authenticate_parser.add_argument('--context', help='Authentication context', default='3G')
 
         @cmd2.with_argparser(authenticate_parser)
         def do_authenticate(self, opts):
             """Perform Authentication and Key Agreement (AKA)."""
-            (data, _sw) = self._cmd.lchan.scc.authenticate(opts.rand, opts.autn)
+            (data, _sw) = self._cmd.lchan.scc.authenticate(opts.RAND, opts.AUTN)
             self._cmd.poutput_json(data)
 
         term_prof_parser = argparse.ArgumentParser()
@@ -1637,7 +1809,8 @@ class ADF_USIM(CardADF):
             self._cmd.poutput('SW: %s, data: %s' % (sw, data))
 
         get_id_parser = argparse.ArgumentParser()
-        get_id_parser.add_argument("--nswo-context", action='store_true')
+        get_id_parser.add_argument("--nswo-context", action='store_true',
+                                   help='use SUCI 5G Non-Seamless WLAN Offload context')
 
         @cmd2.with_argparser(get_id_parser)
         def do_get_identity(self, opts):

pySim/ts_31_102_telecom.py

@@ -28,10 +28,10 @@ Needs to be a separate python module to avoid cyclic imports
 
 from construct import Optional as COptional
 from construct import Struct, Int16ub, Int32ub
+from osmocom.tlv import *
+from osmocom.construct import *
 
-from pySim.tlv import *
 from pySim.filesystem import *
-from pySim.construct import *
 
 # TS 31.102 Section 4.2.8
 class EF_UServiceTable(TransparentEF):
@@ -59,7 +59,7 @@ class EF_UServiceTable(TransparentEF):
                     ret[service_nr]['description'] = self.table[service_nr]
         return ret
 
-    def _encode_bin(self, in_json):
+    def _encode_bin(self, in_json, **kwargs):
         # compute the required binary size
         bin_len = 0
         for srv in in_json.keys():

pySim/ts_31_103.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 
 """
-Various constants from 3GPP TS 31.103 V16.1.0
+Various constants from 3GPP TS 31.103 V18.1.0
 """
 
 #
 # Copyright (C) 2020 Supreeth Herle <herlesupreeth@gmail.com>
-# Copyright (C) 2021 Harald Welte <laforge@osmocom.org>
+# Copyright (C) 2021-2024 Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,15 +23,15 @@ Various constants from 3GPP TS 31.103 V16.1.0
 #
 
 from construct import Struct, Switch, this, Bytes, GreedyString
+from osmocom.utils import *
+from osmocom.tlv import *
+from osmocom.construct import *
 from pySim.filesystem import *
-from pySim.utils import *
-from pySim.tlv import *
-from pySim.ts_51_011 import EF_AD, EF_SMS, EF_SMSS, EF_SMSR, EF_SMSP
+from pySim.profile.ts_51_011 import EF_AD, EF_SMS, EF_SMSS, EF_SMSR, EF_SMSP
 from pySim.ts_31_102 import ADF_USIM, EF_FromPreferred
 from pySim.ts_31_102_telecom import EF_UServiceTable
-import pySim.ts_102_221
-from pySim.ts_102_221 import EF_ARR
-from pySim.construct import *
+from pySim.ts_31_103_shared import *
+from pySim.profile.ts_102_221 import EF_ARR, CardProfileUICC
 
 # Mapping between ISIM Service Number and its description
 EF_IST_map = {
@@ -56,6 +56,7 @@ EF_IST_map = {
     19: 'XCAP Configuration Data',
     20: 'WebRTC URI',
     21: 'MuD and MiD configuration data',
+    22: 'IMS Data Channel indication',
 }
 
 # TS 31.103 Section 4.2.2
@@ -189,96 +190,6 @@ class EF_NAFKCA(LinFixedEF):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_NAFKCA.NafKeyCentreAddress
 
-# TS 31.103 Section 4.2.16
-class EF_UICCIARI(LinFixedEF):
-    class iari(BER_TLV_IE, tag=0x80):
-        _construct = Utf8Adapter(GreedyBytes)
-
-    def __init__(self, fid='6fe7', sfid=None, name='EF.UICCIARI', desc='UICC IARI', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
-        self._tlv = EF_UICCIARI.iari
-
-# TS 31.103 Section 4.2.18
-class EF_IMSConfigData(BerTlvEF):
-    class ImsConfigDataEncoding(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(Bytes(1))
-    class ImsConfigData(BER_TLV_IE, tag=0x81):
-        _construct = GreedyString
-    # pylint: disable=undefined-variable
-    class ImsConfigDataCollection(TLV_IE_Collection, nested=[ImsConfigDataEncoding, ImsConfigData]):
-        pass
-    def __init__(self, fid='6ff8', sfid=None, name='EF.IMSConfigData', desc='IMS Configuration Data', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
-        self._tlv = EF_IMSConfigData.ImsConfigDataCollection
-
-# TS 31.103 Section 4.2.19
-class EF_XCAPConfigData(BerTlvEF):
-    class Access(BER_TLV_IE, tag=0x81):
-        pass
-    class ApplicationName(BER_TLV_IE, tag=0x82):
-        pass
-    class ProviderID(BER_TLV_IE, tag=0x83):
-        pass
-    class URI(BER_TLV_IE, tag=0x84):
-        pass
-    class XcapAuthenticationUserName(BER_TLV_IE, tag=0x85):
-        pass
-    class XcapAuthenticationPassword(BER_TLV_IE, tag=0x86):
-        pass
-    class XcapAuthenticationType(BER_TLV_IE, tag=0x87):
-        pass
-    class AddressType(BER_TLV_IE, tag=0x88):
-        pass
-    class Address(BER_TLV_IE, tag=0x89):
-        pass
-    class PDPAuthenticationType(BER_TLV_IE, tag=0x8a):
-        pass
-    class PDPAuthenticationName(BER_TLV_IE, tag=0x8b):
-        pass
-    class PDPAuthenticationSecret(BER_TLV_IE, tag=0x8c):
-        pass
-
-    class AccessForXCAP(BER_TLV_IE, tag=0x81):
-        pass
-    class NumberOfXcapConnParPolicy(BER_TLV_IE, tag=0x82):
-        _construct = Int8ub
-    # pylint: disable=undefined-variable
-    class XcapConnParamsPolicyPart(BER_TLV_IE, tag=0xa1, nested=[Access, ApplicationName, ProviderID, URI,
-                                 XcapAuthenticationUserName, XcapAuthenticationPassword,
-                                 XcapAuthenticationType, AddressType, Address, PDPAuthenticationType,
-                                 PDPAuthenticationName, PDPAuthenticationSecret]):
-        pass
-    class XcapConnParamsPolicy(BER_TLV_IE, tag=0xa0, nested=[AccessForXCAP, NumberOfXcapConnParPolicy, XcapConnParamsPolicyPart]):
-        pass
-    class XcapConnParamsPolicyDO(BER_TLV_IE, tag=0x80, nested=[XcapConnParamsPolicy]):
-        pass
-    def __init__(self, fid='6ffc', sfid=None, name='EF.XCAPConfigData', desc='XCAP Configuration Data', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
-        self._tlv = EF_XCAPConfigData.XcapConnParamsPolicy
-
-# TS 31.103 Section 4.2.20
-class EF_WebRTCURI(TransparentEF):
-    class uri(BER_TLV_IE, tag=0x80):
-        _construct = Utf8Adapter(GreedyBytes)
-
-    def __init__(self, fid='6ffa', sfid=None, name='EF.WebRTCURI', desc='WebRTC URI', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
-        self._tlv = EF_WebRTCURI.uri
-
-# TS 31.103 Section 4.2.21
-class EF_MuDMiDConfigData(BerTlvEF):
-    class MudMidConfigDataEncoding(BER_TLV_IE, tag=0x80):
-        _construct = HexAdapter(Bytes(1))
-    class MudMidConfigData(BER_TLV_IE, tag=0x81):
-        _construct = GreedyString
-    # pylint: disable=undefined-variable
-    class MudMidConfigDataCollection(TLV_IE_Collection, nested=[MudMidConfigDataEncoding, MudMidConfigData]):
-        pass
-    def __init__(self, fid='6ffe', sfid=None, name='EF.MuDMiDConfigData',
-                 desc='MuD and MiD Configuration Data', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
-        self._tlv = EF_MuDMiDConfigData.MudMidConfigDataCollection
-
 
 class ADF_ISIM(CardADF):
     def __init__(self, aid='a0000000871004', has_fs=True, name='ADF.ISIM', fid=None, sfid=None,
@@ -306,13 +217,15 @@ class ADF_ISIM(CardADF):
             EF_XCAPConfigData(service=19),
             EF_WebRTCURI(service=20),
             EF_MuDMiDConfigData(service=21),
+            EF_AC_GBAUAPI(service=2),
+            EF_IMSDCI(service=22),
         ]
         self.add_files(files)
         # add those commands to the general commands of a TransparentEF
         self.shell_commands += [ADF_USIM.AddlShellCommands()]
 
     def decode_select_response(self, data_hex):
-        return pySim.ts_102_221.CardProfileUICC.decode_select_response(data_hex)
+        return CardProfileUICC.decode_select_response(data_hex)
 
 
 # TS 31.103 Section 7.1

pySim/ts_31_103_shared.py

@@ -0,0 +1,138 @@
+"""
+Definitions from 3GPP TS 31.103 V18.1.0 which are shared by both USIM (31.102) and ISIM (31.103) and
+hence need to be in a separate python module to avoid circular dependencies.
+"""
+
+# Copyright (C) 2021-2024 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from construct import Struct, Switch, Bytes, GreedyString, GreedyBytes, Int8ub, Prefixed, Enum, Byte
+from osmocom.tlv import BER_TLV_IE, TLV_IE_Collection
+from osmocom.construct import HexAdapter, Utf8Adapter
+from pySim.filesystem import *
+
+# TS 31.103 Section 4.2.16
+class EF_UICCIARI(LinFixedEF):
+    class iari(BER_TLV_IE, tag=0x80):
+        _construct = Utf8Adapter(GreedyBytes)
+
+    def __init__(self, fid='6fe7', sfid=None, name='EF.UICCIARI', desc='UICC IARI', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_UICCIARI.iari
+
+# TS 31.103 Section 4.2.18
+class EF_IMSConfigData(BerTlvEF):
+    class ImsConfigDataEncoding(BER_TLV_IE, tag=0x80):
+        _construct = HexAdapter(Bytes(1))
+    class ImsConfigData(BER_TLV_IE, tag=0x81):
+        _construct = GreedyString
+    # pylint: disable=undefined-variable
+    class ImsConfigDataCollection(TLV_IE_Collection, nested=[ImsConfigDataEncoding, ImsConfigData]):
+        pass
+    def __init__(self, fid='6ff8', sfid=None, name='EF.IMSConfigData', desc='IMS Configuration Data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_IMSConfigData.ImsConfigDataCollection
+
+# TS 31.103 Section 4.2.19
+class EF_XCAPConfigData(BerTlvEF):
+    class Access(BER_TLV_IE, tag=0x81):
+        pass
+    class ApplicationName(BER_TLV_IE, tag=0x82):
+        pass
+    class ProviderID(BER_TLV_IE, tag=0x83):
+        pass
+    class URI(BER_TLV_IE, tag=0x84):
+        pass
+    class XcapAuthenticationUserName(BER_TLV_IE, tag=0x85):
+        pass
+    class XcapAuthenticationPassword(BER_TLV_IE, tag=0x86):
+        pass
+    class XcapAuthenticationType(BER_TLV_IE, tag=0x87):
+        pass
+    class AddressType(BER_TLV_IE, tag=0x88):
+        pass
+    class Address(BER_TLV_IE, tag=0x89):
+        pass
+    class PDPAuthenticationType(BER_TLV_IE, tag=0x8a):
+        pass
+    class PDPAuthenticationName(BER_TLV_IE, tag=0x8b):
+        pass
+    class PDPAuthenticationSecret(BER_TLV_IE, tag=0x8c):
+        pass
+
+    class AccessForXCAP(BER_TLV_IE, tag=0x81):
+        pass
+    class NumberOfXcapConnParPolicy(BER_TLV_IE, tag=0x82):
+        _construct = Int8ub
+    # pylint: disable=undefined-variable
+    class XcapConnParamsPolicyPart(BER_TLV_IE, tag=0xa1, nested=[Access, ApplicationName, ProviderID, URI,
+                                 XcapAuthenticationUserName, XcapAuthenticationPassword,
+                                 XcapAuthenticationType, AddressType, Address, PDPAuthenticationType,
+                                 PDPAuthenticationName, PDPAuthenticationSecret]):
+        pass
+    class XcapConnParamsPolicy(BER_TLV_IE, tag=0xa0, nested=[AccessForXCAP, NumberOfXcapConnParPolicy, XcapConnParamsPolicyPart]):
+        pass
+    class XcapConnParamsPolicyDO(BER_TLV_IE, tag=0x80, nested=[XcapConnParamsPolicy]):
+        pass
+    def __init__(self, fid='6ffc', sfid=None, name='EF.XCAPConfigData', desc='XCAP Configuration Data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_XCAPConfigData.XcapConnParamsPolicy
+
+# TS 31.103 Section 4.2.20
+class EF_WebRTCURI(LinFixedEF):
+    class uri(BER_TLV_IE, tag=0x80):
+        _construct = Utf8Adapter(GreedyBytes)
+
+    def __init__(self, fid='6ffa', sfid=None, name='EF.WebRTCURI', desc='WebRTC URI', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_WebRTCURI.uri
+
+# TS 31.103 Section 4.2.21
+class EF_MuDMiDConfigData(BerTlvEF):
+    class MudMidConfigDataEncoding(BER_TLV_IE, tag=0x80):
+        _construct = HexAdapter(Bytes(1))
+    class MudMidConfigData(BER_TLV_IE, tag=0x81):
+        _construct = GreedyString
+    # pylint: disable=undefined-variable
+    class MudMidConfigDataCollection(TLV_IE_Collection, nested=[MudMidConfigDataEncoding, MudMidConfigData]):
+        pass
+    def __init__(self, fid='6ffe', sfid=None, name='EF.MuDMiDConfigData',
+                 desc='MuD and MiD Configuration Data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_MuDMiDConfigData.MudMidConfigDataCollection
+
+# TS 31.103 Section 4.2.22
+class EF_AC_GBAUAPI(LinFixedEF):
+    """The use of this EF is eescribed in 3GPP TS 31.130"""
+    class AppletNafAccessControl(BER_TLV_IE, tag=0x80):
+        # the use of Int8ub as length field in Prefixed is strictly speaking incorrect, as it is a BER-TLV
+        # length field whihc will consume two bytes from length > 127 bytes.  However, AIDs and NAF IDs can
+        # safely be assumed shorter than that
+        _construct = Struct('aid'/Prefixed(Int8ub, GreedyBytes),
+                            'naf_id'/Prefixed(Int8ub, GreedyBytes))
+    def __init__(self, fid='6f0a', sfid=None, name='EF.GBAUAPI',
+                 desc='Access Control to GBA_U_API', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_AC_GBAUAPI.AppletNafAccessControl
+
+# TS 31.103 Section 4.2.23
+class EF_IMSDCI(TransparentEF):
+    """See Management object as defined in 3GPP TS 24.275."""
+    def __init__(self, fid='6f0b', sfid=None, name='EF.IMSDCI', desc='IMS Data Channel Indication', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._construct = Enum(Byte, ims_dc_not_allowed=0x00,
+                                     ims_dc_allowed_after_ims_session=0x01,
+                                     ims_dc_allowed_simultaneous_ims_session=0x02)

pySim/ts_31_104.py

@@ -20,13 +20,12 @@ Support for 3GPP TS 31.104 V17.0.0
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from osmocom.utils import *
+from osmocom.tlv import *
 from pySim.filesystem import *
-from pySim.utils import *
-from pySim.tlv import *
 from pySim.ts_31_102 import ADF_USIM
-from pySim.ts_51_011 import EF_IMSI, EF_AD
-import pySim.ts_102_221
-from pySim.ts_102_221 import EF_ARR
+from pySim.profile.ts_51_011 import EF_IMSI, EF_AD
+from pySim.profile.ts_102_221 import EF_ARR, CardProfileUICC
 
 
 class ADF_HPSIM(CardADF):
@@ -44,7 +43,7 @@ class ADF_HPSIM(CardADF):
         self.shell_commands += [ADF_USIM.AddlShellCommands()]
 
     def decode_select_response(self, data_hex):
-        return pySim.ts_102_221.CardProfileUICC.decode_select_response(data_hex)
+        return CardProfileUICC.decode_select_response(data_hex)
 
 
 # TS 31.104 Section 7.1

pySim/utils.py

@@ -10,6 +10,8 @@ import datetime
 import argparse
 from io import BytesIO
 from typing import Optional, List, Dict, Any, Tuple, NewType, Union
+from osmocom.utils import *
+from osmocom.tlv import bertlv_encode_tag, bertlv_encode_len
 
 # Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
 # Copyright (C) 2021 Harald Welte <laforge@osmocom.org>
@@ -28,374 +30,6 @@ from typing import Optional, List, Dict, Any, Tuple, NewType, Union
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-# just to differentiate strings of hex nibbles from everything else
-Hexstr = NewType('Hexstr', str)
-SwHexstr = NewType('SwHexstr', str)
-SwMatchstr = NewType('SwMatchstr', str)
-ResTuple = Tuple[Hexstr, SwHexstr]
-
-def h2b(s: Hexstr) -> bytearray:
-    """convert from a string of hex nibbles to a sequence of bytes"""
-    return bytearray.fromhex(s)
-
-
-def b2h(b: bytearray) -> Hexstr:
-    """convert from a sequence of bytes to a string of hex nibbles"""
-    return ''.join(['%02x' % (x) for x in b])
-
-
-def h2i(s: Hexstr) -> List[int]:
-    """convert from a string of hex nibbles to a list of integers"""
-    return [(int(x, 16) << 4)+int(y, 16) for x, y in zip(s[0::2], s[1::2])]
-
-
-def i2h(s: List[int]) -> Hexstr:
-    """convert from a list of integers to a string of hex nibbles"""
-    return ''.join(['%02x' % (x) for x in s])
-
-
-def h2s(s: Hexstr) -> str:
-    """convert from a string of hex nibbles to an ASCII string"""
-    return ''.join([chr((int(x, 16) << 4)+int(y, 16)) for x, y in zip(s[0::2], s[1::2])
-                    if int(x + y, 16) != 0xff])
-
-
-def s2h(s: str) -> Hexstr:
-    """convert from an ASCII string to a string of hex nibbles"""
-    b = bytearray()
-    b.extend(map(ord, s))
-    return b2h(b)
-
-
-def i2s(s: List[int]) -> str:
-    """convert from a list of integers to an ASCII string"""
-    return ''.join([chr(x) for x in s])
-
-
-def swap_nibbles(s: Hexstr) -> Hexstr:
-    """swap the nibbles in a hex string"""
-    return ''.join([x+y for x, y in zip(s[1::2], s[0::2])])
-
-
-def rpad(s: str, l: int, c='f') -> str:
-    """pad string on the right side.
-    Args:
-            s : string to pad
-            l : total length to pad to
-            c : padding character
-    Returns:
-            String 's' padded with as many 'c' as needed to reach total length of 'l'
-    """
-    return s + c * (l - len(s))
-
-
-def lpad(s: str, l: int, c='f') -> str:
-    """pad string on the left side.
-    Args:
-            s : string to pad
-            l : total length to pad to
-            c : padding character
-    Returns:
-            String 's' padded with as many 'c' as needed to reach total length of 'l'
-    """
-    return c * (l - len(s)) + s
-
-
-def half_round_up(n: int) -> int:
-    return (n + 1)//2
-
-
-def str_sanitize(s: str) -> str:
-    """replace all non printable chars, line breaks and whitespaces, with ' ', make sure that
-    there are no whitespaces at the end and at the beginning of the string.
-
-    Args:
-            s : string to sanitize
-    Returns:
-            filtered result of string 's'
-    """
-
-    chars_to_keep = string.digits + string.ascii_letters + string.punctuation
-    res = ''.join([c if c in chars_to_keep else ' ' for c in s])
-    return res.strip()
-
-#########################################################################
-# poor man's COMPREHENSION-TLV decoder.
-#########################################################################
-
-
-def comprehensiontlv_parse_tag_raw(binary: bytes) -> Tuple[int, bytes]:
-    """Parse a single Tag according to ETSI TS 101 220 Section 7.1.1"""
-    if binary[0] in [0x00, 0x80, 0xff]:
-        raise ValueError("Found illegal value 0x%02x in %s" %
-                         (binary[0], binary))
-    if binary[0] == 0x7f:
-        # three-byte tag
-        tag = binary[0] << 16 | binary[1] << 8 | binary[2]
-        return (tag, binary[3:])
-    elif binary[0] == 0xff:
-        return None, binary
-    else:
-        # single byte tag
-        tag = binary[0]
-        return (tag, binary[1:])
-
-
-def comprehensiontlv_parse_tag(binary: bytes) -> Tuple[dict, bytes]:
-    """Parse a single Tag according to ETSI TS 101 220 Section 7.1.1"""
-    if binary[0] in [0x00, 0x80, 0xff]:
-        raise ValueError("Found illegal value 0x%02x in %s" %
-                         (binary[0], binary))
-    if binary[0] == 0x7f:
-        # three-byte tag
-        tag = (binary[1] & 0x7f) << 8
-        tag |= binary[2]
-        compr = bool(binary[1] & 0x80)
-        return ({'comprehension': compr, 'tag': tag}, binary[3:])
-    else:
-        # single byte tag
-        tag = binary[0] & 0x7f
-        compr = bool(binary[0] & 0x80)
-        return ({'comprehension': compr, 'tag': tag}, binary[1:])
-
-
-def comprehensiontlv_encode_tag(tag) -> bytes:
-    """Encode a single Tag according to ETSI TS 101 220 Section 7.1.1"""
-    # permit caller to specify tag also as integer value
-    if isinstance(tag, int):
-        compr = bool(tag < 0xff and tag & 0x80)
-        tag = {'tag': tag, 'comprehension': compr}
-    compr = tag.get('comprehension', False)
-    if tag['tag'] in [0x00, 0x80, 0xff] or tag['tag'] > 0xff:
-        # 3-byte format
-        byte3 = tag['tag'] & 0xff
-        byte2 = (tag['tag'] >> 8) & 0x7f
-        if compr:
-            byte2 |= 0x80
-        return b'\x7f' + byte2.to_bytes(1, 'big') + byte3.to_bytes(1, 'big')
-    else:
-        # 1-byte format
-        ret = tag['tag']
-        if compr:
-            ret |= 0x80
-        return ret.to_bytes(1, 'big')
-
-# length value coding is equal to BER-TLV
-
-
-def comprehensiontlv_parse_one(binary: bytes) -> Tuple[dict, int, bytes, bytes]:
-    """Parse a single TLV IE at the start of the given binary data.
-    Args:
-            binary : binary input data of BER-TLV length field
-    Returns:
-            Tuple of (tag:dict, len:int, remainder:bytes)
-    """
-    (tagdict, remainder) = comprehensiontlv_parse_tag(binary)
-    (length, remainder) = bertlv_parse_len(remainder)
-    value = remainder[:length]
-    remainder = remainder[length:]
-    return (tagdict, length, value, remainder)
-
-
-#########################################################################
-# poor man's BER-TLV decoder. To be a more sophisticated OO library later
-#########################################################################
-
-def bertlv_parse_tag_raw(binary: bytes) -> Tuple[int, bytes]:
-    """Get a single raw Tag from start of input according to ITU-T X.690 8.1.2
-    Args:
-            binary : binary input data of BER-TLV length field
-    Returns:
-    Tuple of (tag:int, remainder:bytes)
-    """
-    # check for FF padding at the end, as customary in SIM card files
-    if binary[0] == 0xff and len(binary) == 1 or binary[0] == 0xff and binary[1] == 0xff:
-        return None, binary
-    tag = binary[0] & 0x1f
-    if tag <= 30:
-        return binary[0], binary[1:]
-    else:  # multi-byte tag
-        tag = binary[0]
-        i = 1
-        last = False
-        while not last:
-            last = not bool(binary[i] & 0x80)
-            tag <<= 8
-            tag |= binary[i]
-            i += 1
-        return tag, binary[i:]
-
-
-def bertlv_parse_tag(binary: bytes) -> Tuple[dict, bytes]:
-    """Parse a single Tag value according to ITU-T X.690 8.1.2
-    Args:
-            binary : binary input data of BER-TLV length field
-    Returns:
-            Tuple of ({class:int, constructed:bool, tag:int}, remainder:bytes)
-    """
-    cls = binary[0] >> 6
-    constructed = bool(binary[0] & 0x20)
-    tag = binary[0] & 0x1f
-    if tag <= 30:
-        return ({'class': cls, 'constructed': constructed, 'tag': tag}, binary[1:])
-    else:  # multi-byte tag
-        tag = 0
-        i = 1
-        last = False
-        while not last:
-            last = not bool(binary[i] & 0x80)
-            tag <<= 7
-            tag |= binary[i] & 0x7f
-            i += 1
-        return ({'class': cls, 'constructed': constructed, 'tag': tag}, binary[i:])
-
-
-def bertlv_encode_tag(t) -> bytes:
-    """Encode a single Tag value according to ITU-T X.690 8.1.2
-    """
-    def get_top7_bits(inp: int) -> Tuple[int, int]:
-        """Get top 7 bits of integer. Returns those 7 bits as integer and the remaining LSBs."""
-        remain_bits = inp.bit_length()
-        if remain_bits >= 7:
-            bitcnt = 7
-        else:
-            bitcnt = remain_bits
-        outp = inp >> (remain_bits - bitcnt)
-        remainder = inp & ~ (inp << (remain_bits - bitcnt))
-        return outp, remainder
-
-    def count_int_bytes(inp: int) -> int:
-        """count the number of bytes require to represent the given integer."""
-        i = 1
-        inp = inp >> 8
-        while inp:
-            i += 1
-            inp = inp >> 8
-        return i
-
-    if isinstance(t, int):
-        # first convert to a dict representation
-        tag_size = count_int_bytes(t)
-        t, _remainder = bertlv_parse_tag(t.to_bytes(tag_size, 'big'))
-    tag = t['tag']
-    constructed = t['constructed']
-    cls = t['class']
-    if tag <= 30:
-        t = tag & 0x1f
-        if constructed:
-            t |= 0x20
-        t |= (cls & 3) << 6
-        return bytes([t])
-    else:  # multi-byte tag
-        t = 0x1f
-        if constructed:
-            t |= 0x20
-        t |= (cls & 3) << 6
-        tag_bytes = bytes([t])
-        remain = tag
-        while True:
-            t, remain = get_top7_bits(remain)
-            if remain:
-                t |= 0x80
-            tag_bytes += bytes([t])
-            if not remain:
-                break
-        return tag_bytes
-
-
-def bertlv_parse_len(binary: bytes) -> Tuple[int, bytes]:
-    """Parse a single Length value according to ITU-T X.690 8.1.3;
-    only the definite form is supported here.
-    Args:
-            binary : binary input data of BER-TLV length field
-    Returns:
-            Tuple of (length, remainder)
-    """
-    if binary[0] < 0x80:
-        return (binary[0], binary[1:])
-    else:
-        num_len_oct = binary[0] & 0x7f
-        length = 0
-        if len(binary) < num_len_oct + 1:
-            return (0, b'')
-        for i in range(1, 1+num_len_oct):
-            length <<= 8
-            length |= binary[i]
-        return (length, binary[1+num_len_oct:])
-
-
-def bertlv_encode_len(length: int) -> bytes:
-    """Encode a single Length value according to ITU-T X.690 8.1.3;
-    only the definite form is supported here.
-    Args:
-            length : length value to be encoded
-    Returns:
-            binary output data of BER-TLV length field
-    """
-    if length < 0x80:
-        return length.to_bytes(1, 'big')
-    elif length <= 0xff:
-        return b'\x81' + length.to_bytes(1, 'big')
-    elif length <= 0xffff:
-        return b'\x82' + length.to_bytes(2, 'big')
-    elif length <= 0xffffff:
-        return b'\x83' + length.to_bytes(3, 'big')
-    elif length <= 0xffffffff:
-        return b'\x84' + length.to_bytes(4, 'big')
-    else:
-        raise ValueError("Length > 32bits not supported")
-
-
-def bertlv_parse_one(binary: bytes) -> Tuple[dict, int, bytes, bytes]:
-    """Parse a single TLV IE at the start of the given binary data.
-    Args:
-            binary : binary input data of BER-TLV length field
-    Returns:
-            Tuple of (tag:dict, len:int, remainder:bytes)
-    """
-    (tagdict, remainder) = bertlv_parse_tag(binary)
-    (length, remainder) = bertlv_parse_len(remainder)
-    value = remainder[:length]
-    remainder = remainder[length:]
-    return (tagdict, length, value, remainder)
-
-
-def dgi_parse_tag_raw(binary: bytes) -> Tuple[int, bytes]:
-    # In absence of any clear spec guidance we assume it's always 16 bit
-    return int.from_bytes(binary[:2], 'big'), binary[2:]
-
-def dgi_encode_tag(t: int) -> bytes:
-    return t.to_bytes(2, 'big')
-
-def dgi_encode_len(length: int) -> bytes:
-    """Encode a single Length value according to GlobalPlatform Systems Scripting Language
-    Specification v1.1.0 Annex B.
-    Args:
-            length : length value to be encoded
-    Returns:
-            binary output data of encoded length field
-    """
-    if length < 255:
-        return length.to_bytes(1, 'big')
-    elif length <= 0xffff:
-        return b'\xff' + length.to_bytes(2, 'big')
-    else:
-        raise ValueError("Length > 32bits not supported")
-
-def dgi_parse_len(binary: bytes) -> Tuple[int, bytes]:
-    """Parse a single Length value according to  GlobalPlatform Systems Scripting Language
-    Specification v1.1.0 Annex B.
-    Args:
-            binary : binary input data of BER-TLV length field
-    Returns:
-            Tuple of (length, remainder)
-    """
-    if binary[0] == 255:
-        assert len(binary) >= 3
-        return ((binary[1] << 8) | binary[2]), binary[3:]
-    else:
-        return binary[0], binary[1:]
-
 # IMSI encoded format:
 # For IMSI 0123456789ABCDE:
 #
@@ -411,6 +45,10 @@ def dgi_parse_len(binary: bytes) -> Tuple[int, bytes]:
 # Because of this, an odd length IMSI fits exactly into len(imsi) + 1 // 2 bytes, whereas an
 # even length IMSI only uses half of the last byte.
 
+SwHexstr = NewType('SwHexstr', str)
+SwMatchstr = NewType('SwMatchstr', str)
+ResTuple = Tuple[Hexstr, SwHexstr]
+
 def enc_imsi(imsi: str):
     """Converts a string IMSI into the encoded value of the EF"""
     l = half_round_up(
@@ -703,109 +341,6 @@ def derive_mnc(digit1: int, digit2: int, digit3: int = 0x0f) -> int:
 
     return mnc
 
-
-def dec_msisdn(ef_msisdn: Hexstr) -> Optional[Tuple[int, int, Optional[str]]]:
-    """
-    Decode MSISDN from EF.MSISDN or EF.ADN (same structure).
-    See 3GPP TS 31.102, section 4.2.26 and 4.4.2.3.
-    """
-
-    # Convert from str to (kind of) 'bytes'
-    ef_msisdn = h2b(ef_msisdn)
-
-    # Make sure mandatory fields are present
-    if len(ef_msisdn) < 14:
-        raise ValueError("EF.MSISDN is too short")
-
-    # Skip optional Alpha Identifier
-    xlen = len(ef_msisdn) - 14
-    msisdn_lhv = ef_msisdn[xlen:]
-
-    # Parse the length (in bytes) of the BCD encoded number
-    bcd_len = msisdn_lhv[0]
-    # BCD length = length of dial num (max. 10 bytes) + 1 byte ToN and NPI
-    if bcd_len == 0xff:
-        return None
-    elif bcd_len > 11 or bcd_len < 1:
-        raise ValueError(
-            "Length of MSISDN (%d bytes) is out of range" % bcd_len)
-
-    # Parse ToN / NPI
-    ton = (msisdn_lhv[1] >> 4) & 0x07
-    npi = msisdn_lhv[1] & 0x0f
-    bcd_len -= 1
-
-    # No MSISDN?
-    if not bcd_len:
-        return (npi, ton, None)
-
-    msisdn = swap_nibbles(b2h(msisdn_lhv[2:][:bcd_len])).rstrip('f')
-    # International number 10.5.118/3GPP TS 24.008
-    if ton == 0x01:
-        msisdn = '+' + msisdn
-
-    return (npi, ton, msisdn)
-
-
-def enc_msisdn(msisdn: str, npi: int = 0x01, ton: int = 0x03) -> Hexstr:
-    """
-    Encode MSISDN as LHV so it can be stored to EF.MSISDN.
-    See 3GPP TS 31.102, section 4.2.26 and 4.4.2.3. (The result
-    will not contain the optional Alpha Identifier at the beginning.)
-
-    Default NPI / ToN values:
-      - NPI: ISDN / telephony numbering plan (E.164 / E.163),
-      - ToN: network specific or international number (if starts with '+').
-    """
-
-    # If no MSISDN is supplied then encode the file contents as all "ff"
-    if msisdn in ["", "+"]:
-        return "ff" * 14
-
-    # Leading '+' indicates International Number
-    if msisdn[0] == '+':
-        msisdn = msisdn[1:]
-        ton = 0x01
-
-    # An MSISDN must not exceed 20 digits
-    if len(msisdn) > 20:
-        raise ValueError("msisdn must not be longer than 20 digits")
-
-    # Append 'f' padding if number of digits is odd
-    if len(msisdn) % 2 > 0:
-        msisdn += 'f'
-
-    # BCD length also includes NPI/ToN header
-    bcd_len = len(msisdn) // 2 + 1
-    npi_ton = (npi & 0x0f) | ((ton & 0x07) << 4) | 0x80
-    bcd = rpad(swap_nibbles(msisdn), 10 * 2)  # pad to 10 octets
-
-    return ('%02x' % bcd_len) + ('%02x' % npi_ton) + bcd + ("ff" * 2)
-
-
-def is_hex(string: str, minlen: int = 2, maxlen: Optional[int] = None) -> bool:
-    """
-    Check if a string is a valid hexstring
-    """
-
-    # Filter obviously bad strings
-    if not string:
-        return False
-    if len(string) < minlen or minlen < 2:
-        return False
-    if len(string) % 2:
-        return False
-    if maxlen and len(string) > maxlen:
-        return False
-
-    # Try actual encoding to be sure
-    try:
-        _try_encode = h2b(string)
-        return True
-    except:
-        return False
-
-
 def sanitize_pin_adm(pin_adm, pin_adm_hex=None) -> Hexstr:
     """
     The ADM pin can be supplied either in its hexadecimal form or as
@@ -937,26 +472,6 @@ def tabulate_str_list(str_list, width: int = 79, hspace: int = 2, lspace: int =
     return '\n'.join(table)
 
 
-def auto_int(x):
-    """Helper function for argparse to accept hexadecimal integers."""
-    return int(x, 0)
-
-def _auto_uint(x, max_val: int):
-    """Helper function for argparse to accept hexadecimal or decimal integers."""
-    ret = int(x, 0)
-    if ret < 0 or ret > max_val:
-        raise argparse.ArgumentTypeError('Number exceeds permited value range (0, %u)' %  max_val)
-    return ret
-
-def auto_uint7(x):
-    return _auto_uint(x, 127)
-
-def auto_uint8(x):
-    return _auto_uint(x, 255)
-
-def auto_uint16(x):
-    return _auto_uint(x, 65535)
-
 def expand_hex(hexstring, length):
     """Expand a given hexstring to a specified length by replacing "." or ".."
        with a filler that is derived from the neighboring nibbles respective
@@ -1011,17 +526,6 @@ def expand_hex(hexstring, length):
     return hexstring
 
 
-class JsonEncoder(json.JSONEncoder):
-    """Extend the standard library JSONEncoder with support for more types."""
-
-    def default(self, o):
-        if isinstance(o, (BytesIO, bytes, bytearray)):
-            return b2h(o)
-        elif isinstance(o, datetime.datetime):
-            return o.isoformat()
-        return json.JSONEncoder.default(self, o)
-
-
 def boxed_heading_str(heading, width=80):
     """Generate a string that contains a boxed heading."""
     # Auto-enlarge box if heading exceeds length
@@ -1035,6 +539,52 @@ def boxed_heading_str(heading, width=80):
     return res
 
 
+def parse_command_apdu(apdu: bytes) -> int:
+    """Parse a given command APDU and return case (see also ISO/IEC 7816-3, Table 12 and Figure 26),
+    lc, le and the data field.
+
+    Args:
+            apdu : hexstring that contains the command APDU
+    Returns:
+            tuple containing case, lc and le values of the APDU (case, lc, le, data)
+    """
+
+    if len(apdu) == 4:
+        # Case #1, No command data field, no response data field
+        lc = 0
+        le = 0
+        data = b''
+        return (1, lc, le, data)
+    elif len(apdu) == 5:
+        # Case #2, No command data field, response data field present
+        lc = 0
+        le = apdu[4]
+        if le == 0:
+            le = 256
+        data = b''
+        return (2, lc, le, data)
+    elif len(apdu) > 5:
+        lc = apdu[4];
+        if lc == 0:
+            lc = 256
+        data = apdu[5:lc+5]
+        if len(apdu) == 5 + lc:
+            # Case #3, Command data field present, no response data field
+            le = 0
+            return (3, lc, le, data)
+        elif len(apdu) == 5 + lc + 1:
+            # Case #4, Command data field present, no response data field
+            le = apdu[5 + lc]
+            if le == 0:
+                le = 256
+            return (4, lc, le, data)
+        else:
+            raise ValueError('invalid APDU (%s), Lc=0x%02x (%d) does not match the length (%d) of the data field'
+                             % (b2h(apdu), lc, lc, len(apdu[5:])))
+    else:
+        raise ValueError('invalid APDU (%s), too short!' % b2h(apdu))
+
+
 class DataObject(abc.ABC):
     """A DataObject (DO) in the sense of ISO 7816-4.  Contrary to 'normal' TLVs where one
     simply has any number of different TLVs that may occur in any order at any point, ISO 7816
@@ -1430,35 +980,3 @@ class CardCommandSet:
         if cla and not cmd.match_cla(cla):
             return None
         return cmd
-
-
-def all_subclasses(cls) -> set:
-    """Recursively get all subclasses of a specified class"""
-    return set(cls.__subclasses__()).union([s for c in cls.__subclasses__() for s in all_subclasses(c)])
-
-def is_hexstr_or_decimal(instr: str) -> str:
-    """Method that can be used as 'type' in argparse.add_argument() to validate the value consists of
-    [hexa]decimal digits only."""
-    if instr.isdecimal():
-        return instr
-    if not all(c in string.hexdigits for c in instr):
-        raise ValueError('Input must be [hexa]decimal')
-    if len(instr) & 1:
-        raise ValueError('Input has un-even number of hex digits')
-    return instr
-
-def is_hexstr(instr: str) -> str:
-    """Method that can be used as 'type' in argparse.add_argument() to validate the value consists of
-    an even sequence of hexadecimal digits only."""
-    if not all(c in string.hexdigits for c in instr):
-        raise ValueError('Input must be hexadecimal')
-    if len(instr) & 1:
-        raise ValueError('Input has un-even number of hex digits')
-    return instr
-
-def is_decimal(instr: str) -> str:
-    """Method that can be used as 'type' in argparse.add_argument() to validate the value consists of
-    an even sequence of decimal digits only."""
-    if not instr.isdecimal():
-        raise ValueError('Input must decimal')
-    return instr

pySim/wsrc/__init__.py

@@ -0,0 +1,3 @@
+
+WSRC_DEFAULT_PORT_USER = 4220
+WSRC_DEFAULT_PORT_CARD = 4221

pySim/wsrc/client_blocking.py

@@ -0,0 +1,68 @@
+"""Connect smartcard to a remote server, so the remote server can take control and
+perform commands on it."""
+
+import abc
+import json
+import logging
+from typing import Optional
+from websockets.sync.client import connect
+from osmocom.utils import b2h
+
+logger = logging.getLogger(__name__)
+
+class WsClientBlocking(abc.ABC):
+    """Generalized synchronous/blocking client for the WSRC (Web Socket Remote Card) protocol"""
+
+    def __init__(self, cltype: str, ws_uri: str):
+        self.client_type = cltype
+        self.ws_uri = ws_uri
+        self.ws = None
+
+    def connect(self):
+        self.ws = connect(uri=self.ws_uri)
+        self.perform_outbound_hello()
+
+    def tx_json(self, msg_type: str, d: dict = {}):
+        """JSON-Encode and transmit a message to the given websocket."""
+        d['msg_type'] = msg_type
+        d_js = json.dumps(d)
+        logger.debug("Tx: %s", d_js)
+        self.ws.send(d_js)
+
+    def tx_error(self, message: str):
+        event = {
+            'message': message,
+        }
+        self.tx_json('error', event)
+
+    def rx_json(self):
+        """Receive a single message from the given websocket and JSON-decode it."""
+        rx = self.ws.recv()
+        rx_js = json.loads(rx)
+        logger.debug("Rx: %s", rx_js)
+        assert 'msg_type' in rx
+        return rx_js
+
+    def transceive_json(self, tx_msg_type: str, tx_d: Optional[dict], rx_msg_type: str) -> dict:
+        self.tx_json(tx_msg_type, tx_d)
+        rx = self.rx_json()
+        assert rx['msg_type'] == rx_msg_type
+        return rx
+
+    def perform_outbound_hello(self, tx: dict = {}):
+        if not 'client_type' in tx:
+            tx['client_type'] = self.client_type
+        self.tx_json('hello', tx)
+        rx = self.rx_json()
+        assert rx['msg_type'] == 'hello_ack'
+        return rx
+
+    def rx_and_execute_cmd(self):
+        """Receve and dispatch/execute a single command from the server."""
+        rx = self.rx_json()
+        handler = getattr(self, 'handle_rx_%s' % rx['msg_type'], None)
+        if handler:
+            handler(rx)
+        else:
+            logger.error('Received unknown/unsupported msg_type %s' % rx['msg_type'])
+            self.tx_error('Message type "%s" is not supported' % rx['msg_type'])

requirements.txt

@@ -3,9 +3,9 @@ pyserial
 pytlv
 cmd2>=1.5
 jsonpath-ng
-construct>=2.9.51
+construct>=2.10.70
 bidict
-gsm0338
+pyosmocom>=0.0.6
 pyyaml>=5.1
 termcolor
 colorlog

setup.py

@@ -3,7 +3,15 @@ from setuptools import setup
 setup(
     name='pySim',
     version='1.0',
-    packages=['pySim', 'pySim.legacy', 'pySim.transport', 'pySim.apdu', 'pySim.apdu_source'],
+    packages=[
+        'pySim',
+        'pySim.apdu',
+        'pySim.apdu_source',
+        'pySim.esim',
+        'pySim.global_platform',
+        'pySim.legacy',
+        'pySim.transport',
+    ],
     url='https://osmocom.org/projects/pysim/wiki',
     license='GPLv2',
     author_email='simtrace@lists.osmocom.org',
@@ -14,9 +22,9 @@ setup(
         "pytlv",
         "cmd2 >= 1.5.0",
         "jsonpath-ng",
-        "construct >= 2.9.51",
+        "construct >= 2.10.70",
         "bidict",
-        "gsm0338",
+        "pyosmocom >= 0.0.6",
         "pyyaml >= 5.1",
         "termcolor",
         "colorlog",