Merge branch 'chrysn/for-29033' into chrysn/ota

Change-Id: Ie0d9b0c04fe618d3809e117823c519b28d15ea05

README.md

@@ -16,7 +16,7 @@ network, and want to issue your own SIM/USIM cards for that network.
 Homepage and Manual
 -------------------
 
-Please visit the [official homepage](https://osmocom.org/projects/pysim/wiki) for usage instructions, manual and examples.
+Please visit the [official homepage](https://osmocom.org/projects/pysim/wiki) for usage instructions, manual and examples.  The user manual can also be built locally from this source code by ``cd docs && make html latexpdf`` for HTML and PDF format, respectively.
 
 Git Repository
 --------------
@@ -45,7 +45,7 @@ Please install the following dependencies:
 
 Example for Debian:
 ```
-apt-get install python3-pyscard python3-serial python3-pip python3-yaml
+apt-get install python3-pyscard python3-serial python3-pip python3-yaml python3-termcolor python3-colorlog
 pip3 install -r requirements.txt
 ```
 
@@ -93,46 +93,30 @@ We are using a gerrit-based patch review process explained at
 <https://osmocom.org/projects/cellular-infrastructure/wiki/Gerrit>
 
 
-Usage Examples
---------------
+Documentation
+-------------
 
- * Program customizable SIMs. Two modes are possible:
+The pySim user manual can be built from this very source code by means
+of sphinx (with sphinxcontrib-napoleon and sphinx-argparse).  See the
+Makefile in the 'docs' directory.
 
-  - one where you specify every parameter manually:
-```
-./pySim-prog.py -n 26C3 -c 49 -x 262 -y 42 -i <IMSI> -s <ICCID>
-```
+A pre-rendered HTML user manual of the current pySim 'git master' is
+available from <https://downloads.osmocom.org/docs/latest/pysim/> and
+a downloadable PDF version is published at
+<https://downloads.osmocom.org/docs/latest/osmopysim-usermanual.pdf>.
 
-  - one where they are generated from some minimal set:
-```
-./pySim-prog.py -n 26C3 -c 49 -x 262 -y 42 -z <random_string_of_choice> -j <card_num>
-```
+A slightly dated video presentation about pySim-shell can be found at
+<https://media.ccc.de/v/osmodevcall-20210409-laforge-pysim-shell>.
 
-With ``<random_string_of_choice>`` and ``<card_num>``, the soft will generate
-'predictable' IMSI and ICCID, so make sure you choose them so as not to
-conflict with anyone. (for e.g. your name as ``<random_string_of_choice>`` and
-0 1 2 ... for ``<card num>``).
 
-You also need to enter some parameters to select the device:
+pySim-shell vs. legacy tools
+----------------------------
 
- -t TYPE : type of card (``supersim``, ``magicsim``, ``fakemagicsim`` or try ``auto``)  
- -d DEV  : Serial port device (default ``/dev/ttyUSB0``)  
- -b BAUD : Baudrate (default 9600)  
+While you will find a lot of online resources still describing the use of
+pySim-prog.py and pySim-read.py, those tools are considered legacy by
+now and have by far been superseded by the much more capable
+pySim-shell.  We strongly encourage users to adopt pySim-shell, unless
+they have very specific requirements like batch programming of large
+quantities of cards, which is about the only remaining use case for the
+legacy tools.
 
- * Interact with SIMs from a python interactive shell (e.g. ipython):
-
-```
-from pySim.transport.serial import SerialSimLink
-from pySim.commands import SimCardCommands
-
-sl = SerialSimLink(device='/dev/ttyUSB0', baudrate=9600)
-sc = SimCardCommands(sl)
-
-sl.wait_for_card()
-
-	# Print IMSI
-print(sc.read_binary(['3f00', '7f20', '6f07']))
-
-	# Run A3/A8
-print(sc.run_gsm('00112233445566778899aabbccddeeff'))
-```

contrib/jenkins.sh

@@ -24,6 +24,11 @@ pip install jsonpath-ng
 pip install construct
 pip install bidict
 pip install gsm0338
+pip install termcolor
+pip install colorlog
+pip install pycryptodome
+# we need this direct git install, as pypi only lists the python2.7 only release 0.3 from 2013 :(
+pip install git+https://github.com/hologram-io/smpp.pdu
 
 # Execute automatically discovered unit tests first
 python -m unittest discover -v -s tests/

contrib/sim-rest-server.py

@@ -2,7 +2,7 @@
 
 # RESTful HTTP service for performing authentication against USIM cards
 #
-# (C) 2021 by Harald Welte <laforge@osmocom.org>
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,7 +21,7 @@ import json
 import sys
 import argparse
 
-from klein import run, route
+from klein import Klein
 
 from pySim.transport import ApduTracer
 from pySim.transport.pcsc import PcscSimLink
@@ -50,74 +50,97 @@ def connect_to_card(slot_nr:int):
 
     return tp, scc, card
 
+class ApiError:
+    def __init__(self, msg:str, sw=None):
+        self.msg = msg
+        self.sw = sw
 
-@route('/sim-auth-api/v1/slot/<int:slot>')
-def auth(request, slot):
-    """REST API endpoint for performing authentication against a USIM.
-       Expects a JSON body containing RAND and AUTN.
-       Returns a JSON body containing RES, CK, IK and Kc."""
-    try:
-        # there are two hex-string JSON parameters in the body: rand and autn
-        content = json.loads(request.content.read())
-        rand = content['rand']
-        autn = content['autn']
-    except:
-        request.setResponseCode(400)
-        return "Malformed Request"
+    def __str__(self):
+        d = {'error': {'message':self.msg}}
+        if self.sw:
+            d['error']['status_word'] = self.sw
+        return json.dumps(d)
 
-    try:
-        tp, scc, card = connect_to_card(slot)
-    except ReaderError:
-        request.setResponseCode(404)
-        return "Specified SIM Slot doesn't exist"
-    except ProtocolError:
-        request.setResponseCode(500)
-        return "Error"
-    except NoCardError:
+
+def set_headers(request):
+    request.setHeader('Content-Type', 'application/json')
+
+class SimRestServer:
+    app = Klein()
+
+    @app.handle_errors(NoCardError)
+    def no_card_error(self, request, failure):
+        set_headers(request)
         request.setResponseCode(410)
-        return "No SIM card inserted in slot"
+        return str(ApiError("No SIM card inserted in slot"))
+
+    @app.handle_errors(ReaderError)
+    def reader_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(404)
+        return str(ApiError("Reader Error: Specified SIM Slot doesn't exist"))
+
+    @app.handle_errors(ProtocolError)
+    def protocol_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(500)
+        return str(ApiError("Protocol Error: %s" % failure.value))
+
+    @app.handle_errors(SwMatchError)
+    def sw_match_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(500)
+        sw = failure.value.sw_actual
+        if sw == '9862':
+            return str(ApiError("Card Authentication Error - Incorrect MAC", sw))
+        elif sw == '6982':
+            return str(ApiError("Security Status not satisfied - Card PIN enabled?", sw))
+        else:
+            return str(ApiError("Card Communication Error %s" % failure.value), sw)
+
+
+    @app.route('/sim-auth-api/v1/slot/<int:slot>')
+    def auth(self, request, slot):
+        """REST API endpoint for performing authentication against a USIM.
+           Expects a JSON body containing RAND and AUTN.
+           Returns a JSON body containing RES, CK, IK and Kc."""
+        try:
+            # there are two hex-string JSON parameters in the body: rand and autn
+            content = json.loads(request.content.read())
+            rand = content['rand']
+            autn = content['autn']
+        except:
+            set_headers(request)
+            request.setResponseCode(400)
+            return str(ApiError("Malformed Request"))
+
+        tp, scc, card = connect_to_card(slot)
 
-    try:
         card.select_adf_by_aid(adf='usim')
         res, sw = scc.authenticate(rand, autn)
-    except SwMatchError as e:
-        request.setResponseCode(500)
-        return "Communication Error %s" % e
 
-    tp.disconnect()
+        tp.disconnect()
 
-    return json.dumps(res, indent=4)
+        set_headers(request)
+        return json.dumps(res, indent=4)
 
-@route('/sim-info-api/v1/slot/<int:slot>')
-def info(request, slot):
-    """REST API endpoint for obtaining information about an USIM.
-    Expects empty body in request.
-    Returns a JSON body containing ICCID, IMSI."""
+    @app.route('/sim-info-api/v1/slot/<int:slot>')
+    def info(self, request, slot):
+        """REST API endpoint for obtaining information about an USIM.
+        Expects empty body in request.
+        Returns a JSON body containing ICCID, IMSI."""
 
-    try:
         tp, scc, card = connect_to_card(slot)
-    except ReaderError:
-        request.setResponseCode(404)
-        return "Specified SIM Slot doesn't exist"
-    except ProtocolError:
-        request.setResponseCode(500)
-        return "Error"
-    except NoCardError:
-        request.setResponseCode(410)
-        return "No SIM card inserted in slot"
 
-    try:
         card.select_adf_by_aid(adf='usim')
         iccid, sw = card.read_iccid()
         imsi, sw = card.read_imsi()
         res = {"imsi": imsi, "iccid": iccid }
-    except SwMatchError as e:
-        request.setResponseCode(500)
-        return "Communication Error %s" % e
 
-    tp.disconnect()
+        tp.disconnect()
 
-    return json.dumps(res, indent=4)
+        set_headers(request)
+        return json.dumps(res, indent=4)
 
 
 def main(argv):
@@ -128,7 +151,8 @@ def main(argv):
 
     args = parser.parse_args()
 
-    run(args.host, args.port)
+    srr = SimRestServer()
+    srr.app.run(args.host, args.port)
 
 if __name__ == "__main__":
     main(sys.argv)

docs/conf.py

@@ -18,7 +18,7 @@ sys.path.insert(0, os.path.abspath('..'))
 # -- Project information -----------------------------------------------------
 
 project = 'osmopysim-usermanual'
-copyright = '2009-2021 by Sylvain Munaut, Harald Welte, Philipp Maier, Supreeth Herle'
+copyright = '2009-2022 by Sylvain Munaut, Harald Welte, Philipp Maier, Supreeth Herle'
 author = 'Sylvain Munaut, Harald Welte, Philipp Maier, Supreeth Herle'
 
 

docs/index.rst

@@ -34,7 +34,7 @@ pySim consists of several parts:
 * the [legacy] :ref:`pySim-prog and pySim-read tools<Legacy tools>`
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 3
    :caption: Contents:
 
    shell

docs/legacy.rst

@@ -4,6 +4,9 @@ Legacy tools
 *legacy tools* are the classic ``pySim-prog`` and ``pySim-read`` programs that
 existed long before ``pySim-shell``.
 
+These days, you should primarily use ``pySim-shell`` instead of these
+legacy tools.
+
 pySim-prog
 ----------
 
@@ -45,6 +48,11 @@ pySim-read
 ``pySim-read`` allows you to read some data from a SIM card.  It will only some files
 of the card, and will only read files accessible to a normal user (without any special authentication)
 
+These days, you should use the ``export`` command of ``pySim-shell``
+instead. It performs a much more comprehensive export of all of the
+[standard] files that can be found on the card.  To get a human-readable
+decode instead of the raw hex export, you can use ``export --json``.
+
 Specifically, pySim-read will dump the following:
 
 * MF

docs/shell.rst

@@ -80,9 +80,11 @@ This will
   pySIM-shell (MF)> select ADF.USIM
   {
       "file_descriptor": {
-          "shareable": true,
-          "file_type": "df",
-          "structure": "no_info_given"
+          "file_descriptor_byte": {
+              "shareable": true,
+              "file_type": "df",
+              "structure": "no_info_given"
+          }
       },
       "df_name": "A0000000871002FFFFFFFF8907090000",
       "proprietary_info": {
@@ -96,6 +98,41 @@ This will
   pySIM-shell (MF/ADF.USIM)>
 
 
+status
+~~~~~~
+
+The ``status`` command [re-]obtains the File Control Template of the
+currently-selected file and print its decoded output.
+
+Example:
+
+::
+
+  pySIM-shell (MF/ADF.ISIM)> status
+  {
+      "file_descriptor": {
+          "file_descriptor_byte": {
+              "shareable": true,
+              "file_type": "df",
+              "structure": "no_info_given"
+          },
+          "record_len": null,
+          "num_of_rec": null
+      },
+      "file_identifier": "ff01",
+      "df_name": "a0000000871004ffffffff8907090000",
+      "proprietary_information": {
+          "uicc_characteristics": "71",
+          "available_memory": 101640
+      },
+      "life_cycle_status_integer": "operational_activated",
+      "security_attrib_compact": "00",
+      "pin_status_template_do": {
+          "ps_do": "70",
+          "key_reference": 11
+      }
+  }
+
 
 change_chv
 ~~~~~~~~~~
@@ -127,9 +164,6 @@ unblock_chv
 
 verify_chv
 ~~~~~~~~~~
-This command allows you to verify a CHV (PIN), which is how the specifications call
-it if you authenticate yourself with the said CHV/PIN.
-
 .. argparse::
    :module: pySim-shell
    :func: Iso7816Commands.verify_chv_parser
@@ -141,7 +175,9 @@ Deactivate the currently selected file.  This used to be called INVALIDATE in TS
 
 activate_file
 ~~~~~~~~~~~~~
-Activate the currently selected file.  This used to be called REHABILITATE in TS 11.11.
+.. argparse::
+   :module: pySim-shell
+   :func: Iso7816Commands.activate_file_parser
 
 open_channel
 ~~~~~~~~~~~~
@@ -170,6 +206,7 @@ including the electrical power down.
    :func: Iso7816Commands.suspend_uicc_parser
 
 
+
 pySim commands
 --------------
 
@@ -179,7 +216,6 @@ a complex sequence of card-commands.
 
 desc
 ~~~~
-
 Display human readable file description for the currently selected file.
 
 
@@ -189,6 +225,17 @@ dir
    :module: pySim-shell
    :func: PySimCommands.dir_parser
 
+Example:
+::
+
+  pySIM-shell (MF)> dir
+  MF
+  3f00
+   ..          ADF.USIM    DF.SYSTEM   EF.DIR      EF.UMPC
+   ADF.ARA-M   DF.EIRENE   DF.TELECOM  EF.ICCID    MF
+   ADF.ISIM    DF.GSM      EF.ARR      EF.PL
+  14 files
+
 
 export
 ~~~~~~
@@ -208,15 +255,27 @@ all/most files.
 
 tree
 ~~~~
-
 Display a tree of the card filesystem.  It is important to note that this displays a tree
 of files that might potentially exist (based on the card profile).  In order to determine if
 a given file really exists on a given card, you have to try to select that file.
 
+Example:
+::
+
+  pySIM-shell (MF)> tree --help
+  EF.DIR                    2f00 Application Directory
+  EF.ICCID                  2fe2 ICC Identification
+  EF.PL                     2f05 Preferred Languages
+  EF.ARR                    2f06 Access Rule Reference
+  EF.UMPC                   2f08 UICC Maximum Power Consumption
+  DF.TELECOM                7f10 None
+    EF.ADN                  6f3a Abbreviated Dialing Numbers
+  ...
+
+
 
 verify_adm
 ~~~~~~~~~~
-
 Verify the ADM (Administrator) PIN specified as argument.  This is typically needed in order
 to get write/update permissions to most of the files on SIM cards.
 
@@ -244,8 +303,6 @@ bulk_script
    :module: pySim-shell
    :func: PysimApp.bulk_script_parser
 
-Run a script for bulk-provisioning of multiple cards.
-
 
 echo
 ~~~~
@@ -254,6 +311,13 @@ echo
    :func: PysimApp.echo_parser
 
 
+apdu
+~~~~
+.. argparse::
+   :module: pySim-shell
+   :func: PySimCommands.apdu_cmd_parser
+
+
 
 Linear Fixed EF commands
 ------------------------
@@ -320,6 +384,13 @@ back to the record on the SIM card.
 This allows for easy interactive modification of records.
 
 
+decode_hex
+~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: LinFixedEF.ShellCommands.dec_hex_parser
+
+
 
 Transparent EF commands
 -----------------------
@@ -396,6 +467,13 @@ to the SIM card.
 This allows for easy interactive modification of file contents.
 
 
+decode_hex
+~~~~~~~~~~
+.. argparse::
+   :module: pySim.filesystem
+   :func: TransparentEF.ShellCommands.dec_hex_parser
+
+
 
 BER-TLV EF commands
 -------------------
@@ -442,6 +520,26 @@ authenticate
    :module: pySim.ts_31_102
    :func: ADF_USIM.AddlShellCommands.authenticate_parser
 
+terminal_profile
+~~~~~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.ts_31_102
+   :func: ADF_USIM.AddlShellCommands.term_prof_parser
+
+envelope
+~~~~~~~~
+.. argparse::
+   :module: pySim.ts_31_102
+   :func: ADF_USIM.AddlShellCommands.envelope_parser
+
+envelope_sms
+~~~~~~~~~~~~
+.. argparse::
+   :module: pySim.ts_31_102
+   :func: ADF_USIM.AddlShellCommands.envelope_sms_parser
+
+
+
 
 ARA-M commands
 --------------
@@ -504,21 +602,14 @@ Perform Config handshake with ARA-M applet: Tell it our version and retrieve its
 
 NOTE: Not supported in all ARA-M implementations.
 
-.. argparse::
-   :module: pySim.ara_m
-   :func: ADF_ARAM.AddlShellCommands.get_config_parser
-
 
 aram_store_ref_ar_do
 ~~~~~~~~~~~~~~~~~~~~
-Store a [new] access rule on the ARA-M applet.
-
 .. argparse::
    :module: pySim.ara_m
    :func: ADF_ARAM.AddlShellCommands.store_ref_ar_do_parse
 
 For example, to store an Android UICC carrier privilege rule for the SHA1 hash of the certificate used to sign the CoIMS android app of Supreeth Herle (https://github.com/herlesupreeth/CoIMS_Wiki) you can use the following command:
-
 ::
 
   pySIM-shell (MF/ADF.ARA-M)> aram_store_ref_ar_do --aid FFFFFFFFFFFF --device-app-id E46872F28B350B7E1F140DE535C2A8D5804F0BE3 --android-permissions 0000000000000001 --apdu-always

ota_test.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python3
+
+from pySim.ota import *
+from pySim.sms import SMS_SUBMIT, SMS_DELIVER, AddressField
+from pySim.utils import h2b, h2b
+
+# KIC1 + KID1 of 8988211000000515398
+#KIC1 = h2b('C039ED58F7B81446105E79EBFD373038')
+#KID1 = h2b('1799B93FE53F430BD7FD4810C77E1FDF')
+#KIC3 = h2b('167F2576D64C8D41862954875C8D7979')
+#KID3 = h2b('ECAE122B0E6AE4186D6487D50FDC0922')
+
+# KIC1 + KID1 of 8988211000000467285
+KIC1 = h2b('D0FDA31990D8D64178601317191669B4')
+KID1 = h2b('D24EB461799C5E035C77451FD9404463')
+KIC3 = h2b('C21DD66ACAC13CB3BC8B331B24AFB57B')
+KID3 = h2b('12110C78E678C25408233076AA033615')
+
+od = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3, kic=KIC3,
+               algo_auth='triple_des_cbc2', kid_idx=3, kid=KID3)
+print(od.crypt)
+print(od.auth)
+
+dialect = OtaDialectSms()
+
+# RAM: B00000
+# SIM RFM: B00010
+# USIM RFM: B00011
+tar = h2b('B00011')
+
+spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+       'por_shall_be_ciphered':True, 'por_rc_cc_ds': 'cc', 'por': 'por_required'}
+outp = dialect.encode_cmd(od, tar, spi, apdu=b'\x00\xa4\x00\x04\x02\x3f\x00')
+print("result: %s" % b2h(outp))
+
+with_udh = b'\x02\x70\x00' + outp
+print("with_udh: %s" % b2h(with_udh))
+
+
+da = AddressField('12345678', 'unknown', 'isdn_e164')
+#tpdu = SMS_SUBMIT(tp_udhi=True, tp_mr=0x23, tp_da=da, tp_pid=0x7F, tp_dcs=0xF6, tp_udl=3, tp_ud=with_udh)
+tpdu = SMS_DELIVER(tp_udhi=True, tp_oa=da, tp_pid=0x7F, tp_dcs=0xF6, tp_scts=h2b('22705200000000'), tp_udl=3, tp_ud=with_udh)
+print(tpdu)
+print("tpdu: %s" % b2h(tpdu.toBytes()))
+
+spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+       'por_shall_be_ciphered':True, 'por_rc_cc_ds': 'cc', 'por': 'por_required'}
+dialect.decode_resp(od, spi, '027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c')
+
+spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+       'por_shall_be_ciphered':False, 'por_rc_cc_ds': 'cc', 'por': 'por_required'}
+dialect.decode_resp(od, spi, '027100001612b000110000000000000055f47118381175fb01612f')
+
+spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+       'por_shall_be_ciphered':False, 'por_rc_cc_ds': 'no_rc_cc_ds', 'por': 'por_required'}
+dialect.decode_resp(od, spi, '027100000e0ab000110000000000000001612f')
+

pySim-shell.py

@@ -2,7 +2,7 @@
 
 # Interactive shell for working with SIM / UICC / USIM / ISIM cards
 #
-# (C) 2021 by Harald Welte <laforge@osmocom.org>
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,7 +23,7 @@ import json
 import traceback
 
 import cmd2
-from cmd2 import style, fg, bg
+from cmd2 import style, fg
 from cmd2 import CommandSet, with_default_category, with_argparser
 import argparse
 
@@ -32,27 +32,26 @@ import sys
 from pathlib import Path
 from io import StringIO
 
-from pySim.ts_51_011 import EF, DF, EF_SST_map
-from pySim.ts_31_102 import EF_UST_map, EF_USIM_ADF_map
-from pySim.ts_31_103 import EF_IST_map, EF_ISIM_ADF_map
+from pprint import pprint as pp
 
 from pySim.exceptions import *
 from pySim.commands import SimCardCommands
-from pySim.transport import init_reader, ApduTracer, argparse_add_reader_args
+from pySim.transport import init_reader, ApduTracer, argparse_add_reader_args, ProactiveHandler
 from pySim.cards import card_detect, SimCard
-from pySim.utils import h2b, swap_nibbles, rpad, b2h, h2s, JsonEncoder, bertlv_parse_one
-from pySim.utils import dec_st, sanitize_pin_adm, tabulate_str_list, is_hex, boxed_heading_str
+from pySim.utils import h2b, swap_nibbles, rpad, b2h, JsonEncoder, bertlv_parse_one, sw_match
+from pySim.utils import sanitize_pin_adm, tabulate_str_list, boxed_heading_str, Hexstr
 from pySim.card_handler import CardHandler, CardHandlerAuto
 
-from pySim.filesystem import CardMF, RuntimeState, CardDF, CardADF, CardModel
+from pySim.filesystem import RuntimeState, CardDF, CardADF, CardModel
 from pySim.profile import CardProfile
-from pySim.ts_51_011 import CardProfileSIM, DF_TELECOM, DF_GSM
 from pySim.ts_102_221 import CardProfileUICC
-from pySim.ts_102_221 import CardProfileUICCSIM
+from pySim.ts_102_222 import Ts102222Commands
 from pySim.ts_31_102 import CardApplicationUSIM
 from pySim.ts_31_103 import CardApplicationISIM
 from pySim.ara_m import CardApplicationARAM
+from pySim.global_platform import CardApplicationISD
 from pySim.gsm_r import DF_EIRENE
+from pySim.cat import ProactiveCommand
 
 # we need to import this module so that the SysmocomSJA2 sub-class of
 # CardModel is created, which will add the ATR-based matching and
@@ -81,15 +80,25 @@ def init_card(sl):
         print("Card not readable!")
         return None, None
 
+    generic_card = False
     card = card_detect("auto", scc)
     if card is None:
         print("Warning: Could not detect card type - assuming a generic card type...")
         card = SimCard(scc)
+        generic_card = True
 
     profile = CardProfile.pick(scc)
     if profile is None:
         print("Unsupported card type!")
-        return None, None
+        return None, card
+
+    # ETSI TS 102 221, Table 9.3 specifies a default for the PIN key
+    # references, however card manufactures may still decide to pick an
+    # arbitrary key reference. In case we run on a generic card class that is
+    # detected as an UICC, we will pick the key reference that is officially
+    # specified.
+    if generic_card and isinstance(profile, CardProfileUICC):
+        card._adm_chv_num = 0x0A
 
     print("Info: Card is of type: %s" % str(profile))
 
@@ -102,6 +111,7 @@ def init_card(sl):
         profile.add_application(CardApplicationUSIM())
         profile.add_application(CardApplicationISIM())
         profile.add_application(CardApplicationARAM())
+        profile.add_application(CardApplicationISD())
 
     # Create runtime state with card profile
     rs = RuntimeState(card, profile)
@@ -129,7 +139,8 @@ class PysimApp(cmd2.Cmd):
         self.default_category = 'pySim-shell built-in commands'
         self.card = None
         self.rs = None
-        self.py_locals = {'card': self.card, 'rs': self.rs}
+        self.lchan = None
+        self.py_locals = {'card': self.card, 'rs': self.rs, 'lchan': self.lchan}
         self.sl = sl
         self.ch = ch
 
@@ -158,7 +169,8 @@ class PysimApp(cmd2.Cmd):
 
         # Unequip everything from pySim-shell that would not work in unequipped state
         if self.rs:
-            self.rs.unregister_cmds(self)
+            lchan = self.rs.lchan[0]
+            lchan.unregister_cmds(self)
         for cmds in [Iso7816Commands, PySimCommands]:
             cmd_set = self.find_commandsets(cmds)
             if cmd_set:
@@ -170,13 +182,15 @@ class PysimApp(cmd2.Cmd):
         # When a card object and a runtime state is present, (re)equip pySim-shell with everything that is
         # needed to operate on cards.
         if self.card and self.rs:
+            self.lchan = self.rs.lchan[0]
             self._onchange_conserve_write(
                 'conserve_write', False, self.conserve_write)
             self._onchange_apdu_trace('apdu_trace', False, self.apdu_trace)
             self.register_command_set(Iso7816Commands())
+            self.register_command_set(Ts102222Commands())
             self.register_command_set(PySimCommands())
-            self.iccid, sw = self.card.read_iccid()
-            rs.select('MF', self)
+            #self.iccid, sw = self.card.read_iccid()
+            self.lchan.select('MF', self)
             rc = True
         else:
             self.poutput("pySim-shell not equipped!")
@@ -215,12 +229,14 @@ class PysimApp(cmd2.Cmd):
             self.cmd2.poutput("<- %s: %s" % (sw, resp))
 
     def update_prompt(self):
-        if self.rs:
-            path_list = self.rs.selected_file.fully_qualified_path(
-                not self.numeric_path)
-            self.prompt = 'pySIM-shell (%s)> ' % ('/'.join(path_list))
+        if self.lchan:
+            path_str = self.lchan.selected_file.fully_qualified_path_str(not self.numeric_path)
+            self.prompt = 'pySIM-shell (%s)> ' % (path_str)
         else:
-            self.prompt = 'pySIM-shell (no card)> '
+            if self.card:
+                self.prompt = 'pySIM-shell (no card profile)> '
+            else:
+                self.prompt = 'pySIM-shell (no card)> '
 
     @cmd2.with_category(CUSTOM_CATEGORY)
     def do_intro(self, _):
@@ -237,6 +253,25 @@ class PysimApp(cmd2.Cmd):
         rs, card = init_card(sl)
         self.equip(card, rs)
 
+    apdu_cmd_parser = argparse.ArgumentParser()
+    apdu_cmd_parser.add_argument('APDU', type=str, help='APDU as hex string')
+    apdu_cmd_parser.add_argument('--expect-sw', help='expect a specified status word', type=str, default=None)
+
+    @cmd2.with_argparser(apdu_cmd_parser)
+    def do_apdu(self, opts):
+        """Send a raw APDU to the card, and print SW + Response.
+        DANGEROUS: pySim-shell will not know any card state changes, and
+        not continue to work as expected if you e.g. select a different
+        file."""
+        data, sw = self.card._scc._tp.send_apdu(opts.APDU)
+        if data:
+            self.poutput("SW: %s, RESP: %s" % (sw, data))
+        else:
+            self.poutput("SW: %s" % sw)
+        if opts.expect_sw:
+            if not sw_match(sw, opts.expect_sw):
+                raise SwMatchError(sw, opts.expect_sw)
+
     class InterceptStderr(list):
         def __init__(self):
             self._stderr_backup = sys.stderr
@@ -415,6 +450,11 @@ class PysimApp(cmd2.Cmd):
         """Echo (print) a string on the console"""
         self.poutput(opts.string)
 
+    @cmd2.with_category(CUSTOM_CATEGORY)
+    def do_version(self, opts):
+        """Print the pySim software version."""
+        import pkg_resources
+        self.poutput(pkg_resources.get_distribution('pySim'))
 
 @with_default_category('pySim Commands')
 class PySimCommands(CommandSet):
@@ -447,22 +487,28 @@ class PySimCommands(CommandSet):
         else:
             flags = ['PARENT', 'SELF', 'FNAMES', 'ANAMES']
         selectables = list(
-            self._cmd.rs.selected_file.get_selectable_names(flags=flags))
+            self._cmd.lchan.selected_file.get_selectable_names(flags=flags))
         directory_str = tabulate_str_list(
             selectables, width=79, hspace=2, lspace=1, align_left=True)
-        path_list = self._cmd.rs.selected_file.fully_qualified_path(True)
-        self._cmd.poutput('/'.join(path_list))
-        path_list = self._cmd.rs.selected_file.fully_qualified_path(False)
-        self._cmd.poutput('/'.join(path_list))
+        path = self._cmd.lchan.selected_file.fully_qualified_path_str(True)
+        self._cmd.poutput(path)
+        path = self._cmd.lchan.selected_file.fully_qualified_path_str(False)
+        self._cmd.poutput(path)
         self._cmd.poutput(directory_str)
         self._cmd.poutput("%d files" % len(selectables))
 
-    def walk(self, indent=0, action=None, context=None):
+    def walk(self, indent=0, action_ef=None, action_df=None, context=None, **kwargs):
         """Recursively walk through the file system, starting at the currently selected DF"""
-        files = self._cmd.rs.selected_file.get_selectables(
+
+        if isinstance(self._cmd.lchan.selected_file, CardDF):
+            if action_df:
+                action_df(context, opts)
+
+        files = self._cmd.lchan.selected_file.get_selectables(
             flags=['FNAMES', 'ANAMES'])
         for f in files:
-            if not action:
+            # special case: When no action is performed, just output a directory
+            if not action_ef and not action_df:
                 output_str = "  " * indent + str(f) + (" " * 250)
                 output_str = output_str[0:25]
                 if isinstance(files[f], CardADF):
@@ -475,12 +521,12 @@ class PySimCommands(CommandSet):
             if isinstance(files[f], CardDF):
                 skip_df = False
                 try:
-                    fcp_dec = self._cmd.rs.select(f, self._cmd)
+                    fcp_dec = self._cmd.lchan.select(f, self._cmd)
                 except Exception as e:
                     skip_df = True
-                    df = self._cmd.rs.selected_file
-                    df_path_list = df.fully_qualified_path(True)
-                    df_skip_reason_str = '/'.join(df_path_list) + \
+                    df = self._cmd.lchan.selected_file
+                    df_path = df.fully_qualified_path_str(True)
+                    df_skip_reason_str = df_path + \
                         "/" + str(f) + ", " + str(e)
                     if context:
                         context['DF_SKIP'] += 1
@@ -489,71 +535,89 @@ class PySimCommands(CommandSet):
                 # If the DF was skipped, we never have entered the directory
                 # below, so we must not move up.
                 if skip_df == False:
-                    self.walk(indent + 1, action, context)
-                    fcp_dec = self._cmd.rs.select("..", self._cmd)
+                    self.walk(indent + 1, action_ef, action_df, context, **kwargs)
+                    fcp_dec = self._cmd.lchan.select("..", self._cmd)
 
-            elif action:
-                df_before_action = self._cmd.rs.selected_file
-                action(f, context)
+            elif action_ef:
+                df_before_action = self._cmd.lchan.selected_file
+                action_ef(f, context, **kwargs)
                 # When walking through the file system tree the action must not
                 # always restore the currently selected file to the file that
                 # was selected before executing the action() callback.
-                if df_before_action != self._cmd.rs.selected_file:
+                if df_before_action != self._cmd.lchan.selected_file:
                     raise RuntimeError("inconsistent walk, %s is currently selected but expecting %s to be selected"
-                                       % (str(self._cmd.rs.selected_file), str(df_before_action)))
+                                       % (str(self._cmd.lchan.selected_file), str(df_before_action)))
 
     def do_tree(self, opts):
         """Display a filesystem-tree with all selectable files"""
         self.walk()
 
-    def export(self, filename, context):
-        """ Select and export a single file """
+    def export_ef(self, filename, context, as_json):
+        """ Select and export a single elementary file (EF) """
         context['COUNT'] += 1
-        df = self._cmd.rs.selected_file
+        df = self._cmd.lchan.selected_file
 
+	# The currently selected file (not the file we are going to export)
+	# must always be an ADF or DF. From this starting point we select
+	# the EF we want to export. To maintain consistency we will then
+	# select the current DF again (see comment below).
         if not isinstance(df, CardDF):
             raise RuntimeError(
                 "currently selected file %s is not a DF or ADF" % str(df))
 
         df_path_list = df.fully_qualified_path(True)
-        df_path_list_fid = df.fully_qualified_path(False)
+        df_path = df.fully_qualified_path_str(True)
+        df_path_fid = df.fully_qualified_path_str(False)
 
-        file_str = '/'.join(df_path_list) + "/" + str(filename)
+        file_str = df_path + "/" + str(filename)
         self._cmd.poutput(boxed_heading_str(file_str))
 
-        self._cmd.poutput("# directory: %s (%s)" %
-                          ('/'.join(df_path_list), '/'.join(df_path_list_fid)))
+        self._cmd.poutput("# directory: %s (%s)" % (df_path, df_path_fid))
         try:
-            fcp_dec = self._cmd.rs.select(filename, self._cmd)
+            fcp_dec = self._cmd.lchan.select(filename, self._cmd)
             self._cmd.poutput("# file: %s (%s)" % (
-                self._cmd.rs.selected_file.name, self._cmd.rs.selected_file.fid))
+                self._cmd.lchan.selected_file.name, self._cmd.lchan.selected_file.fid))
 
-            fd = fcp_dec['file_descriptor']
-            structure = fd['structure']
+            structure = self._cmd.lchan.selected_file_structure()
             self._cmd.poutput("# structure: %s" % str(structure))
+            self._cmd.poutput("# RAW FCP Template: %s" % str(self._cmd.lchan.selected_file_fcp_hex))
+            self._cmd.poutput("# Decoded FCP Template: %s" % str(self._cmd.lchan.selected_file_fcp))
 
             for f in df_path_list:
                 self._cmd.poutput("select " + str(f))
-            self._cmd.poutput("select " + self._cmd.rs.selected_file.name)
+            self._cmd.poutput("select " + self._cmd.lchan.selected_file.name)
 
             if structure == 'transparent':
-                result = self._cmd.rs.read_binary()
-                self._cmd.poutput("update_binary " + str(result[0]))
+                if as_json:
+                    result = self._cmd.lchan.read_binary_dec()
+                    self._cmd.poutput("update_binary_decoded '%s'" % json.dumps(result[0], cls=JsonEncoder))
+                else:
+                    result = self._cmd.lchan.read_binary()
+                    self._cmd.poutput("update_binary " + str(result[0]))
             elif structure == 'cyclic' or structure == 'linear_fixed':
                 # Use number of records specified in select response
-                if 'num_of_rec' in fd:
-                    num_of_rec = fd['num_of_rec']
+                num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
+                if num_of_rec:
                     for r in range(1, num_of_rec + 1):
-                        result = self._cmd.rs.read_record(r)
-                        self._cmd.poutput("update_record %d %s" %
-                                          (r, str(result[0])))
+                        if as_json:
+                            result = self._cmd.lchan.read_record_dec(r)
+                            self._cmd.poutput("update_record_decoded %d '%s'" % (r, json.dumps(result[0], cls=JsonEncoder)))
+                        else:
+                            result = self._cmd.lchan.read_record(r)
+                            self._cmd.poutput("update_record %d %s" % (r, str(result[0])))
+
                 # When the select response does not return the number of records, read until we hit the
                 # first record that cannot be read.
                 else:
                     r = 1
                     while True:
                         try:
-                            result = self._cmd.rs.read_record(r)
+                            if as_json:
+                                result = self._cmd.lchan.read_record_dec(r)
+                                self._cmd.poutput("update_record_decoded %d '%s'" % (r, json.dumps(result[0], cls=JsonEncoder)))
+                            else:
+                                result = self._cmd.lchan.read_record(r)
+                                self._cmd.poutput("update_record %d %s" % (r, str(result[0])))
                         except SwMatchError as e:
                             # We are past the last valid record - stop
                             if e.sw_actual == "9402":
@@ -561,21 +625,18 @@ class PySimCommands(CommandSet):
                             # Some other problem occurred
                             else:
                                 raise e
-                        self._cmd.poutput("update_record %d %s" %
-                                          (r, str(result[0])))
                         r = r + 1
             elif structure == 'ber_tlv':
-                tags = self._cmd.rs.retrieve_tags()
+                tags = self._cmd.lchan.retrieve_tags()
                 for t in tags:
-                    result = self._cmd.rs.retrieve_data(t)
+                    result = self._cmd.lchan.retrieve_data(t)
                     (tag, l, val, remainer) = bertlv_parse_one(h2b(result[0]))
                     self._cmd.poutput("set_data 0x%02x %s" % (t, b2h(val)))
             else:
                 raise RuntimeError(
                     'Unsupported structure "%s" of file "%s"' % (structure, filename))
         except Exception as e:
-            bad_file_str = '/'.join(df_path_list) + \
-                "/" + str(filename) + ", " + str(e)
+            bad_file_str = df_path + "/" + str(filename) + ", " + str(e)
             self._cmd.poutput("# bad file: %s" % bad_file_str)
             context['ERR'] += 1
             context['BAD'].append(bad_file_str)
@@ -583,24 +644,34 @@ class PySimCommands(CommandSet):
         # When reading the file is done, make sure the parent file is
         # selected again. This will be the usual case, however we need
         # to check before since we must not select the same DF twice
-        if df != self._cmd.rs.selected_file:
-            self._cmd.rs.select(df.fid or df.aid, self._cmd)
+        if df != self._cmd.lchan.selected_file:
+            self._cmd.lchan.select(df.fid or df.aid, self._cmd)
 
         self._cmd.poutput("#")
 
     export_parser = argparse.ArgumentParser()
     export_parser.add_argument(
         '--filename', type=str, default=None, help='only export specific file')
+    export_parser.add_argument(
+        '--json', action='store_true', help='export as JSON (less reliable)')
 
     @cmd2.with_argparser(export_parser)
     def do_export(self, opts):
         """Export files to script that can be imported back later"""
         context = {'ERR': 0, 'COUNT': 0, 'BAD': [],
                    'DF_SKIP': 0, 'DF_SKIP_REASON': []}
+        kwargs_export = {'as_json': opts.json}
+        exception_str_add = ""
+
         if opts.filename:
-            self.export(opts.filename, context)
+            self.export_ef(opts.filename, context, **kwargs_export)
         else:
-            self.walk(0, self.export, context)
+            try:
+                self.walk(0, self.export_ef, None, context, **kwargs_export)
+            except Exception as e:
+                print("# Stopping early here due to exception: " + str(e))
+                print("#")
+                exception_str_add = ", also had to stop early due to exception:" + str(e)
 
         self._cmd.poutput(boxed_heading_str("Export summary"))
 
@@ -615,24 +686,24 @@ class PySimCommands(CommandSet):
             self._cmd.poutput("#  " + b)
 
         if context['ERR'] and context['DF_SKIP']:
-            raise RuntimeError("unable to export %i elementary file(s) and %i dedicated file(s)" % (
-                context['ERR'], context['DF_SKIP']))
+            raise RuntimeError("unable to export %i elementary file(s) and %i dedicated file(s)%s" % (
+                    context['ERR'], context['DF_SKIP'], exception_str_add))
         elif context['ERR']:
             raise RuntimeError(
-                "unable to export %i elementary file(s)" % context['ERR'])
+                    "unable to export %i elementary file(s)%s" % (context['ERR'], exception_str_add))
         elif context['DF_SKIP']:
             raise RuntimeError(
-                "unable to export %i dedicated files(s)" % context['ERR'])
+                    "unable to export %i dedicated files(s)%s" % (context['ERR'], exception_str_add))
 
     def do_reset(self, opts):
         """Reset the Card."""
-        atr = self._cmd.rs.reset(self._cmd)
+        atr = self._cmd.lchan.reset(self._cmd)
         self._cmd.poutput('Card ATR: %s' % atr)
         self._cmd.update_prompt()
 
     def do_desc(self, opts):
         """Display human readable file description for the currently selected file"""
-        desc = self._cmd.rs.selected_file.desc
+        desc = self._cmd.lchan.selected_file.desc
         if desc:
             self._cmd.poutput(desc)
         else:
@@ -669,21 +740,19 @@ class Iso7816Commands(CommandSet):
     def do_select(self, opts):
         """SELECT a File (ADF/DF/EF)"""
         if len(opts.arg_list) == 0:
-            path_list = self._cmd.rs.selected_file.fully_qualified_path(True)
-            path_list_fid = self._cmd.rs.selected_file.fully_qualified_path(
-                False)
-            self._cmd.poutput("currently selected file: " +
-                              '/'.join(path_list) + " (" + '/'.join(path_list_fid) + ")")
+            path = self._cmd.lchan.selected_file.fully_qualified_path_str(True)
+            path_fid = self._cmd.lchan.selected_file.fully_qualified_path_str(False)
+            self._cmd.poutput("currently selected file: %s (%s)" % (path, path_fid))
             return
 
         path = opts.arg_list[0]
-        fcp_dec = self._cmd.rs.select(path, self._cmd)
+        fcp_dec = self._cmd.lchan.select(path, self._cmd)
         self._cmd.update_prompt()
         self._cmd.poutput_json(fcp_dec)
 
     def complete_select(self, text, line, begidx, endidx) -> List[str]:
         """Command Line tab completion for SELECT"""
-        index_dict = {1: self._cmd.rs.selected_file.get_selectable_names()}
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
         return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
 
     def get_code(self, code):
@@ -712,7 +781,9 @@ class Iso7816Commands(CommandSet):
 
     @cmd2.with_argparser(verify_chv_parser)
     def do_verify_chv(self, opts):
-        """Verify (authenticate) using specified PIN code"""
+        """Verify (authenticate) using specified CHV (PIN) code, which is how the specifications
+        call it if you authenticate yourself using the specified PIN.  There usually is at least PIN1 and
+        PIN2."""
         pin = self.get_code(opts.pin_code)
         (data, sw) = self._cmd.card._scc.verify_chv(opts.pin_nr, h2b(pin))
         self._cmd.poutput("CHV verification successful")
@@ -778,17 +849,20 @@ class Iso7816Commands(CommandSet):
         self._cmd.poutput("CHV enable successful")
 
     def do_deactivate_file(self, opts):
-        """Deactivate the current EF"""
+        """Deactivate the currently selected EF"""
         (data, sw) = self._cmd.card._scc.deactivate_file()
 
+    activate_file_parser = argparse.ArgumentParser()
+    activate_file_parser.add_argument('NAME', type=str, help='File name or FID of file to activate')
+    @cmd2.with_argparser(activate_file_parser)
     def do_activate_file(self, opts):
-        """Activate the specified EF"""
-        path = opts.arg_list[0]
-        (data, sw) = self._cmd.rs.activate_file(path)
+        """Activate the specified EF. This used to be called REHABILITATE in TS 11.11 for classic
+        SIM.  You need to specify the name or FID of the file to activate."""
+        (data, sw) = self._cmd.lchan.activate_file(opts.NAME)
 
     def complete_activate_file(self, text, line, begidx, endidx) -> List[str]:
         """Command Line tab completion for ACTIVATE FILE"""
-        index_dict = {1: self._cmd.rs.selected_file.get_selectable_names()}
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
         return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
 
     open_chan_parser = argparse.ArgumentParser()
@@ -813,7 +887,7 @@ class Iso7816Commands(CommandSet):
 
     def do_status(self, opts):
         """Perform the STATUS command."""
-        fcp_dec = self._cmd.rs.status()
+        fcp_dec = self._cmd.lchan.status()
         self._cmd.poutput_json(fcp_dec)
 
     suspend_uicc_parser = argparse.ArgumentParser()
@@ -831,6 +905,13 @@ class Iso7816Commands(CommandSet):
         self._cmd.poutput(
             'Negotiated Duration: %u secs, Token: %s, SW: %s' % (duration, token, sw))
 
+class Proact(ProactiveHandler):
+    def receive_fetch(self, pcmd: ProactiveCommand):
+        # print its parsed representation
+        print(pcmd.decoded)
+        # TODO: implement the basics, such as SMS Sending, ...
+
+
 
 option_parser = argparse.ArgumentParser(prog='pySim-shell', description='interactive SIM card shell',
                                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -871,7 +952,7 @@ if __name__ == '__main__':
         card_key_provider_register(CardKeyProviderCsv(csv_default))
 
     # Init card reader driver
-    sl = init_reader(opts)
+    sl = init_reader(opts, proactive_handler = Proact())
     if sl is None:
         exit(1)
 
@@ -900,7 +981,7 @@ if __name__ == '__main__':
             " it should also be noted that some readers may behave strangely when no card")
         print(" is inserted.)")
         print("")
-        app = PysimApp(None, None, sl, ch, opts.script)
+        app = PysimApp(card, None, sl, ch, opts.script)
 
     # If the user supplies an ADM PIN at via commandline args authenticate
     # immediately so that the user does not have to use the shell commands

pySim-trace.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+
+import sys
+import logging, colorlog
+import argparse
+from pprint import pprint as pp
+
+from pySim.apdu import *
+from pySim.filesystem import RuntimeState
+
+from pySim.cards import UsimCard
+from pySim.commands import SimCardCommands
+from pySim.profile import CardProfile
+from pySim.ts_102_221 import CardProfileUICCSIM
+from pySim.ts_31_102 import CardApplicationUSIM
+from pySim.ts_31_103 import CardApplicationISIM
+from pySim.transport import LinkBase
+
+from pySim.apdu_source.gsmtap import GsmtapApduSource
+from pySim.apdu_source.pyshark_rspro import PysharkRsproPcap, PysharkRsproLive
+
+from pySim.apdu.ts_102_221 import UiccSelect, UiccStatus
+
+log_format='%(log_color)s%(levelname)-8s%(reset)s %(name)s: %(message)s'
+colorlog.basicConfig(level=logging.INFO, format = log_format)
+logger = colorlog.getLogger()
+
+# merge all of the command sets into one global set. This will override instructions,
+# the one from the 'last' set in the addition below will prevail.
+from pySim.apdu.ts_51_011 import ApduCommands as SimApduCommands
+from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
+from pySim.apdu.global_platform import ApduCommands as GpApduCommands
+ApduCommands = SimApduCommands + UiccApduCommands + UsimApduCommands #+ GpApduCommands
+
+
+class DummySimLink(LinkBase):
+    """A dummy implementation of the LinkBase abstract base class.  Currently required
+    as the UsimCard doesn't work without SimCardCommands, which in turn require
+    a LinkBase implementation talking to a card.
+
+    In the tracer, we don't actually talk to any card, so we simply drop everything
+    and claim it is successful.
+
+    The UsimCard / SimCardCommands should be refactored to make this obsolete later."""
+    def __init__(self, debug: bool = False, **kwargs):
+        super().__init__(**kwargs)
+        self._debug = debug
+        self._atr = h2i('3B9F96801F878031E073FE211B674A4C753034054BA9')
+
+    def _send_apdu_raw(self, pdu):
+        #print("DummySimLink-apdu: %s" % pdu)
+        return [], '9000'
+
+    def connect(self):
+        pass
+
+    def disconnect(self):
+        pass
+
+    def reset_card(self):
+        return 1
+
+    def get_atr(self):
+        return self._atr
+
+    def wait_for_card(self):
+        pass
+
+
+class Tracer:
+    def __init__(self, **kwargs):
+        # we assume a generic SIM + UICC + USIM + ISIM card
+        profile = CardProfileUICCSIM()
+        profile.add_application(CardApplicationUSIM())
+        profile.add_application(CardApplicationISIM())
+        scc = SimCardCommands(transport=DummySimLink())
+        card = UsimCard(scc)
+        self.rs = RuntimeState(card, profile)
+        # APDU Decoder
+        self.ad = ApduDecoder(ApduCommands)
+        # parameters
+        self.suppress_status = kwargs.get('suppress_status', True)
+        self.suppress_select = kwargs.get('suppress_select', True)
+        self.source = kwargs.get('source', None)
+
+    def format_capdu(self, inst: ApduCommand):
+        """Output a single decoded + processed ApduCommand."""
+        print("%02u %-16s %-35s %-8s %s %s" % (inst.lchan_nr, inst._name, inst.path_str, inst.col_id, inst.col_sw, inst.processed))
+        print("===============================")
+
+    def main(self):
+        """Main loop of tracer: Iterates over all Apdu received from source."""
+        while True:
+            # obtain the next APDU from the source (blocking read)
+            apdu = self.source.read()
+            #print(apdu)
+
+            if isinstance(apdu, CardReset):
+                self.rs.reset()
+                continue
+
+            # ask ApduDecoder to look-up (INS,CLA) + instantiate an ApduCommand derived
+            # class like 'UiccSelect'
+            inst = self.ad.input(apdu)
+            # process the APDU (may modify the RuntimeState)
+            inst.process(self.rs)
+
+            # Avoid cluttering the log with too much verbosity
+            if self.suppress_select and isinstance(inst, UiccSelect):
+                continue
+            if self.suppress_status and isinstance(inst, UiccStatus):
+                continue
+            #print(inst)
+            self.format_capdu(inst)
+
+option_parser = argparse.ArgumentParser(prog='pySim-trace', description='Osmocom pySim high-level SIM card trace decoder',
+                                        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+global_group = option_parser.add_argument_group('General Options')
+global_group.add_argument('--no-suppress-select', action='store_false', dest='suppress_select',
+                          help="Don't suppress displaying SELECT APDUs")
+global_group.add_argument('--no-suppress-status', action='store_false', dest='suppress_status',
+                          help="Don't suppress displaying STATUS APDUs")
+
+subparsers = option_parser.add_subparsers(help='APDU Source', dest='source', required=True)
+
+parser_gsmtap = subparsers.add_parser('gsmtap-udp', help='Live capture of GSMTAP-SIM on UDP port')
+parser_gsmtap.add_argument('-i', '--bind-ip', default='127.0.0.1',
+                           help='Local IP address to which to bind the UDP port')
+parser_gsmtap.add_argument('-p', '--bind-port', default=4729,
+                           help='Local UDP port')
+
+parser_rspro_pyshark_pcap = subparsers.add_parser('rspro-pyshark-pcap', help="""
+    PCAP file containing RSPRO (osmo-remsim) communication; processed via pyshark.
+    REQUIRES OSMOCOM PATCHED WIRESHARK!""")
+parser_rspro_pyshark_pcap.add_argument('-f', '--pcap-file', required=True,
+                                       help='Name of the PCAP[ng] file to be read')
+
+parser_rspro_pyshark_live = subparsers.add_parser('rspro-pyshark-live', help="""
+    Live capture of RSPRO (osmo-remsim) communication; processed via pyshark.
+    REQUIRES OSMOCOM PATCHED WIRESHARK!""")
+parser_rspro_pyshark_live.add_argument('-i', '--interface', required=True,
+                                       help='Name of the network interface to capture on')
+
+if __name__ == '__main__':
+
+    opts = option_parser.parse_args()
+
+    logger.info('Opening source %s...' % opts.source)
+    if opts.source == 'gsmtap-udp':
+        s = GsmtapApduSource(opts.bind_ip, opts.bind_port)
+    elif opts.source == 'rspro-pyshark-pcap':
+        s = PysharkRsproPcap(opts.pcap_file)
+    elif opts.source == 'rspro-pyshark-live':
+        s = PysharkRsproLive(opts.interface)
+
+    tracer = Tracer(source=s, suppress_status=opts.suppress_status, suppress_select=opts.suppress_select)
+    logger.info('Entering main loop...')
+    tracer.main()
+

pySim/apdu/__init__.py

@@ -0,0 +1,446 @@
+# coding=utf-8
+"""APDU (and TPDU) parser for UICC/USIM/ISIM cards.
+
+The File (and its classes) represent the structure / hierarchy
+of the APDUs as seen in SIM/UICC/SIM/ISIM cards.  The primary use case
+is to perform a meaningful decode of protocol traces taken between card and UE.
+
+The ancient wirshark dissector developed for GSMTAP generated by SIMtrace
+is far too simplistic, while this decoder can utilize all of the information
+we already know in pySim about the filesystem structure, file encoding, etc.
+"""
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import abc
+from termcolor import colored
+import typing
+from typing import List, Dict, Optional
+
+from construct import *
+from construct import Optional as COptional
+from pySim.construct import *
+from pySim.utils import *
+from pySim.filesystem import RuntimeLchan, RuntimeState, lchan_nr_from_cla
+from pySim.filesystem import CardADF, CardFile, TransparentEF, LinFixedEF
+
+"""There are multiple levels of decode:
+
+    1) pure TPDU / APDU level (no filesystem state required to decode)
+        1a) the raw C-TPDU + R-TPDU
+        1b) the raw C-APDU + R-APDU
+        1c) the C-APDU + R-APDU split in its portions (p1/p2/lc/le/cmd/rsp)
+        1d) the abstract C-APDU + R-APDU (mostly p1/p2 parsing; SELECT response)
+    2) the decoded DATA of command/response APDU
+        * READ/UPDATE: requires state/context: which file is selected? how to decode it?
+"""
+
+class ApduCommandMeta(abc.ABCMeta):
+    """A meta-class that we can use to set some class variables when declaring
+       a derived class of ApduCommand."""
+    def __new__(metacls, name, bases, namespace, **kwargs):
+        x = super().__new__(metacls, name, bases, namespace)
+        x._name = namespace.get('name', kwargs.get('n', None))
+        x._ins = namespace.get('ins', kwargs.get('ins', None))
+        x._cla = namespace.get('cla', kwargs.get('cla', None))
+        return x
+
+BytesOrHex = typing.Union[bytes, Hexstr]
+
+class Tpdu:
+    def __init__(self, cmd: BytesOrHex, rsp: Optional[BytesOrHex] = None):
+        if isinstance(cmd, str):
+            self.cmd = h2b(cmd)
+        else:
+            self.cmd = cmd
+        if  isinstance(rsp, str):
+            self.rsp = h2b(rsp)
+        else:
+            self.rsp = rsp
+
+    def __str__(self):
+        return '%s(%02X %02X %02X %02X %02X %s %s %s)' % (type(self).__name__, self.cla, self.ins, self.p1,
+                self.p2, self.p3, b2h(self.cmd_data), b2h(self.rsp_data), b2h(self.sw))
+
+    @property
+    def cla(self) -> int:
+        """Return CLA of the C-APDU Header."""
+        return self.cmd[0]
+
+    @property
+    def ins(self) -> int:
+        """Return INS of the C-APDU Header."""
+        return self.cmd[1]
+
+    @property
+    def p1(self) -> int:
+        """Return P1 of the C-APDU Header."""
+        return self.cmd[2]
+
+    @property
+    def p2(self) -> int:
+        """Return P2 of the C-APDU Header."""
+        return self.cmd[3]
+
+    @property
+    def p3(self) -> int:
+        """Return P3 of the C-APDU Header."""
+        return self.cmd[4]
+
+    @property
+    def cmd_data(self) -> int:
+        """Return the DATA portion of the C-APDU"""
+        return self.cmd[5:]
+
+    @property
+    def sw(self) -> Optional[bytes]:
+        """Return Status Word (SW) of the R-APDU"""
+        return self.rsp[-2:] if self.rsp else None
+
+    @property
+    def rsp_data(self) -> Optional[bytes]:
+        """Return the DATA portion of the R-APDU"""
+        return self.rsp[:-2] if self.rsp else None
+
+
+class Apdu(Tpdu):
+    @property
+    def lc(self) -> int:
+        """Return Lc; Length of C-APDU body."""
+        return len(self.cmd_data)
+
+    @property
+    def lr(self) -> int:
+        """Return Lr; Length of R-APDU body."""
+        return len(self.rsp_data)
+
+    @property
+    def successful(self) -> bool:
+        """Was the execution of this APDU successful?"""
+        method = getattr(self, '_is_success', None)
+        if callable(method):
+            return method()
+        # default case: only 9000 is success
+        return self.sw == b'\x90\x00'
+
+
+class ApduCommand(Apdu, metaclass=ApduCommandMeta):
+    """Base class from which you would derive individual commands/instructions like SELECT.
+       A derived class represents a decoder for a specific instruction.
+       An instance of such a derived class is one concrete APDU."""
+    # fall-back constructs if the derived class provides no override
+    _construct_p1 = Byte
+    _construct_p2 = Byte
+    _construct = HexAdapter(GreedyBytes)
+    _construct_rsp = HexAdapter(GreedyBytes)
+
+    def __init__(self, cmd: BytesOrHex, rsp: Optional[BytesOrHex] = None):
+        """Instantiate a new ApduCommand from give cmd + resp."""
+        # store raw data
+        super().__init__(cmd, rsp)
+        # default to 'empty' ID column. To be set to useful values (like record number)
+        # by derived class {cmd_rsp}_to_dict() or process() methods
+        self.col_id = '-'
+        # fields only set by process_* methods
+        self.file = None
+        self.lchan = None
+        self.processed = None
+        # the methods below could raise exceptions and those handlers might assume cmd_{dict,resp}
+        self.cmd_dict = None
+        self.rsp_dict = None
+        # interpret the data
+        self.cmd_dict = self.cmd_to_dict()
+        self.rsp_dict = self.rsp_to_dict() if self.rsp else {}
+
+
+    @classmethod
+    def from_apdu(cls, apdu:Apdu, **kwargs) -> 'ApduCommand':
+        """Instantiate an ApduCommand from an existing APDU."""
+        return cls(cmd=apdu.cmd, rsp=apdu.rsp, **kwargs)
+
+    @classmethod
+    def from_bytes(cls, buffer:bytes) -> 'ApduCommand':
+        """Instantiate an ApduCommand from a linear byte buffer containing hdr,cmd,rsp,sw.
+        This is for example used when parsing GSMTAP traces that traditionally contain the
+        full command and response portion in one packet: "CLA INS P1 P2 P3 DATA SW" and we
+        now need to figure out whether the DATA part is part of the CMD or the RSP"""
+        apdu_case = cls.get_apdu_case(buffer)
+        if apdu_case in [1, 2]:
+            # data is part of response
+            return cls(buffer[:5], buffer[5:])
+        elif apdu_case in [3, 4]:
+            # data is part of command
+            lc = buffer[4]
+            return cls(buffer[:5+lc], buffer[5+lc:])
+        else:
+            raise ValueError('%s: Invalid APDU Case %u' % (cls.__name__, apdu_case))
+
+    @property
+    def path(self) -> List[str]:
+        """Return (if known) the path as list of files to the file on which this command operates."""
+        if self.file:
+            return self.file.fully_qualified_path()
+        else:
+            return []
+
+    @property
+    def path_str(self) -> str:
+        """Return (if known) the path as string to the file on which this command operates."""
+        if self.file:
+            return self.file.fully_qualified_path_str()
+        else:
+            return ''
+
+    @property
+    def col_sw(self) -> str:
+        """Return the ansi-colorized status word. Green==OK, Red==Error"""
+        if self.successful:
+            return colored(b2h(self.sw), 'green')
+        else:
+            return colored(b2h(self.sw), 'red')
+
+    @property
+    def lchan_nr(self) -> int:
+        """Logical channel number over which this ApduCommand was transmitted."""
+        if self.lchan:
+            return self.lchan.lchan_nr
+        else:
+            return lchan_nr_from_cla(self.cla)
+
+    def __str__(self) -> str:
+        return '%02u %s(%s): %s' % (self.lchan_nr, type(self).__name__, self.path_str, self.to_dict())
+
+    def __repr__(self) -> str:
+        return '%s(INS=%02x,CLA=%s)' % (self.__class__, self.ins, self.cla)
+
+    def _process_fallback(self, rs: RuntimeState):
+        """Fall-back function to be called if there is no derived-class-specific
+        process_global or process_on_lchan method. Uses information from APDU decode."""
+        self.processed = {}
+        if not 'p1' in self.cmd_dict:
+            self.processed = self.to_dict()
+        else:
+            self.processed['p1'] = self.cmd_dict['p1']
+            self.processed['p2'] = self.cmd_dict['p2']
+            if 'body' in self.cmd_dict and self.cmd_dict['body']:
+                self.processed['cmd'] = self.cmd_dict['body']
+            if 'body' in self.rsp_dict and self.rsp_dict['body']:
+                self.processed['rsp'] = self.rsp_dict['body']
+        return self.processed
+
+    def process(self, rs: RuntimeState):
+        # if there is a global method, use that; else use process_on_lchan
+        method = getattr(self, 'process_global', None)
+        if callable(method):
+            self.processed = method(rs)
+            return self.processed
+        method = getattr(self, 'process_on_lchan', None)
+        if callable(method):
+            self.lchan = rs.get_lchan_by_cla(self.cla)
+            self.processed = method(self.lchan)
+            return self.processed
+        # if none of the two methods exist:
+        return self._process_fallback(rs)
+
+    @classmethod
+    def get_apdu_case(cls, hdr:bytes) -> int:
+        if hasattr(cls, '_apdu_case'):
+            return cls._apdu_case
+        method = getattr(cls, '_get_apdu_case', None)
+        if callable(method):
+            return method(hdr)
+        raise ValueError('%s: Class definition missing _apdu_case attribute or _get_apdu_case method' % cls.__name__)
+
+    @classmethod
+    def match_cla(cls, cla) -> bool:
+        """Does the given CLA match the CLA list of the command?."""
+        if not isinstance(cla, str):
+            cla = '%02X' % cla
+        cla = cla.lower()
+        # see https://github.com/PyCQA/pylint/issues/7219
+        # pylint: disable=no-member
+        for cla_match in cls._cla:
+            cla_masked = ""
+            for i in range(0, 2):
+                if cla_match[i] == 'X':
+                    cla_masked += 'X'
+                else:
+                    cla_masked += cla[i]
+            if cla_masked == cla_match:
+                return True
+        return False
+
+    def cmd_to_dict(self) -> Dict:
+        """Convert the Command part of the APDU to a dict."""
+        method = getattr(self, '_decode_cmd', None)
+        if callable(method):
+            return method()
+        else:
+            r = {}
+            method = getattr(self, '_decode_p1p2', None)
+            if callable(method):
+                r = self._decode_p1p2()
+            else:
+                r['p1'] = parse_construct(self._construct_p1, self.p1.to_bytes(1, 'big'))
+                r['p2'] = parse_construct(self._construct_p2, self.p2.to_bytes(1, 'big'))
+            r['p3'] = self.p3
+            if self.cmd_data:
+                r['body'] = parse_construct(self._construct, self.cmd_data)
+            return r
+
+    def rsp_to_dict(self) -> Dict:
+        """Convert the Response part of the APDU to a dict."""
+        method = getattr(self, '_decode_rsp', None)
+        if callable(method):
+            return method()
+        else:
+            r = {}
+            if self.rsp_data:
+                r['body'] = parse_construct(self._construct_rsp, self.rsp_data)
+            r['sw'] = b2h(self.sw)
+            return r
+
+    def to_dict(self) -> Dict:
+        """Convert the entire APDU to a dict."""
+        return {'cmd': self.cmd_dict, 'rsp': self.rsp_dict}
+
+    def to_json(self) -> str:
+        """Convert the entire APDU to JSON."""
+        d = self.to_dict()
+        return json.dumps(d)
+
+    def _determine_file(self, lchan) -> CardFile:
+        """Helper function for read/update commands that might use SFI instead of selected file.
+        Expects that the self.cmd_dict has already been populated with the 'file' member."""
+        if self.cmd_dict['file'] == 'currently_selected_ef':
+            self.file = lchan.selected_file
+        elif self.cmd_dict['file'] == 'sfi':
+            cwd = lchan.get_cwd()
+            self.file = cwd.lookup_file_by_sfid(self.cmd_dict['sfi'])
+
+
+class ApduCommandSet:
+    """A set of card instructions, typically specified within one spec."""
+
+    def __init__(self, name: str, cmds: List[ApduCommand] =[]):
+        self.name = name
+        self.cmds = {c._ins: c for c in cmds}
+
+    def __str__(self) -> str:
+        return self.name
+
+    def __getitem__(self, idx) -> ApduCommand:
+        return self.cmds[idx]
+
+    def __add__(self, other) -> 'ApduCommandSet':
+        if isinstance(other, ApduCommand):
+            if other.ins in self.cmds:
+                raise ValueError('%s: INS 0x%02x already defined: %s' %
+                                 (self, other.ins, self.cmds[other.ins]))
+            self.cmds[other.ins] = other
+        elif isinstance(other, ApduCommandSet):
+            for c in other.cmds.keys():
+                self.cmds[c] = other.cmds[c]
+        else:
+            raise ValueError(
+                '%s: Unsupported type to add operator: %s' % (self, other))
+        return self
+
+    def lookup(self, ins, cla=None) -> Optional[ApduCommand]:
+        """look-up the command within the CommandSet."""
+        ins = int(ins)
+        if not ins in self.cmds:
+            return None
+        cmd = self.cmds[ins]
+        if cla and not cmd.match_cla(cla):
+            return None
+        return cmd
+
+    def parse_cmd_apdu(self, apdu: Apdu) -> ApduCommand:
+        """Parse a Command-APDU. Returns an instance of an ApduCommand derived class."""
+        # first look-up which of our member classes match CLA + INS
+        a_cls = self.lookup(apdu.ins, apdu.cla)
+        if not a_cls:
+            raise ValueError('Unknown CLA=%02X INS=%02X' % (apdu.cla, apdu.ins))
+        # then create an instance of that class and return it
+        return a_cls.from_apdu(apdu)
+
+    def parse_cmd_bytes(self, buf:bytes) -> ApduCommand:
+        """Parse from a buffer (simtrace style). Returns an instance of an ApduCommand derived class."""
+        # first look-up which of our member classes match CLA + INS
+        cla = buf[0]
+        ins = buf[1]
+        a_cls = self.lookup(ins, cla)
+        if not a_cls:
+            raise ValueError('Unknown CLA=%02X INS=%02X' % (cla, ins))
+        # then create an instance of that class and return it
+        return a_cls.from_bytes(buf)
+
+
+
+class ApduHandler(abc.ABC):
+    @abc.abstractmethod
+    def input(self, cmd: bytes, rsp: bytes):
+        pass
+
+
+class TpduFilter(ApduHandler):
+    """The TpduFilter removes the T=0 specific GET_RESPONSE from the TPDU stream and
+       calls the ApduHandler only with the actual APDU command and response parts."""
+    def __init__(self, apdu_handler: ApduHandler):
+        self.apdu_handler = apdu_handler
+        self.state = 'INIT'
+        self.last_cmd = None
+
+    def input_tpdu(self, tpdu:Tpdu):
+        # handle SW=61xx / 6Cxx
+        if tpdu.sw[0] == 0x61 or tpdu.sw[0] == 0x6C:
+            self.state = 'WAIT_GET_RESPONSE'
+            # handle successive 61/6c responses by stupid phone/modem OS
+            if tpdu.ins != 0xC0:
+                self.last_cmd = tpdu.cmd
+            return None
+        else:
+            if self.last_cmd:
+                icmd = self.last_cmd
+                self.last_cmd = None
+            else:
+                icmd = tpdu.cmd
+            apdu = Apdu(icmd, tpdu.rsp)
+            if self.apdu_handler:
+                return self.apdu_handler.input(apdu)
+            else:
+                return Apdu(icmd, tpdu.rsp)
+
+    def input(self, cmd: bytes, rsp: bytes):
+        if isinstance(cmd, str):
+            cmd = bytes.fromhex(cmd)
+        if isinstance(rsp, str):
+            rsp = bytes.fromhex(rsp)
+        tpdu = Tpdu(cmd, rsp)
+        return self.input_tpdu(tpdu)
+
+class ApduDecoder(ApduHandler):
+    def __init__(self, cmd_set: ApduCommandSet):
+        self.cmd_set = cmd_set
+
+    def input(self, apdu: Apdu):
+        return self.cmd_set.parse_cmd_apdu(apdu)
+
+
+class CardReset:
+    pass

pySim/apdu/global_platform.py

@@ -0,0 +1,57 @@
+# coding=utf-8
+"""APDU definition/decoder of GlobalPLatform Card Spec (currently 2.1.1)
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+
+class GpDelete(ApduCommand, n='DELETE', ins=0xE4, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpStoreData(ApduCommand, n='STORE DATA', ins=0xE2, cla=['8X', 'CX', 'EX']):
+    @classmethod
+    def _get_apdu_case(cls, hdr:bytes) -> int:
+        p1 = hdr[2]
+        if p1 & 0x01:
+            return 4
+        else:
+            return 3
+
+class GpGetDataCA(ApduCommand, n='GET DATA', ins=0xCA, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpGetDataCB(ApduCommand, n='GET DATA', ins=0xCB, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpGetStatus(ApduCommand, n='GET STATUS', ins=0xF2, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpInstall(ApduCommand, n='INSTALL', ins=0xE6, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpLoad(ApduCommand, n='LOAD', ins=0xE8, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpPutKey(ApduCommand, n='PUT KEY', ins=0xD8, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpSetStatus(ApduCommand, n='SET STATUS', ins=0xF0, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+
+ApduCommands = ApduCommandSet('GlobalPlatform v2.3.1', cmds=[GpDelete, GpStoreData,
+                              GpGetDataCA, GpGetDataCB, GpGetStatus, GpInstall,
+                              GpLoad, GpPutKey, GpSetStatus])

pySim/apdu/ts_102_221.py

@@ -0,0 +1,525 @@
+# coding=utf-8
+"""APDU definitions/decoders of ETSI TS 102 221, the core UICC spec.
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import logging
+from pySim.construct import *
+from pySim.filesystem import *
+from pySim.apdu import ApduCommand, ApduCommandSet
+from typing import Optional, Dict, Tuple
+
+logger = logging.getLogger(__name__)
+
+# TS 102 221 Section 11.1.1
+class UiccSelect(ApduCommand, n='SELECT', ins=0xA4, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p1 = Enum(Byte, df_ef_or_mf_by_file_id=0, child_df_of_current_df=1, parent_df_of_current_df=3,
+                         df_name=4, path_from_mf=8, path_from_current_df=9)
+    _construct_p2 = BitStruct(Flag,
+                              'app_session_control'/Enum(BitsInteger(2), activation_reset=0, termination=2),
+                              'return'/Enum(BitsInteger(3), fcp=1, no_data=3),
+                              'aid_control'/Enum(BitsInteger(2), first_or_only=0, last=1, next=2, previous=3))
+
+    @staticmethod
+    def _find_aid_substr(selectables, aid) -> Optional[CardADF]:
+        # full-length match
+        if aid in selectables:
+            return selectables[aid]
+        # sub-string match
+        for s in selectables.keys():
+            if aid[:len(s)] == s:
+                return selectables[s]
+        return None
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        mode = self.cmd_dict['p1']
+        if mode in ['path_from_mf', 'path_from_current_df']:
+            # rewind to MF, if needed
+            if mode == 'path_from_mf':
+                lchan.selected_file = lchan.rs.mf
+            path = [self.cmd_data[i:i+2] for i in range(0, len(self.cmd_data), 2)]
+            for file in path:
+                file_hex = b2h(file)
+                if file_hex == '7fff': # current application
+                    if not lchan.selected_adf:
+                        sels = lchan.rs.mf.get_app_selectables(['ANAMES'])
+                        # HACK: Assume USIM
+                        logger.warning('SELECT relative to current ADF, but no ADF selected. Assuming ADF.USIM')
+                        lchan.selected_adf = sels['ADF.USIM']
+                    lchan.selected_file = lchan.selected_adf
+                    #print("\tSELECT CUR_ADF %s" % lchan.selected_file)
+                    # iterate to next element in path
+                    continue
+                else:
+                    sels = lchan.selected_file.get_selectables(['FIDS','MF','PARENT','SELF'])
+                    if file_hex in sels:
+                        if self.successful:
+                            #print("\tSELECT %s" % sels[file_hex])
+                            lchan.selected_file = sels[file_hex]
+                        else:
+                            #print("\tSELECT %s FAILED" % sels[file_hex])
+                            pass
+                        # iterate to next element in path
+                        continue
+                logger.warning('SELECT UNKNOWN FID %s (%s)' % (file_hex, '/'.join([b2h(x) for x in path])))
+        elif mode == 'df_ef_or_mf_by_file_id':
+            if len(self.cmd_data) != 2:
+                raise ValueError('Expecting a 2-byte FID')
+            sels = lchan.selected_file.get_selectables(['FIDS','MF','PARENT','SELF'])
+            file_hex = b2h(self.cmd_data)
+            if file_hex in sels:
+                if self.successful:
+                    #print("\tSELECT %s" % sels[file_hex])
+                    lchan.selected_file = sels[file_hex]
+                else:
+                    #print("\tSELECT %s FAILED" % sels[file_hex])
+                    pass
+            else:
+                logger.warning('SELECT UNKNOWN FID %s' % (file_hex))
+        elif mode == 'df_name':
+            # Select by AID (can be sub-string!)
+            aid = self.cmd_dict['body']
+            sels = lchan.rs.mf.get_app_selectables(['AIDS'])
+            adf = self._find_aid_substr(sels, aid)
+            if adf:
+                lchan.selected_adf = adf
+                lchan.selected_file = lchan.selected_adf
+                #print("\tSELECT AID %s" % adf)
+            else:
+                logger.warning('SELECT UNKNOWN AID %s' % aid)
+                pass
+        else:
+            raise ValueError('Select Mode %s not implemented' % mode)
+        # decode the SELECT response
+        if self.successful:
+            self.file = lchan.selected_file
+            if 'body' in self.rsp_dict:
+                # not every SELECT is asking for the FCP in response...
+                return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+        return None
+
+
+
+# TS 102 221 Section 11.1.2
+class UiccStatus(ApduCommand, n='STATUS', ins=0xF2, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 2
+    _construct_p1 = Enum(Byte, no_indication=0, current_app_is_initialized=1, terminal_will_terminate_current_app=2)
+    _construct_p2 = Enum(Byte, response_like_select=0, response_df_name_tlv=1, response_no_data=0x0c)
+
+    def process_on_lchan(self, lchan):
+        if self.cmd_dict['p2'] == 'response_like_select':
+            return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+
+def _decode_binary_p1p2(p1, p2) -> Dict:
+    ret = {}
+    if p1 & 0x80:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p1 & 0x1f
+        ret['offset'] = p2
+    else:
+        ret['file'] = 'currently_selected_ef'
+        ret['offset'] = ((p1 & 0x7f) << 8) & p2
+    return ret
+
+# TS 102 221 Section 11.1.3
+class ReadBinary(ApduCommand, n='READ BINARY', ins=0xB0, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short reads
+        if self.cmd_dict['offset'] != 0 or self.lr < self.file.size[0]:
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 102 221 Section 11.1.4
+class UpdateBinary(ApduCommand, n='UPDATE BINARY', ins=0xD6, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short writes
+        if self.cmd_dict['offset'] != 0 or self.lc < self.file.size[0]:
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+def _decode_record_p1p2(p1, p2):
+    ret = {}
+    ret['record_number'] = p1
+    if p2 >> 3 == 0:
+        ret['file'] = 'currently_selected_ef'
+    else:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p2 >> 3
+    mode = p2 & 0x7
+    if mode == 2:
+        ret['mode'] = 'next_record'
+    elif mode == 3:
+        ret['mode'] = 'previous_record'
+    elif mode == 8:
+        ret['mode'] = 'absolute_current'
+    return ret
+
+# TS 102 221 Section 11.1.5
+class ReadRecord(ApduCommand, n='READ RECORD', ins=0xB2, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 102 221 Section 11.1.6
+class UpdateRecord(ApduCommand, n='UPDATE RECORD', ins=0xDC, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+# TS 102 221 Section 11.1.7
+class SearchRecord(ApduCommand, n='SEARCH RECORD', ins=0xA2, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_rsp = GreedyRange(Int8ub)
+
+    def _decode_p1p2(self):
+        ret = {}
+        sfi = self.p2 >> 3
+        if sfi == 0:
+            ret['file'] = 'currently_selected_ef'
+        else:
+            ret['file'] = 'sfi'
+            ret['sfi'] = sfi
+        mode = self.p2 & 0x7
+        if mode in [0x4, 0x5]:
+            if mode == 0x4:
+                ret['mode'] = 'forward_search'
+            else:
+                ret['mode'] = 'backward_search'
+            ret['record_number'] = self.p1
+            self.col_id = '%02u' % ret['record_number']
+        elif mode == 6:
+            ret['mode'] = 'enhanced_search'
+            # TODO: further decode
+        elif mode == 7:
+            ret['mode'] = 'proprietary_search'
+        return ret
+
+    def _decode_cmd(self):
+        ret = self._decode_p1p2()
+        if self.cmd_data:
+            if ret['mode'] == 'enhanced_search':
+                ret['search_indication'] = b2h(self.cmd_data[:2])
+                ret['search_string'] = b2h(self.cmd_data[2:])
+            else:
+                ret['search_string'] = b2h(self.cmd_data)
+        return ret
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        return self.to_dict()
+
+# TS 102 221 Section 11.1.8
+class Increase(ApduCommand, n='INCREASE', ins=0x32, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+PinConstructP2 = BitStruct('scope'/Enum(Flag, global_mf=0, specific_df_adf=1),
+                           BitsInteger(2), 'reference_data_nr'/BitsInteger(5))
+# TS 102 221 Section 11.1.9
+class VerifyPin(ApduCommand, n='VERIFY PIN', ins=0x20, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    @staticmethod
+    def _pin_process(apdu):
+        processed = {
+            'scope': apdu.cmd_dict['p2']['scope'],
+            'referenced_data_nr': apdu.cmd_dict['p2']['reference_data_nr'],
+            }
+        if apdu.lc == 0:
+            # this is just a question on the counters remaining
+            processed['mode'] = 'check_remaining_attempts'
+        else:
+            processed['pin'] = b2h(apdu.cmd_data)
+        if apdu.sw[0] == 0x63:
+            processed['remaining_attempts'] = apdu.sw[1] & 0xf
+        return processed
+
+    @staticmethod
+    def _pin_is_success(sw):
+        if sw[0] == 0x63:
+            return True
+        else:
+            return False
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.10
+class ChangePin(ApduCommand, n='CHANGE PIN', ins=0x24, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.11
+class DisablePin(ApduCommand, n='DISABLE PIN', ins=0x26, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.12
+class EnablePin(ApduCommand, n='ENABLE PIN', ins=0x28, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.13
+class UnblockPin(ApduCommand, n='UNBLOCK PIN', ins=0x2C, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.14
+class DeactivateFile(ApduCommand, n='DEACTIVATE FILE', ins=0x04, cla=['0X', '4X', '6X']):
+    _apdu_case = 1
+    _construct_p1 = BitStruct(BitsInteger(4),
+                              'select_mode'/Enum(BitsInteger(4), ef_by_file_id=0,
+                                                 path_from_mf=8, path_from_current_df=9))
+
+# TS 102 221 Section 11.1.15
+class ActivateFile(ApduCommand, n='ACTIVATE FILE', ins=0x44, cla=['0X', '4X', '6X']):
+    _apdu_case = 1
+    _construct_p1 = DeactivateFile._construct_p1
+
+# TS 102 221 Section 11.1.16
+auth_p2_construct = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              BitsInteger(2),
+                              'reference_data_nr'/BitsInteger(5))
+class Authenticate88(ApduCommand, n='AUTHENTICATE', ins=0x88, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = auth_p2_construct
+
+# TS 102 221 Section 11.1.16
+class Authenticate89(ApduCommand, n='AUTHENTICATE', ins=0x89, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = auth_p2_construct
+
+# TS 102 221 Section 11.1.17
+class ManageChannel(ApduCommand, n='MANAGE CHANNEL', ins=0x70, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+    _construct_p1 = Enum(Flag, open_channel=0, close_channel=1)
+    _construct_p2 = Struct('logical_channel_number'/Int8ub)
+    _construct_rsp = Struct('logical_channel_number'/Int8ub)
+
+    def process_global(self, rs):
+        if not self.successful:
+            return
+        mode = self.cmd_dict['p1']
+        if mode == 'open_channel':
+            created_channel_nr = self.cmd_dict['p2']['logical_channel_number']
+            if created_channel_nr == 0:
+                # auto-assignment by UICC
+                # pylint: disable=unsubscriptable-object
+                created_channel_nr = self.rsp_data[0]
+            manage_channel = rs.get_lchan_by_cla(self.cla)
+            manage_channel.add_lchan(created_channel_nr)
+            self.col_id = '%02u' % created_channel_nr
+        elif mode == 'close_channel':
+            closed_channel_nr = self.cmd_dict['p2']
+            rs.del_lchan(closed_channel_nr)
+            self.col_id = '%02u' % closed_channel_nr
+        else:
+            raise ValueError('Unsupported MANAGE CHANNEL P1=%02X' % self.p1)
+
+# TS 102 221 Section 11.1.18
+class GetChallenge(ApduCommand, n='GET CHALLENGE', ins=0x84, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+
+# TS 102 221 Section 11.1.19
+class TerminalCapability(ApduCommand, n='TERMINAL CAPABILITY', ins=0xAA, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+
+# TS 102 221 Section 11.1.20
+class ManageSecureChannel(ApduCommand, n='MANAGE SECURE CHANNEL', ins=0x73, cla=['0X', '4X', '6X']):
+    @classmethod
+    def _get_apdu_case(cls, hdr:bytes) -> int:
+        p1 = hdr[2]
+        p2 = hdr[3]
+        if p1 & 0x7 == 0:   # retrieve UICC Endpoints
+            return 2
+        elif p1 & 0xf in [1,2,3]:   # establish sa, start secure channel SA
+            p2_cmd = p2 >> 5
+            if p2_cmd in [0,2,4]:   # command data
+                return 3
+            elif p2_cmd in [1,3,5]: # response data
+                return 2
+        elif p1 & 0xf == 4:         # terminate secure channel SA
+            return 3
+        raise ValueError('%s: Unable to detect APDU case for %s' % (cls.__name__, b2h(hdr)))
+
+# TS 102 221 Section 11.1.21
+class TransactData(ApduCommand, n='TRANSACT DATA', ins=0x75, cla=['0X', '4X', '6X']):
+    @classmethod
+    def _get_apdu_case(cls, hdr:bytes) -> int:
+        p1 = hdr[2]
+        if p1 & 0x04:
+            return 3
+        else:
+            return 2
+
+# TS 102 221 Section 11.1.22
+class SuspendUicc(ApduCommand, n='SUSPEND UICC', ins=0x76, cla=['80']):
+    _apdu_case = 4
+    _construct_p1 = BitStruct('rfu'/BitsInteger(7), 'mode'/Enum(Flag, suspend=0, resume=1))
+
+# TS 102 221 Section 11.1.23
+class GetIdentity(ApduCommand, n='GET IDENTITY', ins=0x78, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1), BitsInteger(7))
+
+# TS 102 221 Section 11.1.24
+class ExchangeCapabilities(ApduCommand, n='EXCHANGE CAPABILITIES', ins=0x7A, cla=['80']):
+    _apdu_case = 4
+
+# TS 102 221 Section 11.2.1
+class TerminalProfile(ApduCommand, n='TERMINAL PROFILE', ins=0x10, cla=['80']):
+    _apdu_case = 3
+
+# TS 102 221 Section 11.2.2 / TS 102 223
+class Envelope(ApduCommand, n='ENVELOPE', ins=0xC2, cla=['80']):
+    _apdu_case = 4
+
+# TS 102 221 Section 11.2.3 / TS 102 223
+class Fetch(ApduCommand, n='FETCH', ins=0x12, cla=['80']):
+    _apdu_case = 2
+
+# TS 102 221 Section 11.2.3 / TS 102 223
+class TerminalResponse(ApduCommand, n='TERMINAL RESPONSE', ins=0x14, cla=['80']):
+    _apdu_case = 3
+
+# TS 102 221 Section 11.3.1
+class RetrieveData(ApduCommand, n='RETRIEVE DATA', ins=0xCB, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+    @staticmethod
+    def _tlv_decode_cmd(self : ApduCommand) -> Dict:
+        c = {}
+        if self.p2 & 0xc0 == 0x80:
+            c['mode'] = 'first_block'
+            sfi = self.p2 & 0x1f
+            if sfi == 0:
+                c['file'] = 'currently_selected_ef'
+            else:
+                c['file'] = 'sfi'
+                c['sfi'] = sfi
+            c['tag'] = i2h([self.cmd_data[0]])
+        elif self.p2 & 0xdf == 0x00:
+            c['mode'] = 'next_block'
+        elif self.p2 & 0xdf == 0x40:
+            c['mode'] = 'retransmit_previous_block'
+        else:
+            logger.warning('%s: invalid P2=%02x' % (self, self.p2))
+        return c
+
+    def _decode_cmd(self):
+        return RetrieveData._tlv_decode_cmd(self)
+
+    def _decode_rsp(self):
+        # TODO: parse tag/len/val?
+        return b2h(self.rsp_data)
+
+
+# TS 102 221 Section 11.3.2
+class SetData(ApduCommand, n='SET DATA', ins=0xDB, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+
+    def _decode_cmd(self):
+        c = RetrieveData._tlv_decode_cmd(self)
+        if c['mode'] == 'first_block':
+            if len(self.cmd_data) == 0:
+                c['delete'] = True
+        # TODO: parse tag/len/val?
+        c['data'] = b2h(self.cmd_data)
+        return c
+
+
+# TS 102 221 Section 12.1.1
+class GetResponse(ApduCommand, n='GET RESPONSE', ins=0xC0, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+
+ApduCommands = ApduCommandSet('TS 102 221', cmds=[UiccSelect, UiccStatus, ReadBinary, UpdateBinary, ReadRecord,
+                              UpdateRecord, SearchRecord, Increase, VerifyPin, ChangePin, DisablePin,
+                              EnablePin, UnblockPin, DeactivateFile, ActivateFile, Authenticate88,
+                              Authenticate89, ManageChannel, GetChallenge, TerminalCapability,
+                              ManageSecureChannel, TransactData, SuspendUicc, GetIdentity,
+                              ExchangeCapabilities, TerminalProfile, Envelope, Fetch, TerminalResponse,
+                              RetrieveData, SetData, GetResponse])

pySim/apdu/ts_31_102.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+
+# without this, pylint will fail when inner classes are used
+# within the 'nested' kwarg of our TlvMeta metaclass on python 3.7 :(
+# pylint: disable=undefined-variable
+
+"""
+APDU commands of 3GPP TS 31.102 V16.6.0
+"""
+from typing import Dict
+
+from construct import *
+from construct import Optional as COptional
+from pySim.filesystem import *
+from pySim.construct import *
+from pySim.ts_31_102 import SUCI_TlvDataObject
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+
+# Copyright (C) 2022 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Mapping between USIM Service Number and its description
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+
+# TS 31.102 Section 7.1
+class UsimAuthenticateEven(ApduCommand, n='AUTHENTICATE', ins=0x88, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              BitsInteger(4),
+                              'authentication_context'/Enum(BitsInteger(3), gsm=0, umts=1,
+                                                            vgcs_vbs=2, gba=4))
+    _cs_cmd_gsm_3g =  Struct('_rand_len'/Int8ub, 'rand'/HexAdapter(Bytes(this._rand_len)),
+                         '_autn_len'/COptional(Int8ub), 'autn'/If(this._autn_len, HexAdapter(Bytes(this._autn_len))))
+    _cs_cmd_vgcs = Struct('_vsid_len'/Int8ub, 'vservice_id'/HexAdapter(Bytes(this._vsid_len)),
+                          '_vkid_len'/Int8ub, 'vk_id'/HexAdapter(Bytes(this._vkid_len)),
+                          '_vstk_rand_len'/Int8ub, 'vstk_rand'/HexAdapter(Bytes(this._vstk_rand_len)))
+    _cmd_gba_bs = Struct('_rand_len'/Int8ub, 'rand'/HexAdapter(Bytes(this._rand_len)),
+                         '_autn_len'/Int8ub, 'autn'/HexAdapter(Bytes(this._autn_len)))
+    _cmd_gba_naf = Struct('_naf_id_len'/Int8ub, 'naf_id'/HexAdapter(Bytes(this._naf_id_len)),
+                          '_impi_len'/Int8ub, 'impi'/HexAdapter(Bytes(this._impi_len)))
+    _cs_cmd_gba = Struct('tag'/Int8ub, 'body'/Switch(this.tag, { 0xDD: 'bootstrap'/_cmd_gba_bs,
+                                                                 0xDE: 'naf_derivation'/_cmd_gba_naf }))
+    _cs_rsp_gsm = Struct('_len_sres'/Int8ub, 'sres'/HexAdapter(Bytes(this._len_sres)),
+                         '_len_kc'/Int8ub, 'kc'/HexAdapter(Bytes(this._len_kc)))
+    _rsp_3g_ok = Struct('_len_res'/Int8ub, 'res'/HexAdapter(Bytes(this._len_res)),
+                        '_len_ck'/Int8ub, 'ck'/HexAdapter(Bytes(this._len_ck)),
+                        '_len_ik'/Int8ub, 'ik'/HexAdapter(Bytes(this._len_ik)),
+                        '_len_kc'/COptional(Int8ub), 'kc'/If(this._len_kc, HexAdapter(Bytes(this._len_kc))))
+    _rsp_3g_sync = Struct('_len_auts'/Int8ub, 'auts'/HexAdapter(Bytes(this._len_auts)))
+    _cs_rsp_3g = Struct('tag'/Int8ub, 'body'/Switch(this.tag, { 0xDB: 'success'/_rsp_3g_ok,
+                                                                0xDC: 'sync_fail'/_rsp_3g_sync}))
+    _cs_rsp_vgcs = Struct(Const(b'\xDB'), '_vstk_len'/Int8ub, 'vstk'/HexAdapter(Bytes(this._vstk_len)))
+    _cs_rsp_gba_naf = Struct(Const(b'\xDB'), '_ks_ext_naf_len'/Int8ub, 'ks_ext_naf'/HexAdapter(Bytes(this._ks_ext_naf_len)))
+    def _decode_cmd(self) -> Dict:
+        r = {}
+        r['p1'] = parse_construct(self._construct_p1, self.p1.to_bytes(1, 'big'))
+        r['p2'] = parse_construct(self._construct_p2, self.p2.to_bytes(1, 'big'))
+        auth_ctx = r['p2']['authentication_context']
+        if auth_ctx in ['gsm', 'umts']:
+            r['body'] = parse_construct(self._cs_cmd_gsm_3g, self.cmd_data)
+        elif auth_ctx == 'vgcs_vbs':
+            r['body'] = parse_construct(self._cs_cmd_vgcs, self.cmd_data)
+        elif auth_ctx == 'gba':
+            r['body'] = parse_construct(self._cs_cmd_gba, self.cmd_data)
+        else:
+            raise ValueError('Unsupported authentication_context: %s' % auth_ctx)
+        return r
+
+    def _decode_rsp(self) -> Dict:
+        r = {}
+        auth_ctx = self.cmd_dict['p2']['authentication_context']
+        if auth_ctx == 'gsm':
+            r['body'] = parse_construct(self._cs_rsp_gsm, self.rsp_data)
+        elif auth_ctx == 'umts':
+            r['body'] = parse_construct(self._cs_rsp_3g, self.rsp_data)
+        elif auth_ctx == 'vgcs_vbs':
+            r['body'] = parse_construct(self._cs_rsp_vgcs, self.rsp_data)
+        elif auth_ctx == 'gba':
+            if self.cmd_dict['body']['tag'] == 0xDD:
+                r['body'] = parse_construct(self._cs_rsp_3g, self.rsp_data)
+            else:
+                r['body'] = parse_construct(self._cs_rsp_gba_naf, self.rsp_data)
+        else:
+            raise ValueError('Unsupported authentication_context: %s' % auth_ctx)
+        return r
+
+class UsimAuthenticateOdd(ApduCommand, n='AUTHENTICATE', ins=0x89, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              BitsInteger(4),
+                              'authentication_context'/Enum(BitsInteger(3), mbms=5, local_key=6))
+# TS 31.102 Section 7.5
+class UsimGetIdentity(ApduCommand, n='GET IDENTITY', ins=0x78, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              'identity_context'/Enum(BitsInteger(7), suci=1))
+    _tlv_rsp = SUCI_TlvDataObject
+
+ApduCommands = ApduCommandSet('TS 31.102', cmds=[UsimAuthenticateEven, UsimAuthenticateOdd,
+                                                 UsimGetIdentity])

pySim/apdu/ts_51_011.py

@@ -0,0 +1,337 @@
+# coding=utf-8
+"""APDU definitions/decoders of 3GPP TS 51.011, the classic SIM spec.
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import logging
+from pySim.construct import *
+from pySim.filesystem import *
+from pySim.apdu import ApduCommand, ApduCommandSet
+from typing import Optional, Dict, Tuple
+
+logger = logging.getLogger(__name__)
+
+# TS 51.011 Section 9.2.1
+class SimSelect(ApduCommand, n='SELECT', ins=0xA4, cla=['A0']):
+    _apdu_case = 4
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        path = [self.cmd_data[i:i+2] for i in range(0, len(self.cmd_data), 2)]
+        for file in path:
+            file_hex = b2h(file)
+            sels = lchan.selected_file.get_selectables(['FIDS'])
+            if file_hex in sels:
+                if self.successful:
+                    #print("\tSELECT %s" % sels[file_hex])
+                    lchan.selected_file = sels[file_hex]
+                else:
+                    #print("\tSELECT %s FAILED" % sels[file_hex])
+                    pass
+                continue
+        logger.warning('SELECT UNKNOWN FID %s (%s)' % (file_hex, '/'.join([b2h(x) for x in path])))
+        if len(self.cmd_data) != 2:
+            raise ValueError('Expecting a 2-byte FID')
+
+        # decode the SELECT response
+        if self.successful:
+            self.file = lchan.selected_file
+            if 'body' in self.rsp_dict:
+                # not every SELECT is asking for the FCP in response...
+                return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+        return None
+
+
+# TS 51.011 Section 9.2.2
+class SimStatus(ApduCommand, n='STATUS', ins=0xF2, cla=['A0']):
+    _apdu_case = 2
+
+    def process_on_lchan(self, lchan):
+        if self.successful:
+            if 'body' in self.rsp_dict:
+                return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+
+def _decode_binary_p1p2(p1, p2) -> Dict:
+    ret = {}
+    if p1 & 0x80:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p1 & 0x1f
+        ret['offset'] = p2
+    else:
+        ret['file'] = 'currently_selected_ef'
+        ret['offset'] = ((p1 & 0x7f) << 8) & p2
+    return ret
+
+# TS 51.011 Section 9.2.3 / 31.101
+class ReadBinary(ApduCommand, n='READ BINARY', ins=0xB0, cla=['A0']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short reads
+        if self.cmd_dict['offset'] != 0 or self.lr < self.file.size[0]:
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 51.011 Section 9.2.4 / 31.101
+class UpdateBinary(ApduCommand, n='UPDATE BINARY', ins=0xD6, cla=['A0']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short writes
+        if self.cmd_dict['offset'] != 0 or self.lc < self.file.size[0]:
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+def _decode_record_p1p2(p1, p2):
+    ret = {}
+    ret['record_number'] = p1
+    if p2 >> 3 == 0:
+        ret['file'] = 'currently_selected_ef'
+    else:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p2 >> 3
+    mode = p2 & 0x7
+    if mode == 2:
+        ret['mode'] = 'next_record'
+    elif mode == 3:
+        ret['mode'] = 'previous_record'
+    elif mode == 8:
+        ret['mode'] = 'absolute_current'
+    return ret
+
+# TS 51.011 Section 9.2.5
+class ReadRecord(ApduCommand, n='READ RECORD', ins=0xB2, cla=['A0']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 51.011 Section 9.2.6
+class UpdateRecord(ApduCommand, n='UPDATE RECORD', ins=0xDC, cla=['A0']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+# TS 51.011 Section 9.2.7
+class Seek(ApduCommand, n='SEEK', ins=0xA2, cla=['A0']):
+    _apdu_case = 4
+    _construct_rsp = GreedyRange(Int8ub)
+
+    def _decode_p1p2(self):
+        ret = {}
+        sfi = self.p2 >> 3
+        if sfi == 0:
+            ret['file'] = 'currently_selected_ef'
+        else:
+            ret['file'] = 'sfi'
+            ret['sfi'] = sfi
+        mode = self.p2 & 0x7
+        if mode in [0x4, 0x5]:
+            if mode == 0x4:
+                ret['mode'] = 'forward_search'
+            else:
+                ret['mode'] = 'backward_search'
+            ret['record_number'] = self.p1
+            self.col_id = '%02u' % ret['record_number']
+        elif mode == 6:
+            ret['mode'] = 'enhanced_search'
+            # TODO: further decode
+        elif mode == 7:
+            ret['mode'] = 'proprietary_search'
+        return ret
+
+    def _decode_cmd(self):
+        ret = self._decode_p1p2()
+        if self.cmd_data:
+            if ret['mode'] == 'enhanced_search':
+                ret['search_indication'] = b2h(self.cmd_data[:2])
+                ret['search_string'] = b2h(self.cmd_data[2:])
+            else:
+                ret['search_string'] = b2h(self.cmd_data)
+        return ret
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        return self.to_dict()
+
+# TS 51.011 Section 9.2.8
+class Increase(ApduCommand, n='INCREASE', ins=0x32, cla=['A0']):
+    _apdu_case = 4
+
+PinConstructP2 = BitStruct('scope'/Enum(Flag, global_mf=0, specific_df_adf=1),
+                           BitsInteger(2), 'reference_data_nr'/BitsInteger(5))
+
+# TS 51.011 Section 9.2.9
+class VerifyChv(ApduCommand, n='VERIFY CHV', ins=0x20, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    @staticmethod
+    def _pin_process(apdu):
+        processed = {
+            'scope': apdu.cmd_dict['p2']['scope'],
+            'referenced_data_nr': apdu.cmd_dict['p2']['reference_data_nr'],
+            }
+        if apdu.lc == 0:
+            # this is just a question on the counters remaining
+            processed['mode'] = 'check_remaining_attempts'
+        else:
+            processed['pin'] = b2h(apdu.cmd_data)
+        if apdu.sw[0] == 0x63:
+            processed['remaining_attempts'] = apdu.sw[1] & 0xf
+        return processed
+
+    @staticmethod
+    def _pin_is_success(sw):
+        if sw[0] == 0x63:
+            return True
+        else:
+            return False
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.10
+class ChangeChv(ApduCommand, n='CHANGE CHV', ins=0x24, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.11
+class DisableChv(ApduCommand, n='DISABLE CHV', ins=0x26, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.12
+class EnableChv(ApduCommand, n='ENABLE CHV', ins=0x28, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.13
+class UnblockChv(ApduCommand, n='UNBLOCK CHV', ins=0x2C, cla=['A0']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 51.011 Section 9.2.14
+class Invalidate(ApduCommand, n='INVALIDATE', ins=0x04, cla=['A0']):
+    _apdu_case = 1
+    _construct_p1 = BitStruct(BitsInteger(4),
+                              'select_mode'/Enum(BitsInteger(4), ef_by_file_id=0,
+                                                 path_from_mf=8, path_from_current_df=9))
+
+# TS 51.011 Section 9.2.15
+class Rehabilitate(ApduCommand, n='REHABILITATE', ins=0x44, cla=['A0']):
+    _apdu_case = 1
+    _construct_p1 = Invalidate._construct_p1
+
+# TS 51.011 Section 9.2.16
+class RunGsmAlgorithm(ApduCommand, n='RUN GSM ALGORITHM', ins=0x88, cla=['A0']):
+    _apdu_case = 4
+    _construct = Struct('rand'/HexAdapter(Bytes(16)))
+    _construct_rsp = Struct('sres'/HexAdapter(Bytes(4)), 'kc'/HexAdapter(Bytes(8)))
+
+# TS 51.011 Section 9.2.17
+class Sleep(ApduCommand, n='SLEEP', ins=0xFA, cla=['A0']):
+    _apdu_case = 2
+
+# TS 51.011 Section 9.2.18
+class GetResponse(ApduCommand, n='GET RESPONSE', ins=0xC0, cla=['A0']):
+    _apdu_case = 2
+
+# TS 51.011 Section 9.2.19
+class TerminalProfile(ApduCommand, n='TERMINAL PROFILE', ins=0x10, cla=['A0']):
+    _apdu_case = 3
+
+# TS 51.011 Section 9.2.20
+class Envelope(ApduCommand, n='ENVELOPE', ins=0xC2, cla=['A0']):
+    _apdu_case = 4
+
+# TS 51.011 Section 9.2.21
+class Fetch(ApduCommand, n='FETCH', ins=0x12, cla=['A0']):
+    _apdu_case = 2
+
+# TS 51.011 Section 9.2.22
+class TerminalResponse(ApduCommand, n='TERMINAL RESPONSE', ins=0x14, cla=['A0']):
+    _apdu_case = 3
+
+
+ApduCommands = ApduCommandSet('TS 51.011', cmds=[SimSelect, SimStatus, ReadBinary, UpdateBinary, ReadRecord,
+                              UpdateRecord, Seek, Increase, VerifyChv, ChangeChv, DisableChv,
+                              EnableChv, UnblockChv, Invalidate, Rehabilitate, RunGsmAlgorithm,
+                              Sleep, GetResponse, TerminalProfile, Envelope, Fetch, TerminalResponse])

pySim/apdu_source/__init__.py

@@ -0,0 +1,35 @@
+import abc
+import logging
+from typing import Union
+from pySim.apdu import Apdu, Tpdu, CardReset, TpduFilter
+
+PacketType = Union[Apdu, Tpdu, CardReset]
+
+logger = logging.getLogger(__name__)
+
+class ApduSource(abc.ABC):
+    def __init__(self):
+        self.apdu_filter = TpduFilter(None)
+
+    @abc.abstractmethod
+    def read_packet(self) -> PacketType:
+        """Read one packet from the source."""
+        pass
+
+    def read(self) -> Union[Apdu, CardReset]:
+        """Main function to call by the user: Blocking read, returns Apdu or CardReset."""
+        apdu = None
+        # loop until we actually have an APDU to return
+        while not apdu:
+            r = self.read_packet()
+            if not r:
+                continue
+            if isinstance(r, Tpdu):
+                apdu = self.apdu_filter.input_tpdu(r)
+            elif isinstance(r, Apdu):
+                apdu = r
+            elif isinstance(r, CardReset):
+                apdu = r
+            else:
+                ValueError('Unknown read_packet() return %s' % r)
+        return apdu

pySim/apdu_source/gsmtap.py

@@ -0,0 +1,57 @@
+# coding=utf-8
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from pySim.gsmtap import GsmtapMessage, GsmtapSource
+from . import ApduSource, PacketType, CardReset
+
+from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
+from pySim.apdu.global_platform import ApduCommands as GpApduCommands
+ApduCommands = UiccApduCommands + UsimApduCommands + GpApduCommands
+
+class GsmtapApduSource(ApduSource):
+    """ApduSource for handling GSMTAP-SIM messages received via UDP, such as
+    those generated by simtrace2-sniff.  Note that *if* you use IP loopback
+    and localhost addresses (which is the default), you will need to start
+    this source before starting simtrace2-sniff, as otherwise the latter will
+    claim the GSMTAP UDP port.
+    """
+    def __init__(self, bind_ip:str='127.0.0.1', bind_port:int=4729):
+        """Create a UDP socket for receiving GSMTAP-SIM messages.
+        Args:
+            bind_ip: IP address to which the socket should be bound (default: 127.0.0.1)
+            bind_port: UDP port number to which the socket should be bound (default: 4729)
+        """
+        super().__init__()
+        self.gsmtap = GsmtapSource(bind_ip, bind_port)
+
+    def read_packet(self) -> PacketType:
+        gsmtap_msg, addr = self.gsmtap.read_packet()
+        if gsmtap_msg['type'] != 'sim':
+            raise ValueError('Unsupported GSMTAP type %s' % gsmtap_msg['type'])
+        sub_type = gsmtap_msg['sub_type']
+        if sub_type == 'apdu':
+            return ApduCommands.parse_cmd_bytes(gsmtap_msg['body'])
+        elif sub_type == 'atr':
+            # card has been reset
+            return CardReset()
+        elif sub_type in ['pps_req', 'pps_rsp']:
+            # simply ignore for now
+            pass
+        else:
+            raise ValueError('Unsupported GSMTAP-SIM sub-type %s' % sub_type)

pySim/apdu_source/pyshark_rspro.py

@@ -0,0 +1,159 @@
+# coding=utf-8
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import sys
+import logging
+from pprint import pprint as pp
+from typing import Tuple
+import pyshark
+
+from pySim.utils import h2b, b2h
+from pySim.apdu import Tpdu
+from . import ApduSource, PacketType, CardReset
+
+logger = logging.getLogger(__name__)
+
+class _PysharkRspro(ApduSource):
+    """APDU Source [provider] base class for reading RSPRO (osmo-remsim) via tshark."""
+
+    def __init__(self, pyshark_inst):
+        self.pyshark = pyshark_inst
+        self.bank_id = None
+        self.bank_slot = None
+        self.cmd_tpdu = None
+        super().__init__()
+
+    @staticmethod
+    def get_bank_slot(bank_slot) -> Tuple[int, int]:
+        """Convert a 'bankSlot_element' field into a tuple of bank_id, slot_nr"""
+        bank_id = bank_slot.get_field('bankId')
+        slot_nr = bank_slot.get_field('slotNr')
+        return int(bank_id), int(slot_nr)
+
+    @staticmethod
+    def get_client_slot(client_slot) -> Tuple[int, int]:
+        """Convert a 'clientSlot_element' field into a tuple of client_id, slot_nr"""
+        client_id = client_slot.get_field('clientId')
+        slot_nr = client_slot.get_field('slotNr')
+        return int(client_id), int(slot_nr)
+
+    @staticmethod
+    def get_pstatus(pstatus) -> Tuple[int, int, int]:
+        """Convert a 'slotPhysStatus_element' field into a tuple of vcc, reset, clk"""
+        vccPresent = int(pstatus.get_field('vccPresent'))
+        resetActive = int(pstatus.get_field('resetActive'))
+        clkActive = int(pstatus.get_field('clkActive'))
+        return vccPresent, resetActive, clkActive
+
+    def read_packet(self) -> PacketType:
+        p = self.pyshark.next()
+        return self._parse_packet(p)
+
+    def _set_or_verify_bank_slot(self, bsl: Tuple[int, int]):
+        """Keep track of the bank:slot to make sure we don't mix traces of multiple cards"""
+        if not self.bank_id:
+            self.bank_id = bsl[0]
+            self.bank_slot = bsl[1]
+        else:
+            if self.bank_id != bsl[0] or self.bank_slot != bsl[1]:
+                raise ValueError('Received data for unexpected B(%u:%u)' % (bsl[0], bsl[1]))
+
+    def _parse_packet(self, p) -> PacketType:
+        rspro_layer = p['rspro']
+        #print("Layer: %s" %  rspro_layer)
+        rspro_element = rspro_layer.get_field('RsproPDU_element')
+        #print("Element: %s" % rspro_element)
+        msg_type = rspro_element.get_field('msg')
+        rspro_msg = rspro_element.get_field('msg_tree')
+        if msg_type == '12': # tpduModemToCard
+            modem2card = rspro_msg.get_field('tpduModemToCard_element')
+            #print(modem2card)
+            client_slot = modem2card.get_field('fromClientSlot_element')
+            csl = self.get_client_slot(client_slot)
+            bank_slot = modem2card.get_field('toBankSlot_element')
+            bsl = self.get_bank_slot(bank_slot)
+            self._set_or_verify_bank_slot(bsl)
+            data = modem2card.get_field('data').replace(':','')
+            logger.debug("C(%u:%u) -> B(%u:%u): %s" % (csl[0], csl[1], bsl[0], bsl[1], data))
+            # store the CMD portion until the RSP portion arrives later
+            self.cmd_tpdu = h2b(data)
+        elif msg_type == '13': # tpduCardToModem
+            card2modem = rspro_msg.get_field('tpduCardToModem_element')
+            #print(card2modem)
+            client_slot = card2modem.get_field('toClientSlot_element')
+            csl = self.get_client_slot(client_slot)
+            bank_slot = card2modem.get_field('fromBankSlot_element')
+            bsl = self.get_bank_slot(bank_slot)
+            self._set_or_verify_bank_slot(bsl)
+            data = card2modem.get_field('data').replace(':','')
+            logger.debug("C(%u:%u) <- B(%u:%u): %s" % (csl[0], csl[1], bsl[0], bsl[1], data))
+            rsp_tpdu = h2b(data)
+            if self.cmd_tpdu:
+                # combine this R-TPDU with the C-TPDU we saw earlier
+                r = Tpdu(self.cmd_tpdu, rsp_tpdu)
+                self.cmd_tpdu = False
+                return r
+        elif msg_type == '14': # clientSlotStatus
+            cl_slotstatus = rspro_msg.get_field('clientSlotStatusInd_element')
+            #print(cl_slotstatus)
+            client_slot = cl_slotstatus.get_field('fromClientSlot_element')
+            bank_slot = cl_slotstatus.get_field('toBankSlot_element')
+            slot_pstatus = cl_slotstatus.get_field('slotPhysStatus_element')
+            vccPresent, resetActive, clkActive = self.get_pstatus(slot_pstatus)
+            if vccPresent and clkActive and not resetActive:
+                logger.debug("RESET")
+                return CardReset()
+        else:
+            print("Unhandled msg type %s: %s" % (msg_type, rspro_msg))
+
+
+class PysharkRsproPcap(_PysharkRspro):
+    """APDU Source [provider] class for reading RSPRO (osmo-remsim) from a PCAP
+    file via pyshark, which in turn uses tshark (part of wireshark).
+
+    In order to use this, you need a wireshark patched with RSPRO support,
+    such as can be found at https://gitea.osmocom.org/osmocom/wireshark/src/branch/laforge/rspro
+
+    A STANDARD UPSTREAM WIRESHARK *DOES NOT WORK*.
+    """
+    def __init__(self, pcap_filename):
+        """
+        Args:
+            pcap_filename: File name of the pcap file to be opened
+        """
+        pyshark_inst = pyshark.FileCapture(pcap_filename, display_filter='rspro', use_json=True, keep_packets=False)
+        super().__init__(pyshark_inst)
+
+class PysharkRsproLive(_PysharkRspro):
+    """APDU Source [provider] class for reading RSPRO (osmo-remsim) from a live capture
+    via pyshark, which in turn uses tshark (part of wireshark).
+
+    In order to use this, you need a wireshark patched with RSPRO support,
+    such as can be found at https://gitea.osmocom.org/osmocom/wireshark/src/branch/laforge/rspro
+
+    A STANDARD UPSTREAM WIRESHARK *DOES NOT WORK*.
+    """
+    def __init__(self, interface, bpf_filter='tcp port 9999 or tcp port 9998'):
+        """
+        Args:
+            interface: Network interface name to capture packets on (like "eth0")
+            bfp_filter: libpcap capture filter to use
+        """
+        pyshark_inst = pyshark.LiveCapture(interface=interface, display_filter='rspro', bpf_filter=bpf_filter,
+                                           use_json=True)
+        super().__init__(pyshark_inst)

pySim/ara_m.py

@@ -311,7 +311,7 @@ class ADF_ARAM(CardADF):
                 self._cmd.poutput_json(res_do.to_dict())
 
         def do_aram_get_config(self, opts):
-            """GET DATA [Config] on the ARA-M Applet"""
+            """Perform GET DATA [Config] on the ARA-M Applet: Tell it our version and retrieve its version."""
             res_do = ADF_ARAM.get_config(self._cmd.card._scc._tp)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
@@ -345,7 +345,7 @@ class ADF_ARAM(CardADF):
 
         @cmd2.with_argparser(store_ref_ar_do_parse)
         def do_aram_store_ref_ar_do(self, opts):
-            """Perform STORE DATA [Command-Store-REF-AR-DO] to store a new access rule."""
+            """Perform STORE DATA [Command-Store-REF-AR-DO] to store a (new) access rule."""
             # REF
             ref_do_content = []
             if opts.aid:

pySim/cards.py

@@ -52,7 +52,7 @@ def format_addr(addr: str, addr_type: str) -> str:
     return res
 
 
-class SimCard(object):
+class SimCard:
 
     name = 'SIM'
 
@@ -63,7 +63,7 @@ class SimCard(object):
 
     def reset(self):
         rc = self._scc.reset_card()
-        if rc is 1:
+        if rc == 1:
             return self._scc.get_atr()
         else:
             return None
@@ -348,6 +348,9 @@ class UsimCard(SimCard):
     def __init__(self, ssc):
         super(UsimCard, self).__init__(ssc)
 
+	# See also: ETSI TS 102 221, Table 9.3
+        self._adm_chv_num = 0xA0
+
     def read_ehplmn(self):
         (res, sw) = self._scc.read_binary(EF_USIM_ADF_map['EHPLMN'])
         if sw == '9000':
@@ -425,6 +428,15 @@ class UsimCard(SimCard):
                 EF_USIM_ADF_map['UST'], content)
         return sw
 
+    def update_est(self, service, bit=1):
+        (res, sw) = self._scc.read_binary(EF_USIM_ADF_map['EST'])
+        if sw == '9000':
+            content = enc_st(res, service, bit)
+            (res, sw) = self._scc.update_binary(
+                EF_USIM_ADF_map['EST'], content)
+        return sw
+
+
 
 class IsimCard(SimCard):
 
@@ -566,6 +578,14 @@ class IsimCard(SimCard):
                     sw)
         return uiari_recs
 
+    def update_ist(self, service, bit=1):
+        (res, sw) = self._scc.read_binary(EF_ISIM_ADF_map['IST'])
+        if sw == '9000':
+            content = enc_st(res, service, bit)
+            (res, sw) = self._scc.update_binary(
+                EF_ISIM_ADF_map['IST'], content)
+        return sw
+
 
 class MagicSimBase(abc.ABC, SimCard):
     """

pySim/commands.py

@@ -23,11 +23,11 @@
 
 from construct import *
 from pySim.construct import LV
-from pySim.utils import rpad, b2h, h2b, sw_match, bertlv_encode_len, Hexstr, h2i, str_sanitize
+from pySim.utils import rpad, b2h, h2b, sw_match, bertlv_encode_len, Hexstr, h2i, str_sanitize, expand_hex
 from pySim.exceptions import SwMatchError
 
 
-class SimCardCommands(object):
+class SimCardCommands:
     def __init__(self, transport):
         self._tp = transport
         self.cla_byte = "a0"
@@ -136,6 +136,10 @@ class SimCardCommands(object):
 
         return self._tp.send_apdu_checksw(self.cla_byte + "a4" + self.sel_ctrl + "02" + fid)
 
+    def select_parent_df(self):
+        """Execute SELECT to switch to the parent DF """
+        return self._tp.send_apdu_checksw(self.cla_byte + "a4030400")
+
     def select_adf(self, aid: str):
         """Execute SELECT a given Applicaiton ADF.
 
@@ -186,6 +190,10 @@ class SimCardCommands(object):
                 offset : byte offset in file from which to start writing
                 verify : Whether or not to verify data after write
         """
+
+        file_len = self.binary_size(ef)
+        data = expand_hex(data, file_len)
+
         data_length = len(data) // 2
 
         # Save write cycles by reading+comparing before write
@@ -251,16 +259,17 @@ class SimCardCommands(object):
                 verify : verify data by re-reading the record
                 conserve : read record and compare it with data, skip write on match
         """
+
         res = self.select_path(ef)
+        rec_length = self.__record_len(res)
+        data = expand_hex(data, rec_length)
 
         if force_len:
             # enforce the record length by the actual length of the given data input
             rec_length = len(data) // 2
         else:
-            # determine the record length from the select response of the file and pad
-            # the input data with 0xFF if necessary. In cases where the input data
-            # exceed we throw an exception.
-            rec_length = self.__record_len(res)
+            # make sure the input data is padded to the record length using 0xFF.
+            # In cases where the input data exceed we throw an exception.
             if (len(data) // 2 > rec_length):
                 raise ValueError('Data length exceeds record length (expected max %d, got %d)' % (
                     rec_length, len(data) // 2))
@@ -444,6 +453,26 @@ class SimCardCommands(object):
         """
         return self._tp.send_apdu_checksw(self.cla_byte + '44000002' + fid)
 
+    def create_file(self, payload: Hexstr):
+        """Execute CREEATE FILE command as per TS 102 222 Section 6.3"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'e00000%02x%s' % (len(payload)//2, payload))
+
+    def delete_file(self, fid):
+        """Execute DELETE FILE command as per TS 102 222 Section 6.4"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'e4000002' + fid)
+
+    def terminate_df(self, fid):
+        """Execute TERMINATE DF command as per TS 102 222 Section 6.7"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'e6000002' + fid)
+
+    def terminate_ef(self, fid):
+        """Execute TERMINATE EF command as per TS 102 222 Section 6.8"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'e8000002' + fid)
+
+    def terminate_card_usage(self):
+        """Execute TERMINATE CARD USAGE command as per TS 102 222 Section 6.9"""
+        return self._tp.send_apdu_checksw(self.cla_byte + 'fe000000')
+
     def manage_channel(self, mode='open', lchan_nr=0):
         """Execute MANAGE CHANNEL command as per TS 102 221 Section 11.1.17.
 
@@ -597,3 +626,7 @@ class SimCardCommands(object):
         negotiated_duration_secs = decode_duration(data[:4])
         resume_token = data[4:]
         return (negotiated_duration_secs, resume_token, sw)
+
+    def get_data(self, tag: int, cla: int = 0x00):
+        data, sw = self._tp.send_apdu('%02xca%04x00' % (cla, tag))
+        return (data, sw)

pySim/construct.py

@@ -2,12 +2,14 @@ from construct.lib.containers import Container, ListContainer
 from construct.core import EnumIntegerString
 import typing
 from construct import *
+from construct.core import evaluate, bytes2integer, integer2bytes, BitwisableString
+from construct.lib import integertypes
 from pySim.utils import b2h, h2b, swap_nibbles
 import gsm0338
 
 """Utility code related to the integration of the 'construct' declarative parser."""
 
-# (C) 2021 by Harald Welte <laforge@osmocom.org>
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -42,6 +44,25 @@ class BcdAdapter(Adapter):
     def _encode(self, obj, context, path):
         return h2b(swap_nibbles(obj))
 
+class InvertAdapter(Adapter):
+    """inverse logic (false->true, true->false)."""
+    @staticmethod
+    def _invert_bool_in_obj(obj):
+        for k,v in obj.items():
+            # skip all private entries
+            if k.startswith('_'):
+                continue
+            if v == False:
+                obj[k] = True
+            elif v == True:
+                obj[k] = False
+        return obj
+
+    def _decode(self, obj, context, path):
+        return self._invert_bool_in_obj(obj)
+
+    def _encode(self, obj, context, path):
+        return self._invert_bool_in_obj(obj)
 
 class Rpad(Adapter):
     """
@@ -184,3 +205,61 @@ def GsmString(n):
         n (Integer): Fixed length of the encoded byte string
     '''
     return GsmStringAdapter(Rpad(Bytes(n), pattern=b'\xff'), codec='gsm03.38')
+
+class GreedyInteger(Construct):
+    """A variable-length integer implementation, think of combining GrredyBytes with BytesInteger."""
+    def __init__(self, signed=False, swapped=False, minlen=0):
+        super().__init__()
+        self.signed = signed
+        self.swapped = swapped
+        self.minlen = minlen
+
+    def _parse(self, stream, context, path):
+        data = stream_read_entire(stream, path)
+        if evaluate(self.swapped, context):
+            data = swapbytes(data)
+        try:
+            return bytes2integer(data, self.signed)
+        except ValueError as e:
+            raise IntegerError(str(e), path=path)
+
+    def __bytes_required(self, i, minlen=0):
+        if self.signed:
+            raise NotImplementedError("FIXME: Implement support for encoding signed integer")
+
+        # compute how many bytes we need
+        nbytes = 1
+        while True:
+            i = i >> 8
+            if i == 0:
+                break
+            else:
+                nbytes = nbytes + 1
+
+        # round up to the minimum number
+        # of bytes we anticipate
+        if nbytes < minlen:
+            nbytes = minlen
+
+        return nbytes
+
+    def _build(self, obj, stream, context, path):
+        if not isinstance(obj, integertypes):
+            raise IntegerError(f"value {obj} is not an integer", path=path)
+        length = self.__bytes_required(obj, self.minlen)
+        try:
+            data = integer2bytes(obj, length, self.signed)
+        except ValueError as e:
+            raise IntegerError(str(e), path=path)
+        if evaluate(self.swapped, context):
+            data = swapbytes(data)
+        stream_write(stream, data, length, path)
+        return obj
+
+# merged definitions of 24.008 + 23.040
+TypeOfNumber = Enum(BitsInteger(3), unknown=0, international=1, national=2, network_specific=3,
+                    short_code=4, alphanumeric=5, abbreviated=6, reserved_for_extension=7)
+NumberingPlan = Enum(BitsInteger(4), unknown=0, isdn_e164=1, data_x121=3, telex_f69=4,
+                     sc_specific_5=5, sc_specific_6=6, national=8, private=9,
+                     ermes=10, reserved_cts=11, reserved_for_extension=15)
+TonNpi = BitStruct('ext'/Flag, 'type_of_number'/TypeOfNumber, 'numbering_plan_id'/NumberingPlan)

pySim/exceptions.py

@@ -53,8 +53,8 @@ class SwMatchError(Exception):
         self.rs = rs
 
     def __str__(self):
-        if self.rs:
-            r = self.rs.interpret_sw(self.sw_actual)
+        if self.rs and self.rs.lchan[0]:
+            r = self.rs.lchan[0].interpret_sw(self.sw_actual)
             if r:
                 return "SW match failed! Expected %s and got %s: %s - %s" % (self.sw_expected, self.sw_actual, r[0], r[1])
         return "SW match failed! Expected %s and got %s." % (self.sw_expected, self.sw_actual)

pySim/filesystem.py

@@ -34,7 +34,7 @@ import cmd2
 from cmd2 import CommandSet, with_default_category, with_argparser
 import argparse
 
-from typing import cast, Optional, Iterable, List, Dict, Tuple
+from typing import cast, Optional, Iterable, List, Dict, Tuple, Union
 
 from smartcard.util import toBytes
 
@@ -44,8 +44,26 @@ from pySim.exceptions import *
 from pySim.jsonpath import js_path_find, js_path_modify
 from pySim.commands import SimCardCommands
 
+# int: a single service is associated with this file
+# list: any of the listed services requires this file
+# tuple: logical-and of the listed services requires this file
+CardFileService = Union[int, List[int], Tuple[int, ...]]
 
-class CardFile(object):
+Size = Tuple[int, Optional[int]]
+
+def lchan_nr_from_cla(cla: int) -> int:
+    """Resolve the logical channel number from the CLA byte."""
+    # TS 102 221 10.1.1 Coding of Class Byte
+    if cla >> 4 in [0x0, 0xA, 0x8]:
+        # Table 10.3
+        return cla & 0x03
+    elif cla & 0xD0 in [0x40, 0xC0]:
+        # Table 10.4a
+        return 4 + (cla & 0x0F)
+    else:
+        raise ValueError('Could not determine logical channel for CLA=%2X' % cla)
+
+class CardFile:
     """Base class for all objects in the smart card filesystem.
     Serve as a common ancestor to all other file types; rarely used directly.
     """
@@ -53,7 +71,8 @@ class CardFile(object):
     RESERVED_FIDS = ['3f00']
 
     def __init__(self, fid: str = None, sfid: str = None, name: str = None, desc: str = None,
-                 parent: Optional['CardDF'] = None, profile: Optional['CardProfile'] = None):
+                 parent: Optional['CardDF'] = None, profile: Optional['CardProfile'] = None,
+                 service: Optional[CardFileService] = None):
         """
         Args:
             fid : File Identifier (4 hex digits)
@@ -62,6 +81,7 @@ class CardFile(object):
             desc : Description of the file
             parent : Parent CardFile object within filesystem hierarchy
             profile : Card profile that this file should be part of
+            service : Service (SST/UST/IST) associated with the file
         """
         if not isinstance(self, CardADF) and fid == None:
             raise ValueError("fid is mandatory")
@@ -75,6 +95,7 @@ class CardFile(object):
         if self.parent and self.parent != self and self.fid:
             self.parent.add_file(self)
         self.profile = profile
+        self.service = service
         self.shell_commands = []  # type: List[CommandSet]
 
         # Note: the basic properties (fid, name, ect.) are verified when
@@ -93,6 +114,14 @@ class CardFile(object):
         else:
             return self.fid
 
+    def fully_qualified_path_str(self, prefer_name: bool = True) -> str:
+        """Return fully qualified path to file as string.
+
+        Args:
+            prefer_name : Preferably build path of names; fall-back to FIDs as required
+        """
+        return '/'.join(self.fully_qualified_path(prefer_name))
+
     def fully_qualified_path(self, prefer_name: bool = True) -> List[str]:
         """Return fully qualified path to file as list of FID or name strings.
 
@@ -108,6 +137,34 @@ class CardFile(object):
             ret.append(elem)
         return ret
 
+    def fully_qualified_path_fobj(self) -> List['CardFile']:
+        """Return fully qualified path to file as list of CardFile instance references."""
+        if self.parent and self.parent != self:
+            ret = self.parent.fully_qualified_path_fobj()
+        else:
+            ret = []
+        if self:
+            ret.append(self)
+        return ret
+
+    def build_select_path_to(self, target: 'CardFile') -> Optional[List['CardFile']]:
+        """Build the relative sequence of files we need to traverse to get from us to 'target'."""
+        cur_fqpath = self.fully_qualified_path_fobj()
+        target_fqpath = target.fully_qualified_path_fobj()
+        inter_path = []
+        cur_fqpath.pop()  # drop last element (currently selected file, doesn't need re-selection
+        cur_fqpath.reverse()
+        for ce in cur_fqpath:
+            inter_path.append(ce)
+            for i in range(0, len(target_fqpath)-1):
+                te = target_fqpath[i]
+                if te == ce:
+                    for te2 in target_fqpath[i+1:]:
+                        inter_path.append(te2)
+                    # we found our common ancestor
+                    return inter_path
+        return None
+
     def get_mf(self) -> Optional['CardMF']:
         """Return the MF (root) of the file system."""
         if self.parent == None:
@@ -137,6 +194,21 @@ class CardFile(object):
             sels.update({self.name: self})
         return sels
 
+    def _get_parent_selectables(self, alias: Optional[str] = None, flags=[]) -> Dict[str, 'CardFile']:
+        sels = {}
+        if not self.parent or self.parent == self:
+            return sels
+        # add our immediate parent
+        if alias:
+            sels.update({alias: self.parent})
+        if self.parent.fid and (flags == [] or 'FIDS' in flags):
+            sels.update({self.parent.fid: self.parent})
+        if self.parent.name and (flags == [] or 'FNAMES' in flags):
+            sels.update({self.parent.name: self.parent})
+        # recurse to parents of our parent, but without any alias
+        sels.update(self.parent._get_parent_selectables(None, flags))
+        return sels
+
     def get_selectables(self, flags=[]) -> Dict[str, 'CardFile']:
         """Return a dict of {'identifier': File} that is selectable from the current file.
 
@@ -153,8 +225,7 @@ class CardFile(object):
             sels = self._get_self_selectables('.', flags)
         # we can always select our parent
         if flags == [] or 'PARENT' in flags:
-            if self.parent:
-                sels = self.parent._get_self_selectables('..', flags)
+            sels.update(self._get_parent_selectables('..', flags))
         # if we have a MF, we can always select its applications
         if flags == [] or 'MF' in flags:
             mf = self.get_mf()
@@ -207,6 +278,28 @@ class CardFile(object):
             return self.parent.get_profile()
         return None
 
+    def should_exist_for_services(self, services: List[int]):
+        """Assuming the provided list of activated services, should this file exist and be activated?."""
+        if self.service is None:
+            return None
+        elif isinstance(self.service, int):
+            # a single service determines the result
+            return self.service in services
+        elif isinstance(self.service, list):
+            # any of the services active -> true
+            for s in self.service:
+                if s in services:
+                    return True
+            return False
+        elif isinstance(self.service, tuple):
+            # all of the services active -> true
+            for s in self.service:
+                if not s in services:
+                    return False
+            return True
+        else:
+            raise ValueError("self.service must be either int or list or tuple")
+
 
 class CardDF(CardFile):
     """DF (Dedicated File) in the smart card filesystem.  Those are basically sub-directories."""
@@ -223,10 +316,35 @@ class CardDF(CardFile):
         super().__init__(**kwargs)
         self.children = dict()
         self.shell_commands = [self.ShellCommands()]
+        # dict of CardFile affected by service(int), indexed by service
+        self.files_by_service = {}
 
     def __str__(self):
         return "DF(%s)" % (super().__str__())
 
+    def _add_file_services(self, child):
+        """Add a child (DF/EF) to the files_by_services of the parent."""
+        if not child.service:
+            return
+        if isinstance(child.service, int):
+            self.files_by_service.setdefault(child.service, []).append(child)
+        elif isinstance(child.service, list):
+            for service in child.service:
+                self.files_by_service.setdefault(service, []).append(child)
+        elif isinstance(child.service, tuple):
+            for service in child.service:
+                self.files_by_service.setdefault(service, []).append(child)
+        else:
+            raise ValueError
+
+    def _has_service(self):
+        if self.service:
+            return True
+        for c in self.children.values():
+            if isinstance(c, CardDF):
+                if c._has_service():
+                    return True
+
     def add_file(self, child: CardFile, ignore_existing: bool = False):
         """Add a child (DF/EF) to this DF.
         Args:
@@ -256,6 +374,16 @@ class CardDF(CardFile):
                 "File with given name %s already exists in %s" % (child.name, self))
         self.children[child.fid] = child
         child.parent = self
+        # update the service -> file relationship table
+        self._add_file_services(child)
+        if isinstance(child, CardDF):
+            for c in child.children.values():
+                self._add_file_services(c)
+                if isinstance(c, CardDF):
+                    for gc in c.children.values():
+                        if isinstance(gc, CardDF):
+                            if gc._has_service():
+                                raise ValueError('TODO: implement recursive service -> file mapping')
 
     def add_files(self, children: Iterable[CardFile], ignore_existing: bool = False):
         """Add a list of child (DF/EF) to this DF
@@ -337,7 +465,7 @@ class CardMF(CardDF):
 
     def get_app_names(self):
         """Get list of completions (AID names)"""
-        return [x.name for x in self.applications]
+        return list(self.applications.values())
 
     def get_selectables(self, flags=[]) -> dict:
         """Return a dict of {'identifier': File} that is selectable from the current DF.
@@ -363,7 +491,7 @@ class CardMF(CardDF):
                 {x.name: x for x in self.applications.values() if x.name})
         return sels
 
-    def decode_select_response(self, data_hex: str) -> object:
+    def decode_select_response(self, data_hex: Optional[str]) -> object:
         """Decode the response to a SELECT command.
 
         This is the fall-back method which automatically defers to the standard decoding
@@ -372,6 +500,9 @@ class CardMF(CardDF):
         install specific decoding.
         """
 
+        if not data_hex:
+            return data_hex
+
         profile = self.get_profile()
 
         if profile:
@@ -393,7 +524,7 @@ class CardADF(CardDF):
             mf.add_application_df(self)
 
     def __str__(self):
-        return "ADF(%s)" % (self.aid)
+        return "ADF(%s)" % (self.name if self.name else self.aid)
 
     def _path_element(self, prefer_name: bool):
         if self.name and prefer_name:
@@ -424,8 +555,10 @@ class CardEF(CardFile):
         """
         # global selectable names + those of the parent DF
         sels = super().get_selectables(flags)
-        sels.update(
-            {x.name: x for x in self.parent.children.values() if x != self})
+        if flags == [] or 'FIDS' in flags:
+            sels.update({x.fid: x for x in self.parent.children.values() if x.fid and x != self})
+        if flags == [] or 'FNAMES' in flags:
+            sels.update({x.name: x for x in self.parent.children.values() if x.name and x != self})
         return sels
 
 
@@ -442,6 +575,17 @@ class TransparentEF(CardEF):
         def __init__(self):
             super().__init__()
 
+        dec_hex_parser = argparse.ArgumentParser()
+        dec_hex_parser.add_argument('--oneline', action='store_true',
+                                    help='No JSON pretty-printing, dump as a single line')
+        dec_hex_parser.add_argument('HEXSTR', help='Hex-string of encoded data to decode')
+
+        @cmd2.with_argparser(dec_hex_parser)
+        def do_decode_hex(self, opts):
+            """Decode command-line provided hex-string as if it was read from the file."""
+            data = self._cmd.lchan.selected_file.decode_hex(opts.HEXSTR)
+            self._cmd.poutput_json(data, opts.oneline)
+
         read_bin_parser = argparse.ArgumentParser()
         read_bin_parser.add_argument(
             '--offset', type=int, default=0, help='Byte offset for start of read')
@@ -451,7 +595,7 @@ class TransparentEF(CardEF):
         @cmd2.with_argparser(read_bin_parser)
         def do_read_binary(self, opts):
             """Read binary data from a transparent EF"""
-            (data, sw) = self._cmd.rs.read_binary(opts.length, opts.offset)
+            (data, sw) = self._cmd.lchan.read_binary(opts.length, opts.offset)
             self._cmd.poutput(data)
 
         read_bin_dec_parser = argparse.ArgumentParser()
@@ -461,7 +605,7 @@ class TransparentEF(CardEF):
         @cmd2.with_argparser(read_bin_dec_parser)
         def do_read_binary_decoded(self, opts):
             """Read + decode data from a transparent EF"""
-            (data, sw) = self._cmd.rs.read_binary_dec()
+            (data, sw) = self._cmd.lchan.read_binary_dec()
             self._cmd.poutput_json(data, opts.oneline)
 
         upd_bin_parser = argparse.ArgumentParser()
@@ -473,7 +617,7 @@ class TransparentEF(CardEF):
         @cmd2.with_argparser(upd_bin_parser)
         def do_update_binary(self, opts):
             """Update (Write) data of a transparent EF"""
-            (data, sw) = self._cmd.rs.update_binary(opts.data, opts.offset)
+            (data, sw) = self._cmd.lchan.update_binary(opts.data, opts.offset)
             if data:
                 self._cmd.poutput(data)
 
@@ -487,18 +631,18 @@ class TransparentEF(CardEF):
         def do_update_binary_decoded(self, opts):
             """Encode + Update (Write) data of a transparent EF"""
             if opts.json_path:
-                (data_json, sw) = self._cmd.rs.read_binary_dec()
+                (data_json, sw) = self._cmd.lchan.read_binary_dec()
                 js_path_modify(data_json, opts.json_path,
                                json.loads(opts.data))
             else:
                 data_json = json.loads(opts.data)
-            (data, sw) = self._cmd.rs.update_binary_dec(data_json)
+            (data, sw) = self._cmd.lchan.update_binary_dec(data_json)
             if data:
                 self._cmd.poutput_json(data)
 
         def do_edit_binary_decoded(self, opts):
             """Edit the JSON representation of the EF contents in an editor."""
-            (orig_json, sw) = self._cmd.rs.read_binary_dec()
+            (orig_json, sw) = self._cmd.lchan.read_binary_dec()
             with tempfile.TemporaryDirectory(prefix='pysim_') as dirname:
                 filename = '%s/file' % dirname
                 # write existing data as JSON to file
@@ -511,12 +655,12 @@ class TransparentEF(CardEF):
                 if edited_json == orig_json:
                     self._cmd.poutput("Data not modified, skipping write")
                 else:
-                    (data, sw) = self._cmd.rs.update_binary_dec(edited_json)
+                    (data, sw) = self._cmd.lchan.update_binary_dec(edited_json)
                     if data:
                         self._cmd.poutput_json(data)
 
     def __init__(self, fid: str, sfid: str = None, name: str = None, desc: str = None, parent: CardDF = None,
-                 size={1, None}):
+                 size: Size = (1, None), **kwargs):
         """
         Args:
             fid : File Identifier (4 hex digits)
@@ -526,7 +670,7 @@ class TransparentEF(CardEF):
             parent : Parent CardFile object within filesystem hierarchy
             size : tuple of (minimum_size, recommended_size)
         """
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent)
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent, **kwargs)
         self._construct = None
         self._tlv = None
         self.size = size
@@ -651,8 +795,19 @@ class LinFixedEF(CardEF):
     class ShellCommands(CommandSet):
         """Shell commands specific for Linear Fixed EFs."""
 
-        def __init__(self):
-            super().__init__()
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+
+        dec_hex_parser = argparse.ArgumentParser()
+        dec_hex_parser.add_argument('--oneline', action='store_true',
+                                    help='No JSON pretty-printing, dump as a single line')
+        dec_hex_parser.add_argument('HEXSTR', help='Hex-string of encoded data to decode')
+
+        @cmd2.with_argparser(dec_hex_parser)
+        def do_decode_hex(self, opts):
+            """Decode command-line provided hex-string as if it was read from the file."""
+            data = self._cmd.lchan.selected_file.decode_record_hex(opts.HEXSTR)
+            self._cmd.poutput_json(data, opts.oneline)
 
         read_rec_parser = argparse.ArgumentParser()
         read_rec_parser.add_argument(
@@ -665,7 +820,7 @@ class LinFixedEF(CardEF):
             """Read one or multiple records from a record-oriented EF"""
             for r in range(opts.count):
                 recnr = opts.record_nr + r
-                (data, sw) = self._cmd.rs.read_record(recnr)
+                (data, sw) = self._cmd.lchan.read_record(recnr)
                 if (len(data) > 0):
                     recstr = str(data)
                 else:
@@ -681,7 +836,7 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(read_rec_dec_parser)
         def do_read_record_decoded(self, opts):
             """Read + decode a record from a record-oriented EF"""
-            (data, sw) = self._cmd.rs.read_record_dec(opts.record_nr)
+            (data, sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
             self._cmd.poutput_json(data, opts.oneline)
 
         read_recs_parser = argparse.ArgumentParser()
@@ -689,9 +844,9 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(read_recs_parser)
         def do_read_records(self, opts):
             """Read all records from a record-oriented EF"""
-            num_of_rec = self._cmd.rs.selected_file_fcp['file_descriptor']['num_of_rec']
+            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
             for recnr in range(1, 1 + num_of_rec):
-                (data, sw) = self._cmd.rs.read_record(recnr)
+                (data, sw) = self._cmd.lchan.read_record(recnr)
                 if (len(data) > 0):
                     recstr = str(data)
                 else:
@@ -705,11 +860,11 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(read_recs_dec_parser)
         def do_read_records_decoded(self, opts):
             """Read + decode all records from a record-oriented EF"""
-            num_of_rec = self._cmd.rs.selected_file_fcp['file_descriptor']['num_of_rec']
+            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
             # collect all results in list so they are rendered as JSON list when printing
             data_list = []
             for recnr in range(1, 1 + num_of_rec):
-                (data, sw) = self._cmd.rs.read_record_dec(recnr)
+                (data, sw) = self._cmd.lchan.read_record_dec(recnr)
                 data_list.append(data)
             self._cmd.poutput_json(data_list, opts.oneline)
 
@@ -722,7 +877,7 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(upd_rec_parser)
         def do_update_record(self, opts):
             """Update (write) data to a record-oriented EF"""
-            (data, sw) = self._cmd.rs.update_record(opts.record_nr, opts.data)
+            (data, sw) = self._cmd.lchan.update_record(opts.record_nr, opts.data)
             if data:
                 self._cmd.poutput(data)
 
@@ -738,12 +893,12 @@ class LinFixedEF(CardEF):
         def do_update_record_decoded(self, opts):
             """Encode + Update (write) data to a record-oriented EF"""
             if opts.json_path:
-                (data_json, sw) = self._cmd.rs.read_record_dec(opts.record_nr)
+                (data_json, sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
                 js_path_modify(data_json, opts.json_path,
                                json.loads(opts.data))
             else:
                 data_json = json.loads(opts.data)
-            (data, sw) = self._cmd.rs.update_record_dec(
+            (data, sw) = self._cmd.lchan.update_record_dec(
                 opts.record_nr, data_json)
             if data:
                 self._cmd.poutput(data)
@@ -755,7 +910,7 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(edit_rec_dec_parser)
         def do_edit_record_decoded(self, opts):
             """Edit the JSON representation of one record in an editor."""
-            (orig_json, sw) = self._cmd.rs.read_record_dec(opts.record_nr)
+            (orig_json, sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
             with tempfile.TemporaryDirectory(prefix='pysim_') as dirname:
                 filename = '%s/file' % dirname
                 # write existing data as JSON to file
@@ -768,13 +923,13 @@ class LinFixedEF(CardEF):
                 if edited_json == orig_json:
                     self._cmd.poutput("Data not modified, skipping write")
                 else:
-                    (data, sw) = self._cmd.rs.update_record_dec(
+                    (data, sw) = self._cmd.lchan.update_record_dec(
                         opts.record_nr, edited_json)
                     if data:
                         self._cmd.poutput_json(data)
 
     def __init__(self, fid: str, sfid: str = None, name: str = None, desc: str = None,
-                 parent: Optional[CardDF] = None, rec_len={1, None}):
+                 parent: Optional[CardDF] = None, rec_len: Size = (1, None), **kwargs):
         """
         Args:
             fid : File Identifier (4 hex digits)
@@ -782,9 +937,9 @@ class LinFixedEF(CardEF):
             name : Brief name of the file, lik EF_ICCID
             desc : Description of the file
             parent : Parent CardFile object within filesystem hierarchy
-            rec_len : set of {minimum_length, recommended_length}
+            rec_len : Tuple of (minimum_length, recommended_length)
         """
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent)
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent, **kwargs)
         self.rec_len = rec_len
         self.shell_commands = [self.ShellCommands()]
         self._construct = None
@@ -905,9 +1060,8 @@ class CyclicEF(LinFixedEF):
     # we don't really have any special support for those; just recycling LinFixedEF here
 
     def __init__(self, fid: str, sfid: str = None, name: str = None, desc: str = None, parent: CardDF = None,
-                 rec_len={1, None}):
-        super().__init__(fid=fid, sfid=sfid, name=name,
-                         desc=desc, parent=parent, rec_len=rec_len)
+                 rec_len: Size = (1, None), **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent, rec_len=rec_len, **kwargs)
 
 
 class TransRecEF(TransparentEF):
@@ -921,7 +1075,7 @@ class TransRecEF(TransparentEF):
     """
 
     def __init__(self, fid: str, rec_len: int, sfid: str = None, name: str = None, desc: str = None,
-                 parent: Optional[CardDF] = None, size={1, None}):
+                 parent: Optional[CardDF] = None, size: Size = (1, None), **kwargs):
         """
         Args:
             fid : File Identifier (4 hex digits)
@@ -932,7 +1086,7 @@ class TransRecEF(TransparentEF):
             rec_len : Length of the fixed-length records within transparent EF
             size : tuple of (minimum_size, recommended_size)
         """
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent, size=size)
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent, size=size, **kwargs)
         self.rec_len = rec_len
 
     def decode_record_hex(self, raw_hex_data: str) -> dict:
@@ -1076,12 +1230,12 @@ class BerTlvEF(CardEF):
         @cmd2.with_argparser(retrieve_data_parser)
         def do_retrieve_data(self, opts):
             """Retrieve (Read) data from a BER-TLV EF"""
-            (data, sw) = self._cmd.rs.retrieve_data(opts.tag)
+            (data, sw) = self._cmd.lchan.retrieve_data(opts.tag)
             self._cmd.poutput(data)
 
         def do_retrieve_tags(self, opts):
             """List tags available in a given BER-TLV EF"""
-            tags = self._cmd.rs.retrieve_tags()
+            tags = self._cmd.lchan.retrieve_tags()
             self._cmd.poutput(tags)
 
         set_data_parser = argparse.ArgumentParser()
@@ -1093,7 +1247,7 @@ class BerTlvEF(CardEF):
         @cmd2.with_argparser(set_data_parser)
         def do_set_data(self, opts):
             """Set (Write) data for a given tag in a BER-TLV EF"""
-            (data, sw) = self._cmd.rs.set_data(opts.tag, opts.data)
+            (data, sw) = self._cmd.lchan.set_data(opts.tag, opts.data)
             if data:
                 self._cmd.poutput(data)
 
@@ -1104,12 +1258,12 @@ class BerTlvEF(CardEF):
         @cmd2.with_argparser(del_data_parser)
         def do_delete_data(self, opts):
             """Delete  data for a given tag in a BER-TLV EF"""
-            (data, sw) = self._cmd.rs.set_data(opts.tag, None)
+            (data, sw) = self._cmd.lchan.set_data(opts.tag, None)
             if data:
                 self._cmd.poutput(data)
 
     def __init__(self, fid: str, sfid: str = None, name: str = None, desc: str = None, parent: CardDF = None,
-                 size={1, None}):
+                 size: Size = (1, None), **kwargs):
         """
         Args:
             fid : File Identifier (4 hex digits)
@@ -1119,13 +1273,13 @@ class BerTlvEF(CardEF):
             parent : Parent CardFile object within filesystem hierarchy
             size : tuple of (minimum_size, recommended_size)
         """
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent)
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent, **kwargs)
         self._construct = None
         self.size = size
         self.shell_commands = [self.ShellCommands()]
 
 
-class RuntimeState(object):
+class RuntimeState:
     """Represent the runtime state of a session with a card."""
 
     def __init__(self, card, profile: 'CardProfile'):
@@ -1136,8 +1290,10 @@ class RuntimeState(object):
         """
         self.mf = CardMF(profile=profile)
         self.card = card
-        self.selected_file = self.mf  # type: CardDF
         self.profile = profile
+        self.lchan = {}
+        # the basic logical channel always exists
+        self.lchan[0] = RuntimeLchan(0, self)
 
         # make sure the class and selection control bytes, which are specified
         # by the card profile are used
@@ -1201,11 +1357,76 @@ class RuntimeState(object):
         Args:
             cmd_app : Command Application State (for unregistering old file commands)
         """
+        # delete all lchan != 0 (basic lchan)
+        for lchan_nr in self.lchan.keys():
+            if lchan_nr == 0:
+                continue
+            del self.lchan[lchan_nr]
         atr = i2h(self.card.reset())
         # select MF to reset internal state and to verify card really works
-        self.select('MF', cmd_app)
+        self.lchan[0].select('MF', cmd_app)
+        self.lchan[0].selected_adf = None
         return atr
 
+    def add_lchan(self, lchan_nr: int) -> 'RuntimeLchan':
+        """Add a logical channel to the runtime state.  You shouldn't call this
+        directly but always go through RuntimeLchan.add_lchan()."""
+        if lchan_nr in self.lchan.keys():
+            raise ValueError('Cannot create already-existing lchan %d' % lchan_nr)
+        self.lchan[lchan_nr] = RuntimeLchan(lchan_nr, self)
+        return self.lchan[lchan_nr]
+
+    def del_lchan(self, lchan_nr: int):
+        if lchan_nr in self.lchan.keys():
+            del self.lchan[lchan_nr]
+            return True
+        else:
+            return False
+
+    def get_lchan_by_cla(self, cla) -> Optional['RuntimeLchan']:
+        lchan_nr = lchan_nr_from_cla(cla)
+        if lchan_nr in self.lchan.keys():
+            return self.lchan[lchan_nr]
+        else:
+            return None
+
+
+class RuntimeLchan:
+    """Represent the runtime state of a logical channel with a card."""
+
+    def __init__(self, lchan_nr: int, rs: RuntimeState):
+        self.lchan_nr = lchan_nr
+        self.rs = rs
+        self.selected_file = self.rs.mf
+        self.selected_adf = None
+        self.selected_file_fcp = None
+        self.selected_file_fcp_hex = None
+
+    def add_lchan(self, lchan_nr: int) -> 'RuntimeLchan':
+        """Add a new logical channel from the current logical channel. Just affects
+        internal state, doesn't actually open a channel with the UICC."""
+        new_lchan = self.rs.add_lchan(lchan_nr)
+        # See TS 102 221 Table 8.3
+        if self.lchan_nr != 0:
+            new_lchan.selected_file = self.get_cwd()
+            new_lchan.selected_adf = self.selected_adf
+        return new_lchan
+
+    def selected_file_descriptor_byte(self) -> dict:
+        return self.selected_file_fcp['file_descriptor']['file_descriptor_byte']
+
+    def selected_file_shareable(self) -> bool:
+        return self.selected_file_descriptor_byte()['shareable']
+
+    def selected_file_structure(self) -> str:
+        return self.selected_file_descriptor_byte()['structure']
+
+    def selected_file_type(self) -> str:
+        return self.selected_file_descriptor_byte()['file_type']
+
+    def selected_file_num_of_rec(self) -> Optional[int]:
+        return self.selected_file_fcp['file_descriptor'].get('num_of_rec')
+
     def get_cwd(self) -> CardDF:
         """Obtain the current working directory.
 
@@ -1249,7 +1470,7 @@ class RuntimeState(object):
             # card profile.
             if app and hasattr(app, "interpret_sw"):
                 res = app.interpret_sw(sw)
-        return res or self.profile.interpret_sw(sw)
+        return res or self.rs.profile.interpret_sw(sw)
 
     def probe_file(self, fid: str, cmd_app=None):
         """Blindly try to select a file and automatically add a matching file
@@ -1259,7 +1480,7 @@ class RuntimeState(object):
                 "Cannot select unknown file by name %s, only hexadecimal 4 digit FID is allowed" % fid)
 
         try:
-            (data, sw) = self.card._scc.select_file(fid)
+            (data, sw) = self.rs.card._scc.select_file(fid)
         except SwMatchError as swm:
             k = self.interpret_sw(swm.sw_actual)
             if not k:
@@ -1267,11 +1488,11 @@ class RuntimeState(object):
             raise RuntimeError("%s: %s - %s" % (swm.sw_actual, k[0], k[1]))
 
         select_resp = self.selected_file.decode_select_response(data)
-        if (select_resp['file_descriptor']['file_type'] == 'df'):
+        if (select_resp['file_descriptor']['file_descriptor_byte']['file_type'] == 'df'):
             f = CardDF(fid=fid, sfid=None, name="DF." + str(fid).upper(),
                        desc="dedicated file, manually added at runtime")
         else:
-            if (select_resp['file_descriptor']['structure'] == 'transparent'):
+            if (select_resp['file_descriptor']['file_descriptor_byte']['structure'] == 'transparent'):
                 f = TransparentEF(fid=fid, sfid=None, name="EF." + str(fid).upper(),
                                   desc="elementary file, manually added at runtime")
             else:
@@ -1280,7 +1501,47 @@ class RuntimeState(object):
 
         self.selected_file.add_files([f])
         self.selected_file = f
-        return select_resp
+        return select_resp, data
+
+    def _select_pre(self, cmd_app):
+        # unregister commands of old file
+        if cmd_app and self.selected_file.shell_commands:
+            for c in self.selected_file.shell_commands:
+                cmd_app.unregister_command_set(c)
+
+    def _select_post(self, cmd_app):
+        # register commands of new file
+        if cmd_app and self.selected_file.shell_commands:
+            for c in self.selected_file.shell_commands:
+                cmd_app.register_command_set(c)
+
+    def select_file(self, file: CardFile, cmd_app=None):
+        """Select a file (EF, DF, ADF, MF, ...).
+
+        Args:
+            file : CardFile [or derived class] instance
+            cmd_app : Command Application State (for unregistering old file commands)
+        """
+        # we need to find a path from our self.selected_file to the destination
+        inter_path = self.selected_file.build_select_path_to(file)
+        if not inter_path:
+            raise RuntimeError('Cannot determine path from %s to %s' % (self.selected_file, file))
+
+        self._select_pre(cmd_app)
+
+        for p in inter_path:
+            try:
+                if isinstance(p, CardADF):
+                    (data, sw) = self.rs.card.select_adf_by_aid(p.aid)
+                    self.selected_adf = p
+                else:
+                    (data, sw) = self.rs.card._scc.select_file(p.fid)
+                self.selected_file = p
+            except SwMatchError as swm:
+                self._select_post(cmd_app)
+                raise(swm)
+
+        self._select_post(cmd_app)
 
     def select(self, name: str, cmd_app=None):
         """Select a file (EF, DF, ADF, MF, ...).
@@ -1289,22 +1550,35 @@ class RuntimeState(object):
             name : Name of file to select
             cmd_app : Command Application State (for unregistering old file commands)
         """
+        # handling of entire paths with multiple directories/elements
+        if '/' in name:
+            prev_sel_file = self.selected_file
+            pathlist = name.split('/')
+            # treat /DF.GSM/foo like MF/DF.GSM/foo
+            if pathlist[0] == '':
+                pathlist[0] = 'MF'
+            try:
+                for p in pathlist:
+                    self.select(p, cmd_app)
+                return
+            except Exception as e:
+                # if any intermediate step fails, go back to where we were
+                self.select_file(prev_sel_file, cmd_app)
+                raise e
+
         sels = self.selected_file.get_selectables()
         if is_hex(name):
             name = name.lower()
 
-        # unregister commands of old file
-        if cmd_app and self.selected_file.shell_commands:
-            for c in self.selected_file.shell_commands:
-                cmd_app.unregister_command_set(c)
+        self._select_pre(cmd_app)
 
         if name in sels:
             f = sels[name]
             try:
                 if isinstance(f, CardADF):
-                    (data, sw) = self.card.select_adf_by_aid(f.aid)
+                    (data, sw) = self.rs.card.select_adf_by_aid(f.aid)
                 else:
-                    (data, sw) = self.card._scc.select_file(f.fid)
+                    (data, sw) = self.rs.card._scc.select_file(f.fid)
                 self.selected_file = f
             except SwMatchError as swm:
                 k = self.interpret_sw(swm.sw_actual)
@@ -1313,27 +1587,29 @@ class RuntimeState(object):
                 raise RuntimeError("%s: %s - %s" % (swm.sw_actual, k[0], k[1]))
             select_resp = f.decode_select_response(data)
         else:
-            select_resp = self.probe_file(name, cmd_app)
-        # store the decoded FCP for later reference
+            (select_resp, data) = self.probe_file(name, cmd_app)
+
+        # store the raw + decoded FCP for later reference
+        self.selected_file_fcp_hex = data
         self.selected_file_fcp = select_resp
 
-        # register commands of new file
-        if cmd_app and self.selected_file.shell_commands:
-            for c in self.selected_file.shell_commands:
-                cmd_app.register_command_set(c)
-
+        self._select_post(cmd_app)
         return select_resp
 
     def status(self):
         """Request STATUS (current selected file FCP) from card."""
-        (data, sw) = self.card._scc.status()
+        (data, sw) = self.rs.card._scc.status()
         return self.selected_file.decode_select_response(data)
 
+    def get_file_for_selectable(self, name: str):
+        sels = self.selected_file.get_selectables()
+        return sels[name]
+
     def activate_file(self, name: str):
         """Request ACTIVATE FILE of specified file."""
         sels = self.selected_file.get_selectables()
         f = sels[name]
-        data, sw = self.card._scc.activate_file(f.fid)
+        data, sw = self.rs.card._scc.activate_file(f.fid)
         return data, sw
 
     def read_binary(self, length: int = None, offset: int = 0):
@@ -1347,7 +1623,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, TransparentEF):
             raise TypeError("Only works with TransparentEF")
-        return self.card._scc.read_binary(self.selected_file.fid, length, offset)
+        return self.rs.card._scc.read_binary(self.selected_file.fid, length, offset)
 
     def read_binary_dec(self) -> Tuple[dict, str]:
         """Read [part of] a transparent EF binary data and decode it.
@@ -1371,7 +1647,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, TransparentEF):
             raise TypeError("Only works with TransparentEF")
-        return self.card._scc.update_binary(self.selected_file.fid, data_hex, offset, conserve=self.conserve_write)
+        return self.rs.card._scc.update_binary(self.selected_file.fid, data_hex, offset, conserve=self.rs.conserve_write)
 
     def update_binary_dec(self, data: dict):
         """Update transparent EF from abstract data. Encodes the data to binary and
@@ -1394,7 +1670,7 @@ class RuntimeState(object):
         if not isinstance(self.selected_file, LinFixedEF):
             raise TypeError("Only works with Linear Fixed EF")
         # returns a string of hex nibbles
-        return self.card._scc.read_record(self.selected_file.fid, rec_nr)
+        return self.rs.card._scc.read_record(self.selected_file.fid, rec_nr)
 
     def read_record_dec(self, rec_nr: int = 0) -> Tuple[dict, str]:
         """Read a record and decode it to abstract data.
@@ -1416,7 +1692,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, LinFixedEF):
             raise TypeError("Only works with Linear Fixed EF")
-        return self.card._scc.update_record(self.selected_file.fid, rec_nr, data_hex, conserve=self.conserve_write)
+        return self.rs.card._scc.update_record(self.selected_file.fid, rec_nr, data_hex, conserve=self.rs.conserve_write)
 
     def update_record_dec(self, rec_nr: int, data: dict):
         """Update a record with given abstract data.  Will encode abstract to binary data
@@ -1440,7 +1716,7 @@ class RuntimeState(object):
         if not isinstance(self.selected_file, BerTlvEF):
             raise TypeError("Only works with BER-TLV EF")
         # returns a string of hex nibbles
-        return self.card._scc.retrieve_data(self.selected_file.fid, tag)
+        return self.rs.card._scc.retrieve_data(self.selected_file.fid, tag)
 
     def retrieve_tags(self):
         """Retrieve tags available on BER-TLV EF.
@@ -1450,7 +1726,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, BerTlvEF):
             raise TypeError("Only works with BER-TLV EF")
-        data, sw = self.card._scc.retrieve_data(self.selected_file.fid, 0x5c)
+        data, sw = self.rs.card._scc.retrieve_data(self.selected_file.fid, 0x5c)
         tag, length, value, remainder = bertlv_parse_one(h2b(data))
         return list(value)
 
@@ -1463,7 +1739,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, BerTlvEF):
             raise TypeError("Only works with BER-TLV EF")
-        return self.card._scc.set_data(self.selected_file.fid, tag, data_hex, conserve=self.conserve_write)
+        return self.rs.card._scc.set_data(self.selected_file.fid, tag, data_hex, conserve=self.rs.conserve_write)
 
     def unregister_cmds(self, cmd_app=None):
         """Unregister all file specific commands."""
@@ -1472,7 +1748,7 @@ class RuntimeState(object):
                 cmd_app.unregister_command_set(c)
 
 
-class FileData(object):
+class FileData:
     """Represent the runtime, on-card data."""
 
     def __init__(self, fdesc):
@@ -1500,7 +1776,7 @@ def interpret_sw(sw_data: dict, sw: str):
     return None
 
 
-class CardApplication(object):
+class CardApplication:
     """A card application is represented by an ADF (with contained hierarchy) and optionally
        some SW definitions."""
 

pySim/global_platform.py

@@ -0,0 +1,256 @@
+# coding=utf-8
+"""Partial Support for GlobalPLatform Card Spec (currently 2.1.1)
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from typing import Optional, List, Dict, Tuple
+from construct import Optional as COptional
+from construct import *
+from bidict import bidict
+from pySim.construct import *
+from pySim.utils import *
+from pySim.filesystem import *
+from pySim.tlv import *
+from pySim.profile import CardProfile
+
+sw_table = {
+    'Warnings': {
+        '6200': 'Logical Channel already closed',
+        '6283': 'Card Life Cycle State is CARD_LOCKED',
+        '6310': 'More data available',
+    },
+    'Execution errors': {
+        '6400': 'No specific diagnosis',
+        '6581': 'Memory failure',
+    },
+    'Checking errors': {
+        '6700': 'Wrong length in Lc',
+    },
+    'Functions in CLA not supported': {
+        '6881': 'Logical channel not supported or active',
+        '6882': 'Secure messaging not supported',
+    },
+    'Command not allowed': {
+        '6982': 'Security Status not satisfied',
+        '6985': 'Conditions of use not satisfied',
+    },
+    'Wrong parameters': {
+        '6a80': 'Incorrect values in command data',
+        '6a81': 'Function not supported e.g. card Life Cycle State is CARD_LOCKED',
+        '6a82': 'Application not found',
+        '6a84': 'Not enough memory space',
+        '6a86': 'Incorrect P1 P2',
+        '6a88': 'Referenced data not found',
+    },
+    'GlobalPlatform': {
+        '6d00': 'Invalid instruction',
+        '6e00': 'Invalid class',
+    },
+    'Application errors': {
+        '9484': 'Algorithm not supported',
+        '9485': 'Invalid key check value',
+    },
+}
+
+# GlobalPlatform 2.1.1 Section 9.1.6
+KeyType = Enum(Byte,    des=0x80,
+                        rsa_public_exponent_e_cleartex=0xA0,
+                        rsa_modulus_n_cleartext=0xA1,
+                        rsa_modulus_n=0xA2,
+                        rsa_private_exponent_d=0xA3,
+                        rsa_chines_remainder_p=0xA4,
+                        rsa_chines_remainder_q=0xA5,
+                        rsa_chines_remainder_pq=0xA6,
+                        rsa_chines_remainder_dpi=0xA7,
+                        rsa_chines_remainder_dqi=0xA8,
+                        not_available=0xff)
+
+# GlobalPlatform 2.1.1 Section 9.3.3.1
+# example:
+#   e0 48
+#       c0 04 01708010
+#       c0 04 02708010
+#       c0 04 03708010
+#       c0 04 01018010
+#       c0 04 02018010
+#       c0 04 03018010
+#       c0 04 01028010
+#       c0 04 02028010
+#       c0 04 03028010
+#       c0 04 01038010
+#       c0 04 02038010
+#       c0 04 03038010
+class KeyInformationData(BER_TLV_IE, tag=0xc0):
+    _construct = Struct('key_identifier'/Byte, 'key_version_number'/Byte,
+                        'key_types'/GreedyRange(KeyType))
+class KeyInformation(BER_TLV_IE, tag=0xe0, nested=[KeyInformationData]):
+    pass
+
+# card data sample, returned in response to GET DATA (80ca006600):
+# 66 31
+#    73 2f
+#        06 07
+#            2a864886fc6b01
+#        60 0c
+#            06 0a
+#                2a864886fc6b02020101
+#        63 09
+#            06 07
+#                2a864886fc6b03
+#        64 0b
+#            06 09
+#                2a864886fc6b040215
+
+# GlobalPlatform 2.1.1 Table F-1
+class ObjectIdentifier(BER_TLV_IE, tag=0x06):
+    _construct = GreedyBytes
+class CardManagementTypeAndVersion(BER_TLV_IE, tag=0x60, nested=[ObjectIdentifier]):
+    pass
+class CardIdentificationScheme(BER_TLV_IE, tag=0x63, nested=[ObjectIdentifier]):
+    pass
+class SecureChannelProtocolOfISD(BER_TLV_IE, tag=0x64, nested=[ObjectIdentifier]):
+    pass
+class CardConfigurationDetails(BER_TLV_IE, tag=0x65):
+    _construct = GreedyBytes
+class CardChipDetails(BER_TLV_IE, tag=0x66):
+    _construct = GreedyBytes
+class CardRecognitionData(BER_TLV_IE, tag=0x73, nested=[ObjectIdentifier,
+                                                        CardManagementTypeAndVersion,
+                                                        CardIdentificationScheme,
+                                                        SecureChannelProtocolOfISD,
+                                                        CardConfigurationDetails,
+                                                        CardChipDetails]):
+    pass
+class CardData(BER_TLV_IE, tag=0x66, nested=[CardRecognitionData]):
+    pass
+
+# GlobalPlatform 2.1.1 Table F-2
+class SecureChannelProtocolOfSelectedSD(BER_TLV_IE, tag=0x64, nested=[ObjectIdentifier]):
+    pass
+class SecurityDomainMgmtData(BER_TLV_IE, tag=0x73, nested=[CardManagementTypeAndVersion,
+                                                           CardIdentificationScheme,
+                                                           SecureChannelProtocolOfSelectedSD,
+                                                           CardConfigurationDetails,
+                                                           CardChipDetails]):
+    pass
+
+# GlobalPlatform 2.1.1 Section 9.1.1
+IsdLifeCycleState = Enum(Byte, op_ready=0x01, initialized=0x07, secured=0x0f,
+                         card_locked = 0x7f, terminated=0xff)
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class ApplicationID(BER_TLV_IE, tag=0x84):
+    _construct = GreedyBytes
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class SecurityDomainManagementData(BER_TLV_IE, tag=0x73):
+    _construct = GreedyBytes
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class ApplicationProductionLifeCycleData(BER_TLV_IE, tag=0x9f6e):
+    _construct = GreedyBytes
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class MaximumLengthOfDataFieldInCommandMessage(BER_TLV_IE, tag=0x9f65):
+    _construct = GreedyInteger()
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class ProprietaryData(BER_TLV_IE, tag=0xA5, nested=[SecurityDomainManagementData,
+                                                    ApplicationProductionLifeCycleData,
+                                                    MaximumLengthOfDataFieldInCommandMessage]):
+    pass
+
+# GlobalPlatform 2.1.1 Section 9.9.3.1
+class FciTemplate(BER_TLV_IE, tag=0x6f, nested=[ApplicationID, SecurityDomainManagementData,
+                                                ApplicationProductionLifeCycleData,
+                                                MaximumLengthOfDataFieldInCommandMessage,
+                                                ProprietaryData]):
+    pass
+
+class IssuerIdentificationNumber(BER_TLV_IE, tag=0x42):
+    _construct = BcdAdapter(GreedyBytes)
+
+class CardImageNumber(BER_TLV_IE, tag=0x45):
+    _construct = BcdAdapter(GreedyBytes)
+
+class SequenceCounterOfDefaultKvn(BER_TLV_IE, tag=0xc1):
+    _construct = GreedyInteger()
+
+class ConfirmationCounter(BER_TLV_IE, tag=0xc2):
+    _construct = GreedyInteger()
+
+# Collection of all the data objects we can get from GET DATA
+class DataCollection(TLV_IE_Collection, nested=[IssuerIdentificationNumber,
+                                                CardImageNumber,
+                                                CardData,
+                                                KeyInformation,
+                                                SequenceCounterOfDefaultKvn,
+                                                ConfirmationCounter]):
+    pass
+
+def decode_select_response(resp_hex: str) -> object:
+    t = FciTemplate()
+    t.from_tlv(h2b(resp_hex))
+    d = t.to_dict()
+    return flatten_dict_lists(d['fci_template'])
+
+# Application Dedicated File of a Security Domain
+class ADF_SD(CardADF):
+    def __init__(self, aid: str, name: str, desc: str):
+        super().__init__(aid=aid, fid=None, sfid=None, name=name, desc=desc)
+        self.shell_commands += [self.AddlShellCommands()]
+
+    @staticmethod
+    def decode_select_response(res_hex: str) -> object:
+        return decode_select_response(res_hex)
+
+    @with_default_category('Application-Specific Commands')
+    class AddlShellCommands(CommandSet):
+        def __init__(self):
+            super().__init__()
+
+        def do_get_data(self, opts):
+            tlv_cls_name = opts.arg_list[0]
+            tlv_cls = DataCollection().members_by_name[tlv_cls_name]
+            (data, sw) = self._cmd.card._scc.get_data(cla=0x80, tag=tlv_cls.tag)
+            ie = tlv_cls()
+            ie.from_tlv(h2b(data))
+            self._cmd.poutput_json(ie.to_dict())
+
+        def complete_get_data(self, text, line, begidx, endidx) -> List[str]:
+            #data_dict = {camel_to_snake(str(x.__name__)): x for x in DataCollection.possible_nested}
+            data_dict = {str(x.__name__): x for x in DataCollection.possible_nested}
+            index_dict = {1: data_dict}
+            return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+# Card Application of a Security Domain
+class CardApplicationSD(CardApplication):
+    def __init__(self, aid: str, name: str, desc: str):
+        super().__init__(name, adf=ADF_SD(aid, name, desc), sw=sw_table)
+
+# Card Application of Issuer Security Domain
+class CardApplicationISD(CardApplicationSD):
+    # FIXME: ISD AID is not static, but could be different. One can select the empty
+    # application using '00a4040000' and then parse the response FCI to get the ISD AID
+    def __init__(self, aid='a000000003000000'):
+        super().__init__(aid=aid, name='ADF.ISD', desc='Issuer Security Domain')
+
+#class CardProfileGlobalPlatform(CardProfile):
+#    ORDER = 23
+#
+#    def __init__(self, name='GlobalPlatform'):
+#        super().__init__(name, desc='GlobalPlatfomr 2.1.1', cla=['00','80','84'], sw=sw_table)

pySim/gsm_r.py

@@ -61,7 +61,7 @@ class EF_FN(LinFixedEF):
 
     def __init__(self):
         super().__init__(fid='6ff1', sfid=None, name='EF.EN',
-                         desc='Functional numbers', rec_len={9, 9})
+                         desc='Functional numbers', rec_len=(9, 9))
         self._construct = Struct('functional_number_and_type'/FuncNTypeAdapter(Bytes(8)),
                                  'list_number'/Int8ub)
 
@@ -149,7 +149,7 @@ class EF_CallconfC(TransparentEF):
     """Section 7.3"""
 
     def __init__(self):
-        super().__init__(fid='6ff2', sfid=None, name='EF.CallconfC', size={24, 24},
+        super().__init__(fid='6ff2', sfid=None, name='EF.CallconfC', size=(24, 24),
                          desc='Call Configuration of emergency calls Configuration')
         self._construct = Struct('pl_conf'/PlConfAdapter(Int8ub),
                                  'conf_nr'/BcdAdapter(Bytes(8)),
@@ -166,7 +166,7 @@ class EF_CallconfI(LinFixedEF):
     """Section 7.5"""
 
     def __init__(self):
-        super().__init__(fid='6ff3', sfid=None, name='EF.CallconfI', rec_len={21, 21},
+        super().__init__(fid='6ff3', sfid=None, name='EF.CallconfI', rec_len=(21, 21),
                          desc='Call Configuration of emergency calls Information')
         self._construct = Struct('t_dur'/Int24ub,
                                  't_relcalc'/Int32ub,
@@ -183,9 +183,9 @@ class EF_Shunting(TransparentEF):
 
     def __init__(self):
         super().__init__(fid='6ff4', sfid=None,
-                         name='EF.Shunting', desc='Shunting', size={8, 8})
+                         name='EF.Shunting', desc='Shunting', size=(8, 8))
         self._construct = Struct('common_gid'/Int8ub,
-                                 'shunting_gid'/Bytes(7))
+                                 'shunting_gid'/HexAdapter(Bytes(7)))
 
 
 class EF_GsmrPLMN(LinFixedEF):
@@ -193,7 +193,7 @@ class EF_GsmrPLMN(LinFixedEF):
 
     def __init__(self):
         super().__init__(fid='6ff5', sfid=None, name='EF.GsmrPLMN',
-                         desc='GSM-R network selection', rec_len={9, 9})
+                         desc='GSM-R network selection', rec_len=(9, 9))
         self._construct = Struct('plmn'/BcdAdapter(Bytes(3)),
                                  'class_of_network'/BitStruct('supported'/FlagsEnum(BitsInteger(5), vbs=1, vgcs=2, emlpp=4, fn=8, eirene=16),
                                                               'preference'/BitsInteger(3)),
@@ -207,7 +207,7 @@ class EF_IC(LinFixedEF):
 
     def __init__(self):
         super().__init__(fid='6f8d', sfid=None, name='EF.IC',
-                         desc='International Code', rec_len={7, 7})
+                         desc='International Code', rec_len=(7, 7))
         self._construct = Struct('next_table_type'/NextTableType,
                                  'id_of_next_table'/HexAdapter(Bytes(2)),
                                  'ic_decision_value'/BcdAdapter(Bytes(2)),
@@ -219,7 +219,7 @@ class EF_NW(LinFixedEF):
 
     def __init__(self):
         super().__init__(fid='6f80', sfid=None, name='EF.NW',
-                         desc='Network Name', rec_len={8, 8})
+                         desc='Network Name', rec_len=(8, 8))
         self._construct = GsmString(8)
 
 
@@ -228,7 +228,7 @@ class EF_Switching(LinFixedEF):
 
     def __init__(self, fid, name, desc):
         super().__init__(fid=fid, sfid=None,
-                         name=name, desc=desc, rec_len={6, 6})
+                         name=name, desc=desc, rec_len=(6, 6))
         self._construct = Struct('next_table_type'/NextTableType,
                                  'id_of_next_table'/HexAdapter(Bytes(2)),
                                  'decision_value'/BcdAdapter(Bytes(2)),
@@ -240,7 +240,7 @@ class EF_Predefined(LinFixedEF):
 
     def __init__(self, fid, name, desc):
         super().__init__(fid=fid, sfid=None,
-                         name=name, desc=desc, rec_len={3, 3})
+                         name=name, desc=desc, rec_len=(3, 3))
         # header and other records have different structure. WTF !?!
         self._construct = Struct('next_table_type'/NextTableType,
                                  'id_of_next_table'/HexAdapter(Bytes(2)),
@@ -253,7 +253,7 @@ class EF_DialledVals(TransparentEF):
     """Section 8.6"""
 
     def __init__(self, fid, name, desc):
-        super().__init__(fid=fid, sfid=None, name=name, desc=desc, size={4, 4})
+        super().__init__(fid=fid, sfid=None, name=name, desc=desc, size=(4, 4))
         self._construct = Struct('next_table_type'/NextTableType,
                                  'id_of_next_table'/HexAdapter(Bytes(2)),
                                  'dialed_digits'/BcdAdapter(Bytes(1)))

pySim/gsmtap.py

@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+
+""" Osmocom GSMTAP python implementation.
+GSMTAP is a packet format used for conveying a number of different
+telecom-related protocol traces over UDP.
+"""
+
+#
+# Copyright (C) 2022  Harald Welte <laforge@gnumonks.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import socket
+from typing import List, Dict, Optional
+from construct import Optional as COptional
+from construct import *
+from pySim.construct import *
+
+# The root definition of GSMTAP can be found at
+# https://cgit.osmocom.org/cgit/libosmocore/tree/include/osmocom/core/gsmtap.h
+
+GSMTAP_UDP_PORT = 4729
+
+# GSMTAP_TYPE_*
+gsmtap_type_construct = Enum(Int8ub,
+                             gsm_um = 0x01,
+                             gsm_abis = 0x02,
+                             gsm_um_burst = 0x03,
+                             sim = 0x04,
+                             tetra_i1 = 0x05,
+                             tetra_i1_burst = 0x06,
+                             wimax_burst = 0x07,
+                             gprs_gb_llc = 0x08,
+                             gprs_gb_sndcp = 0x09,
+                             gmr1_um = 0x0a,
+                             umts_rlc_mac = 0x0b,
+                             umts_rrc = 0x0c,
+                             lte_rrc = 0x0d,
+                             lte_mac = 0x0e,
+                             lte_mac_framed = 0x0f,
+                             osmocore_log = 0x10,
+                             qc_diag = 0x11,
+                             lte_nas = 0x12,
+                             e1_t1 = 0x13)
+
+
+# TYPE_UM_BURST
+gsmtap_subtype_burst_construct = Enum(Int8ub,
+                                      unknown = 0x00,
+                                      fcch = 0x01,
+                                      partial_sch = 0x02,
+                                      sch = 0x03,
+                                      cts_sch = 0x04,
+                                      compact_sch = 0x05,
+                                      normal = 0x06,
+                                      dummy = 0x07,
+                                      access = 0x08,
+                                      none = 0x09)
+
+gsmtap_subtype_wimax_burst_construct = Enum(Int8ub,
+                                            cdma_code = 0x10,
+                                            fch = 0x11,
+                                            ffb = 0x12,
+                                            pdu = 0x13,
+                                            hack = 0x14,
+                                            phy_attributes = 0x15)
+
+# GSMTAP_CHANNEL_*
+gsmtap_subtype_um_construct = Enum(Int8ub,
+                                   unknown = 0x00,
+                                   bcch = 0x01,
+                                   ccch = 0x02,
+                                   rach = 0x03,
+                                   agch = 0x04,
+                                   pch = 0x05,
+                                   sdcch = 0x06,
+                                   sdcch4 = 0x07,
+                                   sdcch8 = 0x08,
+                                   facch_f = 0x09,
+                                   facch_h = 0x0a,
+                                   pacch = 0x0b,
+                                   cbch52 = 0x0c,
+                                   pdtch = 0x0d,
+                                   ptcch = 0x0e,
+                                   cbch51 = 0x0f,
+                                   voice_f = 0x10,
+                                   voice_h = 0x11)
+
+
+# GSMTAP_SIM_*
+gsmtap_subtype_sim_construct = Enum(Int8ub,
+                                    apdu = 0x00,
+                                    atr = 0x01,
+                                    pps_req = 0x02,
+                                    pps_rsp = 0x03,
+                                    tpdu_hdr = 0x04,
+                                    tpdu_cmd = 0x05,
+                                    tpdu_rsp = 0x06,
+                                    tpdu_sw = 0x07)
+
+gsmtap_subtype_tetra_construct = Enum(Int8ub,
+                                      bsch = 0x01,
+                                      aach = 0x02,
+                                      sch_hu = 0x03,
+                                      sch_hd = 0x04,
+                                      sch_f = 0x05,
+                                      bnch = 0x06,
+                                      stch = 0x07,
+                                      tch_f = 0x08,
+                                      dmo_sch_s = 0x09,
+                                      dmo_sch_h = 0x0a,
+                                      dmo_sch_f = 0x0b,
+                                      dmo_stch = 0x0c,
+                                      dmo_tch = 0x0d)
+
+gsmtap_subtype_gmr1_construct = Enum(Int8ub,
+                                     unknown = 0x00,
+                                     bcch = 0x01,
+                                     ccch = 0x02,
+                                     pch = 0x03,
+                                     agch = 0x04,
+                                     bach = 0x05,
+                                     rach = 0x06,
+                                     cbch = 0x07,
+                                     sdcch = 0x08,
+                                     tachh = 0x09,
+                                     gbch = 0x0a,
+                                     tch3 = 0x10,
+                                     tch6 = 0x14,
+                                     tch9 = 0x18)
+
+gsmtap_subtype_e1t1_construct = Enum(Int8ub,
+                                     lapd = 0x01,
+                                     fr = 0x02,
+                                     raw = 0x03,
+                                     trau16 = 0x04,
+                                     trau8 = 0x05)
+
+gsmtap_arfcn_construct = BitStruct('pcs'/Flag, 'uplink'/Flag, 'arfcn'/BitsInteger(14))
+
+gsmtap_hdr_construct = Struct('version'/Int8ub,
+                              'hdr_len'/Int8ub,
+                              'type'/gsmtap_type_construct,
+                              'timeslot'/Int8ub,
+                              'arfcn'/gsmtap_arfcn_construct,
+                              'signal_dbm'/Int8sb,
+                              'snr_db'/Int8sb,
+                              'frame_nr'/Int32ub,
+                              'sub_type'/Switch(this.type, {
+                                                'gsm_um': gsmtap_subtype_um_construct,
+                                                'gsm_um_burst': gsmtap_subtype_burst_construct,
+                                                'sim': gsmtap_subtype_sim_construct,
+                                                'tetra_i1': gsmtap_subtype_tetra_construct,
+                                                'tetra_i1_burst': gsmtap_subtype_tetra_construct,
+                                                'wimax_burst': gsmtap_subtype_wimax_burst_construct,
+                                                'gmr1_um': gsmtap_subtype_gmr1_construct,
+                                                'e1_t1': gsmtap_subtype_e1t1_construct,
+                                                }),
+                              'antenna_nr'/Int8ub,
+                              'sub_slot'/Int8ub,
+                              'res'/Int8ub,
+                              'body'/GreedyBytes)
+
+osmocore_log_ts_construct = Struct('sec'/Int32ub, 'usec'/Int32ub)
+osmocore_log_level_construct = Enum(Int8ub, debug=1, info=3, notice=5, error=7, fatal=8)
+gsmtap_osmocore_log_hdr_construct = Struct('ts'/osmocore_log_ts_construct,
+                                           'proc_name'/PaddedString(16, 'ascii'),
+                                           'pid'/Int32ub,
+                                           'level'/osmocore_log_level_construct,
+                                           Bytes(3),
+                                           'subsys'/PaddedString(16, 'ascii'),
+                                           'src_file'/Struct('name'/PaddedString(32, 'ascii'), 'line_nr'/Int32ub))
+
+
+class GsmtapMessage:
+    """Class whose objects represent a single GSMTAP message. Can encode and decode messages."""
+    def __init__(self, encoded = None):
+        self.encoded = encoded
+        self.decoded = None
+
+    def decode(self):
+        self.decoded = parse_construct(gsmtap_hdr_construct, self.encoded)
+        return self.decoded
+
+    def encode(self, decoded):
+        self.encoded = gsmtap_hdr_construct.build(decoded)
+        return self.encoded
+
+class GsmtapSource:
+    def __init__(self, bind_ip:str='127.0.0.1', bind_port:int=4729):
+        self.bind_ip = bind_ip
+        self.bind_port = bind_port
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.sock.bind((self.bind_ip, self.bind_port))
+
+    def read_packet(self) -> GsmtapMessage:
+        data, addr = self.sock.recvfrom(1024)
+        gsmtap_msg = GsmtapMessage(data)
+        gsmtap_msg.decode()
+        if gsmtap_msg.decoded['version'] != 0x02:
+            raise ValueError('Unknown GSMTAP version 0x%02x' % gsmtap_msg.decoded['version'])
+        return gsmtap_msg.decoded, addr

pySim/iso7816_4.py

@@ -24,56 +24,38 @@ from pySim.filesystem import *
 from pySim.tlv import *
 
 # Table 91 + Section 8.2.1.2
-
-
 class ApplicationId(BER_TLV_IE, tag=0x4f):
     _construct = GreedyBytes
 
 # Table 91
-
-
 class ApplicationLabel(BER_TLV_IE, tag=0x50):
     _construct = GreedyBytes
 
 # Table 91 + Section 5.3.1.2
-
-
 class FileReference(BER_TLV_IE, tag=0x51):
     _construct = GreedyBytes
 
 # Table 91
-
-
 class CommandApdu(BER_TLV_IE, tag=0x52):
     _construct = GreedyBytes
 
 # Table 91
-
-
 class DiscretionaryData(BER_TLV_IE, tag=0x53):
     _construct = GreedyBytes
 
 # Table 91
-
-
 class DiscretionaryTemplate(BER_TLV_IE, tag=0x73):
     _construct = GreedyBytes
 
 # Table 91 + RFC1738 / RFC2396
-
-
 class URL(BER_TLV_IE, tag=0x5f50):
     _construct = GreedyString('ascii')
 
 # Table 91
-
-
 class ApplicationRelatedDOSet(BER_TLV_IE, tag=0x61):
     _construct = GreedyBytes
 
 # Section 8.2.1.3 Application Template
-
-
 class ApplicationTemplate(BER_TLV_IE, tag=0x61, nested=[ApplicationId, ApplicationLabel, FileReference,
                           CommandApdu, DiscretionaryData, DiscretionaryTemplate, URL,
                           ApplicationRelatedDOSet]):

pySim/ota.py

@@ -0,0 +1,461 @@
+"""Code related to SIM/UICC OTA according to TS 102 225 + TS 31.115."""
+
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from pySim.construct import *
+from pySim.utils import b2h
+from pySim.sms import UserDataHeader
+from construct import *
+from bidict import bidict
+import zlib
+import abc
+import struct
+from typing import Optional
+
+# ETS TS 102 225 gives the general command structure and the dialects for CAT_TP, TCP/IP and HTTPS
+# 3GPP TS 31.115 gives the dialects for SMS-PP, SMS-CB, USSD and HTTP
+
+# CPI CPL CHI CHL SPI KIc KID TAR CNTR PCNTR RC/CC/DS data
+
+#                                           CAT_TP  TCP/IP          SMS
+# CPI                                       0x01    0x01            =IEIa=70,len=0
+# CHI                                       NULL    NULL            NULL
+# CPI, CPL and CHL included in RC/CC/DS     true    true
+# RPI                                       0x02    0x02            =IEIa=71,len=0
+# RHI                                       NULL    NULL
+# RPI, RPL and RHL included in RC/CC/DS     true    true
+# packet-id                             0-bf,ff     0-bf,ff
+# identification packet                     false   102 225 tbl 6
+
+# KVN 1..f; KI1=KIc, KI2=KID, KI3=DEK
+
+# TS 102 225 Table 5
+ota_status_codes = bidict({
+    0x00:   'PoR OK',
+    0x01:   'RC/CC/DS failed',
+    0x02:   'CNTR low',
+    0x03:   'CNTR high',
+    0x04:   'CNTR blocked',
+    0x05:   'Ciphering error',
+    0x06:   'Unidentified security error',
+    0x07:   'Insufficient memory',
+    0x08:   'more time',
+    0x09:   'TAR unknown',
+    0x0a:   'Insufficient security level',
+    0x0b:   'Actual Response in SMS-SUBMIT',    # 31.115
+    0x0c:   'Actual Response in USSD',          # 31.115
+})
+
+# ETSI TS 102 225 Table 5 + 3GPP TS 31.115 Section 7
+ResponseStatus = Enum(Int8ub, por_ok=0, rc_cc_ds_failed=1, cntr_low=2, cntr_high=3,
+                      cntr_blocked=4, ciphering_error=5, undefined_security_error=6,
+                      insufficient_memory=7, more_time_needed=8, tar_unknown=9,
+                      insufficient_security_level=0x0A,
+                      actual_response_sms_submit=0x0B,
+                      actual_response_ussd=0x0C)
+
+# ETSI TS 102 226 Section 5.1.2
+CompactRemoteResp = Struct('number_of_commands'/Int8ub,
+                           'last_status_word'/HexAdapter(Bytes(2)),
+                           'last_response_data'/HexAdapter(GreedyBytes))
+
+RC_CC_DS = Enum(BitsInteger(2), no_rc_cc_ds=0, rc=1, cc=2, ds=3)
+
+# TS 102 225 Section 5.1.1 + TS 31.115 Section 4.2
+SPI = BitStruct(  # first octet
+    Padding(3),
+    'counter'/Enum(BitsInteger(2), no_counter=0, counter_no_replay_or_seq=1,
+                   counter_must_be_higher=2, counter_must_be_lower=3),
+    'ciphering'/Flag,
+    'rc_cc_ds'/RC_CC_DS,
+    # second octet
+    Padding(2),
+    'por_in_submit'/Flag,
+    'por_shall_be_ciphered'/Flag,
+    'por_rc_cc_ds'/RC_CC_DS,
+    'por'/Enum(BitsInteger(2), no_por=0,
+               por_required=1, por_only_when_error=2)
+)
+
+# TS 102 225 Section 5.1.2
+KIC = BitStruct('key'/BitsInteger(4),
+                'algo'/Enum(BitsInteger(4), implicit=0, single_des=1, triple_des_cbc2=5, triple_des_cbc3=9,
+                            aes_cbc=2)
+                )
+
+# TS 102 225 Section 5.1.3.1
+KID_CC = BitStruct('key'/BitsInteger(4),
+                   'algo'/Enum(BitsInteger(4), implicit=0, single_des=1, triple_des_cbc2=5, triple_des_cbc3=9,
+                               aes_cmac=2)
+                   )
+
+# TS 102 225 Section 5.1.3.2
+KID_RC = BitStruct('key'/BitsInteger(4),
+                   'algo'/Enum(BitsInteger(4), implicit=0, crc16=1, crc32=5, proprietary=3)
+                   )
+
+SmsCommandPacket = Struct('cmd_pkt_len'/Int16ub,
+                          'cmd_hdr_len'/Int8ub,
+                          'spi'/SPI,
+                          'kic'/KIC,
+                          'kid'/Switch(this.spi.rc_cc_ds, {'cc': KID_CC, 'rc': KID_RC }),
+                          'tar'/Bytes(3),
+                          'secured_data'/GreedyBytes)
+
+class OtaKeyset:
+    """The OTA related data (key material, counter) to be used in encrypt/decrypt."""
+    def __init__(self, algo_crypt: str, kic_idx: int, kic: bytes,
+                 algo_auth: str, kid_idx: int, kid: bytes, cntr: int = 0):
+        self.algo_crypt = algo_crypt
+        self.kic = bytes(kic)
+        self.kic_idx = kic_idx
+        self.algo_auth = algo_auth
+        self.kid = bytes(kid)
+        self.kid_idx = kid_idx
+        self.cntr = cntr
+
+    @property
+    def auth(self):
+        """Return an instance of the matching OtaAlgoAuth."""
+        return OtaAlgoAuth.fromKeyset(self)
+
+    @property
+    def crypt(self):
+        """Return an instance of the matching OtaAlgoCrypt."""
+        return OtaAlgoCrypt.fromKeyset(self)
+
+class OtaCheckError(Exception):
+    pass
+
+class OtaDialect(abc.ABC):
+    """Base Class for OTA dialects such as SMS, BIP, ..."""
+
+    def _compute_sig_len(self, spi:SPI):
+        if spi['rc_cc_ds'] == 'no_rc_cc_ds':
+            return 0
+        elif spi['rc_cc_ds'] == 'rc': # CRC-32
+            return 4
+        elif spi['rc_cc_ds'] == 'cc': # Cryptographic Checksum (CC)
+            # TODO: this is not entirely correct, as in AES case it could be 4 or 8
+            return 8
+        else:
+            raise ValueError("Invalid rc_cc_ds: %s" % spi['rc_cc_ds'])
+
+    @abc.abstractmethod
+    def encode_cmd(self, otak: OtaKeyset, tar: bytes, apdu: bytes) -> bytes:
+        pass
+
+    @abc.abstractmethod
+    def decode_resp(self, otak: OtaKeyset, apdu: bytes) -> (object, Optional["CompactRemoteResp"]):
+        """Decode a response into a response packet and, if indicted (by a
+        response status of `"por_ok"`) a decoded response.
+
+        The response packet's common characteristics are not fully determined,
+        and (so far) completely proprietary per dialect."""
+        pass
+
+
+from Crypto.Cipher import DES, DES3, AES
+from Crypto.Hash import CMAC
+
+class OtaAlgo(abc.ABC):
+    iv = b'\x00\x00\x00\x00\x00\x00\x00\x00'
+    blocksize = None
+    enum_name = None
+
+    @staticmethod
+    def _get_padding(in_len: int, multiple: int, padding: int = 0):
+        """Return padding bytes towards multiple of N."""
+        if in_len % multiple == 0:
+            return b''
+        pad_cnt = multiple - (in_len % multiple)
+        return b'\x00' * pad_cnt
+
+    @staticmethod
+    def _pad_to_multiple(indat: bytes, multiple: int, padding: int = 0):
+        """Pad input bytes to multiple of N."""
+        return indat + OtaAlgo._get_padding(len(indat), multiple, padding)
+
+    def pad_to_blocksize(self, indat: bytes, padding: int = 0):
+        """Pad the given input data to multiple of the cipher block size."""
+        return self._pad_to_multiple(indat, self.blocksize, padding)
+
+    def __init__(self, otak: OtaKeyset):
+        self.otak = otak
+
+    def __str__(self):
+        return self.__class__.__name__
+
+class OtaAlgoCrypt(OtaAlgo, abc.ABC):
+    def __init__(self, otak: OtaKeyset):
+        if self.enum_name != otak.algo_crypt:
+            raise ValueError('Cannot use algorithm %s with key for %s' % (self.enum_name, otak.algo_crypt))
+        super().__init__(otak)
+
+    def encrypt(self, data:bytes) -> bytes:
+        """Encrypt given input bytes using the key material given in constructor."""
+        padded_data = self.pad_to_blocksize(data)
+        return self._encrypt(data)
+
+    def decrypt(self, data:bytes) -> bytes:
+        """Decrypt given input bytes using the key material given in constructor."""
+        return self._decrypt(data)
+
+    @abc.abstractmethod
+    def _encrypt(self, data:bytes) -> bytes:
+        """Actual implementation, to be implemented by derived class."""
+        pass
+
+    @abc.abstractmethod
+    def _decrypt(self, data:bytes) -> bytes:
+        """Actual implementation, to be implemented by derived class."""
+        pass
+
+    @classmethod
+    def fromKeyset(cls, otak: OtaKeyset) -> 'OtaAlgoCrypt':
+        """Resolve the class for the encryption algorithm of otak and instantiate it."""
+        for subc in cls.__subclasses__():
+            if subc.enum_name == otak.algo_crypt:
+                return subc(otak)
+        raise ValueError('No implementation for crypt algorithm %s' % otak.algo_auth)
+
+class OtaAlgoAuth(OtaAlgo, abc.ABC):
+    def __init__(self, otak: OtaKeyset):
+        if self.enum_name != otak.algo_auth:
+            raise ValueError('Cannot use algorithm %s with key for %s' % (self.enum_name, otak.algo_crypt))
+        super().__init__(otak)
+
+    def sign(self, data:bytes) -> bytes:
+        """Compute the CC/CR check bytes for the input data using key material
+        given in constructor."""
+        padded_data = self.pad_to_blocksize(data)
+        sig = self._sign(padded_data)
+        return sig
+
+    def check_sig(self, data:bytes, cc_received:bytes):
+        """Compute the CC/CR check bytes for the input data and compare against cc_received."""
+        cc = self.sign(data)
+        if cc_received != cc:
+            raise OtaCheckError('Received CC (%s) != Computed CC (%s)' % (b2h(cc_received), b2h(cc)))
+
+    @abc.abstractmethod
+    def _sign(self, data:bytes) -> bytes:
+        """Actual implementation, to be implemented by derived class."""
+        pass
+
+    @classmethod
+    def fromKeyset(cls, otak: OtaKeyset) -> 'OtaAlgoAuth':
+        """Resolve the class for the authentication algorithm of otak and instantiate it."""
+        for subc in cls.__subclasses__():
+            if subc.enum_name == otak.algo_auth:
+                return subc(otak)
+        raise ValueError('No implementation for auth algorithm %s' % otak.algo_auth)
+
+class OtaAlgoCryptDES(OtaAlgoCrypt):
+    """DES is insecure.  For backwards compatibility with pre-Rel8"""
+    name = 'DES'
+    enum_name = 'single_des'
+    blocksize = 8
+    def _encrypt(self, data:bytes) -> bytes:
+        cipher = DES.new(self.otak.kic, DES.MODE_CBC, self.iv)
+        return cipher.encrypt(data)
+
+    def _decrypt(self, data:bytes) -> bytes:
+        cipher = DES.new(self.otak.kic, DES.MODE_CBC, self.iv)
+        return cipher.decrypt(data)
+
+class OtaAlgoAuthDES(OtaAlgoAuth):
+    """DES is insecure.  For backwards compatibility with pre-Rel8"""
+    name = 'DES'
+    enum_name = 'single_des'
+    blocksize = 8
+    def _sign(self, data:bytes) -> bytes:
+        cipher = DES.new(self.otak.kid, DES.MODE_CBC, self.iv)
+        ciph = cipher.encrypt(data)
+        return ciph[len(ciph) - 8:]
+
+class OtaAlgoCryptDES3(OtaAlgoCrypt):
+    name = '3DES'
+    enum_name = 'triple_des_cbc2'
+    blocksize = 8
+    def _encrypt(self, data:bytes) -> bytes:
+        cipher = DES3.new(self.otak.kic, DES3.MODE_CBC, self.iv)
+        return cipher.encrypt(data)
+
+    def _decrypt(self, data:bytes) -> bytes:
+        cipher = DES3.new(self.otak.kic, DES3.MODE_CBC, self.iv)
+        return cipher.decrypt(data)
+
+class OtaAlgoAuthDES3(OtaAlgoAuth):
+    name = '3DES'
+    enum_name = 'triple_des_cbc2'
+    blocksize = 8
+    def _sign(self, data:bytes) -> bytes:
+        cipher = DES3.new(self.otak.kid, DES3.MODE_CBC, self.iv)
+        ciph = cipher.encrypt(data)
+        return ciph[len(ciph) - 8:]
+
+class OtaAlgoCryptAES(OtaAlgoCrypt):
+    name = 'AES'
+    enum_name = 'aes_cbc'
+    blocksize = 16 # TODO: is this needed?
+    def _encrypt(self, data:bytes) -> bytes:
+        cipher = AES.new(self.otak.kic, AES.MODE_CBC, self.iv)
+        return cipher.encrypt(data)
+
+    def _decrypt(self, data:bytes) -> bytes:
+        cipher = AES.new(self.otak.kic, AES.MODE_CBC, self.iv)
+        return cipher.decrypt(data)
+
+class OtaAlgoAuthAES(OtaAlgoAuth):
+    name = 'AES'
+    enum_name = 'aes_cmac'
+    blocksize = 16 # TODO: is this needed?
+    def _sign(self, data:bytes) -> bytes:
+        cmac = CMAC.new(self.otak.kid, ciphermod=AES, mac_len=8)
+        cmac.update(data)
+        ciph = cmac.digest()
+        return ciph[len(ciph) - 8:]
+
+
+
+class OtaDialectSms(OtaDialect):
+    """OTA dialect for SMS based transport, as described in 3GPP TS 31.115."""
+    SmsResponsePacket = Struct('rpl'/Int16ub,
+                               'rhl'/Int8ub,
+                               'tar'/Bytes(3),
+                               'cntr'/Bytes(5),
+                               'pcntr'/Int8ub,
+                               'response_status'/ResponseStatus,
+                               'cc_rc'/Bytes(this.rhl-10),
+                               'secured_data'/GreedyBytes)
+
+    def encode_cmd(self, otak: OtaKeyset, tar: bytes, spi: dict, apdu: bytes) -> bytes:
+        # length of signature in octets
+        len_sig = self._compute_sig_len(spi)
+        pad_cnt = 0
+        if spi['ciphering']: # ciphering is requested
+            # append padding bytes to end up with blocksize
+            len_cipher = 6 + len_sig + len(apdu)
+            padding = otak.crypt._get_padding(len_cipher, otak.crypt.blocksize)
+            pad_cnt = len(padding)
+            apdu += padding
+
+        kic = {'key': otak.kic_idx, 'algo': otak.algo_crypt}
+        kid = {'key': otak.kid_idx, 'algo': otak.algo_auth}
+
+        # CHL = number of octets from (and including) SPI to the end of RC/CC/DS
+        # 13 == SPI(2) + KIc(1) + KId(1) + TAR(3) + CNTR(5) + PCNTR(1)
+        chl = 13 + len_sig
+
+        # CHL + SPI (+ KIC + KID)
+        c = Struct('chl'/Int8ub, 'spi'/SPI, 'kic'/KIC, 'kid'/KID_CC, 'tar'/Bytes(3))
+        part_head = c.build({'chl': chl, 'spi':spi, 'kic':kic, 'kid':kid, 'tar':tar})
+        #print("part_head: %s" % b2h(part_head))
+
+        # CNTR + PCNTR (CNTR not used)
+        part_cnt = otak.cntr.to_bytes(5, 'big') + pad_cnt.to_bytes(1, 'big')
+        #print("part_cnt: %s" % b2h(part_cnt))
+
+        envelope_data = part_head + part_cnt + apdu
+        #print("envelope_data: %s" % b2h(envelope_data))
+
+        # 2-byte CPL. CPL is part of RC/CC/CPI to end of secured data, including any padding for ciphering
+        # CPL from and including CPI to end of secured data, including any padding for ciphering
+        cpl = len(envelope_data) + len_sig
+        envelope_data = cpl.to_bytes(2, 'big') + envelope_data
+        #print("envelope_data with cpl: %s" % b2h(envelope_data))
+
+        if spi['rc_cc_ds'] == 'cc':
+            cc = otak.auth.sign(envelope_data)
+            envelope_data = part_cnt + cc + apdu
+        elif spi['rc_cc_ds'] == 'rc':
+            # CRC32
+            crc32 = zlib.crc32(envelope_data) & 0xffffffff
+            envelope_data = part_cnt + crc32.to_bytes(4, 'big') + apdu
+        elif spi['rc_cc_ds'] == 'no_rc_cc_ds':
+            envelope_data = part_cnt + apdu
+        else:
+            raise ValueError("Invalid rc_cc_ds: %s" % spi['rc_cc_ds'])
+
+        #print("envelope_data with sig: %s" % b2h(envelope_data))
+
+        # encrypt as needed
+        if spi['ciphering']: # ciphering is requested
+            ciph = otak.crypt.encrypt(envelope_data)
+            envelope_data = part_head + ciph
+            # prefix with another CPL
+            cpl = len(envelope_data)
+            envelope_data = cpl.to_bytes(2, 'big') + envelope_data
+        else:
+            envelope_data = part_head + envelope_data
+
+        #print("envelope_data: %s" % b2h(envelope_data))
+
+        return envelope_data
+
+    def decode_resp(self, otak: OtaKeyset, spi: dict, data: bytes) -> ("OtaDialectSms.SmsResponsePacket", Optional["CompactRemoteResp"]):
+        if isinstance(data, str):
+            data = h2b(data)
+        # plain-text POR:   027100000e0ab000110000000000000001612f
+        # UDHL RPI IEDLa  RPL  RHL TAR    CNTR       PCNTR STS
+        # 02   71  00     000e 0a  b00011 0000000000 00    00  01 612f
+        # POR with CC:      027100001612b000110000000000000055f47118381175fb01612f
+        # POR with CC+CIPH: 027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c
+        if data[0] != 0x02:
+            raise ValueError('Unexpected UDL=0x%02x' % data[0])
+        udhd, remainder = UserDataHeader.fromBytes(data)
+        if not udhd.has_ie(0x71):
+            raise ValueError('RPI 0x71 not found in UDH')
+        rph_rhl_tar = remainder[:6] # RPH+RHL+TAR; not ciphered
+        res = self.SmsResponsePacket.parse(remainder)
+
+        if spi['por_shall_be_ciphered']:
+            # decrypt
+            ciphered_part = remainder[6:]
+            deciph = otak.crypt.decrypt(ciphered_part)
+            temp_data = rph_rhl_tar + deciph
+            res = self.SmsResponsePacket.parse(temp_data)
+            # remove specified number of padding bytes, if any
+            if res['pcntr'] != 0:
+                # this conditional is needed as python [:-0] renders an empty return!
+                res['secured_data'] = res['secured_data'][:-res['pcntr']]
+            remainder = temp_data
+
+        # is there a CC/RC present?
+        len_sig = res['rhl'] - 10
+        if spi['por_rc_cc_ds'] == 'no_rc_cc_ds':
+            if len_sig:
+                raise OtaCheckError('No RC/CC/DS requested, but len_sig=%u' % len_sig)
+        elif spi['por_rc_cc_ds'] == 'cc':
+            # verify signature
+            # UDH is part of CC/RC!
+            udh = data[:3]
+            # RPL, RHL, TAR, CNTR, PCNTR and STSare part of CC/RC
+            rpl_rhl_tar_cntr_pcntr_sts = remainder[:13]
+            # remove the CC/RC bytes
+            temp_data = udh + rpl_rhl_tar_cntr_pcntr_sts + remainder[13+len_sig:]
+            cc = otak.auth.check_sig(temp_data, res['cc_rc'])
+        # TODO: CRC
+        else:
+            raise OtaCheckError('Unknown por_rc_cc_ds: %s' % spi['por_rc_cc_ds'])
+
+        # TODO: ExpandedRemoteResponse according to TS 102 226 5.2.2
+        if res.response_status == 'por_ok':
+            dec = CompactRemoteResp.parse(res['secured_data'])
+        else:
+            dec = None
+        return (res, dec)

pySim/profile.py

@@ -62,7 +62,7 @@ def match_sim(scc: SimCardCommands) -> bool:
     return _mf_select_test(scc, "a0", "0000")
 
 
-class CardProfile(object):
+class CardProfile:
     """A Card Profile describes a card, it's filesystem hierarchy, an [initial] list of
     applications as well as profile-specific SW and shell commands.  Every card has
     one card profile, but there may be multiple applications within that profile."""

pySim/sms.py

@@ -0,0 +1,394 @@
+"""Code related to SMS Encoding/Decoding"""
+# simplistic SMS T-PDU code, as unfortunately nobody bothered to port the python smspdu
+# module to python3, and I gave up after >= 3 hours of trying and failing to do so
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import typing
+import abc
+from pprint import pprint as pp
+from construct import Int8ub, Byte, Bytes, Bit, Flag, BitsInteger, Flag
+from construct import Struct, Enum, Tell, BitStruct, this, Padding
+from construct import Prefixed, GreedyRange, GreedyBytes
+
+from pySim.construct import HexAdapter, BcdAdapter, TonNpi
+from pySim.utils import Hexstr, h2b, b2h
+
+from smpp.pdu import pdu_types
+
+BytesOrHex = typing.Union[Hexstr, bytes]
+
+class UserDataHeader:
+    # a single IE in the user data header
+    ie_c = Struct('iei'/Int8ub, 'length'/Int8ub, 'value'/Bytes(this.length))
+    # parser for the full UDH: Length octet followed by sequence of IEs
+    _construct = Struct('ies'/Prefixed(Int8ub, GreedyRange(ie_c)),
+                        'data'/GreedyBytes)
+
+    def __init__(self, ies=[]):
+        self.ies = ies
+
+    def __repr__(self) -> str:
+        return 'UDH(%r)' % self.ies
+
+    def has_ie(self, iei:int) -> bool:
+        for ie in self.ies:
+            if ie['iei'] == iei:
+                return True
+        return False
+
+    @classmethod
+    def fromBytes(cls, inb: BytesOrHex) -> typing.Tuple['UserDataHeader', bytes]:
+        if isinstance(inb, str):
+            inb = h2b(inb)
+        res = cls._construct.parse(inb)
+        return cls(res['ies']), res['data']
+
+    def toBytes(self) -> bytes:
+        return self._construct.build({'ies':self.ies, 'data':b''})
+
+
+def smpp_dcs_is_8bit(dcs: pdu_types.DataCoding) -> bool:
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED):
+        return True
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED_COMMON):
+        return True
+    if dcs.scheme == pdu_types.DataCodingScheme.GSM_MESSAGE_CLASS and dcs.schemeData['msgCoding'] == pdu_types.DataCodingGsmMsgCoding.DATA_8BIT:
+        return True
+    else:
+        return False
+
+def ensure_smpp_is_8bit(dcs: pdu_types.DataCoding):
+    if not smpp_dcs_is_8bit(smpp_pdu.params['data_coding']):
+        raise ValueError('We only support 8bit coded SMS for now')
+
+class AddressField:
+    """Representation of an address field as used in SMS T-PDU."""
+    _construct = Struct('addr_len'/Int8ub,
+                        'type_of_addr'/TonNpi,
+                        'digits'/BcdAdapter(Bytes(this.addr_len//2 + this.addr_len%2)),
+                        'tell'/Tell)
+
+    def __init__(self, digits, ton='unknown', npi='unknown'):
+        self.ton = ton
+        self.npi = npi
+        self.digits = digits
+
+    def __str__(self):
+        return 'AddressField(TON=%s, NPI=%s, %s)' % (self.ton, self.npi, self.digits)
+
+    @classmethod
+    def fromBytes(cls, inb: BytesOrHex) -> typing.Tuple['AddressField', bytes]:
+        """Construct an AddressField instance from the binary T-PDU address format."""
+        if isinstance(inb, str):
+            inb = h2b(inb)
+        res = cls._construct.parse(inb)
+        #pp(res)
+        #print("size: %s" % cls._construct.sizeof())
+        ton = res['type_of_addr']['type_of_number']
+        npi = res['type_of_addr']['numbering_plan_id']
+        # return resulting instance + remainder bytes
+        return cls(res['digits'][:res['addr_len']], ton, npi), inb[res['tell']:]
+
+    @classmethod
+    def fromSmpp(cls, addr, ton, npi) -> 'AddressField':
+        """Construct an AddressField from {source,dest}_addr_{,ton,npi} attributes of smpp.pdu."""
+        smpp_map_npi = {
+            'UNKNOWN': 'unknown',
+            'ISDN': 'isdn_e164',
+            'DATA': 'data_x121',
+            'TELEX': 'telex_f69',
+            'LAND_MOBILE': 'sc_specific6',
+            'NATIONAL': 'national',
+            'PRIVATE': 'private',
+            'ERMES': 'ermes',
+            }
+        smpp_map_ton = {
+            'UNKNOWN': 'unknown',
+            'INTERNATIONAL': 'international',
+            'NATIONAL': 'national',
+            'NETWORK_SPECIFIC': 'network_specific',
+            'SUBSCRIBER_NUMBER': 'short_code',
+            'ALPHANUMERIC': 'alphanumeric',
+            'ABBREVIATED': 'abbreviated',
+            }
+        # return the resulting instance
+        return cls(addr.decode('ascii'), smpp_map_ton[ton.name], smpp_map_npi[npi.name])
+
+
+    def toBytes(self) -> bytes:
+        """Encode the AddressField into the binary representation as used in T-PDU."""
+        num_digits = len(self.digits)
+        if num_digits % 2:
+            self.digits += 'f'
+        d = {
+            'addr_len': num_digits,
+            'type_of_addr': {
+                'ext': True,
+                'type_of_number': self.ton,
+                'numbering_plan_id': self.npi,
+                },
+            'digits': self.digits,
+            }
+        return self._construct.build(d)
+
+
+class SMS_TPDU(abc.ABC):
+    """Base class for a SMS T-PDU."""
+    def __init__(self, **kwargs):
+        self.tp_mti = kwargs.get('tp_mti', None)
+        self.tp_rp = kwargs.get('tp_rp', False)
+        self.tp_udhi = kwargs.get('tp_udhi', False)
+        self.tp_pid = kwargs.get('tp_pid', None)
+        self.tp_dcs = kwargs.get('tp_dcs', None)
+        self.tp_udl = kwargs.get('tp_udl', None)
+        self.tp_ud = kwargs.get('tp_ud', None)
+
+
+
+class SMS_DELIVER(SMS_TPDU):
+    """Representation of a SMS-DELIVER T-PDU."""
+    flags_construct = BitStruct('tp_rp'/Flag, 'tp_udhi'/Flag, 'tp_rp'/Flag, 'tp_sri'/Flag,
+                                Padding(1), 'tp_mms'/Flag, 'tp_mti'/BitsInteger(2))
+    def __init__(self, **kwargs):
+        kwargs['tp_mti'] = 0
+        super().__init__(**kwargs)
+        self.tp_lp = kwargs.get('tp_lp', False)
+        self.tp_mms = kwargs.get('tp_mms', False)
+        self.tp_oa = kwargs.get('tp_oa', None)
+        self.tp_scts = kwargs.get('tp_scts', None)
+        self.tp_sri = kwargs.get('tp_sri', False)
+
+    def __repr__(self):
+        return '%s(MTI=%s, MMS=%s, LP=%s, RP=%s, UDHI=%s, SRI=%s, OA=%s, PID=%2x, DCS=%x, SCTS=%s, UDL=%u, UD=%s)' % (self.__class__.__name__, self.tp_mti, self.tp_mms, self.tp_lp, self.tp_rp, self.tp_udhi, self.tp_sri, self.tp_oa, self.tp_pid, self.tp_dcs, self.tp_scts, self.tp_udl, self.tp_ud)
+
+    @classmethod
+    def fromBytes(cls, inb: BytesOrHex) -> 'SMS_DELIVER':
+        """Construct a SMS_DELIVER instance from the binary encoded format as used in T-PDU."""
+        if isinstance(inb, str):
+            inb = h2b(inb)
+        flags = inb[0]
+        d = SMS_DELIVER.flags_construct.parse(inb)
+        oa, remainder = AddressField.fromBytes(inb[1:])
+        d['tp_oa'] = oa
+        offset = 0
+        d['tp_pid'] = remainder[offset]
+        offset += 1
+        d['tp_dcs'] = remainder[offset]
+        offset += 1
+        # TODO: further decode
+        d['tp_scts'] = remainder[offset:offset+7]
+        offset += 7
+        d['tp_udl'] = remainder[offset]
+        offset += 1
+        d['tp_ud'] = remainder[offset:]
+        return cls(**d)
+
+    def toBytes(self) -> bytes:
+        """Encode a SMS_DELIVER instance to the binary encoded format as used in T-PDU."""
+        outb = bytearray()
+        d = {
+            'tp_mti': self.tp_mti, 'tp_mms': self.tp_mms, 'tp_lp': self.tp_lp,
+            'tp_rp': self.tp_rp, 'tp_udhi': self.tp_udhi, 'tp_sri': self.tp_sri,
+            }
+        flags = SMS_DELIVER.flags_construct.build(d)
+        outb.extend(flags)
+        outb.extend(self.tp_oa.toBytes())
+        outb.append(self.tp_pid)
+        outb.append(self.tp_dcs)
+        outb.extend(self.tp_scts)
+        outb.append(self.tp_udl)
+        outb.extend(self.tp_ud)
+
+        return outb
+
+    @classmethod
+    def fromSmpp(cls, smpp_pdu) -> 'SMS_DELIVER':
+        """Construct a SMS_DELIVER instance from the deliver format used by smpp.pdu."""
+        if smpp_pdu.id == pdu_types.CommandId.submit_sm:
+            return cls.fromSmppSubmit(cls, smpp_pdu)
+        else:
+            raise ValueError('Unsupported SMPP commandId %s' % smpp_pdu.id)
+ 
+    @classmethod
+    def fromSmppSubmit(cls, smpp_pdu) -> 'SMS_DELIVER':
+        """Construct a SMS_DELIVER instance from the submit format used by smpp.pdu."""
+        ensure_smpp_is_8bit(smpp_pdu.params['data_coding'])
+        tp_oa = AddressField.fromSmpp(smpp_pdu.params['source_addr'],
+                                      smpp_pdu.params['source_addr_ton'],
+                                      smpp_pdu.params['source_addr_npi'])
+        tp_ud = smpp_pdu.params['short_message']
+        d = {
+            'tp_lp': False,
+            'tp_mms': False,
+            'tp_oa': tp_oa,
+            'tp_scts': h2b('22705200000000'), # FIXME
+            'tp_sri': False,
+            'tp_rp': False,
+            'tp_udhi': pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET in smpp_pdu.params['esm_class'].gsmFeatures,
+            'tp_pid': smpp_pdu.params['protocol_id'],
+            'tp_dcs': 0xF6, # we only deal with binary SMS here
+            'tp_udl': len(tp_ud),
+            'tp_ud': tp_ud,
+            }
+        return cls(**d)
+
+
+
+class SMS_SUBMIT(SMS_TPDU):
+    """Representation of a SMS-DELIVER T-PDU."""
+    flags_construct = BitStruct('tp_srr'/Flag, 'tp_udhi'/Flag, 'tp_rp'/Flag,
+                                'tp_vpf'/Enum(BitsInteger(2), none=0, relative=2, enhanced=1, absolute=3),
+                                'tp_rd'/Flag, 'tp_mti'/BitsInteger(2))
+    def __init__(self, **kwargs):
+        kwargs['tp_mti'] = 1
+        super().__init__(**kwargs)
+        self.tp_rd = kwargs.get('tp_rd', False)
+        self.tp_vpf = kwargs.get('tp_vpf', 'none')
+        self.tp_srr = kwargs.get('tp_srr', False)
+        self.tp_mr = kwargs.get('tp_mr', None)
+        self.tp_da = kwargs.get('tp_da', None)
+        self.tp_vp = kwargs.get('tp_vp', None)
+
+    def __repr__(self):
+        return '%s(MTI=%s, RD=%s, VPF=%u, RP=%s, UDHI=%s, SRR=%s, DA=%s, PID=%2x, DCS=%x, VP=%s, UDL=%u, UD=%s)' % (self.__class__.__name__, self.tp_mti, self.tp_rd, self.tp_vpf, self.tp_rp, self.tp_udhi, self.tp_srr, self.tp_da, self.tp_pid, self.tp_dcs, self.tp_vp, self.tp_udl, self.tp_ud)
+
+    @classmethod
+    def fromBytes(cls, inb:BytesOrHex) -> 'SMS_SUBMIT':
+        """Construct a SMS_SUBMIT instance from the binary encoded format as used in T-PDU."""
+        offset = 0
+        if isinstance(inb, str):
+            inb = h2b(inb)
+        d = SMS_SUBMIT.flags_construct.parse(inb)
+        offset += 1
+        d['tp_mr']= inb[offset]
+        offset += 1
+        da, remainder = AddressField.fromBytes(inb[2:])
+        d['tp_da'] = da
+
+        offset = 0
+        d['tp_pid'] = remainder[offset]
+        offset += 1
+        d['tp_dcs'] = remainder[offset]
+        offset += 1
+        if d['tp_vpf'] == 'none':
+            pass
+        elif d['tp_vpf'] == 'relative':
+            # TODO: further decode
+            d['tp_vp'] = remainder[offset:offset+1]
+            offset += 1
+        elif d['tp_vpf'] == 'enhanced':
+            # TODO: further decode
+            d['tp_vp'] = remainder[offset:offset+7]
+            offset += 7
+            pass
+        elif d['tp_vpf'] == 'absolute':
+            # TODO: further decode
+            d['tp_vp'] = remainder[offset:offset+7]
+            offset += 7
+            pass
+        else:
+            raise ValueError('Invalid VPF: %s' % d['tp_vpf'])
+        d['tp_udl'] = remainder[offset]
+        offset += 1
+        d['tp_ud'] = remainder[offset:]
+        return cls(**d)
+
+    def toBytes(self) -> bytes:
+        """Encode a SMS_SUBMIT instance to the binary encoded format as used in T-PDU."""
+        outb = bytearray()
+        d = {
+            'tp_mti': self.tp_mti, 'tp_rd': self.tp_rd, 'tp_vpf': self.tp_vpf,
+            'tp_rp': self.tp_rp, 'tp_udhi': self.tp_udhi, 'tp_srr': self.tp_srr,
+            }
+        flags = SMS_SUBMIT.flags_construct.build(d)
+        outb.extend(flags)
+        outb.append(self.tp_mr)
+        outb.extend(self.tp_da.toBytes())
+        outb.append(self.tp_pid)
+        outb.append(self.tp_dcs)
+        if self.tp_vpf != 'none':
+            outb.extend(self.tp_vp)
+        outb.append(self.tp_udl)
+        outb.extend(self.tp_ud)
+        return outb
+
+    @classmethod
+    def fromSmpp(cls, smpp_pdu) -> 'SMS_SUBMIT':
+        """Construct a SMS_DELIVER instance from the format used by smpp.pdu."""
+        if smpp_pdu.id == pdu_types.CommandId.submit_sm:
+            return cls.fromSmppSubmit(cls, smpp_pdu)
+        else:
+            raise ValueError('Unsupported SMPP commandId %s' % smpp_pdu.id)
+
+    @classmethod
+    def fromSmppSubmit(cls, smpp_pdu) -> 'SMS_SUBMIT':
+        """Construct a SMS_DELIVER instance from the submit format used by smpp.pdu."""
+        ensure_smpp_is_8bit(smpp_pdu.params['data_coding'])
+        tp_da = AddressField.fromSmpp(smpp_pdu.params['destination_addr'],
+                                      smpp_pdu.params['dest_addr_ton'],
+                                      smpp_pdu.params['dest_addr_npi'])
+        tp_ud = smpp_pdu.params['short_message']
+        #vp_smpp = smpp_pdu.params['validity_period']
+        #if not vp_smpp:
+        #    vpf = 'none'
+        d = {
+            'tp_rd': True if smpp_pdu.params['replace_if_present_flag'].name == 'REPLACE' else False,
+            'tp_vpf': None, # vpf,
+            'tp_rp': False, # related to ['registered_delivery'] ?
+            'tp_udhi': pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET in smpp_pdu.params['esm_class'].gsmFeatures,
+            'tp_srr': True if smpp_pdu.params['registered_delivery'] else False,
+            'tp_mr': 0, # FIXME: sm_default_msg_id ?
+            'tp_da': tp_da,
+            'tp_pid': smpp_pdu.params['protocol_id'],
+            'tp_dcs': 0xF6, # FIXME: we only deal with binary SMS here
+            'tp_vp': None, # FIXME: implement VPF conversion
+            'tp_udl': len(tp_ud),
+            'tp_ud': tp_ud,
+            }
+        return cls(**d)
+
+    def toSmpp(self) -> pdu_types.PDU:
+        """Translate a SMS_DELIVER instance to a smpp.pdu.pdu_types.SubmitSM instance."""
+        esm_class = pdu_types.EsmClass(pdu_types.EsmClassMode.DEFAULT, pdu_types.EsmClassType.DEFAULT)
+        reg_del = pdu_types.RegisteredDelivery(pdu_types.RegisteredDeliveryReceipt.NO_SMSC_DELIVERY_RECEIPT_REQUESTED)
+        if self.tp_rp:
+            repl_if = pdu_types.ReplaceIfPresentFlag.REPLACE
+        else:
+            repl_if = pdu_types.ReplaceIfPresentFlag.DO_NOT_REPLACE
+        # we only deal with binary SMS here:
+        if self.tp_dcs != 0xF6:
+            raise ValueError('Unsupported DCS: We only support DCS=0xF6 for now')
+        dc = pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT, pdu_types.DataCodingDefault.OCTET_UNSPECIFIED)
+        return pdu_types.SubmitSM(service_type='',
+                                  source_addr_ton=pdu_types.AddrTon.ALPHANUMERIC,
+                                  source_addr_npi=pdu_types.AddrNpi.UNKNOWN,
+                                  source_addr='simcard',
+                                  dest_addr_ton=FIXME(self.tp_da.ton),
+                                  dest_addr_npi=FIXME(self.tp_da.npi),
+                                  destination_addr=self.tp_da.digits,
+                                  esm_class=esm_class,
+                                  protocol_id=self.tp_pid,
+                                  priority_flag=pdu_types.PriorityFlag.LEVEL_0,
+                                  #schedule_delivery_time,
+                                  #validity_period,
+                                  registered_delivery=reg_del,
+                                  replace_if_present_flag=repl_if,
+                                  data_coding=dc,
+                                  #sm_default_msg_id,
+                                  short_message=self.tp_ud)

pySim/sysmocom_sja2.py

@@ -85,7 +85,7 @@ class EF_MILENAGE_CFG(TransparentEF):
 
 class EF_0348_KEY(LinFixedEF):
     def __init__(self, fid='6f22', name='EF.0348_KEY', desc='TS 03.48 OTA Keys'):
-        super().__init__(fid, name=name, desc=desc, rec_len={27, 35})
+        super().__init__(fid, name=name, desc=desc, rec_len=(27, 35))
 
     def _decode_record_bin(self, raw_bin_data):
         u = unpack('!BBB', raw_bin_data[0:3])
@@ -103,7 +103,7 @@ class EF_0348_KEY(LinFixedEF):
 
 class EF_0348_COUNT(LinFixedEF):
     def __init__(self, fid='6f23', name='EF.0348_COUNT', desc='TS 03.48 OTA Counters'):
-        super().__init__(fid, name=name, desc=desc, rec_len={7, 7})
+        super().__init__(fid, name=name, desc=desc, rec_len=(7, 7))
 
     def _decode_record_bin(self, raw_bin_data):
         u = unpack('!BB5s', raw_bin_data)
@@ -118,7 +118,7 @@ class EF_SIM_AUTH_COUNTER(TransparentEF):
 
 class EF_GP_COUNT(LinFixedEF):
     def __init__(self, fid='6f26', name='EF.GP_COUNT', desc='GP SCP02 Counters'):
-        super().__init__(fid, name=name, desc=desc, rec_len={5, 5})
+        super().__init__(fid, name=name, desc=desc, rec_len=(5, 5))
 
     def _decode_record_bin(self, raw_bin_data):
         u = unpack('!BBHB', raw_bin_data)
@@ -127,7 +127,7 @@ class EF_GP_COUNT(LinFixedEF):
 
 class EF_GP_DIV_DATA(LinFixedEF):
     def __init__(self, fid='6f27', name='EF.GP_DIV_DATA', desc='GP SCP02 key diversification data'):
-        super().__init__(fid, name=name, desc=desc, rec_len={12, 12})
+        super().__init__(fid, name=name, desc=desc, rec_len=(12, 12))
 
     def _decode_record_bin(self, raw_bin_data):
         u = unpack('!BB8s', raw_bin_data)
@@ -137,20 +137,41 @@ class EF_GP_DIV_DATA(LinFixedEF):
 class EF_SIM_AUTH_KEY(TransparentEF):
     def __init__(self, fid='6f20', name='EF.SIM_AUTH_KEY'):
         super().__init__(fid, name=name, desc='USIM authentication key')
-        CfgByte = BitStruct(Bit[2],
+        CfgByte = BitStruct(Padding(2),
                             'use_sres_deriv_func_2'/Bit,
                             'use_opc_instead_of_op'/Bit,
                             'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/Bytes(16),
-                                 'op' /
-                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op, Bytes(
-                                     16)),
+                                 'key'/HexAdapter(Bytes(16)),
+                                 'op'/ If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16))),
                                  'opc' /
-                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op, Bytes(
-                                     16))
+                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16)))
                                  )
 
+class EF_HTTPS_CFG(TransparentEF):
+    def __init__(self, fid='6f2a', name='EF.HTTPS_CFG'):
+        super().__init__(fid, name=name, desc='HTTPS configuration')
+
+class EF_HTTPS_KEYS(TransparentEF):
+    KeyRecord = Struct('security_domain'/Int8ub,
+                       'key_type'/Enum(Int8ub, des=0x80, psk=0x85, aes=0x88),
+                       'key_version'/Int8ub,
+                       'key_id'/Int8ub,
+                       'key_length'/Int8ub,
+                       'key'/HexAdapter(Bytes(this.key_length)))
+    def __init__(self, fid='6f2b', name='EF.HTTPS_KEYS'):
+        super().__init__(fid, name=name, desc='HTTPS PSK and DEK keys')
+        self._construct = GreedyRange(self.KeyRecord)
+
+class EF_HTTPS_POLL(TransparentEF):
+    TimeUnit = Enum(Int8ub, seconds=0, minutes=1, hours=2, days=3, ten_days=4)
+    def __init__(self, fid='6f2c', name='EF.HTTPS_POLL'):
+        super().__init__(fid, name=name, desc='HTTPS polling interval')
+        self._construct = Struct(Const(b'\x82'), 'time_unit'/self.TimeUnit, 'value'/Int8ub,
+                                 'adm_session_triggering_tlv'/HexAdapter(GreedyBytes))
+
 
 class DF_SYSTEM(CardDF):
     def __init__(self):
@@ -169,6 +190,9 @@ class DF_SYSTEM(CardDF):
             EF_0348_COUNT(),
             EF_GP_COUNT(),
             EF_GP_DIV_DATA(),
+            EF_HTTPS_CFG(),
+            EF_HTTPS_KEYS(),
+            EF_HTTPS_POLL(),
         ]
         self.add_files(files)
 
@@ -193,36 +217,36 @@ class EF_USIM_SQN(TransparentEF):
 class EF_USIM_AUTH_KEY(TransparentEF):
     def __init__(self, fid='af20', name='EF.USIM_AUTH_KEY'):
         super().__init__(fid, name=name, desc='USIM authentication key')
-        CfgByte = BitStruct(Bit, 'only_4bytes_res_in_3g'/Bit,
+        CfgByte = BitStruct(Padding(1), 'only_4bytes_res_in_3g'/Bit,
                             'use_sres_deriv_func_2_in_3g'/Bit,
                             'use_opc_instead_of_op'/Bit,
                             'algorithm'/Enum(Nibble, milenage=4, sha1_aka=5, xor=15))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/Bytes(16),
+                                 'key'/HexAdapter(Bytes(16)),
                                  'op' /
-                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op, Bytes(
-                                     16)),
+                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16))),
                                  'opc' /
-                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op, Bytes(
-                                     16))
+                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16)))
                                  )
 
 
 class EF_USIM_AUTH_KEY_2G(TransparentEF):
     def __init__(self, fid='af22', name='EF.USIM_AUTH_KEY_2G'):
         super().__init__(fid, name=name, desc='USIM authentication key in 2G context')
-        CfgByte = BitStruct(Bit, 'only_4bytes_res_in_3g'/Bit,
+        CfgByte = BitStruct(Padding(1), 'only_4bytes_res_in_3g'/Bit,
                             'use_sres_deriv_func_2_in_3g'/Bit,
                             'use_opc_instead_of_op'/Bit,
                             'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/Bytes(16),
+                                 'key'/HexAdapter(Bytes(16)),
                                  'op' /
-                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op, Bytes(
-                                     16)),
+                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16))),
                                  'opc' /
-                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op, Bytes(
-                                     16))
+                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16)))
                                  )
 
 
@@ -242,7 +266,7 @@ class EF_GBA_REC_LIST(TransparentEF):
 class EF_GBA_INT_KEY(LinFixedEF):
     def __init__(self, fid='af33', name='EF.GBA_INT_KEY'):
         super().__init__(fid, name=name,
-                         desc='Secret key for GBA key derivation', rec_len={32, 32})
+                         desc='Secret key for GBA key derivation', rec_len=(32, 32))
         self._construct = GreedyBytes
 
 

pySim/tlv.py

@@ -100,7 +100,7 @@ class Transcodable(abc.ABC):
 
     # not an abstractmethod, as it is only required if no _construct exists
     def _to_bytes(self):
-        raise NotImplementedError
+        raise NotImplementedError('%s._to_bytes' % type(self).__name__)
 
     def from_bytes(self, do: bytes):
         """Convert from binary bytes to internal representation. Store the decoded result
@@ -118,7 +118,7 @@ class Transcodable(abc.ABC):
 
     # not an abstractmethod, as it is only required if no _construct exists
     def _from_bytes(self, do: bytes):
-        raise NotImplementedError
+        raise NotImplementedError('%s._from_bytes' % type(self).__name__)
 
 
 class IE(Transcodable, metaclass=TlvMeta):
@@ -232,9 +232,11 @@ class TLV_IE(IE):
         return self._encode_tag() + self._encode_len(val) + val
 
     def from_tlv(self, do: bytes):
+        if len(do) == 0:
+            return {}, b''
         (rawtag, remainder) = self.__class__._parse_tag_raw(do)
         if rawtag:
-            if rawtag != self.tag:
+            if rawtag != self._compute_tag():
                 raise ValueError("%s: Encountered tag %s doesn't match our supported tag %s" %
                                  (self, rawtag, self.tag))
             (length, remainder) = self.__class__._parse_len(remainder)

pySim/transport/__init__.py

@@ -9,11 +9,12 @@ from typing import Optional, Tuple
 
 from pySim.exceptions import *
 from pySim.construct import filter_dict
-from pySim.utils import sw_match, b2h, h2b, i2h
+from pySim.utils import sw_match, b2h, h2b, i2h, Hexstr
+from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
 
 #
 # Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
-# Copyright (C) 2021 Harald Welte <laforge@osmocom.org>
+# Copyright (C) 2021-2022 Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -37,13 +38,33 @@ class ApduTracer:
     def trace_response(self, cmd, sw, resp):
         pass
 
+class ProactiveHandler(abc.ABC):
+    """Abstract base class representing the interface of some code that handles
+    the proactive commands, as returned by the card in responses to the FETCH
+    command."""
+    def receive_fetch_raw(self, pcmd: ProactiveCommand, parsed: Hexstr):
+        # try to find a generic handler like handle_SendShortMessage
+        handle_name = 'handle_%s' % type(parsed).__name__
+        if hasattr(self, handle_name):
+            handler = getattr(self, handle_name)
+            return handler(pcmd.decoded)
+        # fall back to common handler
+        return self.receive_fetch(pcmd)
+
+    def receive_fetch(self, pcmd: ProactiveCommand):
+        """Default handler for not otherwise handled proactive commands."""
+        raise NotImplementedError('No handler method for %s' % pcmd.decoded)
+
+
 
 class LinkBase(abc.ABC):
     """Base class for link/transport to card."""
 
-    def __init__(self, sw_interpreter=None, apdu_tracer=None):
+    def __init__(self, sw_interpreter=None, apdu_tracer=None,
+                 proactive_handler: Optional[ProactiveHandler]=None):
         self.sw_interpreter = sw_interpreter
         self.apdu_tracer = apdu_tracer
+        self.proactive_handler = proactive_handler
 
     @abc.abstractmethod
     def _send_apdu_raw(self, pdu: str) -> Tuple[str, str]:
@@ -136,11 +157,57 @@ class LinkBase(abc.ABC):
                         sw   : string (in hex) of status word (ex. "9000")
         """
         rv = self.send_apdu(pdu)
+        last_sw = rv[1]
 
-        if sw == '9000' and sw_match(rv[1], '91xx'):
+        while sw == '9000' and sw_match(last_sw, '91xx'):
+            # It *was* successful after all -- the extra pieces FETCH handled
+            # need not concern the caller.
+            rv = (rv[0], '9000')
             # proactive sim as per TS 102 221 Setion 7.4.2
-            rv = self.send_apdu_checksw('80120000' + rv[1][2:], sw)
-            print("FETCH: %s", rv[0])
+            # TODO: Check SW manually to avoid recursing on the stack (provided this piece of code stays in this place)
+            fetch_rv = self.send_apdu_checksw('80120000' + last_sw[2:], sw)
+            # Setting this in case we later decide not to send a terminal
+            # response immediately unconditionally -- the card may still have
+            # something pending even though the last command was not processed
+            # yet.
+            last_sw = fetch_rv[1]
+            # parse the proactive command
+            pcmd = ProactiveCommand()
+            parsed = pcmd.from_tlv(h2b(fetch_rv[0]))
+            print("FETCH: %s (%s)" % (fetch_rv[0], type(parsed).__name__))
+            result = Result()
+            if self.proactive_handler:
+                # Extension point: If this does return a list of TLV objects,
+                # they could be appended after the Result; if the first is a
+                # Result, that cuold replace the one built here.
+                self.proactive_handler.receive_fetch_raw(pcmd, parsed)
+                result.from_dict({'general_result': 'performed_successfully', 'additional_information': ''})
+            else:
+                result.from_dict({'general_result': 'command_beyond_terminal_capability', 'additional_information': ''})
+
+            # Send response immediately, thus also flushing out any further
+            # proactive commands that the card already wants to send
+            #
+            # Structure as per TS 102 223 V4.4.0 Section 6.8
+
+            # The Command Details are echoed from the command that has been processed.
+            (command_details,) = [c for c in pcmd.decoded.children if isinstance(c, CommandDetails)]
+            # The Device Identities are fixed. (TS 102 223 V4.0.0 Section 6.8.2)
+            device_identities = DeviceIdentities()
+            device_identities.from_dict({'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'})
+
+            # Testing hint: The value of tail does not influence the behavior
+            # of an SJA2 that sent ans SMS, so this is implemented only
+            # following TS 102 223, and not fully tested.
+            tail = command_details.to_tlv() + device_identities.to_tlv() + result.to_tlv()
+            # Testing hint: In contrast to the above, this part is positively
+            # essential to get the SJA2 to provide the later parts of a
+            # multipart SMS in response to an OTA RFM command.
+            terminal_response = '80140000' + b2h(len(tail).to_bytes(1, 'big') + tail)
+
+            terminal_response_rv = self.send_apdu(terminal_response)
+            last_sw = terminal_response_rv[1]
+
         if not sw_match(rv[1], sw):
             raise SwMatchError(rv[1], sw.lower(), self.sw_interpreter)
         return rv

pySim/transport/calypso.py

@@ -26,7 +26,7 @@ from pySim.exceptions import *
 from pySim.utils import h2b, b2h
 
 
-class L1CTLMessage(object):
+class L1CTLMessage:
 
     # Every (encoded) L1CTL message has the following structure:
     #  - msg_length (2 bytes, net order)

pySim/transport/pcsc.py

@@ -34,7 +34,7 @@ class PcscSimLink(LinkBase):
         super().__init__(**kwargs)
         r = readers()
         if reader_number >= len(r):
-            raise ReaderError
+            raise ReaderError('No reader found for number %d' % reader_number)
         self._reader = r[reader_number]
         self._con = self._reader.createConnection()
 

pySim/ts_102_221.py

@@ -17,8 +17,8 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
-from pytlv.TLV import *
 from construct import *
+from construct import Optional as COptional
 from pySim.construct import *
 from pySim.utils import *
 from pySim.filesystem import *
@@ -78,131 +78,178 @@ ts_102_22x_cmdset = CardCommandSet('TS 102 22x', [
     CardCommand('RESIZE FILE',              0xD4, ['8X', 'CX']),
 ])
 
+# ETSI TS 102 221 11.1.1.4.2
+class FileSize(BER_TLV_IE, tag=0x80):
+    _construct = GreedyInteger(minlen=2)
 
-FCP_TLV_MAP = {
-    '82': 'file_descriptor',
-    '83': 'file_identifier',
-    '84': 'df_name',
-    'A5': 'proprietary_info',
-    '8A': 'life_cycle_status_int',
-    '8B': 'security_attrib_ref_expanded',
-    '8C': 'security_attrib_compact',
-    'AB': 'security_attrib_espanded',
-    'C6': 'pin_status_template_do',
-    '80': 'file_size',
-    '81': 'total_file_size',
-    '88': 'short_file_id',
-}
-
-# ETSI TS 102 221 11.1.1.4.6
-FCP_Proprietary_TLV_MAP = {
-    '80': 'uicc_characteristics',
-    '81': 'application_power_consumption',
-    '82': 'minimum_app_clock_freq',
-    '83': 'available_memory',
-    '84': 'file_details',
-    '85': 'reserved_file_size',
-    '86': 'maximum_file_size',
-    '87': 'suported_system_commands',
-    '88': 'specific_uicc_env_cond',
-    '89': 'p2p_cat_secured_apdu',
-    # Additional private TLV objects (bits b7 and b8 of the first byte of the tag set to '1')
-}
+# ETSI TS 102 221 11.1.1.4.2
+class TotalFileSize(BER_TLV_IE, tag=0x81):
+    _construct = GreedyInteger(minlen=2)
 
 # ETSI TS 102 221 11.1.1.4.3
+class FileDescriptor(BER_TLV_IE, tag=0x82):
+    class BerTlvAdapter(Adapter):
+        def _parse(self, obj, context, path):
+            if obj == 0x39:
+                return 'ber_tlv'
+            raise ValidationError
+        def _build(self, obj, context, path):
+            if obj == 'ber_tlv':
+                return 0x39
+            raise ValidationError
 
+    FDB = Select(BitStruct(Const(0, Bit), 'shareable'/Flag, 'structure'/BerTlvAdapter(Const(0x39, BitsInteger(6)))),
+                 BitStruct(Const(0, Bit), 'shareable'/Flag, 'file_type'/Enum(BitsInteger(3), working_ef=0, internal_ef=1, df=7),
+                           'structure'/Enum(BitsInteger(3), no_info_given=0, transparent=1, linear_fixed=2, cyclic=6))
+                )
+    _construct = Struct('file_descriptor_byte'/FDB, Const(b'\x21'),
+                        'record_len'/COptional(Int16ub), 'num_of_rec'/COptional(Int8ub))
 
-def interpret_file_descriptor(in_hex):
-    in_bin = h2b(in_hex)
-    out = {}
-    ft_dict = {
-        0: 'working_ef',
-        1: 'internal_ef',
-        7: 'df'
-    }
-    fs_dict = {
-        0: 'no_info_given',
-        1: 'transparent',
-        2: 'linear_fixed',
-        6: 'cyclic',
-        0x39: 'ber_tlv',
-    }
-    fdb = in_bin[0]
-    ftype = (fdb >> 3) & 7
-    if fdb & 0xbf == 0x39:
-        fstruct = 0x39
-    else:
-        fstruct = fdb & 7
-    out['shareable'] = True if fdb & 0x40 else False
-    out['file_type'] = ft_dict[ftype] if ftype in ft_dict else ftype
-    out['structure'] = fs_dict[fstruct] if fstruct in fs_dict else fstruct
-    if len(in_bin) >= 5:
-        out['record_len'] = int.from_bytes(in_bin[2:4], 'big')
-        out['num_of_rec'] = int.from_bytes(in_bin[4:5], 'big')
-    return out
+# ETSI TS 102 221 11.1.1.4.4
+class FileIdentifier(BER_TLV_IE, tag=0x83):
+    _construct = HexAdapter(GreedyBytes)
+
+# ETSI TS 102 221 11.1.1.4.5
+class DfName(BER_TLV_IE, tag=0x84):
+    _construct = HexAdapter(GreedyBytes)
+
+# ETSI TS 102 221 11.1.1.4.6.1
+class UiccCharacteristics(BER_TLV_IE, tag=0x80):
+    _construct = GreedyBytes
+
+# ETSI TS 102 221 11.1.1.4.6.2
+class ApplicationPowerConsumption(BER_TLV_IE, tag=0x81):
+    _construct = Struct('voltage_class'/Int8ub,
+                        'power_consumption_ma'/Int8ub,
+                        'reference_freq_100k'/Int8ub)
+
+# ETSI TS 102 221 11.1.1.4.6.3
+class MinApplicationClockFrequency(BER_TLV_IE, tag=0x82):
+    _construct = Int8ub
+
+# ETSI TS 102 221 11.1.1.4.6.4
+class AvailableMemory(BER_TLV_IE, tag=0x83):
+    _construct = GreedyInteger()
+
+# ETSI TS 102 221 11.1.1.4.6.5
+class FileDetails(BER_TLV_IE, tag=0x84):
+    _construct = FlagsEnum(Byte, der_coding_only=1)
+
+# ETSI TS 102 221 11.1.1.4.6.6
+class ReservedFileSize(BER_TLV_IE, tag=0x85):
+    _construct = GreedyInteger()
+
+# ETSI TS 102 221 11.1.1.4.6.7
+class MaximumFileSize(BER_TLV_IE, tag=0x86):
+    _construct = GreedyInteger()
+
+# ETSI TS 102 221 11.1.1.4.6.8
+class SupportedFilesystemCommands(BER_TLV_IE, tag=0x87):
+    _construct = FlagsEnum(Byte, terminal_capability=1)
+
+# ETSI TS 102 221 11.1.1.4.6.9
+class SpecificUiccEnvironmentConditions(BER_TLV_IE, tag=0x88):
+    _construct = BitStruct('rfu'/BitsRFU(4),
+                           'high_humidity_supported'/Flag,
+                           'temperature_class'/Enum(BitsInteger(3), standard=0, class_A=1, class_B=2, class_C=3))
+
+# ETSI TS 102 221 11.1.1.4.6.10
+class Platform2PlatformCatSecuredApdu(BER_TLV_IE, tag=0x89):
+    _construct = GreedyBytes
+
+# sysmoISIM-SJA2 specific
+class ToolkitAccessConditions(BER_TLV_IE, tag=0xD2):
+    _construct = FlagsEnum(Byte, rfm_create=1, rfm_delete_terminate=2, other_applet_create=4,
+                           other_applet_delete_terminate=8)
+
+# ETSI TS 102 221 11.1.1.4.6.0
+class ProprietaryInformation(BER_TLV_IE, tag=0xA5,
+                             nested=[UiccCharacteristics, ApplicationPowerConsumption,
+                                     MinApplicationClockFrequency, AvailableMemory,
+                                     FileDetails, ReservedFileSize, MaximumFileSize,
+                                     SupportedFilesystemCommands, SpecificUiccEnvironmentConditions,
+                                     ToolkitAccessConditions]):
+    pass
+
+# ETSI TS 102 221 11.1.1.4.7.1
+class SecurityAttribCompact(BER_TLV_IE, tag=0x8c):
+    _construct = GreedyBytes
+
+# ETSI TS 102 221 11.1.1.4.7.2
+class SecurityAttribExpanded(BER_TLV_IE, tag=0xab):
+    _construct = GreedyBytes
+
+# ETSI TS 102 221 11.1.1.4.7.3
+class SecurityAttribReferenced(BER_TLV_IE, tag=0x8b):
+    # TODO: longer format with SEID
+    _construct = Struct('ef_arr_file_id'/HexAdapter(Bytes(2)), 'ef_arr_record_nr'/Int8ub)
+
+# ETSI TS 102 221 11.1.1.4.8
+class ShortFileIdentifier(BER_TLV_IE, tag=0x88):
+    # If the length of the TLV is 1, the SFI value is indicated in the 5 most significant bits (bits b8 to b4)
+    # of the TLV value field. In this case, bits b3 to b1 shall be set to 0
+    class Shift3RAdapter(Adapter):
+        def _decode(self, obj, context, path):
+            return int.from_bytes(obj, 'big') >> 3
+        def _encode(self, obj, context, path):
+            val = int(obj) << 3
+            return val.to_bytes(1, 'big')
+    _construct = COptional(Shift3RAdapter(Bytes(1)))
 
 # ETSI TS 102 221 11.1.1.4.9
+class LifeCycleStatusInteger(BER_TLV_IE, tag=0x8A):
+    def _from_bytes(self, do: bytes):
+        lcsi = int.from_bytes(do, 'big')
+        if lcsi == 0x00:
+            ret = 'no_information'
+        elif lcsi == 0x01:
+            ret = 'creation'
+        elif lcsi == 0x03:
+            ret = 'initialization'
+        elif lcsi & 0x05 == 0x05:
+            ret = 'operational_activated'
+        elif lcsi & 0x05 == 0x04:
+            ret = 'operational_deactivated'
+        elif lcsi & 0xc0 == 0xc0:
+            ret = 'termination'
+        else:
+            ret = lcsi
+        self.decoded = ret
+        return self.decoded
+    def _to_bytes(self):
+        if self.decoded == 'no_information':
+            return b'\x00'
+        elif self.decoded == 'creation':
+            return b'\x01'
+        elif self.decoded == 'initialization':
+            return b'\x03'
+        elif self.decoded == 'operational_activated':
+            return b'\x05'
+        elif self.decoded == 'operational_deactivated':
+            return b'\x04'
+        elif self.decoded == 'termination':
+            return b'\x0c'
+        elif isinstance(self.decoded, int):
+            return self.decoded.to_bytes(1, 'big')
+        else:
+            raise ValueError
 
+# ETSI TS 102 221 11.1.1.4.9
+class PS_DO(BER_TLV_IE, tag=0x90):
+    _construct = GreedyBytes
+class UsageQualifier_DO(BER_TLV_IE, tag=0x95):
+    _construct = GreedyBytes
+class KeyReference(BER_TLV_IE, tag=0x83):
+    _construct = Byte
+class PinStatusTemplate_DO(BER_TLV_IE, tag=0xC6, nested=[PS_DO, UsageQualifier_DO, KeyReference]):
+    pass
 
-def interpret_life_cycle_sts_int(in_hex):
-    lcsi = int(in_hex, 16)
-    if lcsi == 0x00:
-        return 'no_information'
-    elif lcsi == 0x01:
-        return 'creation'
-    elif lcsi == 0x03:
-        return 'initialization'
-    elif lcsi & 0x05 == 0x05:
-        return 'operational_activated'
-    elif lcsi & 0x05 == 0x04:
-        return 'operational_deactivated'
-    elif lcsi & 0xc0 == 0xc0:
-        return 'termination'
-    else:
-        return in_hex
-
-
-# ETSI TS 102 221 11.1.1.4.10
-FCP_Pin_Status_TLV_MAP = {
-    '90': 'ps_do',
-    '95': 'usage_qualifier',
-    '83': 'key_reference',
-}
-
-
-def interpret_ps_templ_do(in_hex):
-    # cannot use the 'TLV' parser due to repeating tags
-    #psdo_tlv = TLV(FCP_Pin_Status_TLV_MAP)
-    # return psdo_tlv.parse(in_hex)
-    return in_hex
-
-
-# 'interpreter' functions for each tag
-FCP_interpreter_map = {
-    '80': lambda x: int(x, 16),
-    '82': interpret_file_descriptor,
-    '8A': interpret_life_cycle_sts_int,
-    'C6': interpret_ps_templ_do,
-}
-
-FCP_prorietary_interpreter_map = {
-    '83': lambda x: int(x, 16),
-}
-
-# pytlv unfortunately doesn't have a setting using which we can make it
-# accept unknown tags.  It also doesn't raise a specific exception type but
-# just the generic ValueError, so we cannot ignore those either.  Instead,
-# we insert a dict entry for every possible proprietary tag permitted
-
-
-def fixup_fcp_proprietary_tlv_map(tlv_map):
-    if 'D0' in tlv_map:
-        return
-    for i in range(0xc0, 0xff):
-        i_hex = i2h([i]).upper()
-        tlv_map[i_hex] = 'proprietary_' + i_hex
-    # Other non-standard TLV objects found on some cards
-    tlv_map['9B'] = 'target_ef'  # for sysmoUSIM-SJS1
+class FcpTemplate(BER_TLV_IE, tag=0x62, nested=[FileSize, TotalFileSize, FileDescriptor, FileIdentifier,
+                                                DfName, ProprietaryInformation, SecurityAttribCompact,
+                                                SecurityAttribExpanded, SecurityAttribReferenced,
+                                                ShortFileIdentifier, LifeCycleStatusInteger,
+                                                PinStatusTemplate_DO]):
+    pass
 
 
 def tlv_key_replace(inmap, indata):
@@ -223,8 +270,6 @@ def tlv_val_interpret(inmap, indata):
     return {d[0]: newval(inmap, d[0], d[1]) for d in indata.items()}
 
 # ETSI TS 102 221 Section 9.2.7 + ISO7816-4 9.3.3/9.3.4
-
-
 class _AM_DO_DF(DataObject):
     def __init__(self):
         super().__init__('access_mode', 'Access Mode', tag=0x80)
@@ -436,8 +481,6 @@ class CRT_DO(DataObject):
         return b'\x83\x01' + pin.to_bytes(1, 'big') + b'\x95\x01\x08'
 
 # ISO7816-4 9.3.3 Table 33
-
-
 class SecCondByte_DO(DataObject):
     def __init__(self, tag=0x9d):
         super().__init__('security_condition_byte', tag=tag)
@@ -530,8 +573,6 @@ SC_DO = DataObjectChoice('security_condition', 'Security Condition',
                                   OR_DO, AND_DO, NOT_DO])
 
 # TS 102 221 Section 13.1
-
-
 class EF_DIR(LinFixedEF):
     class ApplicationLabel(BER_TLV_IE, tag=0x50):
         # TODO: UCS-2 coding option as per Annex A of TS 102 221
@@ -547,15 +588,13 @@ class EF_DIR(LinFixedEF):
         pass
 
     def __init__(self, fid='2f00', sfid=0x1e, name='EF.DIR', desc='Application Directory'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len={5, 54})
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(5, 54))
         self._tlv = EF_DIR.ApplicationTemplate
 
 # TS 102 221 Section 13.2
-
-
 class EF_ICCID(TransparentEF):
     def __init__(self, fid='2fe2', sfid=0x02, name='EF.ICCID', desc='ICC Identification'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size={10, 10})
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(10, 10))
 
     def _decode_hex(self, raw_hex):
         return {'iccid': dec_iccid(raw_hex)}
@@ -564,12 +603,10 @@ class EF_ICCID(TransparentEF):
         return enc_iccid(abstract['iccid'])
 
 # TS 102 221 Section 13.3
-
-
 class EF_PL(TransRecEF):
     def __init__(self, fid='2f05', sfid=0x05, name='EF.PL', desc='Preferred Languages'):
         super().__init__(fid, sfid=sfid, name=name,
-                         desc=desc, rec_len=2, size={2, None})
+                         desc=desc, rec_len=2, size=(2, None))
 
     def _decode_record_bin(self, bin_data):
         if bin_data == b'\xff\xff':
@@ -636,6 +673,11 @@ class EF_ARR(LinFixedEF):
         # 'un-flattening' decoder, and hence would be unable to encode :(
         return dec[0]
 
+    def _encode_record_bin(self, in_json):
+        # we can only guess if we should decode for EF or DF here :(
+        arr_seq = DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        return arr_seq.encode_multi(in_json)
+
     @with_default_category('File-Specific Commands')
     class AddlShellCommands(CommandSet):
         def __init__(self):
@@ -644,19 +686,19 @@ class EF_ARR(LinFixedEF):
         @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
         def do_read_arr_record(self, opts):
             """Read one EF.ARR record in flattened, human-friendly form."""
-            (data, sw) = self._cmd.rs.read_record_dec(opts.record_nr)
-            data = self._cmd.rs.selected_file.flatten(data)
+            (data, sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
+            data = self._cmd.lchan.selected_file.flatten(data)
             self._cmd.poutput_json(data, opts.oneline)
 
         @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
         def do_read_arr_records(self, opts):
             """Read + decode all EF.ARR records in flattened, human-friendly form."""
-            num_of_rec = self._cmd.rs.selected_file_fcp['file_descriptor']['num_of_rec']
+            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
             # collect all results in list so they are rendered as JSON list when printing
             data_list = []
             for recnr in range(1, 1 + num_of_rec):
-                (data, sw) = self._cmd.rs.read_record_dec(recnr)
-                data = self._cmd.rs.selected_file.flatten(data)
+                (data, sw) = self._cmd.lchan.read_record_dec(recnr)
+                data = self._cmd.lchan.selected_file.flatten(data)
                 data_list.append(data)
             self._cmd.poutput_json(data_list, opts.oneline)
 
@@ -664,7 +706,7 @@ class EF_ARR(LinFixedEF):
 # TS 102 221 Section 13.6
 class EF_UMPC(TransparentEF):
     def __init__(self, fid='2f08', sfid=0x08, name='EF.UMPC', desc='UICC Maximum Power Consumption'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size={5, 5})
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(5, 5))
         addl_info = FlagsEnum(Byte, req_inc_idle_current=1,
                               support_uicc_suspend=2)
         self._construct = Struct(
@@ -759,23 +801,10 @@ class CardProfileUICC(CardProfile):
     @staticmethod
     def decode_select_response(resp_hex: str) -> object:
         """ETSI TS 102 221 Section 11.1.1.3"""
-        fixup_fcp_proprietary_tlv_map(FCP_Proprietary_TLV_MAP)
-        resp_hex = resp_hex.upper()
-        # outer layer
-        fcp_base_tlv = TLV(['62'])
-        fcp_base = fcp_base_tlv.parse(resp_hex)
-        # actual FCP
-        fcp_tlv = TLV(FCP_TLV_MAP)
-        fcp = fcp_tlv.parse(fcp_base['62'])
-        # further decode the proprietary information
-        if 'A5' in fcp:
-            prop_tlv = TLV(FCP_Proprietary_TLV_MAP)
-            prop = prop_tlv.parse(fcp['A5'])
-            fcp['A5'] = tlv_val_interpret(FCP_prorietary_interpreter_map, prop)
-            fcp['A5'] = tlv_key_replace(FCP_Proprietary_TLV_MAP, fcp['A5'])
-        # finally make sure we get human-readable keys in the output dict
-        r = tlv_val_interpret(FCP_interpreter_map, fcp)
-        return tlv_key_replace(FCP_TLV_MAP, r)
+        t = FcpTemplate()
+        t.from_tlv(h2b(resp_hex))
+        d = t.to_dict()
+        return flatten_dict_lists(d['fcp_template'])
 
     @staticmethod
     def match_with_card(scc: SimCardCommands) -> bool:

pySim/ts_102_222.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+
+# Interactive shell for working with SIM / UICC / USIM / ISIM cards
+#
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from typing import List
+
+import cmd2
+from cmd2 import CommandSet, with_default_category, with_argparser
+import argparse
+
+from pySim.ts_31_102 import EF_UST_map, EF_USIM_ADF_map
+from pySim.ts_31_103 import EF_IST_map, EF_ISIM_ADF_map
+
+from pySim.exceptions import *
+from pySim.utils import h2b, swap_nibbles, b2h, JsonEncoder
+
+from pySim.ts_102_221 import *
+
+@with_default_category('TS 102 222 Administrative Commands')
+class Ts102222Commands(CommandSet):
+    """Administrative commands for telecommunication applications."""
+
+    def __init__(self):
+        super().__init__()
+
+    delfile_parser = argparse.ArgumentParser()
+    delfile_parser.add_argument('--force-delete', action='store_true',
+            help='I really want to permanently delete the file. I know pySim cannot re-create it yet!')
+    delfile_parser.add_argument('NAME', type=str, help='File name or FID to delete')
+
+    @cmd2.with_argparser(delfile_parser)
+    def do_delete_file(self, opts):
+        """Delete the specified file. DANGEROUS!  See TS 102 222 Section 6.4.
+        This will permanently delete the specified file from the card.
+        pySim has no support to re-create files yet, and even if it did, your card may not allow it!"""
+        if not opts.force_delete:
+            self._cmd.perror("Refusing to permanently delete the file, please read the help text.")
+            return
+        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        (data, sw) = self._cmd.card._scc.delete_file(f.fid)
+
+    def complete_delete_file(self, text, line, begidx, endidx) -> List[str]:
+        """Command Line tab completion for DELETE FILE"""
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
+        return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+    termdf_parser = argparse.ArgumentParser()
+    termdf_parser.add_argument('--force', action='store_true',
+            help='I really want to terminate the file. I know I can not recover from it!')
+    termdf_parser.add_argument('NAME', type=str, help='File name or FID')
+
+    @cmd2.with_argparser(termdf_parser)
+    def do_terminate_df(self, opts):
+        """Terminate the specified DF. DANGEROUS!  See TS 102 222 6.7.
+        This is a permanent, one-way operation on the card. There is no undo, you can not recover
+        a terminated DF.  The only permitted command for a terminated DF is the DLETE FILE command."""
+        if not opts.force:
+            self._cmd.perror("Refusing to terminate the file, please read the help text.")
+            return
+        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        (data, sw) = self._cmd.card._scc.terminate_df(f.fid)
+
+    def complete_terminate_df(self, text, line, begidx, endidx) -> List[str]:
+        """Command Line tab completion for TERMINATE DF"""
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
+        return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+    @cmd2.with_argparser(termdf_parser)
+    def do_terminate_ef(self, opts):
+        """Terminate the specified EF. DANGEROUS!  See TS 102 222 6.8.
+        This is a permanent, one-way operation on the card. There is no undo, you can not recover
+        a terminated EF.  The only permitted command for a terminated EF is the DLETE FILE command."""
+        if not opts.force:
+            self._cmd.perror("Refusing to terminate the file, please read the help text.")
+            return
+        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
+        (data, sw) = self._cmd.card._scc.terminate_ef(f.fid)
+
+    def complete_terminate_ef(self, text, line, begidx, endidx) -> List[str]:
+        """Command Line tab completion for TERMINATE EF"""
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
+        return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
+
+    tcard_parser = argparse.ArgumentParser()
+    tcard_parser.add_argument('--force-terminate-card', action='store_true',
+            help='I really want to permanently terminate the card. It will not be usable afterwards!')
+
+    @cmd2.with_argparser(tcard_parser)
+    def do_terminate_card_usage(self, opts):
+        """Terminate the Card. SUPER DANGEROUS!  See TS 102 222 Section 6.9.
+        This will permanently brick the card and can NOT be recovered from!"""
+        if not opts.force_terminate_card:
+            self._cmd.perror("Refusing to permanently terminate the card, please read the help text.")
+            return
+        (data, sw) = self._cmd.card._scc.terminate_card_usage()
+
+    create_parser = argparse.ArgumentParser()
+    create_parser.add_argument('FILE_ID', type=str, help='File Identifier as 4-character hex string')
+    create_parser._action_groups.pop()
+    create_required = create_parser.add_argument_group('required arguments')
+    create_optional = create_parser.add_argument_group('optional arguments')
+    create_required.add_argument('--ef-arr-file-id', required=True, type=str, help='Referenced Security: File Identifier of EF.ARR')
+    create_required.add_argument('--ef-arr-record-nr', required=True, type=int, help='Referenced Security: Record Number within EF.ARR')
+    create_required.add_argument('--file-size', required=True, type=int, help='Size of file in octets')
+    create_required.add_argument('--structure', required=True, type=str, choices=['transparent', 'linear_fixed', 'ber_tlv'],
+                                 help='Structure of the to-be-created EF')
+    create_optional.add_argument('--short-file-id', type=str, help='Short File Identifier as 2-digit hex string')
+    create_optional.add_argument('--shareable', action='store_true', help='Should the file be shareable?')
+    create_optional.add_argument('--record-length', type=int, help='Length of each record in octets')
+
+    @cmd2.with_argparser(create_parser)
+    def do_create_ef(self, opts):
+        """Create a new EF below the currently selected DF.  Requires related privileges."""
+        file_descriptor = {
+            'file_descriptor_byte': {
+                'shareable': opts.shareable,
+                'file_type': 'working_ef',
+                'structure': opts.structure,
+            }
+        }
+        if opts.structure == 'linear_fixed':
+            if not opts.record_length:
+                self._cmd.perror("you must specify the --record-length for linear fixed EF")
+                return
+            file_descriptor['record_len'] = opts.record_length
+            file_descriptor['num_of_rec'] = opts.file_size // opts.record_length
+            if file_descriptor['num_of_rec'] * file_descriptor['record_len'] != opts.file_size:
+                raise ValueError("File size not evenly divisible by record length")
+        elif opts.structure == 'ber_tlv':
+            self._cmd.perror("BER-TLV creation not yet fully supported, sorry")
+            return
+        ies = [FileDescriptor(decoded=file_descriptor), FileIdentifier(decoded=opts.FILE_ID),
+               LifeCycleStatusInteger(decoded='operational_activated'),
+               SecurityAttribReferenced(decoded={'ef_arr_file_id': opts.ef_arr_file_id,
+                                                 'ef_arr_record_nr': opts.ef_arr_record_nr }),
+               FileSize(decoded=opts.file_size),
+               ShortFileIdentifier(decoded=opts.short_file_id),
+            ]
+        fcp = FcpTemplate(children=ies)
+        (data, sw) = self._cmd.card._scc.create_file(b2h(fcp.to_tlv()))
+        # the newly-created file is automatically selected but our runtime state knows nothing of it
+        self._cmd.lchan.select_file(self._cmd.lchan.selected_file)
+
+    createdf_parser = argparse.ArgumentParser()
+    createdf_parser.add_argument('FILE_ID', type=str, help='File Identifier as 4-character hex string')
+    createdf_parser._action_groups.pop()
+    createdf_required = createdf_parser.add_argument_group('required arguments')
+    createdf_optional = createdf_parser.add_argument_group('optional arguments')
+    createdf_sja_optional = createdf_parser.add_argument_group('sysmoISIM-SJA optional arguments')
+    createdf_required.add_argument('--ef-arr-file-id', required=True, type=str, help='Referenced Security: File Identifier of EF.ARR')
+    createdf_required.add_argument('--ef-arr-record-nr', required=True, type=int, help='Referenced Security: Record Number within EF.ARR')
+    createdf_optional.add_argument('--shareable', action='store_true', help='Should the file be shareable?')
+    createdf_optional.add_argument('--aid', type=str, help='Application ID (creates an ADF, instead of a DF)')
+    # mandatory by spec, but ignored by several OS, so don't force the user
+    createdf_optional.add_argument('--total-file-size', type=int, help='Physical memory allocated for DF/ADi in octets')
+    createdf_sja_optional.add_argument('--permit-rfm-create', action='store_true')
+    createdf_sja_optional.add_argument('--permit-rfm-delete-terminate', action='store_true')
+    createdf_sja_optional.add_argument('--permit-other-applet-create', action='store_true')
+    createdf_sja_optional.add_argument('--permit-other-applet-delete-terminate', action='store_true')
+
+    @cmd2.with_argparser(createdf_parser)
+    def do_create_df(self, opts):
+        """Create a new DF below the currently selected DF.  Requires related privileges."""
+        file_descriptor = {
+            'file_descriptor_byte': {
+                'shareable': opts.shareable,
+                'file_type': 'df',
+                'structure': 'no_info_given',
+            }
+        }
+        ies = []
+        ies.append(FileDescriptor(decoded=file_descriptor))
+        ies.append(FileIdentifier(decoded=opts.FILE_ID))
+        if opts.aid:
+            ies.append(DfName(decoded=opts.aid))
+        ies.append(LifeCycleStatusInteger(decoded='operational_activated'))
+        ies.append(SecurityAttribReferenced(decoded={'ef_arr_file_id': opts.ef_arr_file_id,
+                                                     'ef_arr_record_nr': opts.ef_arr_record_nr }))
+        if opts.total_file_size:
+            ies.append(TotalFileSize(decoded=opts.total_file_size))
+        # TODO: Spec states PIN Status Template DO is mandatory
+        if opts.permit_rfm_create or opts.permit_rfm_delete_terminate or opts.permit_other_applet_create or opts.permit_other_applet_delete_terminate:
+               toolkit_ac = {
+                   'rfm_create': opts.permit_rfm_create,
+                   'rfm_delete_terminate': opts.permit_rfm_delete_terminate,
+                   'other_applet_create': opts.permit_other_applet_create,
+                   'other_applet_delete_terminate': opts.permit_other_applet_delete_terminate,
+                   }
+               ies.append(ProprietaryInformation(children=[ToolkitAccessConditions(decoded=toolkit_ac)]))
+        fcp = FcpTemplate(children=ies)
+        (data, sw) = self._cmd.card._scc.create_file(b2h(fcp.to_tlv()))
+        # the newly-created file is automatically selected but our runtime state knows nothing of it
+        self._cmd.lchan.select_file(self._cmd.lchan.selected_file)

pySim/ts_31_102_telecom.py

@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# without this, pylint will fail when inner classes are used
+# within the 'nested' kwarg of our TlvMeta metaclass on python 3.7 :(
+# pylint: disable=undefined-variable
+
+"""
+DF_PHONEBOOK, DF_MULTIMEDIA, DF_MCS as specified in 3GPP TS 31.102 V16.6.0
+Needs to be a separate python module to avoid cyclic imports
+"""
+
+#
+# Copyright (C) 2022 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from pySim.tlv import *
+from pySim.filesystem import *
+from pySim.construct import *
+from construct import Optional as COptional
+from construct import *
+
+# TS 31.102 Section 4.2.8
+class EF_UServiceTable(TransparentEF):
+    def __init__(self, fid, sfid, name, desc, size, table, **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
+        self.table = table
+
+    @staticmethod
+    def _bit_byte_offset_for_service(service: int) -> Tuple[int, int]:
+        i = service - 1
+        byte_offset = i//8
+        bit_offset = (i % 8)
+        return (byte_offset, bit_offset)
+
+    def _decode_bin(self, in_bin):
+        ret = {}
+        for i in range(0, len(in_bin)):
+            byte = in_bin[i]
+            for bitno in range(0, 8):
+                service_nr = i * 8 + bitno + 1
+                ret[service_nr] = {
+                    'activated': True if byte & (1 << bitno) else False
+                }
+                if service_nr in self.table:
+                    ret[service_nr]['description'] = self.table[service_nr]
+        return ret
+
+    def _encode_bin(self, in_json):
+        # compute the required binary size
+        bin_len = 0
+        for srv in in_json.keys():
+            service_nr = int(srv)
+            (byte_offset, bit_offset) = EF_UServiceTable._bit_byte_offset_for_service(
+                service_nr)
+            if byte_offset >= bin_len:
+                bin_len = byte_offset+1
+        # encode the actual data
+        out = bytearray(b'\x00' * bin_len)
+        for srv in in_json.keys():
+            service_nr = int(srv)
+            (byte_offset, bit_offset) = EF_UServiceTable._bit_byte_offset_for_service(
+                service_nr)
+            if in_json[srv]['activated'] == True:
+                bit = 1
+            else:
+                bit = 0
+            out[byte_offset] |= (bit) << bit_offset
+        return out
+
+    def get_active_services(self, cmd):
+        # obtain list of currently active services
+        (service_data, sw) = cmd.lchan.read_binary_dec()
+        active_services = []
+        for s in service_data.keys():
+            if service_data[s]['activated']:
+                active_services.append(s)
+        return active_services
+
+    def ust_service_check(self, cmd):
+        """Check consistency between services of this file and files present/activated"""
+        num_problems = 0
+        # obtain list of currently active services
+        active_services = self.get_active_services(cmd)
+        # iterate over all the service-constraints we know of
+        files_by_service = self.parent.files_by_service
+        try:
+            for s in sorted(files_by_service.keys()):
+                active_str = 'active' if s in active_services else 'inactive'
+                cmd.poutput("Checking service No %u (%s)" % (s, active_str))
+                for f in files_by_service[s]:
+                    should_exist = f.should_exist_for_services(active_services)
+                    try:
+                        cmd.lchan.select_file(f)
+                        sw = None
+                        exists = True
+                    except SwMatchError as e:
+                        sw = str(e)
+                        exists = False
+                    if exists != should_exist:
+                        num_problems += 1
+                        if exists:
+                            cmd.perror("  ERROR: File %s is selectable but should not!" % f)
+                        else:
+                            cmd.perror("  ERROR: File %s is not selectable (%s) but should!" %  (f, sw))
+        finally:
+            # re-select the EF.UST
+            cmd.lchan.select_file(self)
+        return num_problems
+
+
+
+# TS 31.102 Section 4.4.2.1
+class EF_PBR(LinFixedEF):
+    def __init__(self, fid='4F30', name='EF.PBR', desc='Phone Book Reference', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+        #self._tlv = FIXME
+
+# TS 31.102 Section 4.4.2.12.2
+class EF_PSC(TransparentEF):
+    _construct = Struct('synce_counter'/Int32ub)
+    def __init__(self, fid='4F22', name='EF.PSC', desc='Phone Book Synchronization Counter', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+        #self._tlv = FIXME
+
+# TS 31.102 Section 4.4.2.12.3
+class EF_CC(TransparentEF):
+    _construct = Struct('change_counter'/Int16ub)
+    def __init__(self, fid='4F23', name='EF.CC', desc='Change Counter', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+# TS 31.102 Section 4.4.2.12.4
+class EF_PUID(TransparentEF):
+    _construct = Struct('previous_uid'/Int16ub)
+    def __init__(self, fid='4F24', name='EF.PUID', desc='Previous Unique Identifer', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+# TS 31.102 Section 4.4.2
+class DF_PHONEBOOK(CardDF):
+    def __init__(self, fid='5F3A', name='DF.PHONEBOOK', desc='Phonebook', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_PBR(),
+            EF_PSC(),
+            EF_CC(),
+            EF_PUID(),
+            # FIXME: Those 4Fxx entries with unspecified FID...
+            ]
+        self.add_files(files)
+
+
+
+# TS 31.102 Section 4.6.3.1
+class EF_MML(BerTlvEF):
+    def __init__(self, fid='4F47', name='EF.MML', desc='Multimedia Messages List', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+# TS 31.102 Section 4.6.3.2
+class EF_MMDF(BerTlvEF):
+    def __init__(self, fid='4F48', name='EF.MMDF', desc='Multimedia Messages Data File', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+class DF_MULTIMEDIA(CardDF):
+    def __init__(self, fid='5F3B', name='DF.MULTIMEDIA', desc='Multimedia', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_MML(),
+            EF_MMDF(),
+            ]
+        self.add_files(files)
+
+
+# TS 31.102 Section 4.6.4.1
+EF_MST_map = {
+    1: 'MCPTT UE configuration data',
+    2: 'MCPTT User profile data',
+    3: 'MCS Group configuration data',
+    4: 'MCPTT Service configuration data',
+    5: 'MCS UE initial configuration data',
+    6: 'MCData UE configuration data',
+    7: 'MCData user profile data',
+    8: 'MCData service configuration data',
+    9: 'MCVideo UE configuration data',
+    10: 'MCVideo user profile data',
+    11: 'MCVideo service configuration data',
+    }
+
+# TS 31.102 Section 4.6.4.2
+class EF_MCS_CONFIG(BerTlvEF):
+    class McpttUeConfigurationData(BER_TLV_IE, tag=0x80):
+        pass
+    class McpttUserProfileData(BER_TLV_IE, tag=0x81):
+        pass
+    class McsGroupConfigurationData(BER_TLV_IE, tag=0x82):
+        pass
+    class McpttServiceConfigurationData(BER_TLV_IE, tag=0x83):
+        pass
+    class McsUeInitialConfigurationData(BER_TLV_IE, tag=0x84):
+        pass
+    class McdataUeConfigurationData(BER_TLV_IE, tag=0x85):
+        pass
+    class McdataUserProfileData(BER_TLV_IE, tag=0x86):
+        pass
+    class McdataServiceConfigurationData(BER_TLV_IE, tag=0x87):
+        pass
+    class McvideoUeConfigurationData(BER_TLV_IE, tag=0x88):
+        pass
+    class McvideoUserProfileData(BER_TLV_IE, tag=0x89):
+        pass
+    class McvideoServiceConfigurationData(BER_TLV_IE, tag=0x8a):
+        pass
+    class McsConfigDataCollection(TLV_IE_Collection, nested=[McpttUeConfigurationData,
+                                  McpttUserProfileData, McsGroupConfigurationData,
+                                  McpttServiceConfigurationData, McsUeInitialConfigurationData,
+                                  McdataUeConfigurationData, McdataUserProfileData,
+                                  McdataServiceConfigurationData, McvideoUeConfigurationData,
+                                  McvideoUserProfileData, McvideoServiceConfigurationData]):
+        pass
+    def __init__(self, fid='4F02', sfid=0x02, name='EF.MCS_CONFIG', desc='MCS configuration data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_MCS_CONFIG.McsConfigDataCollection
+
+# TS 31.102 Section 4.6.4.1
+class EF_MST(EF_UServiceTable):
+    def __init__(self, fid='4F01', sfid=0x01, name='EF.MST', desc='MCS Service Table', size=(2,2),
+                 table=EF_MST_map, **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, table=table)
+
+class DF_MCS(CardDF):
+    def __init__(self, fid='5F3D', name='DF.MCS', desc='Mission Critical Services', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_MST(),
+            EF_MCS_CONFIG(),
+            ]
+        self.add_files(files)
+
+
+# TS 31.102 Section 4.6.5.2
+EF_VST_map = {
+    1: 'MCPTT UE configuration data',
+    2: 'MCPTT User profile data',
+    3: 'MCS Group configuration data',
+    4: 'MCPTT Service configuration data',
+    5: 'MCS UE initial configuration data',
+    6: 'MCData UE configuration data',
+    7: 'MCData user profile data',
+    8: 'MCData service configuration data',
+    9: 'MCVideo UE configuration data',
+    10: 'MCVideo user profile data',
+    11: 'MCVideo service configuration data',
+    }
+
+# TS 31.102 Section 4.6.5.2
+class EF_VST(EF_UServiceTable):
+    def __init__(self, fid='4F01', sfid=0x01, name='EF.VST', desc='V2X Service Table', size=(2,2),
+                 table=EF_VST_map, **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, table=table)
+
+# TS 31.102 Section 4.6.5.3
+class EF_V2X_CONFIG(BerTlvEF):
+    class V2xConfigurationData(BER_TLV_IE, tag=0x80):
+        pass
+    class V2xConfigDataCollection(TLV_IE_Collection, nested=[V2xConfigurationData]):
+        pass
+    def __init__(self, fid='4F02', sfid=0x02, name='EF.V2X_CONFIG', desc='V2X configuration data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_V2X_CONFIG.V2xConfigDataCollection
+
+# TS 31.102 Section 4.6.5
+class DF_V2X(CardDF):
+    def __init__(self, fid='5F3E', name='DF.V2X', desc='Vehicle to X', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_VST(),
+            EF_V2X_CONFIG(),
+            ]
+        self.add_files(files)

pySim/ts_31_103.py

@@ -26,7 +26,8 @@ from pySim.filesystem import *
 from pySim.utils import *
 from pySim.tlv import *
 from pySim.ts_51_011 import EF_AD, EF_SMS, EF_SMSS, EF_SMSR, EF_SMSP
-from pySim.ts_31_102 import ADF_USIM, EF_FromPreferred, EF_UServiceTable
+from pySim.ts_31_102 import ADF_USIM, EF_FromPreferred
+from pySim.ts_31_102_telecom import EF_UServiceTable
 import pySim.ts_102_221
 from pySim.ts_102_221 import EF_ARR
 
@@ -78,44 +79,68 @@ EF_ISIM_ADF_map = {
 }
 
 # TS 31.103 Section 4.2.2
-
-
 class EF_IMPI(TransparentEF):
     class nai(BER_TLV_IE, tag=0x80):
         _construct = GreedyString("utf8")
 
-    def __init__(self, fid='6f02', sfid=0x02, name='EF.IMPI', desc='IMS private user identity'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6f02', sfid=0x02, name='EF.IMPI', desc='IMS private user identity', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_IMPI.nai
 
 # TS 31.103 Section 4.2.3
-
-
 class EF_DOMAIN(TransparentEF):
     class domain(BER_TLV_IE, tag=0x80):
         _construct = GreedyString("utf8")
 
-    def __init__(self, fid='6f05', sfid=0x05, name='EF.DOMAIN', desc='Home Network Domain Name'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6f03', sfid=0x05, name='EF.DOMAIN', desc='Home Network Domain Name', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_DOMAIN.domain
 
 # TS 31.103 Section 4.2.4
-
-
 class EF_IMPU(LinFixedEF):
     class impu(BER_TLV_IE, tag=0x80):
         _construct = GreedyString("utf8")
 
-    def __init__(self, fid='6f04', sfid=0x04, name='EF.IMPU', desc='IMS public user identity'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6f04', sfid=0x04, name='EF.IMPU', desc='IMS public user identity', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_IMPU.impu
 
+# TS 31.103 Section 4.2.7
+class EF_IST(EF_UServiceTable):
+    def __init__(self, **kwargs):
+        super().__init__('6f07', 0x07, 'EF.IST', 'ISIM Service Table', (1, None), EF_IST_map)
+        # add those commands to the general commands of a TransparentEF
+        self.shell_commands += [self.AddlShellCommands()]
+
+    @with_default_category('File-Specific Commands')
+    class AddlShellCommands(CommandSet):
+        def __init__(self):
+            super().__init__()
+
+        def do_ist_service_activate(self, arg):
+            """Activate a service within EF.IST"""
+            self._cmd.card.update_ist(int(arg), 1)
+
+        def do_ist_service_deactivate(self, arg):
+            """Deactivate a service within EF.IST"""
+            self._cmd.card.update_ist(int(arg), 0)
+
+        def do_ist_service_check(self, arg):
+            """Check consistency between services of this file and files present/activated.
+
+            Many services determine if one or multiple files shall be present/activated or if they shall be
+            absent/deactivated.  This performs a consistency check to ensure that no services are activated
+            for files that are not - and vice-versa, no files are activated for services that are not.  Error
+            messages are printed for every inconsistency found."""
+            selected_file = self._cmd.lchan.selected_file
+            num_problems = selected_file.ust_service_check(self._cmd)
+            self._cmd.poutput("===> %u service / file inconsistencies detected" % num_problems)
+
+
 # TS 31.103 Section 4.2.8
-
-
 class EF_PCSCF(LinFixedEF):
-    def __init__(self, fid='6f09', sfid=None, name='EF.P-CSCF', desc='P-CSCF Address'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6f09', sfid=None, name='EF.P-CSCF', desc='P-CSCF Address', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
 
     def _decode_record_hex(self, raw_hex):
         addr, addr_type = dec_addr_tlv(raw_hex)
@@ -127,69 +152,109 @@ class EF_PCSCF(LinFixedEF):
         return enc_addr_tlv(addr, addr_type)
 
 # TS 31.103 Section 4.2.9
-
-
 class EF_GBABP(TransparentEF):
-    def __init__(self, fid='6fd5', sfid=None, name='EF.GBABP', desc='GBA Bootstrapping'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6fd5', sfid=None, name='EF.GBABP', desc='GBA Bootstrapping', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
 
 # TS 31.103 Section 4.2.10
-
-
 class EF_GBANL(LinFixedEF):
-    def __init__(self, fid='6fd7', sfid=None, name='EF.GBANL', desc='GBA NAF List'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6fd7', sfid=None, name='EF.GBANL', desc='GBA NAF List', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
 
 # TS 31.103 Section 4.2.11
-
-
 class EF_NAFKCA(LinFixedEF):
-    def __init__(self, fid='6fdd', sfid=None, name='EF.NAFKCA', desc='NAF Key Centre Address'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6fdd', sfid=None, name='EF.NAFKCA', desc='NAF Key Centre Address', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
 
 # TS 31.103 Section 4.2.16
-
-
 class EF_UICCIARI(LinFixedEF):
     class iari(BER_TLV_IE, tag=0x80):
         _construct = GreedyString("utf8")
 
-    def __init__(self, fid='6fe7', sfid=None, name='EF.UICCIARI', desc='UICC IARI'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6fe7', sfid=None, name='EF.UICCIARI', desc='UICC IARI', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_UICCIARI.iari
 
 # TS 31.103 Section 4.2.18
-
-
 class EF_IMSConfigData(BerTlvEF):
-    def __init__(self, fid='6ff8', sfid=None, name='EF.IMSConfigData', desc='IMS Configuration Data'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    class ImsConfigDataEncoding(BER_TLV_IE, tag=0x80):
+        _construct = HexAdapter(Bytes(1))
+    class ImsConfigData(BER_TLV_IE, tag=0x81):
+        _construct = GreedyString
+    # pylint: disable=undefined-variable
+    class ImsConfigDataCollection(TLV_IE_Collection, nested=[ImsConfigDataEncoding, ImsConfigData]):
+        pass
+    def __init__(self, fid='6ff8', sfid=None, name='EF.IMSConfigData', desc='IMS Configuration Data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_IMSConfigData.ImsConfigDataCollection
 
 # TS 31.103 Section 4.2.19
-
-
 class EF_XCAPConfigData(BerTlvEF):
-    def __init__(self, fid='6ffc', sfid=None, name='EF.XCAPConfigData', desc='XCAP Configuration Data'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    class Access(BER_TLV_IE, tag=0x81):
+        pass
+    class ApplicationName(BER_TLV_IE, tag=0x82):
+        pass
+    class ProviderID(BER_TLV_IE, tag=0x83):
+        pass
+    class URI(BER_TLV_IE, tag=0x84):
+        pass
+    class XcapAuthenticationUserName(BER_TLV_IE, tag=0x85):
+        pass
+    class XcapAuthenticationPassword(BER_TLV_IE, tag=0x86):
+        pass
+    class XcapAuthenticationType(BER_TLV_IE, tag=0x87):
+        pass
+    class AddressType(BER_TLV_IE, tag=0x88):
+        pass
+    class Address(BER_TLV_IE, tag=0x89):
+        pass
+    class PDPAuthenticationType(BER_TLV_IE, tag=0x8a):
+        pass
+    class PDPAuthenticationName(BER_TLV_IE, tag=0x8b):
+        pass
+    class PDPAuthenticationSecret(BER_TLV_IE, tag=0x8c):
+        pass
+
+    class AccessForXCAP(BER_TLV_IE, tag=0x81):
+        pass
+    class NumberOfXcapConnParPolicy(BER_TLV_IE, tag=0x82):
+        _construct = Int8ub
+    # pylint: disable=undefined-variable
+    class XcapConnParamsPolicyPart(BER_TLV_IE, tag=0xa1, nested=[Access, ApplicationName, ProviderID, URI,
+                                 XcapAuthenticationUserName, XcapAuthenticationPassword,
+                                 XcapAuthenticationType, AddressType, Address, PDPAuthenticationType,
+                                 PDPAuthenticationName, PDPAuthenticationSecret]):
+        pass
+    class XcapConnParamsPolicy(BER_TLV_IE, tag=0xa0, nested=[AccessForXCAP, NumberOfXcapConnParPolicy, XcapConnParamsPolicyPart]):
+        pass
+    class XcapConnParamsPolicyDO(BER_TLV_IE, tag=0x80, nested=[XcapConnParamsPolicy]):
+        pass
+    def __init__(self, fid='6ffc', sfid=None, name='EF.XCAPConfigData', desc='XCAP Configuration Data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_XCAPConfigData.XcapConnParamsPolicy
 
 # TS 31.103 Section 4.2.20
-
-
 class EF_WebRTCURI(TransparentEF):
     class uri(BER_TLV_IE, tag=0x80):
         _construct = GreedyString("utf8")
 
-    def __init__(self, fid='6ffa', sfid=None, name='EF.WebRTCURI', desc='WebRTC URI'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6ffa', sfid=None, name='EF.WebRTCURI', desc='WebRTC URI', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_WebRTCURI.uri
 
 # TS 31.103 Section 4.2.21
-
-
 class EF_MuDMiDConfigData(BerTlvEF):
+    class MudMidConfigDataEncoding(BER_TLV_IE, tag=0x80):
+        _construct = HexAdapter(Bytes(1))
+    class MudMidConfigData(BER_TLV_IE, tag=0x81):
+        _construct = GreedyString
+    # pylint: disable=undefined-variable
+    class MudMidConfigDataCollection(TLV_IE_Collection, nested=[MudMidConfigDataEncoding, MudMidConfigData]):
+        pass
     def __init__(self, fid='6ffe', sfid=None, name='EF.MuDMiDConfigData',
-                 desc='MuD and MiD Configuration Data'):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc)
+                 desc='MuD and MiD Configuration Data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_MuDMiDConfigData.MudMidConfigDataCollection
 
 
 class ADF_ISIM(CardADF):
@@ -203,22 +268,21 @@ class ADF_ISIM(CardADF):
             EF_IMPU(),
             EF_AD(),
             EF_ARR('6f06', 0x06),
-            EF_UServiceTable('6f07', 0x07, 'EF.IST',
-                             'ISIM Service Table', {1, None}, EF_IST_map),
-            EF_PCSCF(),
-            EF_GBABP(),
-            EF_GBANL(),
-            EF_NAFKCA(),
-            EF_SMS(),
-            EF_SMSS(),
-            EF_SMSR(),
-            EF_SMSP(),
-            EF_UICCIARI(),
-            EF_FromPreferred(),
-            EF_IMSConfigData(),
-            EF_XCAPConfigData(),
-            EF_WebRTCURI(),
-            EF_MuDMiDConfigData(),
+            EF_IST(),
+            EF_PCSCF(service=5),
+            EF_GBABP(service=2),
+            EF_GBANL(service=2),
+            EF_NAFKCA(service=2),
+            EF_SMS(service=(6,8)),
+            EF_SMSS(service=(6,8)),
+            EF_SMSR(service=(7,8)),
+            EF_SMSP(service=8),
+            EF_UICCIARI(service=10),
+            EF_FromPreferred(service=17),
+            EF_IMSConfigData(service=18),
+            EF_XCAPConfigData(service=19),
+            EF_WebRTCURI(service=20),
+            EF_MuDMiDConfigData(service=21),
         ]
         self.add_files(files)
         # add those commands to the general commands of a TransparentEF

pySim/ts_51_011.py

@@ -32,6 +32,7 @@ order to describe the files specified in the relevant ETSI + 3GPP specifications
 from pySim.profile import match_sim
 from pySim.profile import CardProfile
 from pySim.filesystem import *
+from pySim.ts_31_102_telecom import DF_PHONEBOOK, DF_MULTIMEDIA, DF_MCS, DF_V2X
 import enum
 from pySim.construct import *
 from construct import Optional as COptional
@@ -340,26 +341,20 @@ EF_SST_map = {
 ######################################################################
 
 # TS 51.011 Section 10.5.1
-
 class EF_ADN(LinFixedEF):
-    def __init__(self, fid='6f3a', sfid=None, name='EF.ADN', desc='Abbreviated Dialing Numbers'):
-        super().__init__(fid, sfid=sfid, name=name,
-                         desc=desc, rec_len={14, 30})
-
-    def _decode_record_bin(self, raw_bin_data):
-        alpha_id_len = len(raw_bin_data) - 14
-        alpha_id = raw_bin_data[:alpha_id_len]
-        u = unpack('!BB10sBB', raw_bin_data[-14:])
-        return {'alpha_id': alpha_id, 'len_of_bcd': u[0], 'ton_npi': u[1],
-                'dialing_nr': u[2], 'cap_conf_id': u[3], 'ext1_record_id': u[4]}
+    def __init__(self, fid='6f3a', sfid=None, name='EF.ADN', desc='Abbreviated Dialing Numbers', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(14, 30), **kwargs)
+        self._construct = Struct('alpha_id'/COptional(GsmStringAdapter(Rpad(Bytes(this._.total_len-14)), codec='ascii')),
+                                 'len_of_bcd'/Int8ub,
+                                 'ton_npi'/TonNpi,
+                                 'dialing_nr'/BcdAdapter(Rpad(Bytes(10))),
+                                 'cap_conf_id'/Int8ub,
+                                 'ext1_record_id'/Int8ub)
 
 # TS 51.011 Section 10.5.5
-
-
 class EF_SMS(LinFixedEF):
-    def __init__(self, fid='6f3c', sfid=None, name='EF.SMS', desc='Short messages'):
-        super().__init__(fid, sfid=sfid, name=name,
-                         desc=desc, rec_len={176, 176})
+    def __init__(self, fid='6f3c', sfid=None, name='EF.SMS', desc='Short messages', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(176, 176), **kwargs)
 
     def _decode_record_bin(self, raw_bin_data):
         def decode_status(status):
@@ -389,9 +384,8 @@ class EF_SMS(LinFixedEF):
 
 # TS 51.011 Section 10.5.5
 class EF_MSISDN(LinFixedEF):
-    def __init__(self, fid='6f40', sfid=None, name='EF.MSISDN', desc='MSISDN'):
-        super().__init__(fid, sfid=sfid, name=name,
-                         desc=desc, rec_len={15, 34})
+    def __init__(self, fid='6f40', sfid=None, name='EF.MSISDN', desc='MSISDN', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(15, 34), **kwargs)
 
     def _decode_record_hex(self, raw_hex_data):
         return {'msisdn': dec_msisdn(raw_hex_data)}
@@ -407,16 +401,45 @@ class EF_MSISDN(LinFixedEF):
         return alpha_identifier + encoded_msisdn
 
 # TS 51.011 Section 10.5.6
-
-
 class EF_SMSP(LinFixedEF):
-    def __init__(self, fid='6f42', sfid=None, name='EF.SMSP', desc='Short message service parameters'):
-        super().__init__(fid, sfid=sfid, name=name,
-                         desc=desc, rec_len={28, None})
+    class ValidityPeriodAdapter(Adapter):
+        def _decode(self, obj, context, path):
+            if obj <= 143:
+                return obj + 1 * 5
+            elif obj <= 167:
+                return 12 * 60 + ((obj - 143) * 30)
+            elif obj <= 196:
+                return (obj - 166) * (24 * 60)
+            elif obj <= 255:
+                return (obj - 192) * (7 * 24 * 60)
+            else:
+                raise ValueError
+        def _encode(self, obj, context, path):
+            if obj <= 12*60:
+                return obj/5 - 1
+            elif obj <= 24*60:
+                return 143 + ((obj - (12 * 60)) / 30)
+            elif obj <= 30 * 24 * 60:
+                return 166 + (obj / (24 * 60))
+            elif obj <= 63 * 7 * 24 * 60:
+                return 192 + (obj / (7 * 24 * 60))
+            else:
+                raise ValueError
+
+    def __init__(self, fid='6f42', sfid=None, name='EF.SMSP', desc='Short message service parameters', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(28, None), **kwargs)
+        ScAddr = Struct('length'/Int8ub, 'ton_npi'/TonNpi, 'call_number'/BcdAdapter(Rpad(Bytes(10))))
+        self._construct = Struct('alpha_id'/COptional(GsmStringAdapter(Rpad(Bytes(this._.total_len-28)))),
+                                 'parameter_indicators'/InvertAdapter(FlagsEnum(Byte, tp_dest_addr=1, tp_sc_addr=2,
+                                                                                tp_pid=3, tp_dcs=4, tp_vp=5)),
+                                 'tp_dest_addr'/ScAddr,
+                                 'tp_sc_addr'/ScAddr,
+
+                                 'tp_pid'/HexAdapter(Bytes(1)),
+                                 'tp_dcs'/HexAdapter(Bytes(1)),
+                                 'tp_vp_minutes'/EF_SMSP.ValidityPeriodAdapter(Byte))
 
 # TS 51.011 Section 10.5.7
-
-
 class EF_SMSS(TransparentEF):
     class MemCapAdapter(Adapter):
         def _decode(self, obj, context, path):
@@ -425,49 +448,45 @@ class EF_SMSS(TransparentEF):
         def _encode(self, obj, context, path):
             return 0 if obj else 1
 
-    def __init__(self, fid='6f43', sfid=None, name='EF.SMSS', desc='SMS status', size={2, 8}):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6f43', sfid=None, name='EF.SMSS', desc='SMS status', size=(2, 8), **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct(
             'last_used_tpmr'/Int8ub, 'memory_capacity_exceeded'/self.MemCapAdapter(Int8ub))
 
 # TS 51.011 Section 10.5.8
-
-
 class EF_SMSR(LinFixedEF):
-    def __init__(self, fid='6f47', sfid=None, name='EF.SMSR', desc='SMS status reports', rec_len={30, 30}):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len)
+    def __init__(self, fid='6f47', sfid=None, name='EF.SMSR', desc='SMS status reports', rec_len=(30, 30), **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
             'sms_record_id'/Int8ub, 'sms_status_report'/HexAdapter(Bytes(29)))
 
 
 class EF_EXT(LinFixedEF):
-    def __init__(self, fid, sfid=None, name='EF.EXT', desc='Extension', rec_len={13, 13}):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len)
+    def __init__(self, fid, sfid=None, name='EF.EXT', desc='Extension', rec_len=(13, 13), **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
             'record_type'/Int8ub, 'extension_data'/HexAdapter(Bytes(11)), 'identifier'/Int8ub)
 
 # TS 51.011 Section 10.5.16
-
-
 class EF_CMI(LinFixedEF):
-    def __init__(self, fid='6f58', sfid=None, name='EF.CMI', rec_len={2, 21},
-                 desc='Comparison Method Information'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len)
+    def __init__(self, fid='6f58', sfid=None, name='EF.CMI', rec_len=(2, 21),
+                 desc='Comparison Method Information', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
-            'alpha_id'/Bytes(this._.total_len-1), 'comparison_method_id'/Int8ub)
+            'alpha_id'/GsmStringAdapter(Rpad(Bytes(this._.total_len-1))), 'comparison_method_id'/Int8ub)
 
 
 class DF_TELECOM(CardDF):
-    def __init__(self, fid='7f10', name='DF.TELECOM', desc=None):
-        super().__init__(fid=fid, name=name, desc=desc)
+    def __init__(self, fid='7f10', name='DF.TELECOM', desc=None, **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
         files = [
             EF_ADN(),
             EF_ADN(fid='6f3b', name='EF.FDN', desc='Fixed dialling numbers'),
             EF_SMS(),
             LinFixedEF(fid='6f3d', name='EF.CCP',
-                       desc='Capability Configuration Parameters', rec_len={14, 14}),
+                       desc='Capability Configuration Parameters', rec_len=(14, 14)),
             LinFixedEF(fid='6f4f', name='EF.ECCP',
-                       desc='Extended Capability Configuration Parameters', rec_len={15, 32}),
+                       desc='Extended Capability Configuration Parameters', rec_len=(15, 32)),
             EF_MSISDN(),
             EF_SMSP(),
             EF_SMSS(),
@@ -479,6 +498,11 @@ class DF_TELECOM(CardDF):
             EF_EXT('6f4e', None, 'EF.EXT4', 'Extension4 (BDN/SSC)'),
             EF_SMSR(),
             EF_CMI(),
+            # not really part of 51.011 but something that TS 31.102 specifies may exist here.
+            DF_PHONEBOOK(),
+            DF_MULTIMEDIA(),
+            DF_MCS(),
+            DF_V2X(),
         ]
         self.add_files(files)
 
@@ -487,10 +511,8 @@ class DF_TELECOM(CardDF):
 ######################################################################
 
 # TS 51.011 Section 10.3.1
-
-
 class EF_LP(TransRecEF):
-    def __init__(self, fid='6f05', sfid=None, name='EF.LP', size={1, None}, rec_len=1,
+    def __init__(self, fid='6f05', sfid=None, name='EF.LP', size=(1, None), rec_len=1,
                  desc='Language Preference'):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
 
@@ -501,10 +523,8 @@ class EF_LP(TransRecEF):
         return h2b(in_json)
 
 # TS 51.011 Section 10.3.2
-
-
 class EF_IMSI(TransparentEF):
-    def __init__(self, fid='6f07', sfid=None, name='EF.IMSI', desc='IMSI', size={9, 9}):
+    def __init__(self, fid='6f07', sfid=None, name='EF.IMSI', desc='IMSI', size=(9, 9)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         # add those commands to the general commands of a TransparentEF
         self.shell_commands += [self.AddlShellCommands(self)]
@@ -525,15 +545,15 @@ class EF_IMSI(TransparentEF):
             """Change the plmn part of the IMSI"""
             plmn = arg.strip()
             if len(plmn) == 5 or len(plmn) == 6:
-                (data, sw) = self._cmd.rs.read_binary_dec()
+                (data, sw) = self._cmd.lchan.read_binary_dec()
                 if sw == '9000' and len(data['imsi'])-len(plmn) == 10:
                     imsi = data['imsi']
                     msin = imsi[len(plmn):]
-                    (data, sw) = self._cmd.rs.update_binary_dec(
+                    (data, sw) = self._cmd.lchan.update_binary_dec(
                         {'imsi': plmn+msin})
                     if sw == '9000' and data:
                         self._cmd.poutput_json(
-                            self._cmd.rs.selected_file.decode_hex(data))
+                            self._cmd.lchan.selected_file.decode_hex(data))
                 else:
                     raise ValueError("PLMN length does not match IMSI length")
             else:
@@ -543,8 +563,8 @@ class EF_IMSI(TransparentEF):
 # TS 51.011 Section 10.3.4
 class EF_PLMNsel(TransRecEF):
     def __init__(self, fid='6f30', sfid=None, name='EF.PLMNsel', desc='PLMN selector',
-                 size={24, None}, rec_len=3):
-        super().__init__(fid, name=name, sfid=sfid, desc=desc, size=size, rec_len=rec_len)
+                 size=(24, None), rec_len=3, **kwargs):
+        super().__init__(fid, name=name, sfid=sfid, desc=desc, size=size, rec_len=rec_len, **kwargs)
 
     def _decode_record_hex(self, in_hex):
         if in_hex[:6] == "ffffff":
@@ -559,17 +579,13 @@ class EF_PLMNsel(TransRecEF):
             return enc_plmn(in_json['mcc'], in_json['mnc'])
 
 # TS 51.011 Section 10.3.6
-
-
 class EF_ACMmax(TransparentEF):
-    def __init__(self, fid='6f37', sfid=None, name='EF.ACMmax', size={3, 3},
-                 desc='ACM maximum value'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6f37', sfid=None, name='EF.ACMmax', size=(3, 3),
+                 desc='ACM maximum value', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('acm_max'/Int24ub)
 
 # TS 51.011 Section 10.3.7
-
-
 class EF_ServiceTable(TransparentEF):
     def __init__(self, fid, sfid, name, desc, size, table):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
@@ -620,11 +636,10 @@ class EF_ServiceTable(TransparentEF):
         return out
 
 # TS 51.011 Section 10.3.11
-
-
 class EF_SPN(TransparentEF):
-    def __init__(self, fid='6f46', sfid=None, name='EF.SPN', desc='Service Provider Name', size={17, 17}):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6f46', sfid=None, name='EF.SPN',
+                 desc='Service Provider Name', size=(17, 17), **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = BitStruct(
             # Byte 1
             'rfu'/BitsRFU(6),
@@ -635,40 +650,28 @@ class EF_SPN(TransparentEF):
         )
 
 # TS 51.011 Section 10.3.13
-
-
 class EF_CBMI(TransRecEF):
-    def __init__(self, fid='6f45', sfid=None, name='EF.CBMI', size={2, None}, rec_len=2,
-                 desc='Cell Broadcast message identifier selection'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
+    def __init__(self, fid='6f45', sfid=None, name='EF.CBMI', size=(2, None), rec_len=2,
+                 desc='Cell Broadcast message identifier selection', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = GreedyRange(Int16ub)
 
 # TS 51.011 Section 10.3.15
-
-
 class EF_ACC(TransparentEF):
-    def __init__(self, fid='6f78', sfid=None, name='EF.ACC', desc='Access Control Class', size={2, 2}):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-
-    def _decode_bin(self, raw_bin):
-        return {'acc': unpack('!H', raw_bin)[0]}
-
-    def _encode_bin(self, abstract):
-        return pack('!H', abstract['acc'])
+    def __init__(self, fid='6f78', sfid=None, name='EF.ACC',
+                 desc='Access Control Class', size=(2, 2), **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
+        self._construct = HexAdapter(Bytes(2))
 
 # TS 51.011 Section 10.3.16
-
-
 class EF_LOCI(TransparentEF):
-    def __init__(self, fid='6f7e', sfid=None, name='EF.LOCI', desc='Location Information', size={11, 11}):
+    def __init__(self, fid='6f7e', sfid=None, name='EF.LOCI', desc='Location Information', size=(11, 11)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-        self._construct = Struct('tmsi'/Bytes(4), 'lai'/Bytes(5), 'tmsi_time'/Int8ub,
+        self._construct = Struct('tmsi'/HexAdapter(Bytes(4)), 'lai'/HexAdapter(Bytes(5)), 'tmsi_time'/Int8ub,
                                  'lu_status'/Enum(Byte, updated=0, not_updated=1, plmn_not_allowed=2,
                                                   location_area_not_allowed=3))
 
 # TS 51.011 Section 10.3.18
-
-
 class EF_AD(TransparentEF):
     class OP_MODE(enum.IntEnum):
         normal = 0x00
@@ -680,7 +683,7 @@ class EF_AD(TransparentEF):
     #OP_MODE_DICT = {int(v) : str(v) for v in EF_AD.OP_MODE}
     #OP_MODE_DICT_REVERSED = {str(v) : int(v) for v in EF_AD.OP_MODE}
 
-    def __init__(self, fid='6fad', sfid=None, name='EF.AD', desc='Administrative Data', size={3, 4}):
+    def __init__(self, fid='6fad', sfid=None, name='EF.AD', desc='Administrative Data', size=(3, 4)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         self._construct = BitStruct(
             # Byte 1
@@ -700,92 +703,74 @@ class EF_AD(TransparentEF):
         )
 
 # TS 51.011 Section 10.3.20 / 10.3.22
-
-
 class EF_VGCS(TransRecEF):
-    def __init__(self, fid='6fb1', sfid=None, name='EF.VGCS', size={4, 200}, rec_len=4,
-                 desc='Voice Group Call Service'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
+    def __init__(self, fid='6fb1', sfid=None, name='EF.VGCS', size=(4, 200), rec_len=4,
+                 desc='Voice Group Call Service', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = BcdAdapter(Bytes(4))
 
 # TS 51.011 Section 10.3.21 / 10.3.23
-
-
 class EF_VGCSS(TransparentEF):
-    def __init__(self, fid='6fb2', sfid=None, name='EF.VGCSS', size={7, 7},
-                 desc='Voice Group Call Service Status'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6fb2', sfid=None, name='EF.VGCSS', size=(7, 7),
+                 desc='Voice Group Call Service Status', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = BitStruct(
             'flags'/Bit[50], Padding(6, pattern=b'\xff'))
 
 # TS 51.011 Section 10.3.24
-
-
 class EF_eMLPP(TransparentEF):
-    def __init__(self, fid='6fb5', sfid=None, name='EF.eMLPP', size={2, 2},
-                 desc='enhanced Multi Level Pre-emption and Priority'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6fb5', sfid=None, name='EF.eMLPP', size=(2, 2),
+                 desc='enhanced Multi Level Pre-emption and Priority', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         FlagsConstruct = FlagsEnum(
             Byte, A=1, B=2, zero=4, one=8, two=16, three=32, four=64)
         self._construct = Struct(
             'levels'/FlagsConstruct, 'fast_call_setup_cond'/FlagsConstruct)
 
 # TS 51.011 Section 10.3.25
-
-
 class EF_AAeM(TransparentEF):
-    def __init__(self, fid='6fb6', sfid=None, name='EF.AAeM', size={1, 1},
-                 desc='Automatic Answer for eMLPP Service'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6fb6', sfid=None, name='EF.AAeM', size=(1, 1),
+                 desc='Automatic Answer for eMLPP Service', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         FlagsConstruct = FlagsEnum(
             Byte, A=1, B=2, zero=4, one=8, two=16, three=32, four=64)
         self._construct = Struct('auto_answer_prio_levels'/FlagsConstruct)
 
 # TS 51.011 Section 10.3.26
-
-
 class EF_CBMID(EF_CBMI):
-    def __init__(self, fid='6f48', sfid=None, name='EF.CBMID', size={2, None}, rec_len=2,
-                 desc='Cell Broadcast Message Identifier for Data Download'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
+    def __init__(self, fid='6f48', sfid=None, name='EF.CBMID', size=(2, None), rec_len=2,
+                 desc='Cell Broadcast Message Identifier for Data Download', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = GreedyRange(Int16ub)
 
 # TS 51.011 Section 10.3.27
-
-
 class EF_ECC(TransRecEF):
-    def __init__(self, fid='6fb7', sfid=None, name='EF.ECC', size={3, 15}, rec_len=3,
-                 desc='Emergency Call Codes'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
+    def __init__(self, fid='6fb7', sfid=None, name='EF.ECC', size=(3, 15), rec_len=3,
+                 desc='Emergency Call Codes', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = GreedyRange(BcdAdapter(Bytes(3)))
 
 # TS 51.011 Section 10.3.28
-
-
 class EF_CBMIR(TransRecEF):
-    def __init__(self, fid='6f50', sfid=None, name='EF.CBMIR', size={4, None}, rec_len=4,
-                 desc='Cell Broadcast message identifier range selection'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
+    def __init__(self, fid='6f50', sfid=None, name='EF.CBMIR', size=(4, None), rec_len=4,
+                 desc='Cell Broadcast message identifier range selection', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = GreedyRange(Struct('lower'/Int16ub, 'upper'/Int16ub))
 
 # TS 51.011 Section 10.3.29
-
-
 class EF_DCK(TransparentEF):
-    def __init__(self, fid='6f2c', sfid=None, name='EF.DCK', size={16, 16},
-                 desc='Depersonalisation Control Keys'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6f2c', sfid=None, name='EF.DCK', size=(16, 16),
+                 desc='Depersonalisation Control Keys', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('network'/BcdAdapter(Bytes(4)),
                                  'network_subset'/BcdAdapter(Bytes(4)),
                                  'service_provider'/BcdAdapter(Bytes(4)),
                                  'corporate'/BcdAdapter(Bytes(4)))
 # TS 51.011 Section 10.3.30
-
-
 class EF_CNL(TransRecEF):
-    def __init__(self, fid='6f32', sfid=None, name='EF.CNL', size={6, None}, rec_len=6,
-                 desc='Co-operative Network List'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
+    def __init__(self, fid='6f32', sfid=None, name='EF.CNL', size=(6, None), rec_len=6,
+                 desc='Co-operative Network List', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
 
     def _decode_record_hex(self, in_hex):
         (in_plmn, sub, svp, corp) = unpack('!3sBBB', h2b(in_hex))
@@ -804,39 +789,31 @@ class EF_CNL(TransRecEF):
                         in_json['corporate_id']))
 
 # TS 51.011 Section 10.3.31
-
-
 class EF_NIA(LinFixedEF):
-    def __init__(self, fid='6f51', sfid=None, name='EF.NIA', rec_len={1, 32},
-                 desc='Network\'s Indication of Alerting'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len)
+    def __init__(self, fid='6f51', sfid=None, name='EF.NIA', rec_len=(1, 32),
+                 desc='Network\'s Indication of Alerting', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
             'alerting_category'/Int8ub, 'category'/GreedyBytes)
 
 # TS 51.011 Section 10.3.32
-
-
 class EF_Kc(TransparentEF):
-    def __init__(self, fid='6f20', sfid=None, name='EF.Kc', desc='Ciphering key Kc', size={9, 9}):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6f20', sfid=None, name='EF.Kc', desc='Ciphering key Kc', size=(9, 9), **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('kc'/HexAdapter(Bytes(8)), 'cksn'/Int8ub)
 
 # TS 51.011 Section 10.3.33
-
-
 class EF_LOCIGPRS(TransparentEF):
-    def __init__(self, fid='6f53', sfid=None, name='EF.LOCIGPRS', desc='GPRS Location Information', size={14, 14}):
+    def __init__(self, fid='6f53', sfid=None, name='EF.LOCIGPRS', desc='GPRS Location Information', size=(14, 14)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-        self._construct = Struct('ptmsi'/Bytes(4), 'ptmsi_sig'/Int8ub, 'rai'/Bytes(6),
+        self._construct = Struct('ptmsi'/HexAdapter(Bytes(4)), 'ptmsi_sig'/Int8ub, 'rai'/HexAdapter(Bytes(6)),
                                  'rau_status'/Enum(Byte, updated=0, not_updated=1, plmn_not_allowed=2,
                                                    routing_area_not_allowed=3))
 
 # TS 51.011 Section 10.3.35..37
-
-
 class EF_xPLMNwAcT(TransRecEF):
-    def __init__(self, fid, sfid=None, name=None, desc=None, size={40, None}, rec_len=5):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
+    def __init__(self, fid, sfid=None, name=None, desc=None, size=(40, None), rec_len=5, **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
 
     def _decode_record_hex(self, in_hex):
         if in_hex[:6] == "ffffff":
@@ -883,75 +860,64 @@ class EF_xPLMNwAcT(TransRecEF):
         return '%04X' % (u16)
 
 # TS 51.011 Section 10.3.38
-
-
 class EF_CPBCCH(TransRecEF):
-    def __init__(self, fid='6f63', sfid=None, name='EF.CPBCCH', size={2, 14}, rec_len=2,
-                 desc='CPBCCH Information'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
+    def __init__(self, fid='6f63', sfid=None, name='EF.CPBCCH', size=(2, 14), rec_len=2,
+                 desc='CPBCCH Information', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = Struct('cpbcch'/Int16ub)
 
 # TS 51.011 Section 10.3.39
-
-
 class EF_InvScan(TransparentEF):
-    def __init__(self, fid='6f64', sfid=None, name='EF.InvScan', size={1, 1},
-                 desc='IOnvestigation Scan'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6f64', sfid=None, name='EF.InvScan', size=(1, 1),
+                 desc='IOnvestigation Scan', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = FlagsEnum(
             Byte, in_limited_service_mode=1, after_successful_plmn_selection=2)
 
 # TS 51.011 Section 4.2.58
-
-
 class EF_PNN(LinFixedEF):
     class FullNameForNetwork(BER_TLV_IE, tag=0x43):
         # TS 24.008 10.5.3.5a
-        pass
+        # TODO: proper decode
+        _construct = HexAdapter(GreedyBytes)
 
     class ShortNameForNetwork(BER_TLV_IE, tag=0x45):
         # TS 24.008 10.5.3.5a
-        pass
+        # TODO: proper decode
+        _construct = HexAdapter(GreedyBytes)
 
     class NetworkNameCollection(TLV_IE_Collection, nested=[FullNameForNetwork, ShortNameForNetwork]):
         pass
 
-    def __init__(self, fid='6fc5', sfid=None, name='EF.PNN', desc='PLMN Network Name'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc)
+    def __init__(self, fid='6fc5', sfid=None, name='EF.PNN', desc='PLMN Network Name', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_PNN.NetworkNameCollection
 
 # TS 51.011 Section 10.3.42
-
-
 class EF_OPL(LinFixedEF):
-    def __init__(self, fid='6fc6', sfid=None, name='EF.OPL', rec_len={8, 8}, desc='Operator PLMN List'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len)
-        self._construct = Struct('lai'/Bytes(5), 'pnn_record_id'/Int8ub)
+    def __init__(self, fid='6fc6', sfid=None, name='EF.OPL', rec_len=(8, 8), desc='Operator PLMN List', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
+        self._construct = Struct('lai'/Struct('mcc_mnc'/BcdAdapter(Bytes(3)),
+                                 'lac_min'/HexAdapter(Bytes(2)), 'lac_max'/HexAdapter(Bytes(2))), 'pnn_record_id'/Int8ub)
 
 # TS 51.011 Section 10.3.44 + TS 31.102 4.2.62
-
-
 class EF_MBI(LinFixedEF):
-    def __init__(self, fid='6fc9', sfid=None, name='EF.MBI', rec_len={4, 5}, desc='Mailbox Identifier'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len)
+    def __init__(self, fid='6fc9', sfid=None, name='EF.MBI', rec_len=(4, 5), desc='Mailbox Identifier', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct('mbi_voicemail'/Int8ub, 'mbi_fax'/Int8ub, 'mbi_email'/Int8ub,
                                  'mbi_other'/Int8ub, 'mbi_videocall'/COptional(Int8ub))
 
 # TS 51.011 Section 10.3.45 + TS 31.102 4.2.63
-
-
 class EF_MWIS(LinFixedEF):
-    def __init__(self, fid='6fca', sfid=None, name='EF.MWIS', rec_len={5, 6},
-                 desc='Message Waiting Indication Status'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len)
+    def __init__(self, fid='6fca', sfid=None, name='EF.MWIS', rec_len=(5, 6),
+                 desc='Message Waiting Indication Status', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct('mwi_status'/FlagsEnum(Byte, voicemail=1, fax=2, email=4, other=8, videomail=16),
                                  'num_waiting_voicemail'/Int8ub,
                                  'num_waiting_fax'/Int8ub, 'num_waiting_email'/Int8ub,
                                  'num_waiting_other'/Int8ub, 'num_waiting_videomail'/COptional(Int8ub))
 
 # TS 51.011 Section 10.3.66
-
-
 class EF_SPDI(TransparentEF):
     class ServiceProviderPLMN(BER_TLV_IE, tag=0x80):
         # flexible numbers of 3-byte PLMN records
@@ -960,28 +926,22 @@ class EF_SPDI(TransparentEF):
     class SPDI(BER_TLV_IE, tag=0xA3, nested=[ServiceProviderPLMN]):
         pass
     def __init__(self, fid='6fcd', sfid=None, name='EF.SPDI',
-                 desc='Service Provider Display Information'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc)
+                 desc='Service Provider Display Information', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, **kwargs)
         self._tlv = EF_SPDI.SPDI
 
 # TS 51.011 Section 10.3.51
-
-
 class EF_MMSN(LinFixedEF):
-    def __init__(self, fid='6fce', sfid=None, name='EF.MMSN', rec_len={4, 20}, desc='MMS Notification'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len)
-        self._construct = Struct('mms_status'/Bytes(2), 'mms_implementation'/Bytes(1),
-                                 'mms_notification'/Bytes(this._.total_len-4), 'ext_record_nr'/Byte)
+    def __init__(self, fid='6fce', sfid=None, name='EF.MMSN', rec_len=(4, 20), desc='MMS Notification', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
+        self._construct = Struct('mms_status'/HexAdapter(Bytes(2)), 'mms_implementation'/HexAdapter(Bytes(1)),
+                                 'mms_notification'/HexAdapter(Bytes(this._.total_len-4)), 'ext_record_nr'/Byte)
 
 # TS 51.011 Annex K.1
-
-
 class MMS_Implementation(BER_TLV_IE, tag=0x80):
     _construct = FlagsEnum(Byte, WAP=1)
 
 # TS 51.011 Section 10.3.53
-
-
 class EF_MMSICP(TransparentEF):
     class MMS_Relay_Server(BER_TLV_IE, tag=0x81):
         # 3GPP TS 23.140
@@ -998,14 +958,12 @@ class EF_MMSICP(TransparentEF):
     class MMS_ConnectivityParamters(TLV_IE_Collection,
                                     nested=[MMS_Implementation, MMS_Relay_Server, Interface_to_CN, Gateway]):
         pass
-    def __init__(self, fid='6fd0', sfid=None, name='EF.MMSICP', size={1, None},
-                 desc='MMS Issuer Connectivity Parameters'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6fd0', sfid=None, name='EF.MMSICP', size=(1, None),
+                 desc='MMS Issuer Connectivity Parameters', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._tlv = EF_MMSICP.MMS_ConnectivityParamters
 
 # TS 51.011 Section 10.3.54
-
-
 class EF_MMSUP(LinFixedEF):
     class MMS_UserPref_ProfileName(BER_TLV_IE, tag=0x81):
         pass
@@ -1016,18 +974,16 @@ class EF_MMSUP(LinFixedEF):
     class MMS_User_Preferences(TLV_IE_Collection,
                                nested=[MMS_Implementation, MMS_UserPref_ProfileName, MMS_UserPref_Info]):
         pass
-    def __init__(self, fid='6fd1', sfid=None, name='EF.MMSUP', rec_len={1, None},
-                 desc='MMS User Preferences'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len)
+    def __init__(self, fid='6fd1', sfid=None, name='EF.MMSUP', rec_len=(1, None),
+                 desc='MMS User Preferences', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._tlv = EF_MMSUP.MMS_User_Preferences
 
 # TS 51.011 Section 10.3.55
-
-
 class EF_MMSUCP(TransparentEF):
-    def __init__(self, fid='6fd2', sfid=None, name='EF.MMSUCP', size={1, None},
-                 desc='MMS User Connectivity Parameters'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6fd2', sfid=None, name='EF.MMSUCP', size=(1, None),
+                 desc='MMS User Connectivity Parameters', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
 
 
 class DF_GSM(CardDF):
@@ -1042,24 +998,24 @@ class DF_GSM(CardDF):
                           'Higher Priority PLMN search period'),
             EF_ACMmax(),
             EF_ServiceTable('6f38', None, 'EF.SST',
-                            'SIM service table', table=EF_SST_map, size={2, 16}),
+                            'SIM service table', table=EF_SST_map, size=(2, 16)),
             CyclicEF('6f39', None, 'EF.ACM',
-                     'Accumulated call meter', rec_len={3, 3}),
+                     'Accumulated call meter', rec_len=(3, 3)),
             TransparentEF('6f3e', None, 'EF.GID1', 'Group Identifier Level 1'),
             TransparentEF('6f3f', None, 'EF.GID2', 'Group Identifier Level 2'),
             EF_SPN(),
             TransparentEF('6f41', None, 'EF.PUCT',
-                          'Price per unit and currency table', size={5, 5}),
+                          'Price per unit and currency table', size=(5, 5)),
             EF_CBMI(),
             TransparentEF('6f7f', None, 'EF.BCCH',
-                          'Broadcast control channels', size={16, 16}),
+                          'Broadcast control channels', size=(16, 16)),
             EF_ACC(),
             EF_PLMNsel('6f7b', None, 'EF.FPLMN',
-                       'Forbidden PLMNs', size={12, 12}),
+                       'Forbidden PLMNs', size=(12, 12)),
             EF_LOCI(),
             EF_AD(),
             TransparentEF('6fa3', None, 'EF.Phase',
-                          'Phase identification', size={1, 1}),
+                          'Phase identification', size=(1, 1)),
             EF_VGCS(),
             EF_VGCSS(),
             EF_VGCS('6fb3', None, 'EF.VBS', 'Voice Broadcast Service'),
@@ -1162,7 +1118,9 @@ class CardProfileSIM(CardProfile):
             4: 'working_ef'
         }
         ret = {
-            'file_descriptor': {},
+            'file_descriptor': {
+                'file_descriptor_byte': {},
+            },
             'proprietary_info': {},
         }
         ret['file_id'] = b2h(resp_bin[4:6])
@@ -1170,7 +1128,7 @@ class CardProfileSIM(CardProfile):
             resp_bin[2:4], 'big')
         file_type = type_of_file_map[resp_bin[6]
                                      ] if resp_bin[6] in type_of_file_map else resp_bin[6]
-        ret['file_descriptor']['file_type'] = file_type
+        ret['file_descriptor']['file_descriptor_byte']['file_type'] = file_type
         if file_type in ['mf', 'df']:
             ret['file_characteristics'] = b2h(resp_bin[13:14])
             ret['num_direct_child_df'] = resp_bin[14]
@@ -1180,7 +1138,7 @@ class CardProfileSIM(CardProfile):
         elif file_type in ['working_ef']:
             file_struct = struct_of_file_map[resp_bin[13]
                                              ] if resp_bin[13] in struct_of_file_map else resp_bin[13]
-            ret['file_descriptor']['structure'] = file_struct
+            ret['file_descriptor']['file_descriptor_byte']['structure'] = file_struct
             ret['access_conditions'] = b2h(resp_bin[8:10])
             if resp_bin[11] & 0x01 == 0:
                 ret['life_cycle_status_int'] = 'operational_activated'

pySim/utils.py

@@ -1225,6 +1225,60 @@ def auto_int(x):
     return int(x, 0)
 
 
+def expand_hex(hexstring, length):
+    """Expand a given hexstring to a specified length by replacing "." or ".."
+       with a filler that is derived from the neighboring nibbles respective
+       bytes. Usually this will be the nibble respective byte before "." or
+       "..", execpt when the string begins with "." or "..", then the nibble
+       respective byte after "." or ".." is used.". In case the string cannot
+       be expanded for some reason, the input string is returned unmodified.
+
+    Args:
+            hexstring : hexstring to expand
+            length : desired length of the resulting hexstring.
+    Returns:
+            expanded hexstring
+    """
+
+    # expand digit aligned
+    if hexstring.count(".") == 1:
+        pos = hexstring.index(".")
+        if pos > 0:
+            filler = hexstring[pos - 1]
+        else:
+            filler = hexstring[pos + 1]
+
+        missing = length * 2 - (len(hexstring) - 1)
+        if missing <= 0:
+            return hexstring
+
+        return hexstring.replace(".", filler * missing)
+
+    # expand byte aligned
+    elif hexstring.count("..") == 1:
+        if len(hexstring) % 2:
+            return hexstring
+
+        pos = hexstring.index("..")
+
+        if pos % 2:
+            return hexstring
+
+        if pos > 1:
+            filler = hexstring[pos - 2:pos]
+        else:
+            filler = hexstring[pos + 2:pos+4]
+
+        missing = length * 2 - (len(hexstring) - 2)
+        if missing <= 0:
+            return hexstring
+
+        return hexstring.replace("..", filler * (missing // 2))
+
+    # no change
+    return hexstring
+
+
 class JsonEncoder(json.JSONEncoder):
     """Extend the standard library JSONEncoder with support for more types."""
 
@@ -1333,7 +1387,7 @@ class DataObject(abc.ABC):
             bytes encoded in TLV format.
         """
         val = self.to_bytes()
-        return bytes(self._compute_tag()) + bytes(len(val)) + val
+        return bertlv_encode_tag(self._compute_tag()) + bertlv_encode_len(len(val)) + val
 
     # 'codec' interface
     def decode(self, binary: bytes) -> Tuple[dict, bytes]:
@@ -1481,7 +1535,8 @@ class DataObjectChoice(DataObjectCollection):
 
     # 'codec' interface
     def encode(self, decoded) -> bytes:
-        obj = self.members_by_name(decoded[0])
+        obj = self.members_by_name[list(decoded)[0]]
+        obj.decoded = list(decoded.values())[0]
         return obj.to_tlv()
 
 
@@ -1560,6 +1615,18 @@ class DataObjectSequence:
             i += 1
         return encoded
 
+    def encode_multi(self, decoded) -> bytes:
+        """Encode multiple occurrences of the sequence from the decoded input data.
+        Args:
+            decoded : list of json-serializable input data; one sequence per list item
+        Returns:
+            binary encoded output data
+        """
+        encoded = bytearray()
+        for d in decoded:
+            encoded += self.encode(d)
+        return encoded
+
 
 class CardCommand:
     """A single card command / instruction."""

requirements.txt

@@ -7,3 +7,7 @@ construct
 bidict
 gsm0338
 pyyaml>=5.1
+termcolor
+colorlog
+pycryptodome
+git+https://github.com/hologram-io/smpp.pdu

scripts/deactivate-5g.script

@@ -0,0 +1,70 @@
+# script to be used with pySim-shell.py which is part of the Osmocom pysim package,
+# found at https://osmocom.org/projects/pysim/wiki
+set echo true
+
+# this script will deactivate all 5G related services and files.  This can be used
+# in case you do not wish to use any 5G services, or you do not wish to configure
+# the 5G specific files on the USIM card.  The card will then behave like a 3G USIM
+# without any 5G capability, using the default fall-back mechanisms specified by 3GPP.
+
+# TODO: add your card-specific ADM pin at the end of the verify_adm line below
+verify_adm
+
+# deactivate any 5G related services in EF.UST
+select ADF.USIM
+select EF.UST
+ust_service_deactivate 122
+ust_service_deactivate 123
+ust_service_deactivate 124
+ust_service_deactivate 125
+ust_service_deactivate 126
+ust_service_deactivate 127
+ust_service_deactivate 129
+ust_service_deactivate 130
+ust_service_deactivate 132
+ust_service_deactivate 133
+ust_service_deactivate 134
+ust_service_deactivate 135
+
+# deactivate all files in EF.5GS
+select ADF.USIM
+select DF.5GS
+
+select EF.5GAUTHKEYS
+deactivate_file
+
+select EF.5GS3GPPLOCI
+deactivate_file
+
+select EF.5GSN3GPPNSC
+deactivate_file
+
+select EF.5GSN3GPPLOCI
+deactivate_file
+
+select EF.5GS3GPPNSC
+deactivate_file
+
+# only exists on sysmoISIM-SJA2v2
+select EF.OPL5G
+deactivate_file
+
+select EF.Routing_Indicator
+deactivate_file
+
+select EF.SUCI_Calc_Info
+deactivate_file
+
+select EF.SUPI_NAI
+deactivate_file
+
+# only exists on sysmoISIM-SJA2v2
+select EF.TN3GPPSNN
+deactivate_file
+
+select EF.UAC_AIC
+deactivate_file
+
+# only exists on sysmoISIM-SJA2v2
+select EF.URSP
+deactivate_file

scripts/deactivate-ims.script

@@ -0,0 +1,74 @@
+# script to be used with pySim-shell.py which is part of the Osmocom pysim package,
+# found at https://osmocom.org/projects/pysim/wiki
+set echo true
+
+# this script will deactivate all IMS related services and files.  This can be used
+# in case you do not wish to use any IMS services, or you do not wish to configure
+# the IMS specific files on the USIM/ISIM cards.  The card will then behave like a 3G USIM
+# without any IMS capability, using the default fall-back mechanisms specified by 3GPP.
+
+# TODO: add your card-specific ADM pin at the end of the verify_adm line below
+verify_adm
+
+# deactivate any IMS related services in EF.UST
+select ADF.USIM
+select EF.UST
+ust_service_deactivate 93
+ust_service_deactivate 95
+ust_service_deactivate 104
+ust_service_deactivate 105
+ust_service_deactivate 106
+ust_service_deactivate 107
+ust_service_deactivate 108
+ust_service_deactivate 109
+ust_service_deactivate 110
+ust_service_deactivate 112
+ust_service_deactivate 114
+ust_service_deactivate 115
+ust_service_deactivate 118
+ust_service_deactivate 120
+ust_service_deactivate 131
+ust_service_deactivate 134
+
+# deactivate all IMS related files in ADF.USIM
+select ADF.USIM
+
+select EF.UICCIARI
+deactivate_file
+
+select EF.ePDGId
+deactivate_file
+
+select EF.ePDGSelection
+deactivate_file
+
+select EF.ePDGIdEm
+deactivate_file
+
+select EF.ePDGSelectionEm
+deactivate_file
+
+select EF.FromPreferred
+deactivate_file
+
+select EF.IMSConfigData
+deactivate_file
+
+select EF.3GPPPSDATAOFF
+deactivate_file
+
+select EF.3GPPPSDATAOFFservicelist
+deactivate_file
+
+select EF.XCAPConfigData
+deactivate_file
+
+select EF.MuDMiDConfigData
+deactivate_file
+
+echo "Please make sure to manually disable the ISIM applet as described in the end of the script"
+# you can currently only manually do this via GlobalPlatformPro or some other tool using
+# java -jar ./gp.jar --key-enc KIC1 --key-mac KID1 --key-dek KIK1 --lock-applet A0000000871004FFFFFFFF8907090000
+# (substituting KIC1/KID1/KIK1 with the card-specific keys, of course)
+
+quit

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='pySim',
-    version='1.0',
+    version='1.1',
     packages=['pySim', 'pySim.transport'],
     url='https://osmocom.org/projects/pysim/wiki',
     license='GPLv2',
@@ -17,6 +17,10 @@ setup(
         "construct >= 2.9",
         "bidict",
         "gsm0338",
+        "termcolor",
+        "colorlog",
+        "pycryptodome",
+        "smpp.pdu @ git+https://github.com/hologram-io/smpp.pdu",
     ],
     scripts=[
         'pySim-prog.py',

smpp2sim.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+#
+# Program to emulate the entire communication path SMSC-MSC-BSC-BTS-ME
+# that is usually between an OTA backend and the SIM card.  This allows
+# to play with SIM OTA technology without using a mobile network or even
+# a mobile phone.
+#
+# An external application must encode (and encrypt/sign) the OTA SMS
+# and submit them via SMPP to this program, just like it would submit
+# it normally to a SMSC (SMS Service Centre).  The program then re-formats
+# the SMPP-SUBMIT into a SMS DELIVER TPDU and passes it via an ENVELOPE
+# APDU to the SIM card that is locally inserted into a smart card reader.
+#
+# The path from SIM to external OTA application works the opposite way.
+
+import argparse
+import logging
+import colorlog
+from pprint import pprint as pp
+
+from twisted.protocols import basic
+from twisted.internet import defer, endpoints, protocol, reactor, task
+from twisted.cred.portal import IRealm
+from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
+from twisted.cred.portal import Portal
+from zope.interface import implementer
+
+from smpp.twisted.config import SMPPServerConfig
+from smpp.twisted.server import SMPPServerFactory, SMPPBindManager
+from smpp.twisted.protocol import SMPPSessionStates, DataHandlerResponse
+
+from smpp.pdu import pdu_types, operations, pdu_encoding
+
+from pySim.sms import SMS_DELIVER, AddressField
+
+from pySim.transport import LinkBase, ProactiveHandler, argparse_add_reader_args, init_reader
+from pySim.commands import SimCardCommands
+from pySim.cards import UsimCard
+from pySim.exceptions import *
+from pySim.cat import ProactiveCommand, SendShortMessage, SMS_TPDU, SMSPPDownload
+from pySim.cat import DeviceIdentities, Address
+from pySim.utils import b2h, h2b
+
+logger = logging.getLogger(__name__)
+
+# MSISDNs to use when generating proactive SMS messages
+SIM_MSISDN='23'
+ESME_MSISDN='12'
+
+# HACK: we need some kind of mapping table between system_id and card-reader
+# or actually route based on MSISDNs
+hackish_global_smpp = None
+
+class Proact(ProactiveHandler):
+    def __init__(self, smpp_factory):
+        self.smpp_factory = smpp_factory
+
+    @staticmethod
+    def _find_first_element_of_type(instlist, cls):
+        for i in instlist:
+            if isinstance(i, cls):
+                return i
+        return None
+
+    """Call-back which the pySim transport core calls whenever it receives a
+    proactive command from the SIM."""
+    def handle_SendShortMessage(self, data):
+        """Card requests sending a SMS."""
+        pp(data)
+        # Relevant parts in data: Address, SMS_TPDU
+        addr_ie = _find_first_element_of_type(data.children, Address)
+        sms_tpdu_ie = _find_first_element_of_type(data.children, SMS_TPDU)
+        raw_tpdu = sms_tpdu_ie.decoded['tpdu']
+        submit = SMS_SUBMIT.fromBytes(raw_tpdu)
+        self.send_sms_via_smpp(data)
+    def handle_OpenChannel(self, data):
+        """Card requests opening a new channel via a UDP/TCP socket."""
+        pp(data)
+        pass
+    def handle_CloseChannel(self, data):
+        """Close a channel."""
+        pp(data)
+        pass
+    def handleReceiveData(self, data):
+        """Receive/read data from the socket."""
+        pp(data)
+        pass
+    def handleSendData(self, data):
+        """Send/write data to the socket."""
+        pp(data)
+        pass
+    def getChannelStatus(self, data):
+        pp(data)
+        pass
+
+    def send_sms_via_smpp(self, data):
+        # while in a normal network the phone/ME would *submit* a message to the SMSC,
+        # we are actually emulating the SMSC itself, so we must *deliver* the message
+        # to the ESME
+        dcs = pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED)
+        esm_class = pdu_types.EsmClass(pdu_types.EsmClassMode.DEFAULT, pdu_types.EsmClassType.DEFAULT,
+                                       gsmFeatures=[pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET])
+        deliver = operations.DeliverSM(source_addr=SIM_MSISDN,
+                                       destination_addr=ESME_MSISDN,
+                                       esm_class=esm_class,
+                                       protocol_id=0x7F,
+                                       data_coding=dcs,
+                                       short_message=h2b(data))
+        hackish_global_smpp.sendDataRequest(deliver)
+#       # obtain the connection/binding of system_id to be used for delivering MO-SMS to the ESME
+#       connection = smpp_server.getBoundConnections[system_id].getNextBindingForDelivery()
+#       connection.sendDataRequest(deliver)
+
+
+
+def dcs_is_8bit(dcs):
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED):
+        return True
+    if dcs == pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED_COMMON):
+        return True
+    if dcs.scheme == pdu_types.DataCodingScheme.GSM_MESSAGE_CLASS and dcs.schemeData['msgCoding'] == pdu_types.DataCodingGsmMsgCoding.DATA_8BIT:
+        return True
+    else:
+        return False
+
+
+class MyServer:
+
+    @implementer(IRealm)
+    class SmppRealm:
+        def requestAvatar(self, avatarId, mind, *interfaces):
+            return ('SMPP', avatarId, lambda: None)
+
+    def __init__(self, tcp_port:int = 2775, bind_ip = '::'):
+        smpp_config = SMPPServerConfig(msgHandler=self._msgHandler,
+                                       systems={'test': {'max_bindings': 2}})
+        portal = Portal(self.SmppRealm())
+        credential_checker = InMemoryUsernamePasswordDatabaseDontUse()
+        credential_checker.addUser('test', 'test')
+        portal.registerChecker(credential_checker)
+        self.factory = SMPPServerFactory(smpp_config, auth_portal=portal)
+        logger.info('Binding Virtual SMSC to TCP Port %u at %s' % (tcp_port, bind_ip))
+        smppEndpoint = endpoints.TCP6ServerEndpoint(reactor, tcp_port, interface=bind_ip)
+        smppEndpoint.listen(self.factory)
+        self.tp = self.scc = self.card = None
+
+    def connect_to_card(self, tp: LinkBase):
+        self.tp = tp
+        self.scc = SimCardCommands(self.tp)
+        self.card = UsimCard(self.scc)
+        # this should be part of UsimCard, but FairewavesSIM breaks with that :/
+        self.scc.cla_byte = "00"
+        self.scc.sel_ctrl = "0004"
+        self.card.read_aids()
+        self.card.select_adf_by_aid(adf='usim')
+        # FIXME: create a more realistic profile than ffffff
+        self.scc.terminal_profile('ffffff')
+
+    def _msgHandler(self, system_id, smpp, pdu):
+        # HACK: we need some kind of mapping table between system_id and card-reader
+        # or actually route based on MSISDNs
+        global hackish_global_smpp
+        hackish_global_smpp = smpp
+        #pp(pdu)
+        if pdu.id == pdu_types.CommandId.submit_sm:
+            return self.handle_submit_sm(system_id, smpp, pdu)
+        else:
+            logging.warning('Rejecting non-SUBMIT commandID')
+            return pdu_types.CommandStatus.ESME_RINVCMDID
+
+    def handle_submit_sm(self, system_id, smpp, pdu):
+        # check for valid data coding scheme + PID
+        if not dcs_is_8bit(pdu.params['data_coding']):
+            logging.warning('Rejecting non-8bit DCS')
+            return pdu_types.CommandStatus.ESME_RINVDCS
+        if pdu.params['protocol_id'] != 0x7f:
+            logging.warning('Rejecting non-SIM PID')
+            return pdu_types.CommandStatus.ESME_RINVDCS
+
+        # 1) build a SMS-DELIVER (!) from the SMPP-SUBMIT
+        tpdu = SMS_DELIVER.fromSmppSubmit(pdu)
+        print(tpdu)
+        # 2) wrap into the CAT ENVELOPE for SMS-PP-Download
+        tpdu_ie = SMS_TPDU(decoded={'tpdu': b2h(tpdu.toBytes())})
+        dev_ids = DeviceIdentities(decoded={'source_dev_id': 'network', 'dest_dev_id': 'uicc'})
+        sms_dl = SMSPPDownload(children=[dev_ids, tpdu_ie])
+        # 3) send to the card
+        envelope_hex = b2h(sms_dl.to_tlv())
+        print("ENVELOPE: %s" % envelope_hex)
+        (data, sw) = self.scc.envelope(envelope_hex)
+        print("SW %s: %s" % (sw, data))
+        if sw == '9300':
+            # TODO send back RP-ERROR message with TP-FCS == 'SIM Application Toolkit Busy'
+            return pdu_types.CommandStatus.ESME_RSUBMITFAIL
+        elif sw == '9000' or sw[0:2] in ['6f', '62', '63']:
+            # data something like 027100000e0ab000110000000000000001612f or
+            # 027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c
+            # which is the user-data portion of the SMS starting with the UDH (027100)
+            # TODO: return the response back to the sender in an RP-ACK; PID/DCS like in CMD
+            deliver = operations.DeliverSM(service_type=pdu.params['service_type'],
+                                           source_addr_ton=pdu.params['dest_addr_ton'],
+                                           source_addr_npi=pdu.params['dest_addr_npi'],
+                                           source_addr=pdu.params['destination_addr'],
+                                           dest_addr_ton=pdu.params['source_addr_ton'],
+                                           dest_addr_npi=pdu.params['source_addr_npi'],
+                                           destination_addr=pdu.params['source_addr'],
+                                           esm_class=pdu.params['esm_class'],
+                                           protocol_id=pdu.params['protocol_id'],
+                                           priority_flag=pdu.params['priority_flag'],
+                                           data_coding=pdu.params['data_coding'],
+                                           short_message=h2b(data))
+            smpp.sendDataRequest(deliver)
+            return pdu_types.CommandStatus.ESME_ROK
+        else:
+            return pdu_types.CommandStatus.ESME_RSUBMITFAIL
+
+
+option_parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+argparse_add_reader_args(option_parser)
+smpp_group = option_parser.add_argument_group('SMPP Options')
+smpp_group.add_argument('--smpp-bind-port', type=int, default=2775,
+                        help='TCP Port to bind the SMPP socket to')
+smpp_group.add_argument('--smpp-bind-ip', default='::',
+                        help='IPv4/IPv6 address to bind the SMPP socket to')
+
+if __name__ == '__main__':
+    log_format='%(log_color)s%(levelname)-8s%(reset)s %(name)s: %(message)s'
+    colorlog.basicConfig(level=logging.INFO, format = log_format)
+    logger = colorlog.getLogger()
+
+    opts = option_parser.parse_args()
+
+    #tp = init_reader(opts, proactive_handler = Proact())
+    tp = init_reader(opts)
+    if tp is None:
+        exit(1)
+    tp.connect()
+
+    ms = MyServer(opts.smpp_bind_port, opts.smpp_bind_ip)
+    ms.connect_to_card(tp)
+    reactor.run()
+

sms_test.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+from pySim.sms import *
+from pprint import pprint as pp
+from construct import setGlobalPrintPrivateEntries
+
+
+print(UserDataHeader.fromBytes('027100'))
+print(UserDataHeader.fromBytes('027100abcdef'))
+print(UserDataHeader.fromBytes('03710110'))
+print(UserDataHeader.fromBytes('0571007001ffabcd'))
+
+setGlobalPrintPrivateEntries(True)
+pp(AddressField.fromBytes('0480214399'))
+
+s = SMS_SUBMIT.fromBytes('550d0b911614261771f000f5a78c0b050423f423f40003010201424547494e3a56434152440d0a56455253494f4e3a322e310d0a4e3a4d650d0a54454c3b505245463b43454c4c3b564f4943453a2b36313431363237313137300d0a54454c3b484f4d453b564f4943453a2b36313339353337303437310d0a54454c3b574f524b3b564f4943453a2b36313339363734373031350d0a454e443a')
+pp(s)
+print(s.tp_da)
+pp(b2h(s.toBytes()))
+
+d = SMS_DELIVER.fromBytes('0408D0E5759A0E7FF6907090307513000824010101BB400101')
+pp(d)
+pp(b2h(d.toBytes()))

tests/test_apdu.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim.utils import h2b, b2h
+from pySim.construct import filter_dict
+from pySim.apdu import Apdu
+from pySim.apdu.ts_31_102 import UsimAuthenticateEven
+
+class TestApdu(unittest.TestCase):
+    def test_successful(self):
+        apdu = Apdu('00a40400023f00', '9000')
+        self.assertEqual(apdu.successful, True)
+        apdu = Apdu('00a40400023f00', '6733')
+        self.assertEqual(apdu.successful, False)
+
+    def test_successful_method(self):
+        """Test overloading of the success property with a custom method."""
+        class SwApdu(Apdu):
+            def _is_success(self):
+                return False
+        apdu = SwApdu('00a40400023f00', '9000')
+        self.assertEqual(apdu.successful, False)
+
+# TODO: Tests for TS 102 221 / 31.102 ApduCommands
+
+class TestUsimAuth(unittest.TestCase):
+    """Test decoding of the rather complex USIM AUTHENTICATE command."""
+    def test_2g(self):
+        apdu = ('80880080' + '09' + '080001020304050607',
+                '04a0a1a2a308b0b1b2b3b4b5b6b79000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'gsm'},
+                    'body': {'rand': '0001020304050607', 'autn': None}},
+            'rsp': {'body': {'sres': 'a0a1a2a3', 'kc': 'b0b1b2b3b4b5b6b7'}}
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+    def test_3g(self):
+        apdu = ('80880081' + '12' + '080001020304050607081011121314151617',
+                'DB' + '08' + 'a0a1a2a3a4a5a6a7' +
+                       '10' + 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf' +
+                       '10' + 'c0c1c2c3c4c5c6c7c8c9cacbcccdcecf' + '9000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'umts'},
+                    'body': {'rand': '0001020304050607', 'autn': '1011121314151617'}},
+            'rsp': {'body': {'tag': 219,
+                             'body': {
+                                 'res': 'a0a1a2a3a4a5a6a7',
+                                 'ck': 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf',
+                                 'ik': 'c0c1c2c3c4c5c6c7c8c9cacbcccdcecf',
+                                 'kc': None
+                                 }
+                             }
+                    }
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+    def test_3g_sync(self):
+        apdu = ('80880081' + '12' + '080001020304050607081011121314151617',
+                'DC' + '08' + 'a0a1a2a3a4a5a6a7' + '9000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'umts'},
+                    'body': {'rand': '0001020304050607', 'autn': '1011121314151617'}},
+            'rsp': {'body': {'tag': 220, 'body': {'auts': 'a0a1a2a3a4a5a6a7' }}}
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+    def test_vgcs(self):
+        apdu = ('80880082' + '0E' + '04' + '00010203' +
+                             '01' + '10' +
+                             '08' + '2021222324252627',
+                'DB' + '10' + 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf' + '9000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'vgcs_vbs'},
+                    'body': { 'vk_id': '10', 'vservice_id': '00010203', 'vstk_rand': '2021222324252627'}},
+            'rsp': {'body': {'vstk': 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf'}}
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_construct.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim.construct import *
+
+tests = [
+        ( b'\x80', 0x80 ),
+        ( b'\x80\x01', 0x8001 ),
+        ( b'\x80\x00\x01', 0x800001 ),
+        ( b'\x80\x23\x42\x01', 0x80234201 ),
+        ]
+
+class TestGreedyInt(unittest.TestCase):
+    def test_GreedyInt_decoder(self):
+        gi = GreedyInteger()
+        for t in tests:
+            self.assertEqual(gi.parse(t[0]), t[1])
+    def test_GreedyInt_encoder(self):
+        gi = GreedyInteger()
+        for t in tests:
+            self.assertEqual(t[0], gi.build(t[1]))
+        pass
+
+class TestUtils(unittest.TestCase):
+    def test_filter_dict(self):
+        inp = {'foo': 0xf00, '_bar' : 0xba5, 'baz': 0xba2 }
+        out = {'foo': 0xf00, 'baz': 0xba2 }
+        self.assertEqual(filter_dict(inp), out)
+
+    def test_filter_dict_nested(self):
+        inp = {'foo': 0xf00, 'nest': {'_bar' : 0xba5}, 'baz': 0xba2 }
+        out = {'foo': 0xf00, 'nest': {}, 'baz': 0xba2 }
+        self.assertEqual(filter_dict(inp), out)
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_ota.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim.utils import h2b, b2h
+from pySim.ota import *
+
+class Test_SMS_3DES(unittest.TestCase):
+    tar = h2b('b00000')
+    """Test the OtaDialectSms for 3DES algorithms."""
+    def __init__(self, foo, **kwargs):
+        super().__init__(foo, **kwargs)
+        # KIC1 + KID1 of 8988211000000467285
+        KIC1 = h2b('D0FDA31990D8D64178601317191669B4')
+        KID1 = h2b('D24EB461799C5E035C77451FD9404463')
+        KIC3 = h2b('C21DD66ACAC13CB3BC8B331B24AFB57B')
+        KID3 = h2b('12110C78E678C25408233076AA033615')
+        self.od = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3, kic=KIC3,
+                            algo_auth='triple_des_cbc2', kid_idx=3, kid=KID3)
+        self.dialect = OtaDialectSms()
+        self.spi_base = {
+                'counter':'no_counter',
+                'ciphering': True,
+                'rc_cc_ds': 'cc',
+                'por_in_submit':False,
+                'por': 'por_required',
+                'por_shall_be_ciphered': True,
+                'por_rc_cc_ds': 'cc',
+            }
+
+    def _check_response(self, r):
+        self.assertEqual(r['number_of_commands'], 1)
+        self.assertEqual(r['last_status_word'], '612f')
+        self.assertEqual(r['last_response_data'], u'')
+        self.assertEqual(r['response_status'], 'por_ok')
+
+    def test_resp_3des_ciphered(self):
+        spi = self.spi_base
+        spi['por_shall_be_ciphered'] = True
+        spi['por_rc_cc_ds'] = 'cc'
+        r = self.dialect.decode_resp(self.od, spi, '027100001c12b000119660ebdb81be189b5e4389e9e7ab2bc0954f963ad869ed7c')
+        self._check_response(r)
+
+    def test_resp_3des_signed(self):
+        spi = self.spi_base
+        spi['por_shall_be_ciphered'] = False
+        spi['por_rc_cc_ds'] = 'cc'
+        r = self.dialect.decode_resp(self.od, spi, '027100001612b000110000000000000055f47118381175fb01612f')
+        self._check_response(r)
+
+    def test_resp_3des_signed_err(self):
+        """Expect an OtaCheckError exception if the computed CC != received CC"""
+        spi = self.spi_base
+        spi['por_shall_be_ciphered'] = False
+        spi['por_rc_cc_ds'] = 'cc'
+        with self.assertRaises(OtaCheckError) as context:
+            r = self.dialect.decode_resp(self.od, spi, '027100001612b000110000000000000055f47118381175fb02612f')
+        self.assertTrue('!= Computed CC' in str(context.exception))
+
+    def test_resp_3des_none(self):
+        spi = self.spi_base
+        spi['por_shall_be_ciphered'] = False
+        spi['por_rc_cc_ds'] = 'no_rc_cc_ds'
+        r = self.dialect.decode_resp(self.od, spi, '027100000e0ab000110000000000000001612f')
+        self._check_response(r)
+
+    def test_cmd_3des_ciphered(self):
+        spi = self.spi_base
+        spi['ciphering'] = True
+        spi['rc_cc_ds'] = 'no_rc_cc_ds'
+        r = self.dialect.encode_cmd(self.od, self.tar, spi, h2b('00a40000023f00'))
+        self.assertEqual(b2h(r), '00180d04193535b000000c8478b552a4ffc5a8f099b83cad7123')
+
+    def test_cmd_3des_signed(self):
+        spi = self.spi_base
+        spi['ciphering'] = False
+        spi['rc_cc_ds'] = 'cc'
+        r = self.dialect.encode_cmd(self.od, self.tar, spi, h2b('00a40000023f00'))
+        self.assertEqual(b2h(r), '1502193535b00000000000000000072ea17bdb72060e00a40000023f00')
+
+    def test_cmd_3des_none(self):
+        spi = self.spi_base
+        spi['ciphering'] = False
+        spi['rc_cc_ds'] = 'no_rc_cc_ds'
+        r = self.dialect.encode_cmd(self.od, self.tar, spi, h2b('00a40000023f00'))
+        self.assertEqual(b2h(r), '0d00193535b0000000000000000000a40000023f00')

tests/test_sms.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim.utils import h2b, b2h
+from pySim.sms import *
+
+class Test_SMS_UDH(unittest.TestCase):
+    def test_single_ie(self):
+        udh, tail = UserDataHeader.fromBytes('027100')
+        self.assertEqual(len(udh.ies), 1)
+        ie = udh.ies[0]
+        self.assertEqual(ie.iei, 0x71)
+        self.assertEqual(ie.length, 0)
+        self.assertEqual(ie.value, b'')
+        self.assertEqual(tail, b'')
+
+    def test_single_ie_tail(self):
+        udh, tail = UserDataHeader.fromBytes('027100abcdef')
+        self.assertEqual(len(udh.ies), 1)
+        ie = udh.ies[0]
+        self.assertEqual(ie.iei, 0x71)
+        self.assertEqual(ie.length, 0)
+        self.assertEqual(ie.value, b'')
+        self.assertEqual(tail, b'\xab\xcd\xef')
+
+    def test_single_ie_value(self):
+        udh, tail = UserDataHeader.fromBytes('03710110')
+        self.assertEqual(len(udh.ies), 1)
+        ie = udh.ies[0]
+        self.assertEqual(ie.iei, 0x71)
+        self.assertEqual(ie.length, 1)
+        self.assertEqual(ie.value, b'\x10')
+        self.assertEqual(tail, b'')
+
+    def test_two_ie_data_tail(self):
+        udh, tail = UserDataHeader.fromBytes('0571007001ffabcd')
+        self.assertEqual(len(udh.ies), 2)
+        ie = udh.ies[0]
+        self.assertEqual(ie.iei, 0x71)
+        self.assertEqual(ie.length, 0)
+        self.assertEqual(ie.value, b'')
+        ie = udh.ies[1]
+        self.assertEqual(ie.iei, 0x70)
+        self.assertEqual(ie.length, 1)
+        self.assertEqual(ie.value, b'\xff')
+        self.assertEqual(tail, b'\xab\xcd')
+
+    def test_toBytes(self):
+        indata = h2b('0571007001ff')
+        udh, tail = UserDataHeader.fromBytes(indata)
+        encoded = udh.toBytes()
+        self.assertEqual(encoded, indata)
+
+class Test_AddressField(unittest.TestCase):
+    def test_fromBytes(self):
+        encoded = h2b('0480214399')
+        af, trailer = AddressField.fromBytes(encoded)
+        self.assertEqual(trailer, b'\x99')
+        self.assertEqual(af.ton, 'unknown')
+        self.assertEqual(af.npi, 'unknown')
+        self.assertEqual(af.digits, '1234')
+
+    def test_fromBytes_odd(self):
+        af, trailer = AddressField.fromBytes('038021f399')
+        self.assertEqual(trailer, b'\x99')
+        self.assertEqual(af.ton, 'unknown')
+        self.assertEqual(af.npi, 'unknown')
+        self.assertEqual(af.digits, '123')
+
+    def test_toBytes(self):
+        encoded = h2b('04802143')
+        af, trailer = AddressField.fromBytes(encoded)
+        self.assertEqual(af.toBytes(), encoded)
+
+    def test_toBytes_odd(self):
+        af = AddressField('12345', 'international', 'isdn_e164')
+        encoded = af.toBytes()
+        self.assertEqual(encoded, h2b('05912143f5'))
+
+
+class Test_SUBMIT(unittest.TestCase):
+    def test_fromBytes(self):
+        s = SMS_SUBMIT.fromBytes('550d0b911614261771f000f5a78c0b050423f423f40003010201424547494e3a56434152440d0a56455253494f4e3a322e310d0a4e3a4d650d0a54454c3b505245463b43454c4c3b564f4943453a2b36313431363237313137300d0a54454c3b484f4d453b564f4943453a2b36313339353337303437310d0a54454c3b574f524b3b564f4943453a2b36313339363734373031350d0a454e443a')
+        self.assertEqual(s.tp_mti, 1)
+        self.assertEqual(s.tp_rd, True)
+        self.assertEqual(s.tp_vpf, 'relative')
+        self.assertEqual(s.tp_rp, False)
+        self.assertEqual(s.tp_udhi, True)
+        self.assertEqual(s.tp_srr, False)
+        self.assertEqual(s.tp_pid, 0)
+        self.assertEqual(s.tp_dcs, 0xf5)
+        self.assertEqual(s.tp_udl, 140)
+
+class Test_DELIVER(unittest.TestCase):
+    def test_fromBytes(self):
+        d = SMS_DELIVER.fromBytes('0408D0E5759A0E7FF6907090307513000824010101BB400101')
+        self.assertEqual(d.tp_mti, 0)
+        self.assertEqual(d.tp_mms, True)
+        self.assertEqual(d.tp_lp, False)
+        self.assertEqual(d.tp_rp, False)
+        self.assertEqual(d.tp_udhi, False)
+        self.assertEqual(d.tp_sri, False)
+        self.assertEqual(d.tp_pid, 0x7f)
+        self.assertEqual(d.tp_dcs, 0xf6)
+        self.assertEqual(d.tp_udl, 8)

tests/test_tlv.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from pySim.tlv import *
+
+class TestUtils(unittest.TestCase):
+    def test_camel_to_snake(self):
+        cases = [
+            ('CamelCase', 'camel_case'),
+            ('CamelCaseUPPER', 'camel_case_upper'),
+            ('Camel_CASE_underSCORE', 'camel_case_under_score'),
+        ]
+        for c in cases:
+            self.assertEqual(camel_to_snake(c[0]), c[1])
+
+    def test_flatten_dict_lists(self):
+        inp = [
+                { 'first': 1 },
+                { 'second': 2 },
+                { 'third': 3 },
+                ]
+        out = { 'first': 1, 'second':2, 'third': 3}
+        self.assertEqual(flatten_dict_lists(inp), out)
+
+    def test_flatten_dict_lists_nodict(self):
+        inp = [
+                { 'first': 1 },
+                { 'second': 2 },
+                { 'third': 3 },
+                4,
+                ]
+        self.assertEqual(flatten_dict_lists(inp), inp)
+
+    def test_flatten_dict_lists_nested(self):
+        inp = {'top': [
+                { 'first': 1 },
+                { 'second': 2 },
+                { 'third': 3 },
+                ] }
+        out = {'top': { 'first': 1, 'second':2, 'third': 3 } }
+        self.assertEqual(flatten_dict_lists(inp), out)
+
+class TestTranscodable(unittest.TestCase):
+    class XC_constr_class(Transcodable):
+        _construct = Int8ub
+        def __init__(self):
+            super().__init__();
+
+    def test_XC_constr_class(self):
+        """Transcodable derived class with _construct class variable"""
+        xc = TestTranscodable.XC_constr_class()
+        self.assertEqual(xc.from_bytes(b'\x23'), 35)
+        self.assertEqual(xc.to_bytes(), b'\x23')
+
+    class XC_constr_instance(Transcodable):
+        def __init__(self):
+            super().__init__();
+            self._construct = Int8ub
+
+    def test_XC_constr_instance(self):
+        """Transcodable derived class with _construct instance variable"""
+        xc = TestTranscodable.XC_constr_instance()
+        self.assertEqual(xc.from_bytes(b'\x23'), 35)
+        self.assertEqual(xc.to_bytes(), b'\x23')
+
+    class XC_method_instance(Transcodable):
+        def __init__(self):
+            super().__init__();
+        def _from_bytes(self, do):
+            return ('decoded', do)
+        def _to_bytes(self):
+            return self.decoded[1]
+
+    def test_XC_method_instance(self):
+        """Transcodable derived class with _{from,to}_bytes() methods"""
+        xc = TestTranscodable.XC_method_instance()
+        self.assertEqual(xc.to_bytes(), b'')
+        self.assertEqual(xc.from_bytes(b''), None)
+        self.assertEqual(xc.from_bytes(b'\x23'), ('decoded', b'\x23'))
+        self.assertEqual(xc.to_bytes(), b'\x23')
+
+class TestIE(unittest.TestCase):
+    class MyIE(IE, tag=0x23, desc='My IE description'):
+        _construct = Int8ub
+        def to_ie(self):
+            return self.to_bytes()
+
+    def test_IE_empty(self):
+        ie = TestIE.MyIE()
+        self.assertEqual(ie.to_dict(), {'my_ie': None})
+        self.assertEqual(repr(ie), 'MyIE(None)')
+        self.assertEqual(ie.is_constructed(), False)
+
+    def test_IE_from_bytes(self):
+        ie = TestIE.MyIE()
+        ie.from_bytes(b'\x42')
+        self.assertEqual(ie.to_dict(), {'my_ie': 66})
+        self.assertEqual(repr(ie), 'MyIE(66)')
+        self.assertEqual(ie.is_constructed(), False)
+        self.assertEqual(ie.to_bytes(), b'\x42')
+        self.assertEqual(ie.to_ie(), b'\x42')
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_utils.py

@@ -4,6 +4,22 @@ import unittest
 from pySim import utils
 from pySim.ts_31_102 import EF_SUCI_Calc_Info
 
+# we don't really want to thest TS 102 221, but the underlying DataObject codebase
+from pySim.ts_102_221 import AM_DO_EF, AM_DO_DF, SC_DO
+
+class DoTestCase(unittest.TestCase):
+
+    def testSeqOfChoices(self):
+        """A sequence of two choices with each a variety of DO/TLVs"""
+        arr_seq = utils.DataObjectSequence('arr', sequence=[AM_DO_EF, SC_DO])
+        # input data
+        dec_in = [{'access_mode': ['update_erase', 'read_search_compare']}, {'control_reference_template':'PIN1'}]
+        # encode it once
+        encoded = arr_seq.encode(dec_in)
+        # decode again
+        re_decoded = arr_seq.decode(encoded)
+        self.assertEqual(dec_in, re_decoded[0])
+
 class DecTestCase(unittest.TestCase):
 	# TS33.501 Annex C.4 test keys
 	hnet_pubkey_profile_b = "0272DA71976234CE833A6907425867B82E074D44EF907DFB4B3E21C1C2256EBCD1" # ID 27 in test file

vpcd2smpp.py

@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+#
+# This program receive APDUs via the VPCD protocol of Frank Morgner's
+# virtualsmartcard, encrypts them with OTA (over the air) keys and
+# forwards them via SMPP to a SMSC (SMS service centre).
+#
+# In other words, you can use it as a poor man's OTA server, to enable
+# you to use unmodified application software with PC/SC support to talk
+# securely via OTA with a remote SMS card.
+#
+# This is very much a work in progress at this point.
+
+#######################################################################
+# twisted VPCD Library
+#######################################################################
+
+import logging
+import struct
+import abc
+from typing import Union, Optional
+from construct import Struct, Int8ub, Int16ub, If, Enum, Bytes, this, len_, Rebuild
+from twisted.internet.protocol import Protocol, ReconnectingClientFactory
+from pySim.utils import b2h, h2b
+
+logger = logging.getLogger(__name__)
+
+class VirtualCard(abc.ABC):
+    """Abstract base class for a virtual smart card."""
+    def __init__(self, atr: Union[str, bytes]):
+        if isinstance(atr, str):
+            atr = h2b(atr)
+        self.atr = atr
+
+    @abc.abstractmethod
+    def power_change(self, new_state: bool):
+        """Power the card on or off."""
+        pass
+
+    @abc.abstractmethod
+    def reset(self):
+        """Reset the card."""
+        pass
+
+    @abc.abstractmethod
+    def rx_c_apdu(self, apdu: bytes):
+        """Receive a C-APDU from the reader/application."""
+        pass
+
+    def tx_r_apdu(self, apdu: Union[str, bytes]):
+        if isinstance(apdu, str):
+            apdu = h2b(apdu)
+        logger.info("R-APDU: %s" % b2h(apdu))
+        self.protocol.send_data(apdu)
+
+class VpcdProtocolBase(Protocol):
+    # Prefixed couldn't be used as the this.length wouldn't be available in this case
+    construct = Struct('length'/Rebuild(Int16ub, len_(this.data) + len_(this.ctrl)),
+                       'data'/If(this.length > 1, Bytes(this.length)),
+                       'ctrl'/If(this.length == 1, Enum(Int8ub, off=0, on=1, reset=2, atr=4)))
+    def __init__(self, vcard: VirtualCard):
+        self.recvBuffer = b''
+        self.connectionCorrupted = False
+        self.pduReadTimer = None
+        self.pduReadTimerSecs = 10
+        self.callLater = reactor.callLater
+        self.on = False
+        self.vcard = vcard
+        self.vcard.protocol = self
+
+    def dataReceived(self, data: bytes):
+        """entry point where twisted tells us data was received."""
+        #logger.debug('Data received: %s' % b2h(data))
+        self.recvBuffer = self.recvBuffer + data
+        while True:
+            if self.connectionCorrupted:
+                return
+            msg = self.readMessage()
+            if msg is None:
+                break
+            self.endPDURead()
+            self.rawMessageReceived(msg)
+
+        if len(self.recvBuffer) > 0:
+            self.incompletePDURead()
+
+    def incompletePDURead(self):
+        """We have an incomplete PDU in readBuffer, schedule pduReadTimer"""
+        if self.pduReadTimer and self.pduReadTimer.active():
+            return
+        self.pduReadTimer = self.callLater(self.pduReadTimerSecs, self.onPDUReadTimeout)
+
+    def endPDURead(self):
+        """We completed reading a PDU, cancel the pduReadTimer."""
+        if self.pduReadTimer and self.pduReadTimer.active():
+            self.pduReadTimer.cancel()
+
+    def readMessage(self) -> Optional[bytes]:
+        """read an entire [raw] message."""
+        pduLen = self._getMessageLength()
+        if pduLen is None:
+            return None
+        return self._getMessage(pduLen)
+
+    def _getMessageLength(self) -> Optional[int]:
+        if len(self.recvBuffer) < 2:
+            return None
+        return struct.unpack('!H', self.recvBuffer[:2])[0]
+
+    def _getMessage(self, pduLen: int) -> Optional[bytes]:
+        if len(self.recvBuffer) < pduLen+2:
+            return None
+
+        message = self.recvBuffer[:pduLen+2]
+        self.recvBuffer = self.recvBuffer[pduLen+2:]
+        return message
+
+    def onPDUReadTimeout(self):
+        logger.error('PDU read timed out. Buffer is now considered corrupt')
+        #self.coruptDataReceived
+
+    def rawMessageReceived(self, message: bytes):
+        """Called once a complete binary vpcd message has been received."""
+        pdu = None
+        try:
+            pdu = VpcdProtocolBase.construct.parse(message)
+        except Exception as e:
+            logger.exception(e)
+            logger.critical('Received corrupt PDU %s' % b2h(message))
+            #self.corupDataRecvd()
+        else:
+            self.PDUReceived(pdu)
+
+    def PDUReceived(self, pdu):
+        logger.debug("Rx PDU: %s" % pdu)
+        if pdu['data']:
+            return self.on_rx_data(pdu)
+        else:
+            method = getattr(self, 'on_rx_' + pdu['ctrl'])
+            return method(pdu)
+
+    def on_rx_atr(self, pdu):
+        self.send_data(self.vcard.atr)
+
+    def on_rx_on(self, pdu):
+        if self.on:
+            return
+        else:
+            self.on = True
+            self.vcard.power_change(self.on)
+
+    def on_rx_reset(self, pdu):
+        self.vcard.reset()
+
+    def on_rx_off(self, pdu):
+        if not self.on:
+            return
+        else:
+            self.on = False
+            self.vcard.power_change(self.on)
+
+    def on_rx_data(self, pdu):
+        self.vcard.rx_c_apdu(pdu['data'])
+
+    def send_pdu(self, pdu):
+        logger.debug("Sending PDU: %s" % pdu)
+        encoded = VpcdProtocolBase.construct.build(pdu)
+        #logger.debug("Sending binary: %s" % b2h(encoded))
+        self.transport.write(encoded)
+
+    def send_data(self, data: Union[str, bytes]):
+        if isinstance(data, str):
+            data = h2b(data)
+        return self.send_pdu({'length': 0, 'ctrl': '', 'data': data})
+
+    def send_ctrl(self, ctrl: str):
+        return self.send_pdu({'length': 0, 'ctrl': ctrl, 'data': ''})
+
+
+class VpcdProtocolClient(VpcdProtocolBase):
+    pass
+
+
+class VpcdClientFactory(ReconnectingClientFactory):
+    def __init__(self, vcard_class: VirtualCard):
+        self.vcard_class = vcard_class
+
+    def startedConnecting(self, connector):
+        logger.debug('Started to connect')
+
+    def buildProtocol(self, addr):
+        logger.info('Connection established to %s' % addr)
+        self.resetDelay()
+        return VpcdProtocolClient(vcard = self.vcard_class())
+
+    def clientConnectionLost(self, connector, reason):
+        logger.warning('Connection lost (reason: %s)' % reason)
+        super().clientConnectionLost(connector, reason)
+
+    def clientConnectionFailed(self, connector, reason):
+        logger.warning('Connection failed (reason: %s)' % reason)
+        super().clientConnectionFailed(connector, reason)
+
+#######################################################################
+# Application
+#######################################################################
+
+from pprint import pprint as pp
+
+from twisted.internet.protocol import Protocol, ReconnectingClientFactory, ClientCreator
+from twisted.internet import reactor
+
+from smpp.twisted.client import SMPPClientTransceiver, SMPPClientService
+from smpp.twisted.protocol import SMPPClientProtocol
+from smpp.twisted.config import SMPPClientConfig
+from smpp.pdu.operations import SubmitSM, DeliverSM
+from smpp.pdu import pdu_types
+
+from pySim.ota import OtaKeyset, OtaDialectSms
+from pySim.utils import b2h, h2b
+
+
+class MyVcard(VirtualCard):
+    def __init__(self, **kwargs):
+        super().__init__(atr='3B9F96801FC78031A073BE21136743200718000001A5', **kwargs)
+        self.smpp_client = None
+        # KIC1 + KID1 of 8988211000000467285
+        KIC1 = h2b('D0FDA31990D8D64178601317191669B4')
+        KID1 = h2b('D24EB461799C5E035C77451FD9404463')
+        KIC3 = h2b('C21DD66ACAC13CB3BC8B331B24AFB57B')
+        KID3 = h2b('12110C78E678C25408233076AA033615')
+        self.ota_keyset = OtaKeyset(algo_crypt='triple_des_cbc2', kic_idx=3, kic=KIC3,
+                                    algo_auth='triple_des_cbc2', kid_idx=3, kid=KID3)
+        self.ota_dialect = OtaDialectSms()
+        self.tar = h2b('B00011')
+        self.spi = {'counter':'no_counter', 'ciphering':True, 'rc_cc_ds': 'cc', 'por_in_submit':False,
+                    'por_shall_be_ciphered':True, 'por_rc_cc_ds': 'cc', 'por': 'por_required'}
+
+    def ensure_smpp(self):
+        config = SMPPClientConfig(host='localhost', port=2775, username='test', password='test')
+        if self.smpp_client:
+            return
+        self.smpp_client = SMPPClientTransceiver(config, self.handleSmpp)
+        smpp = self.smpp_client.connectAndBind()
+        #self.smpp = ClientCreator(reactor, SMPPClientProtocol, config, self.handleSmpp)
+        #d = self.smpp.connectTCP(config.host, config.port)
+        #d = self.smpp.connectAndBind()
+        #d.addCallback(self.forwardToClient, self.smpp)
+
+    def power_change(self, new_state: bool):
+        if new_state:
+            logger.info("POWER ON")
+            self.ensure_smpp()
+        else:
+            logger.info("POWER OFF")
+
+    def reset(self):
+        logger.info("RESET")
+
+    def rx_c_apdu(self, apdu: bytes):
+        pp(self.smpp_client.smpp)
+        logger.info("C-APDU: %s" % b2h(apdu))
+        # translate to Secured OTA RFM
+        secured = self.ota_dialect.encode_cmd(self.ota_keyset, self.tar, self.spi, apdu=apdu)
+        # add user data header
+        tpdu = b'\x02\x70\x00' + secured
+        # send via SMPP
+        self.tx_sms_tpdu(tpdu)
+        #self.tx_r_apdu('9000')
+
+    def tx_sms_tpdu(self, tpdu: bytes):
+        """Send a SMS TPDU via SMPP SubmitSM."""
+        dcs = pdu_types.DataCoding(pdu_types.DataCodingScheme.DEFAULT,
+                                   pdu_types.DataCodingDefault.OCTET_UNSPECIFIED)
+        esm_class = pdu_types.EsmClass(pdu_types.EsmClassMode.DEFAULT, pdu_types.EsmClassType.DEFAULT,
+                                       gsmFeatures=[pdu_types.EsmClassGsmFeatures.UDHI_INDICATOR_SET])
+        submit = SubmitSM(source_addr='12',destination_addr='23', data_coding=dcs, esm_class=esm_class,
+                          protocol_id=0x7f, short_message=tpdu)
+        self.smpp_client.smpp.sendDataRequest(submit)
+
+    def handleSmpp(self, smpp, pdu):
+        #logger.info("Received SMPP %s" % pdu)
+        data = pdu.params['short_message']
+        #logger.info("Received SMS Data %s" % b2h(data))
+        r = self.ota_dialect.decode_resp(self.ota_keyset, self.spi, data)
+        logger.info("Decoded SMPP %s" % r)
+        self.tx_r_apdu(r['last_response_data'] + r['last_status_word'])
+
+
+if __name__ == '__main__':
+    import logging
+    logger = logging.getLogger(__name__)
+    import colorlog
+    log_format='%(log_color)s%(levelname)-8s%(reset)s %(name)s: %(message)s'
+    colorlog.basicConfig(level=logging.INFO, format = log_format)
+    logger = colorlog.getLogger()
+
+    from twisted.internet import reactor
+    host = 'localhost'
+    port = 35963
+    reactor.connectTCP(host, port, VpcdClientFactory(vcard_class=MyVcard))
+    reactor.run()