pySim-shell.py: make it work with cmd2 >= v2.4.0

In v2.3.0 both cmd2.{fg,bg} have been deprecated in favour of cmd2.{Fg,Bg}.
In v2.4.0 both cmd2.{fg,bg} have been removed.

See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md

Change-Id: I7ca95e85fc45ba66fd9ba6bea1fec2bae0e892c0

README.md

@@ -16,17 +16,17 @@ network, and want to issue your own SIM/USIM cards for that network.
 Homepage and Manual
 -------------------
 
-Please visit the [official homepage](https://osmocom.org/projects/pysim/wiki) for usage instructions, manual and examples.
+Please visit the [official homepage](https://osmocom.org/projects/pysim/wiki) for usage instructions, manual and examples.  The user manual can also be built locally from this source code by ``cd docs && make html latexpdf`` for HTML and PDF format, respectively.
 
 Git Repository
 --------------
 
 You can clone from the official Osmocom  git repository using
 ```
-git clone git://git.osmocom.org/pysim.git
+git clone https://gitea.osmocom.org/sim-card/pysim.git
 ```
 
-There is a cgit interface at <https://git.osmocom.org/pysim>
+There is a web interface at <https://gitea.osmocom.org/sim-card/pysim>.
 
 
 Installation
@@ -35,18 +35,26 @@ Installation
 Please install the following dependencies:
 
  - pyscard
- - serial
+ - pyserial
  - pytlv
  - cmd2 >= 1.3.0 but < 2.0.0
  - jsonpath-ng
- - construct
+ - construct >= 2.9.51
  - bidict
  - gsm0338
+ - pyyaml >= 5.1
+ - termcolor
+ - colorlog
 
 Example for Debian:
-```
-apt-get install python3-pyscard python3-serial python3-pip python3-yaml
-pip3 install -r requirements.txt
+```sh
+sudo apt-get install --no-install-recommends \
+	pcscd libpcsclite-dev \
+	python3 \
+	python3-setuptools \
+	python3-pyscard \
+	python3-pip
+pip3 install --user -r requirements.txt
 ```
 
 After installing all dependencies, the pySim applications ``pySim-read.py``, ``pySim-prog.py`` and ``pySim-shell.py`` may be started directly from the cloned repository.

contrib/jenkins.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -xe
 # jenkins build helper script for pysim.  This is how we build on jenkins.osmocom.org
 #
 # environment variables:
@@ -6,8 +6,6 @@
 # * PUBLISH: upload manuals after building if set to "1" (ignored without WITH_MANUALS = "1")
 #
 
-set -e
-
 if [ ! -d "./pysim-testdata/" ] ; then
 	echo "###############################################"
 	echo "Please call from pySim-prog top directory"
@@ -17,13 +15,7 @@ fi
 
 virtualenv -p python3 venv --system-site-packages
 . venv/bin/activate
-pip install pytlv
-pip install 'pyyaml>=5.1'
-pip install cmd2==1.5
-pip install jsonpath-ng
-pip install construct
-pip install bidict
-pip install gsm0338
+pip install -r requirements.txt
 
 # Execute automatically discovered unit tests first
 python -m unittest discover -v -s tests/
@@ -34,8 +26,8 @@ python -m unittest discover -v -s tests/
 # Ignore E0401: import-error
 #   pySim/utils.py:276: E0401: Unable to import 'Crypto.Cipher' (import-error)
 #   pySim/utils.py:277: E0401: Unable to import 'Crypto.Util.strxor' (import-error)
-pip install pylint
-python -m pylint --errors-only \
+pip install pylint==2.14.5 # FIXME: 2.15 is crashing, see OS#5668
+python -m pylint -j0 --errors-only \
 	--disable E1102 \
 	--disable E0401 \
 	--enable W0301 \

contrib/sim-rest-server.py

@@ -2,7 +2,7 @@
 
 # RESTful HTTP service for performing authentication against USIM cards
 #
-# (C) 2021 by Harald Welte <laforge@osmocom.org>
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,7 +21,7 @@ import json
 import sys
 import argparse
 
-from klein import run, route
+from klein import Klein
 
 from pySim.transport import ApduTracer
 from pySim.transport.pcsc import PcscSimLink
@@ -50,74 +50,97 @@ def connect_to_card(slot_nr:int):
 
     return tp, scc, card
 
+class ApiError:
+    def __init__(self, msg:str, sw=None):
+        self.msg = msg
+        self.sw = sw
 
-@route('/sim-auth-api/v1/slot/<int:slot>')
-def auth(request, slot):
-    """REST API endpoint for performing authentication against a USIM.
-       Expects a JSON body containing RAND and AUTN.
-       Returns a JSON body containing RES, CK, IK and Kc."""
-    try:
-        # there are two hex-string JSON parameters in the body: rand and autn
-        content = json.loads(request.content.read())
-        rand = content['rand']
-        autn = content['autn']
-    except:
-        request.setResponseCode(400)
-        return "Malformed Request"
+    def __str__(self):
+        d = {'error': {'message':self.msg}}
+        if self.sw:
+            d['error']['status_word'] = self.sw
+        return json.dumps(d)
 
-    try:
-        tp, scc, card = connect_to_card(slot)
-    except ReaderError:
-        request.setResponseCode(404)
-        return "Specified SIM Slot doesn't exist"
-    except ProtocolError:
-        request.setResponseCode(500)
-        return "Error"
-    except NoCardError:
+
+def set_headers(request):
+    request.setHeader('Content-Type', 'application/json')
+
+class SimRestServer:
+    app = Klein()
+
+    @app.handle_errors(NoCardError)
+    def no_card_error(self, request, failure):
+        set_headers(request)
         request.setResponseCode(410)
-        return "No SIM card inserted in slot"
+        return str(ApiError("No SIM card inserted in slot"))
+
+    @app.handle_errors(ReaderError)
+    def reader_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(404)
+        return str(ApiError("Reader Error: Specified SIM Slot doesn't exist"))
+
+    @app.handle_errors(ProtocolError)
+    def protocol_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(500)
+        return str(ApiError("Protocol Error: %s" % failure.value))
+
+    @app.handle_errors(SwMatchError)
+    def sw_match_error(self, request, failure):
+        set_headers(request)
+        request.setResponseCode(500)
+        sw = failure.value.sw_actual
+        if sw == '9862':
+            return str(ApiError("Card Authentication Error - Incorrect MAC", sw))
+        elif sw == '6982':
+            return str(ApiError("Security Status not satisfied - Card PIN enabled?", sw))
+        else:
+            return str(ApiError("Card Communication Error %s" % failure.value), sw)
+
+
+    @app.route('/sim-auth-api/v1/slot/<int:slot>')
+    def auth(self, request, slot):
+        """REST API endpoint for performing authentication against a USIM.
+           Expects a JSON body containing RAND and AUTN.
+           Returns a JSON body containing RES, CK, IK and Kc."""
+        try:
+            # there are two hex-string JSON parameters in the body: rand and autn
+            content = json.loads(request.content.read())
+            rand = content['rand']
+            autn = content['autn']
+        except:
+            set_headers(request)
+            request.setResponseCode(400)
+            return str(ApiError("Malformed Request"))
+
+        tp, scc, card = connect_to_card(slot)
 
-    try:
         card.select_adf_by_aid(adf='usim')
         res, sw = scc.authenticate(rand, autn)
-    except SwMatchError as e:
-        request.setResponseCode(500)
-        return "Communication Error %s" % e
 
-    tp.disconnect()
+        tp.disconnect()
 
-    return json.dumps(res, indent=4)
+        set_headers(request)
+        return json.dumps(res, indent=4)
 
-@route('/sim-info-api/v1/slot/<int:slot>')
-def info(request, slot):
-    """REST API endpoint for obtaining information about an USIM.
-    Expects empty body in request.
-    Returns a JSON body containing ICCID, IMSI."""
+    @app.route('/sim-info-api/v1/slot/<int:slot>')
+    def info(self, request, slot):
+        """REST API endpoint for obtaining information about an USIM.
+        Expects empty body in request.
+        Returns a JSON body containing ICCID, IMSI."""
 
-    try:
         tp, scc, card = connect_to_card(slot)
-    except ReaderError:
-        request.setResponseCode(404)
-        return "Specified SIM Slot doesn't exist"
-    except ProtocolError:
-        request.setResponseCode(500)
-        return "Error"
-    except NoCardError:
-        request.setResponseCode(410)
-        return "No SIM card inserted in slot"
 
-    try:
         card.select_adf_by_aid(adf='usim')
         iccid, sw = card.read_iccid()
         imsi, sw = card.read_imsi()
         res = {"imsi": imsi, "iccid": iccid }
-    except SwMatchError as e:
-        request.setResponseCode(500)
-        return "Communication Error %s" % e
 
-    tp.disconnect()
+        tp.disconnect()
 
-    return json.dumps(res, indent=4)
+        set_headers(request)
+        return json.dumps(res, indent=4)
 
 
 def main(argv):
@@ -128,7 +151,8 @@ def main(argv):
 
     args = parser.parse_args()
 
-    run(args.host, args.port)
+    srr = SimRestServer()
+    srr.app.run(args.host, args.port)
 
 if __name__ == "__main__":
     main(sys.argv)

pySim-shell.py

@@ -2,7 +2,7 @@
 
 # Interactive shell for working with SIM / UICC / USIM / ISIM cards
 #
-# (C) 2021 by Harald Welte <laforge@osmocom.org>
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,7 +23,7 @@ import json
 import traceback
 
 import cmd2
-from cmd2 import style, fg, bg
+from cmd2 import style, Fg
 from cmd2 import CommandSet, with_default_category, with_argparser
 import argparse
 
@@ -32,29 +32,26 @@ import sys
 from pathlib import Path
 from io import StringIO
 
-from pySim.ts_51_011 import EF, DF, EF_SST_map
-from pySim.ts_31_102 import EF_UST_map, EF_USIM_ADF_map
-from pySim.ts_31_103 import EF_IST_map, EF_ISIM_ADF_map
+from pprint import pprint as pp
 
 from pySim.exceptions import *
 from pySim.commands import SimCardCommands
-from pySim.transport import init_reader, ApduTracer, argparse_add_reader_args
+from pySim.transport import init_reader, ApduTracer, argparse_add_reader_args, ProactiveHandler
 from pySim.cards import card_detect, SimCard
-from pySim.utils import h2b, swap_nibbles, rpad, b2h, h2s, JsonEncoder, bertlv_parse_one
-from pySim.utils import dec_st, sanitize_pin_adm, tabulate_str_list, is_hex, boxed_heading_str
+from pySim.utils import h2b, swap_nibbles, rpad, b2h, JsonEncoder, bertlv_parse_one, sw_match
+from pySim.utils import sanitize_pin_adm, tabulate_str_list, boxed_heading_str, Hexstr
 from pySim.card_handler import CardHandler, CardHandlerAuto
 
-from pySim.filesystem import CardMF, RuntimeState, CardDF, CardADF, CardModel
+from pySim.filesystem import RuntimeState, CardDF, CardADF, CardModel
 from pySim.profile import CardProfile
-from pySim.ts_51_011 import CardProfileSIM, DF_TELECOM, DF_GSM
 from pySim.ts_102_221 import CardProfileUICC
-from pySim.ts_102_221 import CardProfileUICCSIM
 from pySim.ts_102_222 import Ts102222Commands
 from pySim.ts_31_102 import CardApplicationUSIM
 from pySim.ts_31_103 import CardApplicationISIM
 from pySim.ara_m import CardApplicationARAM
 from pySim.global_platform import CardApplicationISD
 from pySim.gsm_r import DF_EIRENE
+from pySim.cat import ProactiveCommand
 
 # we need to import this module so that the SysmocomSJA2 sub-class of
 # CardModel is created, which will add the ATR-based matching and
@@ -83,15 +80,25 @@ def init_card(sl):
         print("Card not readable!")
         return None, None
 
+    generic_card = False
     card = card_detect("auto", scc)
     if card is None:
         print("Warning: Could not detect card type - assuming a generic card type...")
         card = SimCard(scc)
+        generic_card = True
 
     profile = CardProfile.pick(scc)
     if profile is None:
         print("Unsupported card type!")
-        return None, None
+        return None, card
+
+    # ETSI TS 102 221, Table 9.3 specifies a default for the PIN key
+    # references, however card manufactures may still decide to pick an
+    # arbitrary key reference. In case we run on a generic card class that is
+    # detected as an UICC, we will pick the key reference that is officially
+    # specified.
+    if generic_card and isinstance(profile, CardProfileUICC):
+        card._adm_chv_num = 0x0A
 
     print("Info: Card is of type: %s" % str(profile))
 
@@ -127,26 +134,27 @@ class PysimApp(cmd2.Cmd):
 
     def __init__(self, card, rs, sl, ch, script=None):
         super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
-                         use_ipython=True, auto_load_commands=False, startup_script=script)
-        self.intro = style('Welcome to pySim-shell!', fg=fg.red)
+                         auto_load_commands=False, startup_script=script)
+        self.intro = style('Welcome to pySim-shell!', fg=Fg.RED)
         self.default_category = 'pySim-shell built-in commands'
         self.card = None
         self.rs = None
-        self.py_locals = {'card': self.card, 'rs': self.rs}
+        self.lchan = None
+        self.py_locals = {'card': self.card, 'rs': self.rs, 'lchan': self.lchan}
         self.sl = sl
         self.ch = ch
 
         self.numeric_path = False
-        self.add_settable(cmd2.Settable('numeric_path', bool, 'Print File IDs instead of names',
+        self.add_settable(cmd2.Settable('numeric_path', bool, 'Print File IDs instead of names', self,
                                         onchange_cb=self._onchange_numeric_path))
         self.conserve_write = True
-        self.add_settable(cmd2.Settable('conserve_write', bool, 'Read and compare before write',
+        self.add_settable(cmd2.Settable('conserve_write', bool, 'Read and compare before write', self,
                                         onchange_cb=self._onchange_conserve_write))
         self.json_pretty_print = True
         self.add_settable(cmd2.Settable('json_pretty_print',
-                          bool, 'Pretty-Print JSON output'))
+                          bool, 'Pretty-Print JSON output', self))
         self.apdu_trace = False
-        self.add_settable(cmd2.Settable('apdu_trace', bool, 'Trace and display APDUs exchanged with card',
+        self.add_settable(cmd2.Settable('apdu_trace', bool, 'Trace and display APDUs exchanged with card', self,
                                         onchange_cb=self._onchange_apdu_trace))
 
         self.equip(card, rs)
@@ -161,7 +169,8 @@ class PysimApp(cmd2.Cmd):
 
         # Unequip everything from pySim-shell that would not work in unequipped state
         if self.rs:
-            self.rs.unregister_cmds(self)
+            lchan = self.rs.lchan[0]
+            lchan.unregister_cmds(self)
         for cmds in [Iso7816Commands, PySimCommands]:
             cmd_set = self.find_commandsets(cmds)
             if cmd_set:
@@ -173,6 +182,7 @@ class PysimApp(cmd2.Cmd):
         # When a card object and a runtime state is present, (re)equip pySim-shell with everything that is
         # needed to operate on cards.
         if self.card and self.rs:
+            self.lchan = self.rs.lchan[0]
             self._onchange_conserve_write(
                 'conserve_write', False, self.conserve_write)
             self._onchange_apdu_trace('apdu_trace', False, self.apdu_trace)
@@ -180,7 +190,7 @@ class PysimApp(cmd2.Cmd):
             self.register_command_set(Ts102222Commands())
             self.register_command_set(PySimCommands())
             self.iccid, sw = self.card.read_iccid()
-            rs.select('MF', self)
+            self.lchan.select('MF', self)
             rc = True
         else:
             self.poutput("pySim-shell not equipped!")
@@ -219,12 +229,14 @@ class PysimApp(cmd2.Cmd):
             self.cmd2.poutput("<- %s: %s" % (sw, resp))
 
     def update_prompt(self):
-        if self.rs:
-            path_list = self.rs.selected_file.fully_qualified_path(
-                not self.numeric_path)
-            self.prompt = 'pySIM-shell (%s)> ' % ('/'.join(path_list))
+        if self.lchan:
+            path_str = self.lchan.selected_file.fully_qualified_path_str(not self.numeric_path)
+            self.prompt = 'pySIM-shell (%s)> ' % (path_str)
         else:
-            self.prompt = 'pySIM-shell (no card)> '
+            if self.card:
+                self.prompt = 'pySIM-shell (no card profile)> '
+            else:
+                self.prompt = 'pySIM-shell (no card)> '
 
     @cmd2.with_category(CUSTOM_CATEGORY)
     def do_intro(self, _):
@@ -241,6 +253,25 @@ class PysimApp(cmd2.Cmd):
         rs, card = init_card(sl)
         self.equip(card, rs)
 
+    apdu_cmd_parser = argparse.ArgumentParser()
+    apdu_cmd_parser.add_argument('APDU', type=str, help='APDU as hex string')
+    apdu_cmd_parser.add_argument('--expect-sw', help='expect a specified status word', type=str, default=None)
+
+    @cmd2.with_argparser(apdu_cmd_parser)
+    def do_apdu(self, opts):
+        """Send a raw APDU to the card, and print SW + Response.
+        DANGEROUS: pySim-shell will not know any card state changes, and
+        not continue to work as expected if you e.g. select a different
+        file."""
+        data, sw = self.card._scc._tp.send_apdu(opts.APDU)
+        if data:
+            self.poutput("SW: %s, RESP: %s" % (sw, data))
+        else:
+            self.poutput("SW: %s" % sw)
+        if opts.expect_sw:
+            if not sw_match(sw, opts.expect_sw):
+                raise SwMatchError(sw, opts.expect_sw)
+
     class InterceptStderr(list):
         def __init__(self):
             self._stderr_backup = sys.stderr
@@ -256,23 +287,23 @@ class PysimApp(cmd2.Cmd):
             sys.stderr = self._stderr_backup
 
     def _show_failure_sign(self):
-        self.poutput(style("  +-------------+", fg=fg.bright_red))
-        self.poutput(style("  +   ##   ##   +", fg=fg.bright_red))
-        self.poutput(style("  +    ## ##    +", fg=fg.bright_red))
-        self.poutput(style("  +     ###     +", fg=fg.bright_red))
-        self.poutput(style("  +    ## ##    +", fg=fg.bright_red))
-        self.poutput(style("  +   ##   ##   +", fg=fg.bright_red))
-        self.poutput(style("  +-------------+", fg=fg.bright_red))
+        self.poutput(style("  +-------------+", fg=Fg.LIGHT_RED))
+        self.poutput(style("  +   ##   ##   +", fg=Fg.LIGHT_RED))
+        self.poutput(style("  +    ## ##    +", fg=Fg.LIGHT_RED))
+        self.poutput(style("  +     ###     +", fg=Fg.LIGHT_RED))
+        self.poutput(style("  +    ## ##    +", fg=Fg.LIGHT_RED))
+        self.poutput(style("  +   ##   ##   +", fg=Fg.LIGHT_RED))
+        self.poutput(style("  +-------------+", fg=Fg.LIGHT_RED))
         self.poutput("")
 
     def _show_success_sign(self):
-        self.poutput(style("  +-------------+", fg=fg.bright_green))
-        self.poutput(style("  +          ## +", fg=fg.bright_green))
-        self.poutput(style("  +         ##  +", fg=fg.bright_green))
-        self.poutput(style("  +  #    ##    +", fg=fg.bright_green))
-        self.poutput(style("  +   ## #      +", fg=fg.bright_green))
-        self.poutput(style("  +    ##       +", fg=fg.bright_green))
-        self.poutput(style("  +-------------+", fg=fg.bright_green))
+        self.poutput(style("  +-------------+", fg=Fg.LIGHT_GREEN))
+        self.poutput(style("  +          ## +", fg=Fg.LIGHT_GREEN))
+        self.poutput(style("  +         ##  +", fg=Fg.LIGHT_GREEN))
+        self.poutput(style("  +  #    ##    +", fg=Fg.LIGHT_GREEN))
+        self.poutput(style("  +   ## #      +", fg=Fg.LIGHT_GREEN))
+        self.poutput(style("  +    ##       +", fg=Fg.LIGHT_GREEN))
+        self.poutput(style("  +-------------+", fg=Fg.LIGHT_GREEN))
         self.poutput("")
 
     def _process_card(self, first, script_path):
@@ -419,6 +450,11 @@ class PysimApp(cmd2.Cmd):
         """Echo (print) a string on the console"""
         self.poutput(opts.string)
 
+    @cmd2.with_category(CUSTOM_CATEGORY)
+    def do_version(self, opts):
+        """Print the pySim software version."""
+        import pkg_resources
+        self.poutput(pkg_resources.get_distribution('pySim'))
 
 @with_default_category('pySim Commands')
 class PySimCommands(CommandSet):
@@ -451,22 +487,28 @@ class PySimCommands(CommandSet):
         else:
             flags = ['PARENT', 'SELF', 'FNAMES', 'ANAMES']
         selectables = list(
-            self._cmd.rs.selected_file.get_selectable_names(flags=flags))
+            self._cmd.lchan.selected_file.get_selectable_names(flags=flags))
         directory_str = tabulate_str_list(
             selectables, width=79, hspace=2, lspace=1, align_left=True)
-        path_list = self._cmd.rs.selected_file.fully_qualified_path(True)
-        self._cmd.poutput('/'.join(path_list))
-        path_list = self._cmd.rs.selected_file.fully_qualified_path(False)
-        self._cmd.poutput('/'.join(path_list))
+        path = self._cmd.lchan.selected_file.fully_qualified_path_str(True)
+        self._cmd.poutput(path)
+        path = self._cmd.lchan.selected_file.fully_qualified_path_str(False)
+        self._cmd.poutput(path)
         self._cmd.poutput(directory_str)
         self._cmd.poutput("%d files" % len(selectables))
 
-    def walk(self, indent=0, action=None, context=None, as_json=False):
+    def walk(self, indent=0, action_ef=None, action_df=None, context=None, **kwargs):
         """Recursively walk through the file system, starting at the currently selected DF"""
-        files = self._cmd.rs.selected_file.get_selectables(
+
+        if isinstance(self._cmd.lchan.selected_file, CardDF):
+            if action_df:
+                action_df(context, opts)
+
+        files = self._cmd.lchan.selected_file.get_selectables(
             flags=['FNAMES', 'ANAMES'])
         for f in files:
-            if not action:
+            # special case: When no action is performed, just output a directory
+            if not action_ef and not action_df:
                 output_str = "  " * indent + str(f) + (" " * 250)
                 output_str = output_str[0:25]
                 if isinstance(files[f], CardADF):
@@ -479,12 +521,12 @@ class PySimCommands(CommandSet):
             if isinstance(files[f], CardDF):
                 skip_df = False
                 try:
-                    fcp_dec = self._cmd.rs.select(f, self._cmd)
+                    fcp_dec = self._cmd.lchan.select(f, self._cmd)
                 except Exception as e:
                     skip_df = True
-                    df = self._cmd.rs.selected_file
-                    df_path_list = df.fully_qualified_path(True)
-                    df_skip_reason_str = '/'.join(df_path_list) + \
+                    df = self._cmd.lchan.selected_file
+                    df_path = df.fully_qualified_path_str(True)
+                    df_skip_reason_str = df_path + \
                         "/" + str(f) + ", " + str(e)
                     if context:
                         context['DF_SKIP'] += 1
@@ -493,71 +535,75 @@ class PySimCommands(CommandSet):
                 # If the DF was skipped, we never have entered the directory
                 # below, so we must not move up.
                 if skip_df == False:
-                    self.walk(indent + 1, action, context, as_json)
-                    fcp_dec = self._cmd.rs.select("..", self._cmd)
+                    self.walk(indent + 1, action_ef, action_df, context, **kwargs)
+                    fcp_dec = self._cmd.lchan.select("..", self._cmd)
 
-            elif action:
-                df_before_action = self._cmd.rs.selected_file
-                action(f, context, as_json)
+            elif action_ef:
+                df_before_action = self._cmd.lchan.selected_file
+                action_ef(f, context, **kwargs)
                 # When walking through the file system tree the action must not
                 # always restore the currently selected file to the file that
                 # was selected before executing the action() callback.
-                if df_before_action != self._cmd.rs.selected_file:
+                if df_before_action != self._cmd.lchan.selected_file:
                     raise RuntimeError("inconsistent walk, %s is currently selected but expecting %s to be selected"
-                                       % (str(self._cmd.rs.selected_file), str(df_before_action)))
+                                       % (str(self._cmd.lchan.selected_file), str(df_before_action)))
 
     def do_tree(self, opts):
         """Display a filesystem-tree with all selectable files"""
         self.walk()
 
-    def export(self, filename, context, as_json=False):
-        """ Select and export a single file """
+    def export_ef(self, filename, context, as_json):
+        """ Select and export a single elementary file (EF) """
         context['COUNT'] += 1
-        df = self._cmd.rs.selected_file
+        df = self._cmd.lchan.selected_file
 
+	# The currently selected file (not the file we are going to export)
+	# must always be an ADF or DF. From this starting point we select
+	# the EF we want to export. To maintain consistency we will then
+	# select the current DF again (see comment below).
         if not isinstance(df, CardDF):
             raise RuntimeError(
                 "currently selected file %s is not a DF or ADF" % str(df))
 
         df_path_list = df.fully_qualified_path(True)
-        df_path_list_fid = df.fully_qualified_path(False)
+        df_path = df.fully_qualified_path_str(True)
+        df_path_fid = df.fully_qualified_path_str(False)
 
-        file_str = '/'.join(df_path_list) + "/" + str(filename)
+        file_str = df_path + "/" + str(filename)
         self._cmd.poutput(boxed_heading_str(file_str))
 
-        self._cmd.poutput("# directory: %s (%s)" %
-                          ('/'.join(df_path_list), '/'.join(df_path_list_fid)))
+        self._cmd.poutput("# directory: %s (%s)" % (df_path, df_path_fid))
         try:
-            fcp_dec = self._cmd.rs.select(filename, self._cmd)
+            fcp_dec = self._cmd.lchan.select(filename, self._cmd)
             self._cmd.poutput("# file: %s (%s)" % (
-                self._cmd.rs.selected_file.name, self._cmd.rs.selected_file.fid))
+                self._cmd.lchan.selected_file.name, self._cmd.lchan.selected_file.fid))
 
-            structure = self._cmd.rs.selected_file_structure()
+            structure = self._cmd.lchan.selected_file_structure()
             self._cmd.poutput("# structure: %s" % str(structure))
-            self._cmd.poutput("# RAW FCP Template: %s" % str(self._cmd.rs.selected_file_fcp_hex))
-            self._cmd.poutput("# Decoded FCP Template: %s" % str(self._cmd.rs.selected_file_fcp))
+            self._cmd.poutput("# RAW FCP Template: %s" % str(self._cmd.lchan.selected_file_fcp_hex))
+            self._cmd.poutput("# Decoded FCP Template: %s" % str(self._cmd.lchan.selected_file_fcp))
 
             for f in df_path_list:
                 self._cmd.poutput("select " + str(f))
-            self._cmd.poutput("select " + self._cmd.rs.selected_file.name)
+            self._cmd.poutput("select " + self._cmd.lchan.selected_file.name)
 
             if structure == 'transparent':
                 if as_json:
-                    result = self._cmd.rs.read_binary_dec()
+                    result = self._cmd.lchan.read_binary_dec()
                     self._cmd.poutput("update_binary_decoded '%s'" % json.dumps(result[0], cls=JsonEncoder))
                 else:
-                    result = self._cmd.rs.read_binary()
+                    result = self._cmd.lchan.read_binary()
                     self._cmd.poutput("update_binary " + str(result[0]))
             elif structure == 'cyclic' or structure == 'linear_fixed':
                 # Use number of records specified in select response
-                num_of_rec = self._cmd.rs.selected_file_num_of_rec()
+                num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
                 if num_of_rec:
                     for r in range(1, num_of_rec + 1):
                         if as_json:
-                            result = self._cmd.rs.read_record_dec(r)
+                            result = self._cmd.lchan.read_record_dec(r)
                             self._cmd.poutput("update_record_decoded %d '%s'" % (r, json.dumps(result[0], cls=JsonEncoder)))
                         else:
-                            result = self._cmd.rs.read_record(r)
+                            result = self._cmd.lchan.read_record(r)
                             self._cmd.poutput("update_record %d %s" % (r, str(result[0])))
 
                 # When the select response does not return the number of records, read until we hit the
@@ -567,10 +613,10 @@ class PySimCommands(CommandSet):
                     while True:
                         try:
                             if as_json:
-                                result = self._cmd.rs.read_record_dec(r)
+                                result = self._cmd.lchan.read_record_dec(r)
                                 self._cmd.poutput("update_record_decoded %d '%s'" % (r, json.dumps(result[0], cls=JsonEncoder)))
                             else:
-                                result = self._cmd.rs.read_record(r)
+                                result = self._cmd.lchan.read_record(r)
                                 self._cmd.poutput("update_record %d %s" % (r, str(result[0])))
                         except SwMatchError as e:
                             # We are past the last valid record - stop
@@ -581,17 +627,16 @@ class PySimCommands(CommandSet):
                                 raise e
                         r = r + 1
             elif structure == 'ber_tlv':
-                tags = self._cmd.rs.retrieve_tags()
+                tags = self._cmd.lchan.retrieve_tags()
                 for t in tags:
-                    result = self._cmd.rs.retrieve_data(t)
+                    result = self._cmd.lchan.retrieve_data(t)
                     (tag, l, val, remainer) = bertlv_parse_one(h2b(result[0]))
                     self._cmd.poutput("set_data 0x%02x %s" % (t, b2h(val)))
             else:
                 raise RuntimeError(
                     'Unsupported structure "%s" of file "%s"' % (structure, filename))
         except Exception as e:
-            bad_file_str = '/'.join(df_path_list) + \
-                "/" + str(filename) + ", " + str(e)
+            bad_file_str = df_path + "/" + str(filename) + ", " + str(e)
             self._cmd.poutput("# bad file: %s" % bad_file_str)
             context['ERR'] += 1
             context['BAD'].append(bad_file_str)
@@ -599,8 +644,8 @@ class PySimCommands(CommandSet):
         # When reading the file is done, make sure the parent file is
         # selected again. This will be the usual case, however we need
         # to check before since we must not select the same DF twice
-        if df != self._cmd.rs.selected_file:
-            self._cmd.rs.select(df.fid or df.aid, self._cmd)
+        if df != self._cmd.lchan.selected_file:
+            self._cmd.lchan.select(df.fid or df.aid, self._cmd)
 
         self._cmd.poutput("#")
 
@@ -615,10 +660,18 @@ class PySimCommands(CommandSet):
         """Export files to script that can be imported back later"""
         context = {'ERR': 0, 'COUNT': 0, 'BAD': [],
                    'DF_SKIP': 0, 'DF_SKIP_REASON': []}
+        kwargs_export = {'as_json': opts.json}
+        exception_str_add = ""
+
         if opts.filename:
-            self.export(opts.filename, context, opts.json)
+            self.export_ef(opts.filename, context, **kwargs_export)
         else:
-            self.walk(0, self.export, context, opts.json)
+            try:
+                self.walk(0, self.export_ef, None, context, **kwargs_export)
+            except Exception as e:
+                print("# Stopping early here due to exception: " + str(e))
+                print("#")
+                exception_str_add = ", also had to stop early due to exception:" + str(e)
 
         self._cmd.poutput(boxed_heading_str("Export summary"))
 
@@ -633,24 +686,24 @@ class PySimCommands(CommandSet):
             self._cmd.poutput("#  " + b)
 
         if context['ERR'] and context['DF_SKIP']:
-            raise RuntimeError("unable to export %i elementary file(s) and %i dedicated file(s)" % (
-                context['ERR'], context['DF_SKIP']))
+            raise RuntimeError("unable to export %i elementary file(s) and %i dedicated file(s)%s" % (
+                    context['ERR'], context['DF_SKIP'], exception_str_add))
         elif context['ERR']:
             raise RuntimeError(
-                "unable to export %i elementary file(s)" % context['ERR'])
+                    "unable to export %i elementary file(s)%s" % (context['ERR'], exception_str_add))
         elif context['DF_SKIP']:
             raise RuntimeError(
-                "unable to export %i dedicated files(s)" % context['ERR'])
+                    "unable to export %i dedicated files(s)%s" % (context['ERR'], exception_str_add))
 
     def do_reset(self, opts):
         """Reset the Card."""
-        atr = self._cmd.rs.reset(self._cmd)
+        atr = self._cmd.lchan.reset(self._cmd)
         self._cmd.poutput('Card ATR: %s' % atr)
         self._cmd.update_prompt()
 
     def do_desc(self, opts):
         """Display human readable file description for the currently selected file"""
-        desc = self._cmd.rs.selected_file.desc
+        desc = self._cmd.lchan.selected_file.desc
         if desc:
             self._cmd.poutput(desc)
         else:
@@ -678,18 +731,6 @@ class PySimCommands(CommandSet):
         else:
             raise ValueError("error: cannot authenticate, no adm-pin!")
 
-    apdu_cmd_parser = argparse.ArgumentParser()
-    apdu_cmd_parser.add_argument('APDU', type=str, help='APDU as hex string')
-
-    @cmd2.with_argparser(apdu_cmd_parser)
-    def do_apdu(self, opts):
-        """Send a raw APDU to the card, and print SW + Response.
-        DANGEROUS: pySim-shell will not know any card state changes, and
-        not continue to work as expected if you e.g. select a different
-        file."""
-        data, sw = self._cmd.card._scc._tp.send_apdu(opts.APDU)
-        self._cmd.poutput("SW: %s %s, RESP: %s" % (sw, self._cmd.rs.interpret_sw(sw), data))
-
 
 @with_default_category('ISO7816 Commands')
 class Iso7816Commands(CommandSet):
@@ -699,21 +740,19 @@ class Iso7816Commands(CommandSet):
     def do_select(self, opts):
         """SELECT a File (ADF/DF/EF)"""
         if len(opts.arg_list) == 0:
-            path_list = self._cmd.rs.selected_file.fully_qualified_path(True)
-            path_list_fid = self._cmd.rs.selected_file.fully_qualified_path(
-                False)
-            self._cmd.poutput("currently selected file: " +
-                              '/'.join(path_list) + " (" + '/'.join(path_list_fid) + ")")
+            path = self._cmd.lchan.selected_file.fully_qualified_path_str(True)
+            path_fid = self._cmd.lchan.selected_file.fully_qualified_path_str(False)
+            self._cmd.poutput("currently selected file: %s (%s)" % (path, path_fid))
             return
 
         path = opts.arg_list[0]
-        fcp_dec = self._cmd.rs.select(path, self._cmd)
+        fcp_dec = self._cmd.lchan.select(path, self._cmd)
         self._cmd.update_prompt()
         self._cmd.poutput_json(fcp_dec)
 
     def complete_select(self, text, line, begidx, endidx) -> List[str]:
         """Command Line tab completion for SELECT"""
-        index_dict = {1: self._cmd.rs.selected_file.get_selectable_names()}
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
         return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
 
     def get_code(self, code):
@@ -819,11 +858,11 @@ class Iso7816Commands(CommandSet):
     def do_activate_file(self, opts):
         """Activate the specified EF. This used to be called REHABILITATE in TS 11.11 for classic
         SIM.  You need to specify the name or FID of the file to activate."""
-        (data, sw) = self._cmd.rs.activate_file(opts.NAME)
+        (data, sw) = self._cmd.lchan.activate_file(opts.NAME)
 
     def complete_activate_file(self, text, line, begidx, endidx) -> List[str]:
         """Command Line tab completion for ACTIVATE FILE"""
-        index_dict = {1: self._cmd.rs.selected_file.get_selectable_names()}
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
         return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
 
     open_chan_parser = argparse.ArgumentParser()
@@ -848,7 +887,7 @@ class Iso7816Commands(CommandSet):
 
     def do_status(self, opts):
         """Perform the STATUS command."""
-        fcp_dec = self._cmd.rs.status()
+        fcp_dec = self._cmd.lchan.status()
         self._cmd.poutput_json(fcp_dec)
 
     suspend_uicc_parser = argparse.ArgumentParser()
@@ -866,6 +905,13 @@ class Iso7816Commands(CommandSet):
         self._cmd.poutput(
             'Negotiated Duration: %u secs, Token: %s, SW: %s' % (duration, token, sw))
 
+class Proact(ProactiveHandler):
+    def receive_fetch(self, pcmd: ProactiveCommand):
+        # print its parsed representation
+        print(pcmd.decoded)
+        # TODO: implement the basics, such as SMS Sending, ...
+
+
 
 option_parser = argparse.ArgumentParser(prog='pySim-shell', description='interactive SIM card shell',
                                         formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -906,7 +952,7 @@ if __name__ == '__main__':
         card_key_provider_register(CardKeyProviderCsv(csv_default))
 
     # Init card reader driver
-    sl = init_reader(opts)
+    sl = init_reader(opts, proactive_handler = Proact())
     if sl is None:
         exit(1)
 
@@ -935,7 +981,7 @@ if __name__ == '__main__':
             " it should also be noted that some readers may behave strangely when no card")
         print(" is inserted.)")
         print("")
-        app = PysimApp(None, None, sl, ch, opts.script)
+        app = PysimApp(card, None, sl, ch, opts.script)
 
     # If the user supplies an ADM PIN at via commandline args authenticate
     # immediately so that the user does not have to use the shell commands

pySim-trace.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+
+import sys
+import logging, colorlog
+import argparse
+from pprint import pprint as pp
+
+from pySim.apdu import *
+from pySim.filesystem import RuntimeState
+
+from pySim.cards import UsimCard
+from pySim.commands import SimCardCommands
+from pySim.profile import CardProfile
+from pySim.ts_102_221 import CardProfileUICCSIM
+from pySim.ts_31_102 import CardApplicationUSIM
+from pySim.ts_31_103 import CardApplicationISIM
+from pySim.transport import LinkBase
+
+from pySim.apdu_source.gsmtap import GsmtapApduSource
+from pySim.apdu_source.pyshark_rspro import PysharkRsproPcap, PysharkRsproLive
+
+from pySim.apdu.ts_102_221 import UiccSelect, UiccStatus
+
+log_format='%(log_color)s%(levelname)-8s%(reset)s %(name)s: %(message)s'
+colorlog.basicConfig(level=logging.INFO, format = log_format)
+logger = colorlog.getLogger()
+
+# merge all of the command sets into one global set. This will override instructions,
+# the one from the 'last' set in the addition below will prevail.
+from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
+from pySim.apdu.global_platform import ApduCommands as GpApduCommands
+ApduCommands = UiccApduCommands + UsimApduCommands #+ GpApduCommands
+
+
+class DummySimLink(LinkBase):
+    """A dummy implementation of the LinkBase abstract base class.  Currently required
+    as the UsimCard doesn't work without SimCardCommands, which in turn require
+    a LinkBase implementation talking to a card.
+
+    In the tracer, we don't actually talk to any card, so we simply drop everything
+    and claim it is successful.
+
+    The UsimCard / SimCardCommands should be refactored to make this obsolete later."""
+    def __init__(self, debug: bool = False, **kwargs):
+        super().__init__(**kwargs)
+        self._debug = debug
+        self._atr = h2i('3B9F96801F878031E073FE211B674A4C753034054BA9')
+
+    def _send_apdu_raw(self, pdu):
+        #print("DummySimLink-apdu: %s" % pdu)
+        return [], '9000'
+
+    def connect(self):
+        pass
+
+    def disconnect(self):
+        pass
+
+    def reset_card(self):
+        return 1
+
+    def get_atr(self):
+        return self._atr
+
+    def wait_for_card(self):
+        pass
+
+
+class Tracer:
+    def __init__(self, **kwargs):
+        # we assume a generic SIM + UICC + USIM + ISIM card
+        profile = CardProfileUICCSIM()
+        profile.add_application(CardApplicationUSIM())
+        profile.add_application(CardApplicationISIM())
+        scc = SimCardCommands(transport=DummySimLink())
+        card = UsimCard(scc)
+        self.rs = RuntimeState(card, profile)
+        # APDU Decoder
+        self.ad = ApduDecoder(ApduCommands)
+        # parameters
+        self.suppress_status = kwargs.get('suppress_status', True)
+        self.suppress_select = kwargs.get('suppress_select', True)
+        self.source = kwargs.get('source', None)
+
+    def format_capdu(self, inst: ApduCommand):
+        """Output a single decoded + processed ApduCommand."""
+        print("%02u %-16s %-35s %-8s %s %s" % (inst.lchan_nr, inst._name, inst.path_str, inst.col_id, inst.col_sw, inst.processed))
+        print("===============================")
+
+    def main(self):
+        """Main loop of tracer: Iterates over all Apdu received from source."""
+        while True:
+            # obtain the next APDU from the source (blocking read)
+            apdu = self.source.read()
+            #print(apdu)
+
+            if isinstance(apdu, CardReset):
+                self.rs.reset()
+                continue
+
+            # ask ApduDecoder to look-up (INS,CLA) + instantiate an ApduCommand derived
+            # class like 'UiccSelect'
+            inst = self.ad.input(apdu)
+            # process the APDU (may modify the RuntimeState)
+            inst.process(self.rs)
+
+            # Avoid cluttering the log with too much verbosity
+            if self.suppress_select and isinstance(inst, UiccSelect):
+                continue
+            if self.suppress_status and isinstance(inst, UiccStatus):
+                continue
+            #print(inst)
+            self.format_capdu(inst)
+
+option_parser = argparse.ArgumentParser(prog='pySim-trace', description='Osmocom pySim high-level SIM card trace decoder',
+                                        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+global_group = option_parser.add_argument_group('General Options')
+global_group.add_argument('--no-suppress-select', action='store_false', dest='suppress_select',
+                          help="Don't suppress displaying SELECT APDUs")
+global_group.add_argument('--no-suppress-status', action='store_false', dest='suppress_status',
+                          help="Don't suppress displaying STATUS APDUs")
+
+subparsers = option_parser.add_subparsers(help='APDU Source', dest='source', required=True)
+
+parser_gsmtap = subparsers.add_parser('gsmtap-udp', help='Live capture of GSMTAP-SIM on UDP port')
+parser_gsmtap.add_argument('-i', '--bind-ip', default='127.0.0.1',
+                           help='Local IP address to which to bind the UDP port')
+parser_gsmtap.add_argument('-p', '--bind-port', default=4729,
+                           help='Local UDP port')
+
+parser_rspro_pyshark_pcap = subparsers.add_parser('rspro-pyshark-pcap', help="""
+    PCAP file containing RSPRO (osmo-remsim) communication; processed via pyshark.
+    REQUIRES OSMOCOM PATCHED WIRESHARK!""")
+parser_rspro_pyshark_pcap.add_argument('-f', '--pcap-file', required=True,
+                                       help='Name of the PCAP[ng] file to be read')
+
+parser_rspro_pyshark_live = subparsers.add_parser('rspro-pyshark-live', help="""
+    Live capture of RSPRO (osmo-remsim) communication; processed via pyshark.
+    REQUIRES OSMOCOM PATCHED WIRESHARK!""")
+parser_rspro_pyshark_live.add_argument('-i', '--interface', required=True,
+                                       help='Name of the network interface to capture on')
+
+if __name__ == '__main__':
+
+    opts = option_parser.parse_args()
+
+    logger.info('Opening source %s...' % opts.source)
+    if opts.source == 'gsmtap-udp':
+        s = GsmtapApduSource(opts.bind_ip, opts.bind_port)
+    elif opts.source == 'rspro-pyshark-pcap':
+        s = PysharkRsproPcap(opts.pcap_file)
+    elif opts.source == 'rspro-pyshark-live':
+        s = PysharkRsproLive(opts.interface)
+
+    tracer = Tracer(source=s, suppress_status=opts.suppress_status, suppress_select=opts.suppress_select)
+    logger.info('Entering main loop...')
+    tracer.main()
+

pySim/apdu/__init__.py

@@ -0,0 +1,446 @@
+# coding=utf-8
+"""APDU (and TPDU) parser for UICC/USIM/ISIM cards.
+
+The File (and its classes) represent the structure / hierarchy
+of the APDUs as seen in SIM/UICC/SIM/ISIM cards.  The primary use case
+is to perform a meaningful decode of protocol traces taken between card and UE.
+
+The ancient wirshark dissector developed for GSMTAP generated by SIMtrace
+is far too simplistic, while this decoder can utilize all of the information
+we already know in pySim about the filesystem structure, file encoding, etc.
+"""
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import abc
+from termcolor import colored
+import typing
+from typing import List, Dict, Optional
+
+from construct import *
+from construct import Optional as COptional
+from pySim.construct import *
+from pySim.utils import *
+from pySim.filesystem import RuntimeLchan, RuntimeState, lchan_nr_from_cla
+from pySim.filesystem import CardADF, CardFile, TransparentEF, LinFixedEF
+
+"""There are multiple levels of decode:
+
+    1) pure TPDU / APDU level (no filesystem state required to decode)
+        1a) the raw C-TPDU + R-TPDU
+        1b) the raw C-APDU + R-APDU
+        1c) the C-APDU + R-APDU split in its portions (p1/p2/lc/le/cmd/rsp)
+        1d) the abstract C-APDU + R-APDU (mostly p1/p2 parsing; SELECT response)
+    2) the decoded DATA of command/response APDU
+        * READ/UPDATE: requires state/context: which file is selected? how to decode it?
+"""
+
+class ApduCommandMeta(abc.ABCMeta):
+    """A meta-class that we can use to set some class variables when declaring
+       a derived class of ApduCommand."""
+    def __new__(metacls, name, bases, namespace, **kwargs):
+        x = super().__new__(metacls, name, bases, namespace)
+        x._name = namespace.get('name', kwargs.get('n', None))
+        x._ins = namespace.get('ins', kwargs.get('ins', None))
+        x._cla = namespace.get('cla', kwargs.get('cla', None))
+        return x
+
+BytesOrHex = typing.Union[bytes, Hexstr]
+
+class Tpdu:
+    def __init__(self, cmd: BytesOrHex, rsp: Optional[BytesOrHex] = None):
+        if isinstance(cmd, str):
+            self.cmd = h2b(cmd)
+        else:
+            self.cmd = cmd
+        if  isinstance(rsp, str):
+            self.rsp = h2b(rsp)
+        else:
+            self.rsp = rsp
+
+    def __str__(self):
+        return '%s(%02X %02X %02X %02X %02X %s %s %s)' % (type(self).__name__, self.cla, self.ins, self.p1,
+                self.p2, self.p3, b2h(self.cmd_data), b2h(self.rsp_data), b2h(self.sw))
+
+    @property
+    def cla(self) -> int:
+        """Return CLA of the C-APDU Header."""
+        return self.cmd[0]
+
+    @property
+    def ins(self) -> int:
+        """Return INS of the C-APDU Header."""
+        return self.cmd[1]
+
+    @property
+    def p1(self) -> int:
+        """Return P1 of the C-APDU Header."""
+        return self.cmd[2]
+
+    @property
+    def p2(self) -> int:
+        """Return P2 of the C-APDU Header."""
+        return self.cmd[3]
+
+    @property
+    def p3(self) -> int:
+        """Return P3 of the C-APDU Header."""
+        return self.cmd[4]
+
+    @property
+    def cmd_data(self) -> int:
+        """Return the DATA portion of the C-APDU"""
+        return self.cmd[5:]
+
+    @property
+    def sw(self) -> Optional[bytes]:
+        """Return Status Word (SW) of the R-APDU"""
+        return self.rsp[-2:] if self.rsp else None
+
+    @property
+    def rsp_data(self) -> Optional[bytes]:
+        """Return the DATA portion of the R-APDU"""
+        return self.rsp[:-2] if self.rsp else None
+
+
+class Apdu(Tpdu):
+    @property
+    def lc(self) -> int:
+        """Return Lc; Length of C-APDU body."""
+        return len(self.cmd_data)
+
+    @property
+    def lr(self) -> int:
+        """Return Lr; Length of R-APDU body."""
+        return len(self.rsp_data)
+
+    @property
+    def successful(self) -> bool:
+        """Was the execution of this APDU successful?"""
+        method = getattr(self, '_is_success', None)
+        if callable(method):
+            return method()
+        # default case: only 9000 is success
+        return self.sw == b'\x90\x00'
+
+
+class ApduCommand(Apdu, metaclass=ApduCommandMeta):
+    """Base class from which you would derive individual commands/instructions like SELECT.
+       A derived class represents a decoder for a specific instruction.
+       An instance of such a derived class is one concrete APDU."""
+    # fall-back constructs if the derived class provides no override
+    _construct_p1 = Byte
+    _construct_p2 = Byte
+    _construct = HexAdapter(GreedyBytes)
+    _construct_rsp = HexAdapter(GreedyBytes)
+
+    def __init__(self, cmd: BytesOrHex, rsp: Optional[BytesOrHex] = None):
+        """Instantiate a new ApduCommand from give cmd + resp."""
+        # store raw data
+        super().__init__(cmd, rsp)
+        # default to 'empty' ID column. To be set to useful values (like record number)
+        # by derived class {cmd_rsp}_to_dict() or process() methods
+        self.col_id = '-'
+        # fields only set by process_* methods
+        self.file = None
+        self.lchan = None
+        self.processed = None
+        # the methods below could raise exceptions and those handlers might assume cmd_{dict,resp}
+        self.cmd_dict = None
+        self.rsp_dict = None
+        # interpret the data
+        self.cmd_dict = self.cmd_to_dict()
+        self.rsp_dict = self.rsp_to_dict() if self.rsp else {}
+
+
+    @classmethod
+    def from_apdu(cls, apdu:Apdu, **kwargs) -> 'ApduCommand':
+        """Instantiate an ApduCommand from an existing APDU."""
+        return cls(cmd=apdu.cmd, rsp=apdu.rsp, **kwargs)
+
+    @classmethod
+    def from_bytes(cls, buffer:bytes) -> 'ApduCommand':
+        """Instantiate an ApduCommand from a linear byte buffer containing hdr,cmd,rsp,sw.
+        This is for example used when parsing GSMTAP traces that traditionally contain the
+        full command and response portion in one packet: "CLA INS P1 P2 P3 DATA SW" and we
+        now need to figure out whether the DATA part is part of the CMD or the RSP"""
+        apdu_case = cls.get_apdu_case(buffer)
+        if apdu_case in [1, 2]:
+            # data is part of response
+            return cls(buffer[:5], buffer[5:])
+        elif apdu_case in [3, 4]:
+            # data is part of command
+            lc = buffer[4]
+            return cls(buffer[:5+lc], buffer[5+lc:])
+        else:
+            raise ValueError('%s: Invalid APDU Case %u' % (cls.__name__, apdu_case))
+
+    @property
+    def path(self) -> List[str]:
+        """Return (if known) the path as list of files to the file on which this command operates."""
+        if self.file:
+            return self.file.fully_qualified_path()
+        else:
+            return []
+
+    @property
+    def path_str(self) -> str:
+        """Return (if known) the path as string to the file on which this command operates."""
+        if self.file:
+            return self.file.fully_qualified_path_str()
+        else:
+            return ''
+
+    @property
+    def col_sw(self) -> str:
+        """Return the ansi-colorized status word. Green==OK, Red==Error"""
+        if self.successful:
+            return colored(b2h(self.sw), 'green')
+        else:
+            return colored(b2h(self.sw), 'red')
+
+    @property
+    def lchan_nr(self) -> int:
+        """Logical channel number over which this ApduCommand was transmitted."""
+        if self.lchan:
+            return self.lchan.lchan_nr
+        else:
+            return lchan_nr_from_cla(self.cla)
+
+    def __str__(self) -> str:
+        return '%02u %s(%s): %s' % (self.lchan_nr, type(self).__name__, self.path_str, self.to_dict())
+
+    def __repr__(self) -> str:
+        return '%s(INS=%02x,CLA=%s)' % (self.__class__, self.ins, self.cla)
+
+    def _process_fallback(self, rs: RuntimeState):
+        """Fall-back function to be called if there is no derived-class-specific
+        process_global or process_on_lchan method. Uses information from APDU decode."""
+        self.processed = {}
+        if not 'p1' in self.cmd_dict:
+            self.processed = self.to_dict()
+        else:
+            self.processed['p1'] = self.cmd_dict['p1']
+            self.processed['p2'] = self.cmd_dict['p2']
+            if 'body' in self.cmd_dict and self.cmd_dict['body']:
+                self.processed['cmd'] = self.cmd_dict['body']
+            if 'body' in self.rsp_dict and self.rsp_dict['body']:
+                self.processed['rsp'] = self.rsp_dict['body']
+        return self.processed
+
+    def process(self, rs: RuntimeState):
+        # if there is a global method, use that; else use process_on_lchan
+        method = getattr(self, 'process_global', None)
+        if callable(method):
+            self.processed = method(rs)
+            return self.processed
+        method = getattr(self, 'process_on_lchan', None)
+        if callable(method):
+            self.lchan = rs.get_lchan_by_cla(self.cla)
+            self.processed = method(self.lchan)
+            return self.processed
+        # if none of the two methods exist:
+        return self._process_fallback(rs)
+
+    @classmethod
+    def get_apdu_case(cls, hdr:bytes) -> int:
+        if hasattr(cls, '_apdu_case'):
+            return cls._apdu_case
+        method = getattr(cls, '_get_apdu_case', None)
+        if callable(method):
+            return method(hdr)
+        raise ValueError('%s: Class definition missing _apdu_case attribute or _get_apdu_case method' % cls.__name__)
+
+    @classmethod
+    def match_cla(cls, cla) -> bool:
+        """Does the given CLA match the CLA list of the command?."""
+        if not isinstance(cla, str):
+            cla = '%02X' % cla
+        cla = cla.lower()
+        # see https://github.com/PyCQA/pylint/issues/7219
+        # pylint: disable=no-member
+        for cla_match in cls._cla:
+            cla_masked = ""
+            for i in range(0, 2):
+                if cla_match[i] == 'X':
+                    cla_masked += 'X'
+                else:
+                    cla_masked += cla[i]
+            if cla_masked == cla_match:
+                return True
+        return False
+
+    def cmd_to_dict(self) -> Dict:
+        """Convert the Command part of the APDU to a dict."""
+        method = getattr(self, '_decode_cmd', None)
+        if callable(method):
+            return method()
+        else:
+            r = {}
+            method = getattr(self, '_decode_p1p2', None)
+            if callable(method):
+                r = self._decode_p1p2()
+            else:
+                r['p1'] = parse_construct(self._construct_p1, self.p1.to_bytes(1, 'big'))
+                r['p2'] = parse_construct(self._construct_p2, self.p2.to_bytes(1, 'big'))
+            r['p3'] = self.p3
+            if self.cmd_data:
+                r['body'] = parse_construct(self._construct, self.cmd_data)
+            return r
+
+    def rsp_to_dict(self) -> Dict:
+        """Convert the Response part of the APDU to a dict."""
+        method = getattr(self, '_decode_rsp', None)
+        if callable(method):
+            return method()
+        else:
+            r = {}
+            if self.rsp_data:
+                r['body'] = parse_construct(self._construct_rsp, self.rsp_data)
+            r['sw'] = b2h(self.sw)
+            return r
+
+    def to_dict(self) -> Dict:
+        """Convert the entire APDU to a dict."""
+        return {'cmd': self.cmd_dict, 'rsp': self.rsp_dict}
+
+    def to_json(self) -> str:
+        """Convert the entire APDU to JSON."""
+        d = self.to_dict()
+        return json.dumps(d)
+
+    def _determine_file(self, lchan) -> CardFile:
+        """Helper function for read/update commands that might use SFI instead of selected file.
+        Expects that the self.cmd_dict has already been populated with the 'file' member."""
+        if self.cmd_dict['file'] == 'currently_selected_ef':
+            self.file = lchan.selected_file
+        elif self.cmd_dict['file'] == 'sfi':
+            cwd = lchan.get_cwd()
+            self.file = cwd.lookup_file_by_sfid(self.cmd_dict['sfi'])
+
+
+class ApduCommandSet:
+    """A set of card instructions, typically specified within one spec."""
+
+    def __init__(self, name: str, cmds: List[ApduCommand] =[]):
+        self.name = name
+        self.cmds = {c._ins: c for c in cmds}
+
+    def __str__(self) -> str:
+        return self.name
+
+    def __getitem__(self, idx) -> ApduCommand:
+        return self.cmds[idx]
+
+    def __add__(self, other) -> 'ApduCommandSet':
+        if isinstance(other, ApduCommand):
+            if other.ins in self.cmds:
+                raise ValueError('%s: INS 0x%02x already defined: %s' %
+                                 (self, other.ins, self.cmds[other.ins]))
+            self.cmds[other.ins] = other
+        elif isinstance(other, ApduCommandSet):
+            for c in other.cmds.keys():
+                self.cmds[c] = other.cmds[c]
+        else:
+            raise ValueError(
+                '%s: Unsupported type to add operator: %s' % (self, other))
+        return self
+
+    def lookup(self, ins, cla=None) -> Optional[ApduCommand]:
+        """look-up the command within the CommandSet."""
+        ins = int(ins)
+        if not ins in self.cmds:
+            return None
+        cmd = self.cmds[ins]
+        if cla and not cmd.match_cla(cla):
+            return None
+        return cmd
+
+    def parse_cmd_apdu(self, apdu: Apdu) -> ApduCommand:
+        """Parse a Command-APDU. Returns an instance of an ApduCommand derived class."""
+        # first look-up which of our member classes match CLA + INS
+        a_cls = self.lookup(apdu.ins, apdu.cla)
+        if not a_cls:
+            raise ValueError('Unknown CLA=%02X INS=%02X' % (apdu.cla, apdu.ins))
+        # then create an instance of that class and return it
+        return a_cls.from_apdu(apdu)
+
+    def parse_cmd_bytes(self, buf:bytes) -> ApduCommand:
+        """Parse from a buffer (simtrace style). Returns an instance of an ApduCommand derived class."""
+        # first look-up which of our member classes match CLA + INS
+        cla = buf[0]
+        ins = buf[1]
+        a_cls = self.lookup(ins, cla)
+        if not a_cls:
+            raise ValueError('Unknown CLA=%02X INS=%02X' % (cla, ins))
+        # then create an instance of that class and return it
+        return a_cls.from_bytes(buf)
+
+
+
+class ApduHandler(abc.ABC):
+    @abc.abstractmethod
+    def input(self, cmd: bytes, rsp: bytes):
+        pass
+
+
+class TpduFilter(ApduHandler):
+    """The TpduFilter removes the T=0 specific GET_RESPONSE from the TPDU stream and
+       calls the ApduHandler only with the actual APDU command and response parts."""
+    def __init__(self, apdu_handler: ApduHandler):
+        self.apdu_handler = apdu_handler
+        self.state = 'INIT'
+        self.last_cmd = None
+
+    def input_tpdu(self, tpdu:Tpdu):
+        # handle SW=61xx / 6Cxx
+        if tpdu.sw[0] == 0x61 or tpdu.sw[0] == 0x6C:
+            self.state = 'WAIT_GET_RESPONSE'
+            # handle successive 61/6c responses by stupid phone/modem OS
+            if tpdu.ins != 0xC0:
+                self.last_cmd = tpdu.cmd
+            return None
+        else:
+            if self.last_cmd:
+                icmd = self.last_cmd
+                self.last_cmd = None
+            else:
+                icmd = tpdu.cmd
+            apdu = Apdu(icmd, tpdu.rsp)
+            if self.apdu_handler:
+                return self.apdu_handler.input(apdu)
+            else:
+                return Apdu(icmd, tpdu.rsp)
+
+    def input(self, cmd: bytes, rsp: bytes):
+        if isinstance(cmd, str):
+            cmd = bytes.fromhex(cmd)
+        if isinstance(rsp, str):
+            rsp = bytes.fromhex(rsp)
+        tpdu = Tpdu(cmd, rsp)
+        return self.input_tpdu(tpdu)
+
+class ApduDecoder(ApduHandler):
+    def __init__(self, cmd_set: ApduCommandSet):
+        self.cmd_set = cmd_set
+
+    def input(self, apdu: Apdu):
+        return self.cmd_set.parse_cmd_apdu(apdu)
+
+
+class CardReset:
+    pass

pySim/apdu/global_platform.py

@@ -0,0 +1,57 @@
+# coding=utf-8
+"""APDU definition/decoder of GlobalPLatform Card Spec (currently 2.1.1)
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+
+class GpDelete(ApduCommand, n='DELETE', ins=0xE4, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpStoreData(ApduCommand, n='STORE DATA', ins=0xE2, cla=['8X', 'CX', 'EX']):
+    @classmethod
+    def _get_apdu_case(cls, hdr:bytes) -> int:
+        p1 = hdr[2]
+        if p1 & 0x01:
+            return 4
+        else:
+            return 3
+
+class GpGetDataCA(ApduCommand, n='GET DATA', ins=0xCA, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpGetDataCB(ApduCommand, n='GET DATA', ins=0xCB, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpGetStatus(ApduCommand, n='GET STATUS', ins=0xF2, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpInstall(ApduCommand, n='INSTALL', ins=0xE6, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpLoad(ApduCommand, n='LOAD', ins=0xE8, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpPutKey(ApduCommand, n='PUT KEY', ins=0xD8, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+class GpSetStatus(ApduCommand, n='SET STATUS', ins=0xF0, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+
+ApduCommands = ApduCommandSet('GlobalPlatform v2.3.1', cmds=[GpDelete, GpStoreData,
+                              GpGetDataCA, GpGetDataCB, GpGetStatus, GpInstall,
+                              GpLoad, GpPutKey, GpSetStatus])

pySim/apdu/ts_102_221.py

@@ -0,0 +1,525 @@
+# coding=utf-8
+"""APDU definitions/decoders of ETSI TS 102 221, the core UICC spec.
+
+(C) 2022 by Harald Welte <laforge@osmocom.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import logging
+from pySim.construct import *
+from pySim.filesystem import *
+from pySim.apdu import ApduCommand, ApduCommandSet
+from typing import Optional, Dict, Tuple
+
+logger = logging.getLogger(__name__)
+
+# TS 102 221 Section 11.1.1
+class UiccSelect(ApduCommand, n='SELECT', ins=0xA4, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p1 = Enum(Byte, df_ef_or_mf_by_file_id=0, child_df_of_current_df=1, parent_df_of_current_df=3,
+                         df_name=4, path_from_mf=8, path_from_current_df=9)
+    _construct_p2 = BitStruct(Flag,
+                              'app_session_control'/Enum(BitsInteger(2), activation_reset=0, termination=2),
+                              'return'/Enum(BitsInteger(3), fcp=1, no_data=3),
+                              'aid_control'/Enum(BitsInteger(2), first_or_only=0, last=1, next=2, previous=3))
+
+    @staticmethod
+    def _find_aid_substr(selectables, aid) -> Optional[CardADF]:
+        # full-length match
+        if aid in selectables:
+            return selectables[aid]
+        # sub-string match
+        for s in selectables.keys():
+            if aid[:len(s)] == s:
+                return selectables[s]
+        return None
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        mode = self.cmd_dict['p1']
+        if mode in ['path_from_mf', 'path_from_current_df']:
+            # rewind to MF, if needed
+            if mode == 'path_from_mf':
+                lchan.selected_file = lchan.rs.mf
+            path = [self.cmd_data[i:i+2] for i in range(0, len(self.cmd_data), 2)]
+            for file in path:
+                file_hex = b2h(file)
+                if file_hex == '7fff': # current application
+                    if not lchan.selected_adf:
+                        sels = lchan.rs.mf.get_app_selectables(['ANAMES'])
+                        # HACK: Assume USIM
+                        logger.warning('SELECT relative to current ADF, but no ADF selected. Assuming ADF.USIM')
+                        lchan.selected_adf = sels['ADF.USIM']
+                    lchan.selected_file = lchan.selected_adf
+                    #print("\tSELECT CUR_ADF %s" % lchan.selected_file)
+                    # iterate to next element in path
+                    continue
+                else:
+                    sels = lchan.selected_file.get_selectables(['FIDS','MF','PARENT','SELF'])
+                    if file_hex in sels:
+                        if self.successful:
+                            #print("\tSELECT %s" % sels[file_hex])
+                            lchan.selected_file = sels[file_hex]
+                        else:
+                            #print("\tSELECT %s FAILED" % sels[file_hex])
+                            pass
+                        # iterate to next element in path
+                        continue
+                logger.warning('SELECT UNKNOWN FID %s (%s)' % (file_hex, '/'.join([b2h(x) for x in path])))
+        elif mode == 'df_ef_or_mf_by_file_id':
+            if len(self.cmd_data) != 2:
+                raise ValueError('Expecting a 2-byte FID')
+            sels = lchan.selected_file.get_selectables(['FIDS','MF','PARENT','SELF'])
+            file_hex = b2h(self.cmd_data)
+            if file_hex in sels:
+                if self.successful:
+                    #print("\tSELECT %s" % sels[file_hex])
+                    lchan.selected_file = sels[file_hex]
+                else:
+                    #print("\tSELECT %s FAILED" % sels[file_hex])
+                    pass
+            else:
+                logger.warning('SELECT UNKNOWN FID %s' % (file_hex))
+        elif mode == 'df_name':
+            # Select by AID (can be sub-string!)
+            aid = self.cmd_dict['body']
+            sels = lchan.rs.mf.get_app_selectables(['AIDS'])
+            adf = self._find_aid_substr(sels, aid)
+            if adf:
+                lchan.selected_adf = adf
+                lchan.selected_file = lchan.selected_adf
+                #print("\tSELECT AID %s" % adf)
+            else:
+                logger.warning('SELECT UNKNOWN AID %s' % aid)
+                pass
+        else:
+            raise ValueError('Select Mode %s not implemented' % mode)
+        # decode the SELECT response
+        if self.successful:
+            self.file = lchan.selected_file
+            if 'body' in self.rsp_dict:
+                # not every SELECT is asking for the FCP in response...
+                return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+        return None
+
+
+
+# TS 102 221 Section 11.1.2
+class UiccStatus(ApduCommand, n='STATUS', ins=0xF2, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 2
+    _construct_p1 = Enum(Byte, no_indication=0, current_app_is_initialized=1, terminal_will_terminate_current_app=2)
+    _construct_p2 = Enum(Byte, response_like_select=0, response_df_name_tlv=1, response_no_data=0x0c)
+
+    def process_on_lchan(self, lchan):
+        if self.cmd_dict['p2'] == 'response_like_select':
+            return lchan.selected_file.decode_select_response(self.rsp_dict['body'])
+
+def _decode_binary_p1p2(p1, p2) -> Dict:
+    ret = {}
+    if p1 & 0x80:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p1 & 0x1f
+        ret['offset'] = p2
+    else:
+        ret['file'] = 'currently_selected_ef'
+        ret['offset'] = ((p1 & 0x7f) << 8) & p2
+    return ret
+
+# TS 102 221 Section 11.1.3
+class ReadBinary(ApduCommand, n='READ BINARY', ins=0xB0, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short reads
+        if self.cmd_dict['offset'] != 0 or self.lr < self.file.size[0]:
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 102 221 Section 11.1.4
+class UpdateBinary(ApduCommand, n='UPDATE BINARY', ins=0xD6, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        return _decode_binary_p1p2(self.p1, self.p2)
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, TransparentEF):
+            return b2h(self.rsp_data)
+        # our decoders don't work for non-zero offsets / short writes
+        if self.cmd_dict['offset'] != 0 or self.lc < self.file.size[0]:
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+def _decode_record_p1p2(p1, p2):
+    ret = {}
+    ret['record_number'] = p1
+    if p2 >> 3 == 0:
+        ret['file'] = 'currently_selected_ef'
+    else:
+        ret['file'] = 'sfi'
+        ret['sfi'] = p2 >> 3
+    mode = p2 & 0x7
+    if mode == 2:
+        ret['mode'] = 'next_record'
+    elif mode == 3:
+        ret['mode'] = 'previous_record'
+    elif mode == 8:
+        ret['mode'] = 'absolute_current'
+    return ret
+
+# TS 102 221 Section 11.1.5
+class ReadRecord(ApduCommand, n='READ RECORD', ins=0xB2, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.rsp_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.rsp_data)
+
+# TS 102 221 Section 11.1.6
+class UpdateRecord(ApduCommand, n='UPDATE RECORD', ins=0xDC, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    def _decode_p1p2(self):
+        r = _decode_record_p1p2(self.p1, self.p2)
+        self.col_id = '%02u' % r['record_number']
+        return r
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        if not isinstance(self.file, LinFixedEF):
+            return b2h(self.cmd_data)
+        method = getattr(self.file, 'decode_record_bin', None)
+        if self.successful and callable(method):
+            return method(self.cmd_data)
+
+# TS 102 221 Section 11.1.7
+class SearchRecord(ApduCommand, n='SEARCH RECORD', ins=0xA2, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_rsp = GreedyRange(Int8ub)
+
+    def _decode_p1p2(self):
+        ret = {}
+        sfi = self.p2 >> 3
+        if sfi == 0:
+            ret['file'] = 'currently_selected_ef'
+        else:
+            ret['file'] = 'sfi'
+            ret['sfi'] = sfi
+        mode = self.p2 & 0x7
+        if mode in [0x4, 0x5]:
+            if mode == 0x4:
+                ret['mode'] = 'forward_search'
+            else:
+                ret['mode'] = 'backward_search'
+            ret['record_number'] = self.p1
+            self.col_id = '%02u' % ret['record_number']
+        elif mode == 6:
+            ret['mode'] = 'enhanced_search'
+            # TODO: further decode
+        elif mode == 7:
+            ret['mode'] = 'proprietary_search'
+        return ret
+
+    def _decode_cmd(self):
+        ret = self._decode_p1p2()
+        if self.cmd_data:
+            if ret['mode'] == 'enhanced_search':
+                ret['search_indication'] = b2h(self.cmd_data[:2])
+                ret['search_string'] = b2h(self.cmd_data[2:])
+            else:
+                ret['search_string'] = b2h(self.cmd_data)
+        return ret
+
+    def process_on_lchan(self, lchan):
+        self._determine_file(lchan)
+        return self.to_dict()
+
+# TS 102 221 Section 11.1.8
+class Increase(ApduCommand, n='INCREASE', ins=0x32, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+PinConstructP2 = BitStruct('scope'/Enum(Flag, global_mf=0, specific_df_adf=1),
+                           BitsInteger(2), 'reference_data_nr'/BitsInteger(5))
+# TS 102 221 Section 11.1.9
+class VerifyPin(ApduCommand, n='VERIFY PIN', ins=0x20, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    @staticmethod
+    def _pin_process(apdu):
+        processed = {
+            'scope': apdu.cmd_dict['p2']['scope'],
+            'referenced_data_nr': apdu.cmd_dict['p2']['reference_data_nr'],
+            }
+        if apdu.lc == 0:
+            # this is just a question on the counters remaining
+            processed['mode'] = 'check_remaining_attempts'
+        else:
+            processed['pin'] = b2h(apdu.cmd_data)
+        if apdu.sw[0] == 0x63:
+            processed['remaining_attempts'] = apdu.sw[1] & 0xf
+        return processed
+
+    @staticmethod
+    def _pin_is_success(sw):
+        if sw[0] == 0x63:
+            return True
+        else:
+            return False
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.10
+class ChangePin(ApduCommand, n='CHANGE PIN', ins=0x24, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.11
+class DisablePin(ApduCommand, n='DISABLE PIN', ins=0x26, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.12
+class EnablePin(ApduCommand, n='ENABLE PIN', ins=0x28, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.13
+class UnblockPin(ApduCommand, n='UNBLOCK PIN', ins=0x2C, cla=['0X', '4X', '6X']):
+    _apdu_case = 3
+    _construct_p2 = PinConstructP2
+
+    def process_on_lchan(self, lchan: RuntimeLchan):
+        return VerifyPin._pin_process(self)
+
+    def _is_success(self):
+        return VerifyPin._pin_is_success(self.sw)
+
+
+# TS 102 221 Section 11.1.14
+class DeactivateFile(ApduCommand, n='DEACTIVATE FILE', ins=0x04, cla=['0X', '4X', '6X']):
+    _apdu_case = 1
+    _construct_p1 = BitStruct(BitsInteger(4),
+                              'select_mode'/Enum(BitsInteger(4), ef_by_file_id=0,
+                                                 path_from_mf=8, path_from_current_df=9))
+
+# TS 102 221 Section 11.1.15
+class ActivateFile(ApduCommand, n='ACTIVATE FILE', ins=0x44, cla=['0X', '4X', '6X']):
+    _apdu_case = 1
+    _construct_p1 = DeactivateFile._construct_p1
+
+# TS 102 221 Section 11.1.16
+auth_p2_construct = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              BitsInteger(2),
+                              'reference_data_nr'/BitsInteger(5))
+class Authenticate88(ApduCommand, n='AUTHENTICATE', ins=0x88, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = auth_p2_construct
+
+# TS 102 221 Section 11.1.16
+class Authenticate89(ApduCommand, n='AUTHENTICATE', ins=0x89, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = auth_p2_construct
+
+# TS 102 221 Section 11.1.17
+class ManageChannel(ApduCommand, n='MANAGE CHANNEL', ins=0x70, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+    _construct_p1 = Enum(Flag, open_channel=0, close_channel=1)
+    _construct_p2 = Struct('logical_channel_number'/Int8ub)
+    _construct_rsp = Struct('logical_channel_number'/Int8ub)
+
+    def process_global(self, rs):
+        if not self.successful:
+            return
+        mode = self.cmd_dict['p1']
+        if mode == 'open_channel':
+            created_channel_nr = self.cmd_dict['p2']['logical_channel_number']
+            if created_channel_nr == 0:
+                # auto-assignment by UICC
+                # pylint: disable=unsubscriptable-object
+                created_channel_nr = self.rsp_data[0]
+            manage_channel = rs.get_lchan_by_cla(self.cla)
+            manage_channel.add_lchan(created_channel_nr)
+            self.col_id = '%02u' % created_channel_nr
+        elif mode == 'close_channel':
+            closed_channel_nr = self.cmd_dict['p2']
+            rs.del_lchan(closed_channel_nr)
+            self.col_id = '%02u' % closed_channel_nr
+        else:
+            raise ValueError('Unsupported MANAGE CHANNEL P1=%02X' % self.p1)
+
+# TS 102 221 Section 11.1.18
+class GetChallenge(ApduCommand, n='GET CHALLENGE', ins=0x84, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+
+# TS 102 221 Section 11.1.19
+class TerminalCapability(ApduCommand, n='TERMINAL CAPABILITY', ins=0xAA, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+
+# TS 102 221 Section 11.1.20
+class ManageSecureChannel(ApduCommand, n='MANAGE SECURE CHANNEL', ins=0x73, cla=['0X', '4X', '6X']):
+    @classmethod
+    def _get_apdu_case(cls, hdr:bytes) -> int:
+        p1 = hdr[2]
+        p2 = hdr[3]
+        if p1 & 0x7 == 0:   # retrieve UICC Endpoints
+            return 2
+        elif p1 & 0xf in [1,2,3]:   # establish sa, start secure channel SA
+            p2_cmd = p2 >> 5
+            if p2_cmd in [0,2,4]:   # command data
+                return 3
+            elif p2_cmd in [1,3,5]: # response data
+                return 2
+        elif p1 & 0xf == 4:         # terminate secure channel SA
+            return 3
+        raise ValueError('%s: Unable to detect APDU case for %s' % (cls.__name__, b2h(hdr)))
+
+# TS 102 221 Section 11.1.21
+class TransactData(ApduCommand, n='TRANSACT DATA', ins=0x75, cla=['0X', '4X', '6X']):
+    @classmethod
+    def _get_apdu_case(cls, hdr:bytes) -> int:
+        p1 = hdr[2]
+        if p1 & 0x04:
+            return 3
+        else:
+            return 2
+
+# TS 102 221 Section 11.1.22
+class SuspendUicc(ApduCommand, n='SUSPEND UICC', ins=0x76, cla=['80']):
+    _apdu_case = 4
+    _construct_p1 = BitStruct('rfu'/BitsInteger(7), 'mode'/Enum(Flag, suspend=0, resume=1))
+
+# TS 102 221 Section 11.1.23
+class GetIdentity(ApduCommand, n='GET IDENTITY', ins=0x78, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1), BitsInteger(7))
+
+# TS 102 221 Section 11.1.24
+class ExchangeCapabilities(ApduCommand, n='EXCHANGE CAPABILITIES', ins=0x7A, cla=['80']):
+    _apdu_case = 4
+
+# TS 102 221 Section 11.2.1
+class TerminalProfile(ApduCommand, n='TERMINAL PROFILE', ins=0x10, cla=['80']):
+    _apdu_case = 3
+
+# TS 102 221 Section 11.2.2 / TS 102 223
+class Envelope(ApduCommand, n='ENVELOPE', ins=0xC2, cla=['80']):
+    _apdu_case = 4
+
+# TS 102 221 Section 11.2.3 / TS 102 223
+class Fetch(ApduCommand, n='FETCH', ins=0x12, cla=['80']):
+    _apdu_case = 2
+
+# TS 102 221 Section 11.2.3 / TS 102 223
+class TerminalResponse(ApduCommand, n='TERMINAL RESPONSE', ins=0x14, cla=['80']):
+    _apdu_case = 3
+
+# TS 102 221 Section 11.3.1
+class RetrieveData(ApduCommand, n='RETRIEVE DATA', ins=0xCB, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+
+    @staticmethod
+    def _tlv_decode_cmd(self : ApduCommand) -> Dict:
+        c = {}
+        if self.p2 & 0xc0 == 0x80:
+            c['mode'] = 'first_block'
+            sfi = self.p2 & 0x1f
+            if sfi == 0:
+                c['file'] = 'currently_selected_ef'
+            else:
+                c['file'] = 'sfi'
+                c['sfi'] = sfi
+            c['tag'] = i2h([self.cmd_data[0]])
+        elif self.p2 & 0xdf == 0x00:
+            c['mode'] = 'next_block'
+        elif self.p2 & 0xdf == 0x40:
+            c['mode'] = 'retransmit_previous_block'
+        else:
+            logger.warning('%s: invalid P2=%02x' % (self, self.p2))
+        return c
+
+    def _decode_cmd(self):
+        return RetrieveData._tlv_decode_cmd(self)
+
+    def _decode_rsp(self):
+        # TODO: parse tag/len/val?
+        return b2h(self.rsp_data)
+
+
+# TS 102 221 Section 11.3.2
+class SetData(ApduCommand, n='SET DATA', ins=0xDB, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 3
+
+    def _decode_cmd(self):
+        c = RetrieveData._tlv_decode_cmd(self)
+        if c['mode'] == 'first_block':
+            if len(self.cmd_data) == 0:
+                c['delete'] = True
+        # TODO: parse tag/len/val?
+        c['data'] = b2h(self.cmd_data)
+        return c
+
+
+# TS 102 221 Section 12.1.1
+class GetResponse(ApduCommand, n='GET RESPONSE', ins=0xC0, cla=['0X', '4X', '6X']):
+    _apdu_case = 2
+
+ApduCommands = ApduCommandSet('TS 102 221', cmds=[UiccSelect, UiccStatus, ReadBinary, UpdateBinary, ReadRecord,
+                              UpdateRecord, SearchRecord, Increase, VerifyPin, ChangePin, DisablePin,
+                              EnablePin, UnblockPin, DeactivateFile, ActivateFile, Authenticate88,
+                              Authenticate89, ManageChannel, GetChallenge, TerminalCapability,
+                              ManageSecureChannel, TransactData, SuspendUicc, GetIdentity,
+                              ExchangeCapabilities, TerminalProfile, Envelope, Fetch, TerminalResponse,
+                              RetrieveData, SetData, GetResponse])

pySim/apdu/ts_31_102.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+
+# without this, pylint will fail when inner classes are used
+# within the 'nested' kwarg of our TlvMeta metaclass on python 3.7 :(
+# pylint: disable=undefined-variable
+
+"""
+APDU commands of 3GPP TS 31.102 V16.6.0
+"""
+from typing import Dict
+
+from construct import *
+from construct import Optional as COptional
+from pySim.filesystem import *
+from pySim.construct import *
+from pySim.ts_31_102 import SUCI_TlvDataObject
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+
+# Copyright (C) 2022 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Mapping between USIM Service Number and its description
+
+from pySim.apdu import ApduCommand, ApduCommandSet
+
+# TS 31.102 Section 7.1
+class UsimAuthenticateEven(ApduCommand, n='AUTHENTICATE', ins=0x88, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              BitsInteger(4),
+                              'authentication_context'/Enum(BitsInteger(3), gsm=0, umts=1,
+                                                            vgcs_vbs=2, gba=4))
+    _cs_cmd_gsm_3g =  Struct('_rand_len'/Int8ub, 'rand'/HexAdapter(Bytes(this._rand_len)),
+                         '_autn_len'/COptional(Int8ub), 'autn'/If(this._autn_len, HexAdapter(Bytes(this._autn_len))))
+    _cs_cmd_vgcs = Struct('_vsid_len'/Int8ub, 'vservice_id'/HexAdapter(Bytes(this._vsid_len)),
+                          '_vkid_len'/Int8ub, 'vk_id'/HexAdapter(Bytes(this._vkid_len)),
+                          '_vstk_rand_len'/Int8ub, 'vstk_rand'/HexAdapter(Bytes(this._vstk_rand_len)))
+    _cmd_gba_bs = Struct('_rand_len'/Int8ub, 'rand'/HexAdapter(Bytes(this._rand_len)),
+                         '_autn_len'/Int8ub, 'autn'/HexAdapter(Bytes(this._autn_len)))
+    _cmd_gba_naf = Struct('_naf_id_len'/Int8ub, 'naf_id'/HexAdapter(Bytes(this._naf_id_len)),
+                          '_impi_len'/Int8ub, 'impi'/HexAdapter(Bytes(this._impi_len)))
+    _cs_cmd_gba = Struct('tag'/Int8ub, 'body'/Switch(this.tag, { 0xDD: 'bootstrap'/_cmd_gba_bs,
+                                                                 0xDE: 'naf_derivation'/_cmd_gba_naf }))
+    _cs_rsp_gsm = Struct('_len_sres'/Int8ub, 'sres'/HexAdapter(Bytes(this._len_sres)),
+                         '_len_kc'/Int8ub, 'kc'/HexAdapter(Bytes(this._len_kc)))
+    _rsp_3g_ok = Struct('_len_res'/Int8ub, 'res'/HexAdapter(Bytes(this._len_res)),
+                        '_len_ck'/Int8ub, 'ck'/HexAdapter(Bytes(this._len_ck)),
+                        '_len_ik'/Int8ub, 'ik'/HexAdapter(Bytes(this._len_ik)),
+                        '_len_kc'/COptional(Int8ub), 'kc'/If(this._len_kc, HexAdapter(Bytes(this._len_kc))))
+    _rsp_3g_sync = Struct('_len_auts'/Int8ub, 'auts'/HexAdapter(Bytes(this._len_auts)))
+    _cs_rsp_3g = Struct('tag'/Int8ub, 'body'/Switch(this.tag, { 0xDB: 'success'/_rsp_3g_ok,
+                                                                0xDC: 'sync_fail'/_rsp_3g_sync}))
+    _cs_rsp_vgcs = Struct(Const(b'\xDB'), '_vstk_len'/Int8ub, 'vstk'/HexAdapter(Bytes(this._vstk_len)))
+    _cs_rsp_gba_naf = Struct(Const(b'\xDB'), '_ks_ext_naf_len'/Int8ub, 'ks_ext_naf'/HexAdapter(Bytes(this._ks_ext_naf_len)))
+    def _decode_cmd(self) -> Dict:
+        r = {}
+        r['p1'] = parse_construct(self._construct_p1, self.p1.to_bytes(1, 'big'))
+        r['p2'] = parse_construct(self._construct_p2, self.p2.to_bytes(1, 'big'))
+        auth_ctx = r['p2']['authentication_context']
+        if auth_ctx in ['gsm', 'umts']:
+            r['body'] = parse_construct(self._cs_cmd_gsm_3g, self.cmd_data)
+        elif auth_ctx == 'vgcs_vbs':
+            r['body'] = parse_construct(self._cs_cmd_vgcs, self.cmd_data)
+        elif auth_ctx == 'gba':
+            r['body'] = parse_construct(self._cs_cmd_gba, self.cmd_data)
+        else:
+            raise ValueError('Unsupported authentication_context: %s' % auth_ctx)
+        return r
+
+    def _decode_rsp(self) -> Dict:
+        r = {}
+        auth_ctx = self.cmd_dict['p2']['authentication_context']
+        if auth_ctx == 'gsm':
+            r['body'] = parse_construct(self._cs_rsp_gsm, self.rsp_data)
+        elif auth_ctx == 'umts':
+            r['body'] = parse_construct(self._cs_rsp_3g, self.rsp_data)
+        elif auth_ctx == 'vgcs_vbs':
+            r['body'] = parse_construct(self._cs_rsp_vgcs, self.rsp_data)
+        elif auth_ctx == 'gba':
+            if self.cmd_dict['body']['tag'] == 0xDD:
+                r['body'] = parse_construct(self._cs_rsp_3g, self.rsp_data)
+            else:
+                r['body'] = parse_construct(self._cs_rsp_gba_naf, self.rsp_data)
+        else:
+            raise ValueError('Unsupported authentication_context: %s' % auth_ctx)
+        return r
+
+class UsimAuthenticateOdd(ApduCommand, n='AUTHENTICATE', ins=0x89, cla=['0X', '4X', '6X']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              BitsInteger(4),
+                              'authentication_context'/Enum(BitsInteger(3), mbms=5, local_key=6))
+# TS 31.102 Section 7.5
+class UsimGetIdentity(ApduCommand, n='GET IDENTITY', ins=0x78, cla=['8X', 'CX', 'EX']):
+    _apdu_case = 4
+    _construct_p2 = BitStruct('scope'/Enum(Flag, mf=0, df_adf_specific=1),
+                              'identity_context'/Enum(BitsInteger(7), suci=1))
+    _tlv_rsp = SUCI_TlvDataObject
+
+ApduCommands = ApduCommandSet('TS 31.102', cmds=[UsimAuthenticateEven, UsimAuthenticateOdd,
+                                                 UsimGetIdentity])

pySim/apdu_source/__init__.py

@@ -0,0 +1,35 @@
+import abc
+import logging
+from typing import Union
+from pySim.apdu import Apdu, Tpdu, CardReset, TpduFilter
+
+PacketType = Union[Apdu, Tpdu, CardReset]
+
+logger = logging.getLogger(__name__)
+
+class ApduSource(abc.ABC):
+    def __init__(self):
+        self.apdu_filter = TpduFilter(None)
+
+    @abc.abstractmethod
+    def read_packet(self) -> PacketType:
+        """Read one packet from the source."""
+        pass
+
+    def read(self) -> Union[Apdu, CardReset]:
+        """Main function to call by the user: Blocking read, returns Apdu or CardReset."""
+        apdu = None
+        # loop until we actually have an APDU to return
+        while not apdu:
+            r = self.read_packet()
+            if not r:
+                continue
+            if isinstance(r, Tpdu):
+                apdu = self.apdu_filter.input_tpdu(r)
+            elif isinstance(r, Apdu):
+                apdu = r
+            elif isinstance(r, CardReset):
+                apdu = r
+            else:
+                ValueError('Unknown read_packet() return %s' % r)
+        return apdu

pySim/apdu_source/gsmtap.py

@@ -0,0 +1,57 @@
+# coding=utf-8
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from pySim.gsmtap import GsmtapMessage, GsmtapSource
+from . import ApduSource, PacketType, CardReset
+
+from pySim.apdu.ts_102_221 import ApduCommands as UiccApduCommands
+from pySim.apdu.ts_31_102 import ApduCommands as UsimApduCommands
+from pySim.apdu.global_platform import ApduCommands as GpApduCommands
+ApduCommands = UiccApduCommands + UsimApduCommands + GpApduCommands
+
+class GsmtapApduSource(ApduSource):
+    """ApduSource for handling GSMTAP-SIM messages received via UDP, such as
+    those generated by simtrace2-sniff.  Note that *if* you use IP loopback
+    and localhost addresses (which is the default), you will need to start
+    this source before starting simtrace2-sniff, as otherwise the latter will
+    claim the GSMTAP UDP port.
+    """
+    def __init__(self, bind_ip:str='127.0.0.1', bind_port:int=4729):
+        """Create a UDP socket for receiving GSMTAP-SIM messages.
+        Args:
+            bind_ip: IP address to which the socket should be bound (default: 127.0.0.1)
+            bind_port: UDP port number to which the socket should be bound (default: 4729)
+        """
+        super().__init__()
+        self.gsmtap = GsmtapSource(bind_ip, bind_port)
+
+    def read_packet(self) -> PacketType:
+        gsmtap_msg, addr = self.gsmtap.read_packet()
+        if gsmtap_msg['type'] != 'sim':
+            raise ValueError('Unsupported GSMTAP type %s' % gsmtap_msg['type'])
+        sub_type = gsmtap_msg['sub_type']
+        if sub_type == 'apdu':
+            return ApduCommands.parse_cmd_bytes(gsmtap_msg['body'])
+        elif sub_type == 'atr':
+            # card has been reset
+            return CardReset()
+        elif sub_type in ['pps_req', 'pps_rsp']:
+            # simply ignore for now
+            pass
+        else:
+            raise ValueError('Unsupported GSMTAP-SIM sub-type %s' % sub_type)

pySim/apdu_source/pyshark_rspro.py

@@ -0,0 +1,159 @@
+# coding=utf-8
+
+# (C) 2022 by Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import sys
+import logging
+from pprint import pprint as pp
+from typing import Tuple
+import pyshark
+
+from pySim.utils import h2b, b2h
+from pySim.apdu import Tpdu
+from . import ApduSource, PacketType, CardReset
+
+logger = logging.getLogger(__name__)
+
+class _PysharkRspro(ApduSource):
+    """APDU Source [provider] base class for reading RSPRO (osmo-remsim) via tshark."""
+
+    def __init__(self, pyshark_inst):
+        self.pyshark = pyshark_inst
+        self.bank_id = None
+        self.bank_slot = None
+        self.cmd_tpdu = None
+        super().__init__()
+
+    @staticmethod
+    def get_bank_slot(bank_slot) -> Tuple[int, int]:
+        """Convert a 'bankSlot_element' field into a tuple of bank_id, slot_nr"""
+        bank_id = bank_slot.get_field('bankId')
+        slot_nr = bank_slot.get_field('slotNr')
+        return int(bank_id), int(slot_nr)
+
+    @staticmethod
+    def get_client_slot(client_slot) -> Tuple[int, int]:
+        """Convert a 'clientSlot_element' field into a tuple of client_id, slot_nr"""
+        client_id = client_slot.get_field('clientId')
+        slot_nr = client_slot.get_field('slotNr')
+        return int(client_id), int(slot_nr)
+
+    @staticmethod
+    def get_pstatus(pstatus) -> Tuple[int, int, int]:
+        """Convert a 'slotPhysStatus_element' field into a tuple of vcc, reset, clk"""
+        vccPresent = int(pstatus.get_field('vccPresent'))
+        resetActive = int(pstatus.get_field('resetActive'))
+        clkActive = int(pstatus.get_field('clkActive'))
+        return vccPresent, resetActive, clkActive
+
+    def read_packet(self) -> PacketType:
+        p = self.pyshark.next()
+        return self._parse_packet(p)
+
+    def _set_or_verify_bank_slot(self, bsl: Tuple[int, int]):
+        """Keep track of the bank:slot to make sure we don't mix traces of multiple cards"""
+        if not self.bank_id:
+            self.bank_id = bsl[0]
+            self.bank_slot = bsl[1]
+        else:
+            if self.bank_id != bsl[0] or self.bank_slot != bsl[1]:
+                raise ValueError('Received data for unexpected B(%u:%u)' % (bsl[0], bsl[1]))
+
+    def _parse_packet(self, p) -> PacketType:
+        rspro_layer = p['rspro']
+        #print("Layer: %s" %  rspro_layer)
+        rspro_element = rspro_layer.get_field('RsproPDU_element')
+        #print("Element: %s" % rspro_element)
+        msg_type = rspro_element.get_field('msg')
+        rspro_msg = rspro_element.get_field('msg_tree')
+        if msg_type == '12': # tpduModemToCard
+            modem2card = rspro_msg.get_field('tpduModemToCard_element')
+            #print(modem2card)
+            client_slot = modem2card.get_field('fromClientSlot_element')
+            csl = self.get_client_slot(client_slot)
+            bank_slot = modem2card.get_field('toBankSlot_element')
+            bsl = self.get_bank_slot(bank_slot)
+            self._set_or_verify_bank_slot(bsl)
+            data = modem2card.get_field('data').replace(':','')
+            logger.debug("C(%u:%u) -> B(%u:%u): %s" % (csl[0], csl[1], bsl[0], bsl[1], data))
+            # store the CMD portion until the RSP portion arrives later
+            self.cmd_tpdu = h2b(data)
+        elif msg_type == '13': # tpduCardToModem
+            card2modem = rspro_msg.get_field('tpduCardToModem_element')
+            #print(card2modem)
+            client_slot = card2modem.get_field('toClientSlot_element')
+            csl = self.get_client_slot(client_slot)
+            bank_slot = card2modem.get_field('fromBankSlot_element')
+            bsl = self.get_bank_slot(bank_slot)
+            self._set_or_verify_bank_slot(bsl)
+            data = card2modem.get_field('data').replace(':','')
+            logger.debug("C(%u:%u) <- B(%u:%u): %s" % (csl[0], csl[1], bsl[0], bsl[1], data))
+            rsp_tpdu = h2b(data)
+            if self.cmd_tpdu:
+                # combine this R-TPDU with the C-TPDU we saw earlier
+                r = Tpdu(self.cmd_tpdu, rsp_tpdu)
+                self.cmd_tpdu = False
+                return r
+        elif msg_type == '14': # clientSlotStatus
+            cl_slotstatus = rspro_msg.get_field('clientSlotStatusInd_element')
+            #print(cl_slotstatus)
+            client_slot = cl_slotstatus.get_field('fromClientSlot_element')
+            bank_slot = cl_slotstatus.get_field('toBankSlot_element')
+            slot_pstatus = cl_slotstatus.get_field('slotPhysStatus_element')
+            vccPresent, resetActive, clkActive = self.get_pstatus(slot_pstatus)
+            if vccPresent and clkActive and not resetActive:
+                logger.debug("RESET")
+                return CardReset()
+        else:
+            print("Unhandled msg type %s: %s" % (msg_type, rspro_msg))
+
+
+class PysharkRsproPcap(_PysharkRspro):
+    """APDU Source [provider] class for reading RSPRO (osmo-remsim) from a PCAP
+    file via pyshark, which in turn uses tshark (part of wireshark).
+
+    In order to use this, you need a wireshark patched with RSPRO support,
+    such as can be found at https://gitea.osmocom.org/osmocom/wireshark/src/branch/laforge/rspro
+
+    A STANDARD UPSTREAM WIRESHARK *DOES NOT WORK*.
+    """
+    def __init__(self, pcap_filename):
+        """
+        Args:
+            pcap_filename: File name of the pcap file to be opened
+        """
+        pyshark_inst = pyshark.FileCapture(pcap_filename, display_filter='rspro', use_json=True, keep_packets=False)
+        super().__init__(pyshark_inst)
+
+class PysharkRsproLive(_PysharkRspro):
+    """APDU Source [provider] class for reading RSPRO (osmo-remsim) from a live capture
+    via pyshark, which in turn uses tshark (part of wireshark).
+
+    In order to use this, you need a wireshark patched with RSPRO support,
+    such as can be found at https://gitea.osmocom.org/osmocom/wireshark/src/branch/laforge/rspro
+
+    A STANDARD UPSTREAM WIRESHARK *DOES NOT WORK*.
+    """
+    def __init__(self, interface, bpf_filter='tcp port 9999 or tcp port 9998'):
+        """
+        Args:
+            interface: Network interface name to capture packets on (like "eth0")
+            bfp_filter: libpcap capture filter to use
+        """
+        pyshark_inst = pyshark.LiveCapture(interface=interface, display_filter='rspro', bpf_filter=bpf_filter,
+                                           use_json=True)
+        super().__init__(pyshark_inst)

pySim/cards.py

@@ -52,7 +52,7 @@ def format_addr(addr: str, addr_type: str) -> str:
     return res
 
 
-class SimCard(object):
+class SimCard:
 
     name = 'SIM'
 
@@ -63,7 +63,7 @@ class SimCard(object):
 
     def reset(self):
         rc = self._scc.reset_card()
-        if rc is 1:
+        if rc == 1:
             return self._scc.get_atr()
         else:
             return None
@@ -348,6 +348,9 @@ class UsimCard(SimCard):
     def __init__(self, ssc):
         super(UsimCard, self).__init__(ssc)
 
+	# See also: ETSI TS 102 221, Table 9.3
+        self._adm_chv_num = 0xA0
+
     def read_ehplmn(self):
         (res, sw) = self._scc.read_binary(EF_USIM_ADF_map['EHPLMN'])
         if sw == '9000':

pySim/cat.py

@@ -1,7 +1,8 @@
 """Code related to the Card Application Toolkit (CAT) as described in
-mainly) ETSI TS 102 223, ETSI TS 101 220 and 3GPP TS 31.111."""
+mainly) ETSI TS 102 223, ETSI TS 101 220 and USIM Application Toolkit (SAT)
+as described in 3GPP TS 31.111."""
 
-# (C) 2021 by Harald Welte <laforge@osmocom.org>
+# (C) 2021-2022 by Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,9 +18,14 @@ mainly) ETSI TS 102 223, ETSI TS 101 220 and 3GPP TS 31.111."""
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-from pySim.tlv import *
-from pySim.construct import *
-from construct import *
+from bidict import bidict
+from typing import List
+from pySim.utils import b2h, h2b, dec_xplmn_w_act
+from pySim.tlv import TLV_IE, COMPR_TLV_IE, BER_TLV_IE, TLV_IE_Collection
+from pySim.construct import BcdAdapter, HexAdapter, GsmStringAdapter, TonNpi
+from construct import Int8ub, Int16ub, Byte, Bytes, Bit, Flag, BitsInteger
+from construct import Struct, Enum, Tell, BitStruct, this, Padding, RepeatUntil
+from construct import GreedyBytes, Switch, GreedyRange, FlagsEnum
 
 # Tag values as per TS 101 220 Table 7.23
 
@@ -37,7 +43,7 @@ class AlphaIdentifier(COMPR_TLV_IE, tag=0x05):
 class Subaddress(COMPR_TLV_IE, tag=0x08):
     pass
 
-# TS 102 223 Section 8.4
+# TS 102 223 Section 8.4 + TS 31.111 Section 8.4
 class CapabilityConfigParams(COMPR_TLV_IE, tag=0x07):
     pass
 
@@ -45,8 +51,8 @@ class CapabilityConfigParams(COMPR_TLV_IE, tag=0x07):
 class CBSPage(COMPR_TLV_IE, tag=0x0C):
     pass
 
-# TS 102 223 Section 8.6
-class CommandDetails(COMPR_TLV_IE, tag=0x01):
+# TS 102 223 Section 8.6 + TS 31.111 Section 8.6
+class CommandDetails(COMPR_TLV_IE, tag=0x81):
     _construct = Struct('command_number'/Int8ub,
                         'type_of_command'/Int8ub,
                         'command_qualifier'/Int8ub)
@@ -102,7 +108,7 @@ class DeviceIdentities(COMPR_TLV_IE, tag=0x82):
 
 # TS 102 223 Section 8.8
 class Duration(COMPR_TLV_IE, tag=0x04):
-    _construct = Struct('time_unit'/Int8ub,
+    _construct = Struct('time_unit'/Enum(Int8ub, minutes=0, seconds=1, tenths_of_seconds=2),
                         'time_interval'/Int8ub)
 
 # TS 102 223 Section 8.9
@@ -121,33 +127,586 @@ class ResponseLength(COMPR_TLV_IE, tag=0x11):
 
 # TS 102 223 Section 8.12
 class Result(COMPR_TLV_IE, tag=0x03):
-    _construct = Struct('general_result'/Int8ub,
-                        'additional_information'/HexAdapter(GreedyBytes))
+    GeneralResult = Enum(Int8ub,
+                         # '0X' and '1X' indicate that the command has been performed
+                         performed_successfully=0,
+                         performed_with_partial_comprehension=1,
+                         performed_with_missing_information=2,
+                         refresh_performed_with_addl_efs_read=3,
+                         porformed_successfully_but_reqd_item_not_displayed=4,
+                         performed_but_modified_by_call_control_by_naa=5,
+                         performed_successfully_limited_service=6,
+                         performed_with_modification=7,
+                         refresh_performed_but_indicated_naa_was_not_active=8,
+                         performed_successfully_tone_not_played=9,
+                         proactive_uicc_session_terminated_by_user=0x10,
+                         backward_move_in_proactive_uicc_session_requested_by_user=0x11,
+                         no_response_from_user=0x12,
+                         help_information_required_by_user=0x13,
+                         # '2X' indicate to the UICC that it may be worth re-trying later
+                         terminal_currently_unable_to_process=0x20,
+                         network_currently_unable_to_process=0x21,
+                         user_did_not_accept_proactive_cmd=0x22,
+                         user_cleared_down_call_before_release=0x23,
+                         action_in_contradiction_with_current_timer_state=0x24,
+                         interaction_with_call_control_by_naa_temporary=0x25,
+                         launch_browser_generic_error=0x26,
+                         mms_temporary_problem=0x27,
+                         # '3X' indicate that it is not worth the UICC re-trying with an identical command
+                         command_beyond_terminal_capability=0x30,
+                         command_type_not_understood_by_terminal=0x31,
+                         command_data_not_understood_by_terminal=0x32,
+                         command_number_not_known_by_terminal=0x33,
+                         error_required_values_missing=0x36,
+                         multiple_card_commands_error=0x38,
+                         #interaction_with_call_control_by_naa_permanent=0x39, # see below 3GPP
+                         bearer_independent_protocol_error=0x3a,
+                         access_technology_unable_to_process_command=0x3b,
+                         frames_error=0x3c,
+                         mms_error=0x3d,
+                         # 3GPP TS 31.111 Section 8.12
+                         ussd_or_ss_transaction_terminated_by_user=0x14,
+                         ss_return_error=0x34,
+                         sms_rp_error=0x35,
+                         ussd_return_error=0x37,
+                         interaction_with_cc_by_usim_or_mo_sm_by_usim_permanent=0x39)
+    # TS 102 223 Section 8.12.2
+    AddlInfoTermProblem = Enum(Int8ub,
+                                no_specific_cause=0x00,
+                                screen_is_busy=0x01,
+                                terminal_currently_busy_on_call=0x02,
+                                no_service=0x04,
+                                access_control_class_bar=0x05,
+                                radio_resource_not_granted=0x06,
+                                not_in_speech_call=0x07,
+                                terminal_currently_busy_on_send_dtmf=0x09,
+                                no_naa_active=0x10,
+                                # TS 31.111 section 8.12
+                                me_currently_busy_on_ss_transaction=0x03,
+                                me_currently_busy_on_ussd_transaction=0x08)
+    # TS 102 223 Section 8.12.8 / TS 31.111 Section 8.12.8
+    AddlInfoCallControl= Enum(Int8ub,
+                                no_specific_cause=0x00,
+                                action_not_allowed=0x01,
+                                the_type_of_request_has_changed=0x02)
+    # TS 102 223 Section 8.12.9
+    AddlInfoMultipleCard = Enum(Int8ub,
+                                no_specific_cause=0x00,
+                                card_reader_removed_or_not_present=0x01,
+                                card_removed_or_not_present=0x02,
+                                card_reader_busy=0x03,
+                                card_powered_off=0x04,
+                                capdu_format_error=0x05,
+                                mute_card=0x06,
+                                transmission_error=0x07,
+                                protocol_not_supported=0x08,
+                                specified_reader_not_valid=0x09)
+    # TS 102 223 Section 8.12.10
+    AddlInfoLaunchBrowser = Enum(Int8ub,
+                                no_specific_cause=0x00,
+                                bearer_unavailable=0x01,
+                                browser_unavailable=0x02,
+                                terminal_unable_to_read_provisioning_data=0x03,
+                                default_url_unavailable=0x04)
+    # TS 102 223 Section 8.12.11
+    AddlInfoBip = Enum(Int8ub,  no_specific_cause=0x00,
+                                no_channel_availabile=0x01,
+                                channel_closed=0x02,
+                                channel_id_not_valid=0x03,
+                                requested_buffer_size_not_available=0x04,
+                                security_error=0x05,
+                                requested_uicc_if_transp_level_not_available=0x06,
+                                remote_device_not_reachable=0x07,
+                                service_error=0x08,
+                                service_identifer_unknown=0x09,
+                                port_not_available=0x10,
+                                launch_parameters_missing_or_incorrect=0x11,
+                                application_launch_failed=0x12)
+    # TS 102 223 Section 8.12.11
+    AddlInfoFrames = Enum(Int8ub,
+                                no_specific_cause=0x00,
+                                frame_identifier_not_valid=0x01,
+                                num_of_frames_beyond_terminal_capabilities=0x02,
+                                no_frame_defined=0x03,
+                                requested_size_not_supported=0x04,
+                                default_active_frame_not_valid=0x05)
 
+    _construct = Struct('general_result'/GeneralResult,
+                        'additional_information'/Switch(this.general_result,
+                        {
+                            'terminal_currently_unable_to_process': AddlInfoTermProblem,
+                            'interaction_with_cc_by_usim_or_mo_sm_by_usim_permanent': AddlInfoCallControl,
+                            'multiple_card_commands_error': AddlInfoMultipleCard,
+                            'launch_browser_generic_error': AddlInfoLaunchBrowser,
+                            'bearer_independent_protocol_error': AddlInfoBip,
+                            'frames_error': AddlInfoFrames
+                        }, default=HexAdapter(GreedyBytes)))
 
 # TS 102 223 Section 8.13  + TS 31.111 Section 8.13
 class SMS_TPDU(COMPR_TLV_IE, tag=0x8B):
     _construct = Struct('tpdu'/HexAdapter(GreedyBytes))
 
+# TS 31.111 Section 8.14
+class SsString(COMPR_TLV_IE, tag=0x89):
+    _construct = Struct('ton_npi'/TonNpi, 'ss_string'/HexAdapter(GreedyBytes))
+
+
 # TS 102 223 Section 8.15
 class TextString(COMPR_TLV_IE, tag=0x0d):
-    _construct = Struct('dcs'/Int8ub,
+    _construct = Struct('dcs'/Int8ub, # TS 03.38
                         'text_string'/HexAdapter(GreedyBytes))
 
 # TS 102 223 Section 8.16
 class Tone(COMPR_TLV_IE, tag=0x0e):
-    _construct = Struct('tone'/Int8ub)
+    _construct = Struct('tone'/Enum(Int8ub, dial_tone=0x01,
+                                            called_subscriber_busy=0x02,
+                                            congestion=0x03,
+                                            radio_path_acknowledge=0x04,
+                                            radio_path_not_available=0x05,
+                                            error_special_info=0x06,
+                                            call_waiting_tone=0x07,
+                                            ringing_tone=0x08,
+                                            general_beep=0x10,
+                                            positive_ack_tone=0x11,
+                                            negative_ack_or_error_tone=0x12,
+                                            ringing_tone_speech=0x13,
+                                            alert_tone_sms=0x14,
+                                            critical_alert=0x15,
+                                            vibrate_only=0x20,
+                                            happy_tone=0x30,
+                                            sad_tone=0x31,
+                                            urgent_action_tone=0x32,
+                                            question_tone=0x33,
+                                            message_received_tone=0x34,
+                                            melody_1=0x40,
+                                            melody_2=0x41,
+                                            melody_3=0x42,
+                                            melody_4=0x43,
+                                            melody_5=0x44,
+                                            melody_6=0x45,
+                                            melody_7=0x46,
+                                            melody_8=0x47))
 
 # TS 31 111 Section 8.17
 class USSDString(COMPR_TLV_IE, tag=0x0a):
     _construct = Struct('dcs'/Int8ub,
                         'ussd_string'/HexAdapter(GreedyBytes))
 
+# TS 102 223 Section 8.18
+class FileList(COMPR_TLV_IE, tag=0x12):
+    FileId=HexAdapter(Bytes(2))
+    _construct = Struct('number_of_files'/Int8ub,
+                        'files'/GreedyRange(FileId))
 
-# TS 101 220 Table 7.17
-class ProactiveCommand(BER_TLV_IE, tag=0xD0):
+# TS 102 223 Secton 8.19
+class LocationInformation(COMPR_TLV_IE, tag=0x93):
     pass
 
+# TS 102 223 Secton 8.20
+class IMEI(COMPR_TLV_IE, tag=0x94):
+    _construct = BcdAdapter(GreedyBytes)
+
+# TS 102 223 Secton 8.21
+class HelpRequest(COMPR_TLV_IE, tag=0x95):
+    pass
+
+# TS 102 223 Secton 8.22
+class NetworkMeasurementResults(COMPR_TLV_IE, tag=0x96):
+    _construct = BcdAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.23
+class DefaultText(COMPR_TLV_IE, tag=0x97):
+    _construct = Struct('dcs'/Int8ub,
+                        'text_string'/HexAdapter(GreedyBytes))
+
+# TS 102 223 Section 8.24
+class ItemsNextActionIndicator(COMPR_TLV_IE, tag=0x18):
+    _construct = GreedyRange(Int8ub)
+
+class EventList(COMPR_TLV_IE, tag=0x99):
+    Event = Enum(Int8ub,    mt_call=0x00,
+                            call_connected=0x01,
+                            call_disconnected=0x02,
+                            location_status=0x03,
+                            user_activity=0x04,
+                            idle_screen_available=0x05,
+                            card_reader_status=0x06,
+                            language_selection=0x07,
+                            browser_termination=0x08,
+                            data_available=0x09,
+                            channel_status=0x0a,
+                            access_technology_change=0x0b,
+                            display_parameters_changed=0x0c,
+                            local_connection=0x0d,
+                            network_search_mode_change=0x0e,
+                            browsing_status=0x0f,
+                            frames_informtion_change=0x10,
+                            hci_connectivity_event=0x13,
+                            access_technology_change_multiple=0x14,
+                            contactless_state_request=0x16,
+                            profile_container=0x19,
+                            secured_profile_container=0x1b,
+                            poll_interval_negotation=0x1c,
+                            # TS 31.111 Section 8.25
+                            wlan_access_status=0x11,
+                            network_rejection=0x12,
+                            csg_cell_selection=0x15,
+                            ims_registration=0x17,
+                            incoming_ims_data=0x18,
+                            data_connection_status_change=0x1d)
+    _construct = GreedyRange(Event)
+
+# TS 102 223 Section 8.26
+class Cause(COMPR_TLV_IE, tag=0x9a):
+    pass
+
+# TS 102 223 Section 8.27
+class LocationStatus(COMPR_TLV_IE, tag=0x9b):
+    _construct = Enum(Int8ub, normal_service=0, limited_service=1, no_service=2)
+
+# TS 102 223 Section 8.31
+class IconIdentifier(COMPR_TLV_IE, tag=0x1e):
+    _construct = Struct('icon_qualifier'/FlagsEnum(Int8ub, not_self_explanatory=1),
+                        'icon_identifier'/Int8ub)
+
+# TS 102 223 Section 8.32
+class ItemIconIdentifierList(COMPR_TLV_IE, tag=0x9f):
+    _construct = Struct('icon_list_qualifier'/FlagsEnum(Int8ub, not_self_explanatory=1),
+                        'icon_identifiers'/GreedyRange(Int8ub))
+
+# TS 102 223 Section 8.35
+class CApdu(COMPR_TLV_IE, tag=0xA2):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.37
+class TimerIdentifier(COMPR_TLV_IE, tag=0xA4):
+    _construct = Int8ub
+
+# TS 102 223 Section 8.38
+class TimerValue(COMPR_TLV_IE, tag=0xA5):
+    _construct = Struct('hour'/Int8ub, 'minute'/Int8ub, 'second'/Int8ub)
+
+# TS 102 223 Section 8.40
+class AtCommand(COMPR_TLV_IE, tag=0xA8):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.43
+class ImmediateResponse(COMPR_TLV_IE, tag=0x2b):
+    pass
+
+# TS 102 223 Section 8.45
+class Language(COMPR_TLV_IE, tag=0xAD):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 31.111 Section 8.46
+class TimingAdvance(COMPR_TLV_IE, tag=0x46):
+    _construct = Struct('me_status'/Enum(Int8ub, in_idle_state=0, not_in_idle_state=1),
+                        'timing_advance'/Int8ub)
+
+# TS 31.111 Section 8.49
+class Bearer(COMPR_TLV_IE, tag=0xB2):
+    SingleBearer = Enum(Int8ub, sms=0, csd=1, ussd=2, packet_Service=3)
+    _construct = GreedyRange(SingleBearer)
+
+# TS 102 223 Section 8.52
+class BearerDescription(COMPR_TLV_IE, tag=0xB5):
+    # TS 31.111 Section 8.52.1
+    BearerParsCs = Struct('data_rate'/Int8ub,
+                          'bearer_service'/Int8ub,
+                          'connection_element'/Int8ub)
+    # TS 31.111 Section 8.52.2
+    BearerParsPacket = Struct('precendence_class'/Int8ub,
+                              'delay'/Int8ub,
+                              'reliability'/Int8ub,
+                              'peak_throughput'/Int8ub,
+                              'mean_throughput'/Int8ub,
+                              'pdp_type'/Enum(Int8ub, ip=0x02, non_ip=0x07))
+    # TS 31.111 Section 8.52.3
+    BearerParsPacketExt = Struct('traffic_class'/Int8ub,
+                                 'max_bitrate_ul'/Int16ub,
+                                 'max_bitrate_dl'/Int16ub,
+                                 'guaranteed_bitrate_ul'/Int16ub,
+                                 'guaranteed_bitrate_dl'/Int16ub,
+                                 'delivery_order'/Int8ub,
+                                 'max_sdu_size'/Int8ub,
+                                 'sdu_err_ratio'/Int8ub,
+                                 'residual_ber'/Int8ub,
+                                 'delivery_of_erroneous_sdu'/Int8ub,
+                                 'transfer_delay'/Int8ub,
+                                 'traffic_handling_priority'/Int8ub,
+                                 'pdp_type'/Enum(Int8ub, ip=0x02, non_ip=0x07)) # 24.008
+    # TODO: TS 31.111 Section 8.52.4 I-WLAN
+    # TODO: TS 31.111 Section 8.52.5 E-UTRAN / mapped UTRAN packet service
+    # TS 31.111 Section 8.52.6
+    BearerParsNgRan = Struct('pdu_session_type'/Int8ub)
+    _construct = Struct('bearer_type'/Enum(Int8ub,
+                            # TS 31.111 section 8.52
+                            csd=1, packet_grps_utran_eutran=2, packet_with_extd_params=9, wlan=0x0a,
+                            packet_eutran_mapped_utran=0x0b, ng_ran=0x0c,
+                            # TS 102 223 Section 8.52
+                            default=3, local_link=4, bluetooth=5, irda=6, rs232=7, cdma2000=8,
+                            usb=10),
+                        'bearer_parameters'/Switch(this.bearer_type,{
+                                                        'csd': BearerParsCs,
+                                                        'packet_grps_utran_eutran': BearerParsPacket,
+                                                        'packet_with_extd_params': BearerParsPacketExt,
+                                                        'ng_ran': BearerParsNgRan,
+                                                    }, default=HexAdapter(GreedyBytes)))
+
+# TS 102 223 Section 8.53
+class ChannelData(COMPR_TLV_IE, tag = 0xB6):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.54
+class ChannelDataLength(COMPR_TLV_IE, tag = 0xB7):
+    _construct = Int8ub
+
+# TS 102 223 Section 8.55
+class BufferSize(COMPR_TLV_IE, tag = 0xB9):
+    _construct = Int16ub
+
+# TS 31.111 Section 8.56
+class ChannelStatus(COMPR_TLV_IE, tag = 0xB8):
+    # complex decoding, depends on out-of-band context/knowledge :(
+    pass
+
+# TS 102 223 Section 8.58
+class OtherAddress(COMPR_TLV_IE, tag = 0xBE):
+    _construct = Struct('type_of_address'/Enum(Int8ub, ipv4=0x21, ipv6=0x57),
+                        'address'/HexAdapter(GreedyBytes))
+
+# TS 102 223 Section 8.59
+class UiccTransportLevel(COMPR_TLV_IE, tag = 0xBC):
+    _construct = Struct('protocol_type'/Enum(Int8ub, udp_uicc_client_remote=1, tcp_uicc_client_remote=2,
+                                             tcp_uicc_server=3, udp_uicc_client_local=4,
+                                             tcp_uicc_client_local=5, direct_channel=6),
+                        'port_number'/Int16ub)
+
+# TS 102 223 Section 8.60
+class Aid(COMPR_TLV_IE, tag=0x2f):
+    _construct = Struct('aid'/HexAdapter(GreedyBytes))
+
+# TS 102 223 Section 8.61
+class AccessTechnology(COMPR_TLV_IE, tag=0xBF):
+    SingleAccessTech = Enum(Int8ub, gsm=0, tia_eia_533=1, tia_eia_136_270=2, utran=3, tetra=4,
+                                    tia_eia_95_b=5, cdma1000_1x=6, cdma2000_hrpd=7, eutran=8,
+                                    ehrpd=9, nr=0x0a)
+    _construct = GreedyRange(SingleAccessTech)
+
+# TS 102 223 Section 8.63
+class ServiceRecord(COMPR_TLV_IE, tag=0xC1):
+    BearerTechId = Enum(Int8ub, technology_independent=0, bluetooth=1, irda=2, rs232=3, usb=4)
+    _construct = Struct('local_bearer_technology'/BearerTechId,
+                        'service_identifier'/Int8ub,
+                        'service_record'/HexAdapter(GreedyBytes))
+
+# TS 102 223 Section 8.64
+class DeviceFilter(COMPR_TLV_IE, tag=0xC2):
+    _construct = Struct('local_bearer_technology'/ServiceRecord.BearerTechId,
+                        'device_filter'/HexAdapter(GreedyBytes))
+
+# TS 102 223 Section 8.65
+class ServiceSearchIE(COMPR_TLV_IE, tag=0xC3):
+    _construct = Struct('local_bearer_technology'/ServiceRecord.BearerTechId,
+                        'service_search'/HexAdapter(GreedyBytes))
+
+# TS 102 223 Section 8.66
+class AttributeInformation(COMPR_TLV_IE, tag=0xC4):
+    _construct = Struct('local_bearer_technology'/ServiceRecord.BearerTechId,
+                        'attribute_information'/HexAdapter(GreedyBytes))
+
+
+# TS 102 223 Section 8.68
+class RemoteEntityAddress(COMPR_TLV_IE, tag=0xC9):
+    _construct = Struct('coding_type'/Enum(Int8ub, ieee802_16=0, irda=1),
+                        'address'/HexAdapter(GreedyBytes))
+
+# TS 102 223 Section 8.70
+class NetworkAccessName(COMPR_TLV_IE, tag=0xC7):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.72
+class TextAttribute(COMPR_TLV_IE, tag=0x50):
+    pass
+
+# TS 31.111 Section 8.72
+class PdpContextActivationParams(COMPR_TLV_IE, tag=0xD2):
+    pass
+
+# TS 31.111 Section 8.73
+class UtranEutranMeasurementQualifier(COMPR_TLV_IE, tag=0xE9):
+    _construct = Enum(Int8ub,   utran_intra_freq=0x01,
+                                utran_inter_freq=0x02,
+                                utran_inter_rat_geran=0x03,
+                                utran_inter_rat_eutran=0x04,
+                                eutran_intra_freq=0x05,
+                                eutran_inter_freq=0x06,
+                                eutran_inter_rat_geran=0x07,
+                                eutran_inter_rat_utran=0x08,
+                                eutran_inter_rat_nr=0x09)
+
+# TS 102 223 Section 8.75
+class NetworkSearchMode(COMPR_TLV_IE, tag=0xE5):
+    _construct = Enum(Int8ub, manual=0, automatic=1)
+
+# TS 102 223 Section 8.76
+class BatteryState(COMPR_TLV_IE, tag=0xE3):
+    _construct = Enum(Int8ub, very_low=0, low=1, average=2, good=3, full=5)
+
+# TS 102 223 Section 8.78
+class FrameLayout(COMPR_TLV_IE, tag=0xE6):
+    _construct = Struct('layout'/Enum(Int8ub, horizontal=1, vertical=2),
+                        'relative_sized_frame'/GreedyRange(Int8ub))
+
+class ItemTextAttributeList(COMPR_TLV_IE, tag=0xD1):
+    _construct = GreedyRange(Int8ub)
+
+# TS 102 223 Section 8.80
+class FrameIdentifier(COMPR_TLV_IE, tag=0x68):
+    _construct = Struct('identifier'/Int8ub)
+
+# TS 102 223 Section 8.82
+class MultimediaMessageReference(COMPR_TLV_IE, tag=0xEA):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.83
+class MultimediaMessageIdentifier(COMPR_TLV_IE, tag=0xEB):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.85
+class MmContentIdentifier(COMPR_TLV_IE, tag=0xEE):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.89
+class ActivateDescriptor(COMPR_TLV_IE, tag=0xFB):
+    _construct = Struct('target'/Int8ub)
+
+# TS 31.111 Section 8.90
+class PlmnWactList(COMPR_TLV_IE, tag=0xF2):
+    def _from_bytes(self, x):
+        r = []
+        i = 0
+        while i < len(x):
+            r.append(dec_xplmn_w_act(b2h(x[i:i+5])))
+            i += 5
+        return r
+
+# TS 102 223 Section 8.92
+class ContactlessFunctionalityState(COMPR_TLV_IE, tag=0xD4):
+    _construct = Enum(Int8ub, enabled=0, disabled=1)
+
+# TS 31.111 Section 8.91
+class RoutingAreaIdentification(COMPR_TLV_IE, tag=0xF3):
+    _construct = Struct('mcc_mnc'/BcdAdapter(Bytes(3)),
+                        'lac'/HexAdapter(Bytes(2)),
+                        'rac'/Int8ub)
+
+# TS 31.111 Section 8.92
+class UpdateAttachRegistrationType(COMPR_TLV_IE, tag=0xF4):
+    _construct = Enum(Int8ub,   normal_location_updating_lu=0x00,
+                                periodic_updating_lu=0x01,
+                                imsi_attach_lu=0x02,
+                                gprs_attach=0x03,
+                                combined_gprs_imsi_attach=0x04,
+                                ra_updating_rau=0x05,
+                                combined_ra_la_updting_rau=0x06,
+                                combined_ra_la_updting_with_imsi_attach_rau=0x07,
+                                periodic_updating_rau=0x08,
+                                eps_attach_emm=0x09,
+                                combined_eps_imsi_attach_emm=0x0a,
+                                ta_updating_tau=0x0b,
+                                combined_ta_la_updating_tau=0x0c,
+                                combined_ta_la_updating_with_imsi_attach_tau=0x0d,
+                                periodic_updating_tau=0x0e,
+                                initial_registration_5grr=0x0f,
+                                mobility_registration_updating_5grr=0x10,
+                                periodic_registration_updating_5grr=0x11)
+
+# TS 31.111 Section 8.93
+class RejectionCauseCode(COMPR_TLV_IE, tag=0xF5):
+    _construct = Int8ub
+
+# TS 31.111 Section 8.94
+class GeographicalLocationParameters(COMPR_TLV_IE, tag=0xF6):
+    _construct = Struct('horizontal_accuracy'/Int8ub,
+                        'vertical_coordinate'/Int8ub,
+                        'velocity'/FlagsEnum(Int8ub, horizontal_requested=0, vertical_requested=1,
+                                             horizontal_uncertainty_requested=2,
+                                             vertical_uncertainty_requested=4),
+                        'preferred_gad_shapes'/FlagsEnum(Int8ub, ellipsoid_point=0,
+                                                         ellipsoid_point_with_uncertainty_circle=1,
+                                                         ellipsoid_point_with_uncertainty_ellipse=2,
+                                                         ellipsoid_point_with_altitude=3,
+                                                         polygon=4,
+                                                         ellipsoid_point_with_altitude_and_uncertainty_ellipsoid=5,
+                                                         ellipsoid_arc=6),
+                        'preferred_nmea_sentences'/FlagsEnum(Int8ub, rmc=0, gga=1, gll=2, gns=3),
+                        'preferred_maximum_response_time'/Int8ub)
+
+# TS 31.111 Section 8.97
+class PlmnList(COMPR_TLV_IE, tag=0xF9):
+    _construct = GreedyRange('mcc_mnc'/HexAdapter(Bytes(3)))
+
+# TS 102 223 Section 8.98
+class EcatSequenceNumber(COMPR_TLV_IE, tag=0xA1):
+    CmdTypeIndicator = Enum(BitsInteger(2), command_container=0,
+                                            terminal_response=1,
+                                            envelope_profile_container_event=2,
+                                            envelope_profile_container_response=3)
+    _construct = BitStruct('command_type_indicator'/CmdTypeIndicator,
+                           'counter'/BitsInteger(22))
+
+# TS 102 223 Section 8.99
+class EncryptedTlvList(COMPR_TLV_IE, tag=0xA2):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.100
+class Mac(COMPR_TLV_IE, tag=0xE0):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.101
+class SaTemplate(COMPR_TLV_IE, tag=0xA3):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.103
+class RefreshEnforcementPolicy(COMPR_TLV_IE, tag=0x3A):
+    _construct = FlagsEnum(Byte, even_if_navigating_menus=0, even_if_data_call=1, even_if_voice_call=2)
+
+# TS 102 223 Section 8.104
+class DnsServerAddress(COMPR_TLV_IE, tag=0xC0):
+    _construct = HexAdapter(GreedyBytes)
+
+# TS 102 223 Section 8.105
+class SupportedRadioAccessTechnologies(COMPR_TLV_IE, tag=0xB4):
+    AccessTechTuple = Struct('technology'/AccessTechnology.SingleAccessTech,
+                             'state'/FlagsEnum(Int8ub, enabled=0))
+    _construct = GreedyRange(AccessTechTuple)
+
+# TS 102 223 Section 8.107
+class ApplicationSpecificRefreshData(COMPR_TLV_IE, tag=0x3B):
+    pass
+
+# TS 31.111 Section 8.108
+class ImsUri(COMPR_TLV_IE, tag=0xB1):
+    pass
+
+# TS 31.111 Section 8.132
+class MediaType(COMPR_TLV_IE, tag=0xFE):
+    _construct = FlagsEnum(Int8ub, voice=0, video=1)
+
+# TS 31.111 Section 8.137
+class DataConnectionStatus(COMPR_TLV_IE, tag=0x9D):
+    _construct = Enum(Int8ub, successful=0, rejected=1, dropped_or_deactivated=2)
+
+# TS 31.111 Section 8.138
+class DataConnectionType(COMPR_TLV_IE, tag=0xAA):
+    _construct = Enum(Int8ub, pdp=0, pdn=1, pdu=2)
+
+# TS 31.111 Section 8.139
+class SmCause(COMPR_TLV_IE, tag=0xAE):
+    _construct = Int8ub
+
+
 # TS 101 220 Table 7.17 + 31.111 7.1.1.2
 class SMSPPDownload(BER_TLV_IE, tag=0xD1,
                     nested=[DeviceIdentities, Address, SMS_TPDU]):
@@ -164,6 +723,306 @@ class USSDDownload(BER_TLV_IE, tag=0xD9,
     pass
 
 
+
+
+class ProactiveCmd(BER_TLV_IE):
+    def _compute_tag(self) -> int:
+        return 0xD0
+
+
+# TS 101 220 Table 7.17 + 102 223 6.6.13/9.4 + TS 31.111 6.6.13
+class Refresh(ProactiveCmd, tag=0x01,
+              nested=[CommandDetails, DeviceIdentities, FileList, Aid, AlphaIdentifier,
+                      IconIdentifier, TextAttribute, FrameIdentifier, RefreshEnforcementPolicy,
+                      ApplicationSpecificRefreshData, PlmnWactList, PlmnList]):
+    pass
+
+class MoreTime(ProactiveCmd, tag=0x02,
+        nested=[CommandDetails]):
+    pass
+
+class PollInterval(ProactiveCmd, tag=0x03,
+        nested=[CommandDetails]):
+    pass
+
+class PollingOff(ProactiveCmd, tag=0x04,
+        nested=[CommandDetails]):
+    pass
+
+class SetUpEventList(ProactiveCmd, tag=0x05,
+        nested=[CommandDetails]):
+    pass
+
+# TS 31.111 Section 6.6.12
+class SetUpCall(ProactiveCmd, tag=0x10,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, Address, ImsUri,
+                CapabilityConfigParams, Subaddress, Duration, IconIdentifier, AlphaIdentifier,
+                TextAttribute, FrameIdentifier, MediaType]):
+    pass
+
+# TS 31.111 Section 6.6.10
+class SendSS(ProactiveCmd, tag=0x11,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, SsString, IconIdentifier,
+                TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 31.111 Section 6.6.11
+class SendUSSD(ProactiveCmd, tag=0x12,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, USSDString, IconIdentifier,
+                TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 101 220 Table 7.17 + 102 223 6.6.9/9.4 + TS 31.111 Section 6.6.9
+class SendShortMessage(ProactiveCmd, tag=0x13,
+                       nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, Address,
+                               SMS_TPDU, IconIdentifier, TextAttribute, FrameIdentifier]):
+    pass
+
+class SendDTMF(ProactiveCmd, tag=0x14,
+        nested=[CommandDetails]):
+    pass
+
+class LaunchBrowser(ProactiveCmd, tag=0x15,
+        nested=[CommandDetails]):
+    pass
+
+class GeographicalLocationRequest(ProactiveCmd, tag=0x16,
+        nested=[CommandDetails]):
+    pass
+
+class PlayTone(ProactiveCmd, tag=0x20,
+        nested=[CommandDetails]):
+    pass
+
+# TS 101 220 Table 7.17 + 102 223 6.6.1/9.4 CMD=0x21
+class DisplayText(ProactiveCmd, tag=0x21,
+                  nested=[CommandDetails, DeviceIdentities, TextString, IconIdentifier,
+                          ImmediateResponse, Duration, TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.2
+class GetInkey(ProactiveCmd, tag=0x22,
+        nested=[CommandDetails, DeviceIdentities, TextString, IconIdentifier, Duration,
+                TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.3
+class GetInput(ProactiveCmd, tag=0x23,
+        nested=[CommandDetails, DeviceIdentities, TextString, ResponseLength, DefaultText,
+                IconIdentifier, TextAttribute, FrameIdentifier, Duration]):
+    pass
+
+# TS 102 223 6.6.8
+class SelectItem(ProactiveCmd, tag=0x24,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, #
+                ItemsNextActionIndicator, ItemIdentifier, IconIdentifier, ItemIconIdentifierList,
+                TextAttribute, ItemTextAttributeList, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.7
+class SetUpMenu(ProactiveCmd, tag=0x25,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, #
+                ItemsNextActionIndicator, IconIdentifier, ItemIconIdentifierList,
+                TextAttribute, ItemTextAttributeList]):
+    pass
+
+# TS 102 223 6.6.15
+class ProvideLocalInformation(ProactiveCmd, tag=0x26,
+        nested=[CommandDetails, DeviceIdentities]):
+    pass
+
+# TS 102 223 6.6.21
+class TimerManagement(ProactiveCmd, tag=0x27,
+        nested=[CommandDetails, DeviceIdentities, TimerIdentifier, TimerValue]):
+    pass
+
+# TS 102 223 6.6.22
+class SetUpIdleModeText(ProactiveCmd, tag=0x28,
+        nested=[CommandDetails, DeviceIdentities, TextString, IconIdentifier, TextAttribute,
+                FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.17
+class PerformCardApdu(ProactiveCmd, tag=0x30,
+        nested=[CommandDetails, DeviceIdentities, CApdu]):
+    pass
+
+# TS 102 223 6.6.19
+class PowerOnCard(ProactiveCmd, tag=0x31,
+        nested=[CommandDetails, DeviceIdentities]):
+    pass
+
+# TS 102 223 6.6.18
+class PowerOffCard(ProactiveCmd, tag=0x32,
+        nested=[CommandDetails, DeviceIdentities]):
+    pass
+
+# TS 102 223 6.6.20
+class GetReaderStatus(ProactiveCmd, tag=0x33,
+        nested=[CommandDetails, DeviceIdentities]):
+    pass
+
+# TS 102 223 6.6.23
+class RunAtCommand(ProactiveCmd, tag=0x34,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, AtCommand, IconIdentifier,
+                TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.25
+class LanguageNotification(ProactiveCmd, tag=0x35,
+        nested=[CommandDetails, DeviceIdentities, Language]):
+    pass
+
+# TS 102 223 6.6.27
+class OpenChannel(ProactiveCmd, tag=0x40,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, IconIdentifier, Address, Subaddress,
+                Duration, BearerDescription, BufferSize, NetworkAccessName, OtherAddress,
+                TextString, UiccTransportLevel, RemoteEntityAddress, TextAttribute,
+                FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.28
+class CloseChannel(ProactiveCmd, tag=0x41,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, IconIdentifier, TextAttribute,
+                FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.29
+class ReceiveData(ProactiveCmd, tag=0x42,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, IconIdentifier, ChannelDataLength,
+                TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.30
+class SendData(ProactiveCmd, tag=0x43,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, IconIdentifier, ChannelData,
+                TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.31
+class GetChannelStatus(ProactiveCmd, tag=0x44,
+        nested=[CommandDetails, DeviceIdentities]):
+    pass
+
+# TS 102 223 6.6.32
+class ServiceSearch(ProactiveCmd, tag=0x45,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, IconIdentifier, ServiceSearchIE,
+                DeviceFilter, TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.33
+class GetServiceInformation(ProactiveCmd, tag=0x46,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, IconIdentifier, AttributeInformation,
+                TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.34
+class DeclareService(ProactiveCmd, tag=0x47,
+        nested=[CommandDetails, DeviceIdentities, ServiceRecord, UiccTransportLevel]):
+    pass
+
+# TS 102 223 6.6.35
+class SetFrames(ProactiveCmd, tag=0x50,
+        nested=[CommandDetails, DeviceIdentities, FrameIdentifier, FrameLayout]):
+    pass
+
+# TS 102 223 6.6.36
+class GetFramesStatus(ProactiveCmd, tag=0x51,
+        nested=[CommandDetails, DeviceIdentities]):
+    pass
+
+# TS 102 223 6.6.37
+class RetrieveMultimediaMessage(ProactiveCmd, tag=0x60,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, MultimediaMessageReference,
+                FileList, MmContentIdentifier, MultimediaMessageIdentifier, TextAttribute,
+                FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.38
+class SubmitMultimediaMessage(ProactiveCmd, tag=0x61,
+        nested=[CommandDetails, DeviceIdentities, AlphaIdentifier, IconIdentifier, FileList,
+                MultimediaMessageIdentifier, TextAttribute, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.39
+class DisplayMultimediaMessage(ProactiveCmd, tag=0x62,
+        nested=[CommandDetails, DeviceIdentities, FileList, MultimediaMessageIdentifier,
+                ImmediateResponse, FrameIdentifier]):
+    pass
+
+# TS 102 223 6.6.40
+class Activate(ProactiveCmd, tag=0x70,
+        nested=[CommandDetails, DeviceIdentities, ActivateDescriptor]):
+    pass
+
+# TS 102 223 6.6.41
+class ContactlessStateChanged(ProactiveCmd, tag=0x71,
+        nested=[CommandDetails, DeviceIdentities, ContactlessFunctionalityState]):
+    pass
+
+# TS 102 223 6.6.42
+class CommandContainer(ProactiveCmd, tag=0x72,
+        nested=[CommandDetails, DeviceIdentities, EcatSequenceNumber, Mac, EncryptedTlvList]):
+    pass
+
+# TS 102 223 6.6.43
+class EncapsulatedSessionControl(ProactiveCmd, tag=0x73,
+        nested=[CommandDetails, DeviceIdentities, SaTemplate]):
+    pass
+
+
+
+# TS 101 220 Table 7.17: FIXME: Merge all nested?
+class ProactiveCommandBase(BER_TLV_IE, tag=0xD0, nested=[CommandDetails]):
+    def find_cmd_details(self):
+        for c in self.children:
+            if type(c).__name__ == 'CommandDetails':
+                return c
+        else:
+            return None
+
+class ProactiveCommand(TLV_IE_Collection,
+                       nested=[Refresh, MoreTime, PollInterval, PollingOff, SetUpEventList, SetUpCall,
+                               SendSS, SendUSSD, SendShortMessage, SendDTMF, LaunchBrowser,
+                               GeographicalLocationRequest, PlayTone, DisplayText, GetInkey, GetInput,
+                               SelectItem, SetUpMenu, ProvideLocalInformation, TimerManagement,
+                               SetUpIdleModeText, PerformCardApdu, PowerOnCard, PowerOffCard,
+                               GetReaderStatus, RunAtCommand, LanguageNotification, OpenChannel,
+                               CloseChannel, ReceiveData, SendData, GetChannelStatus, ServiceSearch,
+                               GetServiceInformation, DeclareService, SetFrames, GetFramesStatus,
+                               RetrieveMultimediaMessage, SubmitMultimediaMessage, DisplayMultimediaMessage,
+                               Activate, ContactlessStateChanged, CommandContainer,
+                               EncapsulatedSessionControl]):
+    """Class representing a CAT proactive command, as (for example) sent via a FETCH response. Parsing this is
+    more difficult than any normal TLV IE Collection, because the content of one of the IEs defines the
+    definitions of all the other IEs.  So we first need to find the CommandDetails, and then parse according
+    to the command type indicated in that IE data."""
+    def from_bytes(self, binary: bytes) -> List[TLV_IE]:
+        # do a first parse step to get the CommandDetails
+        pcmd = ProactiveCommandBase()
+        pcmd.from_tlv(binary)
+        cmd_details = pcmd.find_cmd_details()
+        # then do a second decode stage for the specific
+        cmd_type = cmd_details.decoded['type_of_command']
+        if cmd_type in self.members_by_tag:
+            cls = self.members_by_tag[cmd_type]
+            inst = cls()
+            dec, remainder = inst.from_tlv(binary)
+            self.decoded = inst
+        else:
+            self.decoded = pcmd
+        return self.decoded
+
+    #def from_dict(self, decoded):
+    #    pass
+
+    def to_dict(self):
+        return self.decoded.to_dict()
+
+    def to_bytes(self):
+        return self.decoded.to_tlv()
+
+
 # reasonable default for playing with OTA
 # 010203040506070809101112131415161718192021222324252627282930313233
 # '7fe1e10e000000000000001f43000000ff00000000000000000000000000000000'

pySim/commands.py

@@ -23,11 +23,11 @@
 
 from construct import *
 from pySim.construct import LV
-from pySim.utils import rpad, b2h, h2b, sw_match, bertlv_encode_len, Hexstr, h2i, str_sanitize
+from pySim.utils import rpad, b2h, h2b, sw_match, bertlv_encode_len, Hexstr, h2i, str_sanitize, expand_hex
 from pySim.exceptions import SwMatchError
 
 
-class SimCardCommands(object):
+class SimCardCommands:
     def __init__(self, transport):
         self._tp = transport
         self.cla_byte = "a0"
@@ -190,6 +190,10 @@ class SimCardCommands(object):
                 offset : byte offset in file from which to start writing
                 verify : Whether or not to verify data after write
         """
+
+        file_len = self.binary_size(ef)
+        data = expand_hex(data, file_len)
+
         data_length = len(data) // 2
 
         # Save write cycles by reading+comparing before write
@@ -255,16 +259,17 @@ class SimCardCommands(object):
                 verify : verify data by re-reading the record
                 conserve : read record and compare it with data, skip write on match
         """
+
         res = self.select_path(ef)
+        rec_length = self.__record_len(res)
+        data = expand_hex(data, rec_length)
 
         if force_len:
             # enforce the record length by the actual length of the given data input
             rec_length = len(data) // 2
         else:
-            # determine the record length from the select response of the file and pad
-            # the input data with 0xFF if necessary. In cases where the input data
-            # exceed we throw an exception.
-            rec_length = self.__record_len(res)
+            # make sure the input data is padded to the record length using 0xFF.
+            # In cases where the input data exceed we throw an exception.
             if (len(data) // 2 > rec_length):
                 raise ValueError('Data length exceeds record length (expected max %d, got %d)' % (
                     rec_length, len(data) // 2))

pySim/construct.py

@@ -2,7 +2,7 @@ from construct.lib.containers import Container, ListContainer
 from construct.core import EnumIntegerString
 import typing
 from construct import *
-from construct.core import evaluate, bytes2integer, integer2bytes, BitwisableString
+from construct.core import evaluate, BitwisableString
 from construct.lib import integertypes
 from pySim.utils import b2h, h2b, swap_nibbles
 import gsm0338
@@ -208,39 +208,47 @@ def GsmString(n):
 
 class GreedyInteger(Construct):
     """A variable-length integer implementation, think of combining GrredyBytes with BytesInteger."""
-    def __init__(self, signed=False, swapped=False):
+    def __init__(self, signed=False, swapped=False, minlen=0):
         super().__init__()
         self.signed = signed
         self.swapped = swapped
+        self.minlen = minlen
 
     def _parse(self, stream, context, path):
         data = stream_read_entire(stream, path)
         if evaluate(self.swapped, context):
             data = swapbytes(data)
         try:
-            return bytes2integer(data, self.signed)
+            return int.from_bytes(data, byteorder='big', signed=self.signed)
         except ValueError as e:
             raise IntegerError(str(e), path=path)
 
-    def __bytes_required(self, i):
+    def __bytes_required(self, i, minlen=0):
         if self.signed:
             raise NotImplementedError("FIXME: Implement support for encoding signed integer")
+
+        # compute how many bytes we need
         nbytes = 1
         while True:
             i = i >> 8
             if i == 0:
-                return nbytes
+                break
             else:
                 nbytes = nbytes + 1
-        # this should never happen, above loop must return eventually...
-        raise IntegerError(f"value {i} is out of range")
+
+        # round up to the minimum number
+        # of bytes we anticipate
+        if nbytes < minlen:
+            nbytes = minlen
+
+        return nbytes
 
     def _build(self, obj, stream, context, path):
         if not isinstance(obj, integertypes):
             raise IntegerError(f"value {obj} is not an integer", path=path)
-        length = self.__bytes_required(obj)
+        length = self.__bytes_required(obj, self.minlen)
         try:
-            data = integer2bytes(obj, length, self.signed)
+            data = obj.to_bytes(length, byteorder='big', signed=self.signed)
         except ValueError as e:
             raise IntegerError(str(e), path=path)
         if evaluate(self.swapped, context):

pySim/exceptions.py

@@ -53,8 +53,8 @@ class SwMatchError(Exception):
         self.rs = rs
 
     def __str__(self):
-        if self.rs:
-            r = self.rs.interpret_sw(self.sw_actual)
+        if self.rs and self.rs.lchan[0]:
+            r = self.rs.lchan[0].interpret_sw(self.sw_actual)
             if r:
                 return "SW match failed! Expected %s and got %s: %s - %s" % (self.sw_expected, self.sw_actual, r[0], r[1])
         return "SW match failed! Expected %s and got %s." % (self.sw_expected, self.sw_actual)

pySim/filesystem.py

@@ -49,7 +49,21 @@ from pySim.commands import SimCardCommands
 # tuple: logical-and of the listed services requires this file
 CardFileService = Union[int, List[int], Tuple[int, ...]]
 
-class CardFile(object):
+Size = Tuple[int, Optional[int]]
+
+def lchan_nr_from_cla(cla: int) -> int:
+    """Resolve the logical channel number from the CLA byte."""
+    # TS 102 221 10.1.1 Coding of Class Byte
+    if cla >> 4 in [0x0, 0xA, 0x8]:
+        # Table 10.3
+        return cla & 0x03
+    elif cla & 0xD0 in [0x40, 0xC0]:
+        # Table 10.4a
+        return 4 + (cla & 0x0F)
+    else:
+        raise ValueError('Could not determine logical channel for CLA=%2X' % cla)
+
+class CardFile:
     """Base class for all objects in the smart card filesystem.
     Serve as a common ancestor to all other file types; rarely used directly.
     """
@@ -100,6 +114,14 @@ class CardFile(object):
         else:
             return self.fid
 
+    def fully_qualified_path_str(self, prefer_name: bool = True) -> str:
+        """Return fully qualified path to file as string.
+
+        Args:
+            prefer_name : Preferably build path of names; fall-back to FIDs as required
+        """
+        return '/'.join(self.fully_qualified_path(prefer_name))
+
     def fully_qualified_path(self, prefer_name: bool = True) -> List[str]:
         """Return fully qualified path to file as list of FID or name strings.
 
@@ -172,6 +194,21 @@ class CardFile(object):
             sels.update({self.name: self})
         return sels
 
+    def _get_parent_selectables(self, alias: Optional[str] = None, flags=[]) -> Dict[str, 'CardFile']:
+        sels = {}
+        if not self.parent or self.parent == self:
+            return sels
+        # add our immediate parent
+        if alias:
+            sels.update({alias: self.parent})
+        if self.parent.fid and (flags == [] or 'FIDS' in flags):
+            sels.update({self.parent.fid: self.parent})
+        if self.parent.name and (flags == [] or 'FNAMES' in flags):
+            sels.update({self.parent.name: self.parent})
+        # recurse to parents of our parent, but without any alias
+        sels.update(self.parent._get_parent_selectables(None, flags))
+        return sels
+
     def get_selectables(self, flags=[]) -> Dict[str, 'CardFile']:
         """Return a dict of {'identifier': File} that is selectable from the current file.
 
@@ -188,8 +225,7 @@ class CardFile(object):
             sels = self._get_self_selectables('.', flags)
         # we can always select our parent
         if flags == [] or 'PARENT' in flags:
-            if self.parent:
-                sels = self.parent._get_self_selectables('..', flags)
+            sels.update(self._get_parent_selectables('..', flags))
         # if we have a MF, we can always select its applications
         if flags == [] or 'MF' in flags:
             mf = self.get_mf()
@@ -301,6 +337,14 @@ class CardDF(CardFile):
         else:
             raise ValueError
 
+    def _has_service(self):
+        if self.service:
+            return True
+        for c in self.children.values():
+            if isinstance(c, CardDF):
+                if c._has_service():
+                    return True
+
     def add_file(self, child: CardFile, ignore_existing: bool = False):
         """Add a child (DF/EF) to this DF.
         Args:
@@ -336,7 +380,10 @@ class CardDF(CardFile):
             for c in child.children.values():
                 self._add_file_services(c)
                 if isinstance(c, CardDF):
-                    raise ValueError('TODO: implement recursive service -> file mapping')
+                    for gc in c.children.values():
+                        if isinstance(gc, CardDF):
+                            if gc._has_service():
+                                raise ValueError('TODO: implement recursive service -> file mapping')
 
     def add_files(self, children: Iterable[CardFile], ignore_existing: bool = False):
         """Add a list of child (DF/EF) to this DF
@@ -477,7 +524,7 @@ class CardADF(CardDF):
             mf.add_application_df(self)
 
     def __str__(self):
-        return "ADF(%s)" % (self.aid)
+        return "ADF(%s)" % (self.name if self.name else self.aid)
 
     def _path_element(self, prefer_name: bool):
         if self.name and prefer_name:
@@ -508,8 +555,10 @@ class CardEF(CardFile):
         """
         # global selectable names + those of the parent DF
         sels = super().get_selectables(flags)
-        sels.update(
-            {x.name: x for x in self.parent.children.values() if x != self})
+        if flags == [] or 'FIDS' in flags:
+            sels.update({x.fid: x for x in self.parent.children.values() if x.fid and x != self})
+        if flags == [] or 'FNAMES' in flags:
+            sels.update({x.name: x for x in self.parent.children.values() if x.name and x != self})
         return sels
 
 
@@ -534,7 +583,7 @@ class TransparentEF(CardEF):
         @cmd2.with_argparser(dec_hex_parser)
         def do_decode_hex(self, opts):
             """Decode command-line provided hex-string as if it was read from the file."""
-            data = self._cmd.rs.selected_file.decode_hex(opts.HEXSTR)
+            data = self._cmd.lchan.selected_file.decode_hex(opts.HEXSTR)
             self._cmd.poutput_json(data, opts.oneline)
 
         read_bin_parser = argparse.ArgumentParser()
@@ -546,7 +595,7 @@ class TransparentEF(CardEF):
         @cmd2.with_argparser(read_bin_parser)
         def do_read_binary(self, opts):
             """Read binary data from a transparent EF"""
-            (data, sw) = self._cmd.rs.read_binary(opts.length, opts.offset)
+            (data, sw) = self._cmd.lchan.read_binary(opts.length, opts.offset)
             self._cmd.poutput(data)
 
         read_bin_dec_parser = argparse.ArgumentParser()
@@ -556,7 +605,7 @@ class TransparentEF(CardEF):
         @cmd2.with_argparser(read_bin_dec_parser)
         def do_read_binary_decoded(self, opts):
             """Read + decode data from a transparent EF"""
-            (data, sw) = self._cmd.rs.read_binary_dec()
+            (data, sw) = self._cmd.lchan.read_binary_dec()
             self._cmd.poutput_json(data, opts.oneline)
 
         upd_bin_parser = argparse.ArgumentParser()
@@ -568,7 +617,7 @@ class TransparentEF(CardEF):
         @cmd2.with_argparser(upd_bin_parser)
         def do_update_binary(self, opts):
             """Update (Write) data of a transparent EF"""
-            (data, sw) = self._cmd.rs.update_binary(opts.data, opts.offset)
+            (data, sw) = self._cmd.lchan.update_binary(opts.data, opts.offset)
             if data:
                 self._cmd.poutput(data)
 
@@ -582,18 +631,18 @@ class TransparentEF(CardEF):
         def do_update_binary_decoded(self, opts):
             """Encode + Update (Write) data of a transparent EF"""
             if opts.json_path:
-                (data_json, sw) = self._cmd.rs.read_binary_dec()
+                (data_json, sw) = self._cmd.lchan.read_binary_dec()
                 js_path_modify(data_json, opts.json_path,
                                json.loads(opts.data))
             else:
                 data_json = json.loads(opts.data)
-            (data, sw) = self._cmd.rs.update_binary_dec(data_json)
+            (data, sw) = self._cmd.lchan.update_binary_dec(data_json)
             if data:
                 self._cmd.poutput_json(data)
 
         def do_edit_binary_decoded(self, opts):
             """Edit the JSON representation of the EF contents in an editor."""
-            (orig_json, sw) = self._cmd.rs.read_binary_dec()
+            (orig_json, sw) = self._cmd.lchan.read_binary_dec()
             with tempfile.TemporaryDirectory(prefix='pysim_') as dirname:
                 filename = '%s/file' % dirname
                 # write existing data as JSON to file
@@ -606,12 +655,12 @@ class TransparentEF(CardEF):
                 if edited_json == orig_json:
                     self._cmd.poutput("Data not modified, skipping write")
                 else:
-                    (data, sw) = self._cmd.rs.update_binary_dec(edited_json)
+                    (data, sw) = self._cmd.lchan.update_binary_dec(edited_json)
                     if data:
                         self._cmd.poutput_json(data)
 
     def __init__(self, fid: str, sfid: str = None, name: str = None, desc: str = None, parent: CardDF = None,
-                 size={1, None}, **kwargs):
+                 size: Size = (1, None), **kwargs):
         """
         Args:
             fid : File Identifier (4 hex digits)
@@ -757,7 +806,7 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(dec_hex_parser)
         def do_decode_hex(self, opts):
             """Decode command-line provided hex-string as if it was read from the file."""
-            data = self._cmd.rs.selected_file.decode_record_hex(opts.HEXSTR)
+            data = self._cmd.lchan.selected_file.decode_record_hex(opts.HEXSTR)
             self._cmd.poutput_json(data, opts.oneline)
 
         read_rec_parser = argparse.ArgumentParser()
@@ -771,7 +820,7 @@ class LinFixedEF(CardEF):
             """Read one or multiple records from a record-oriented EF"""
             for r in range(opts.count):
                 recnr = opts.record_nr + r
-                (data, sw) = self._cmd.rs.read_record(recnr)
+                (data, sw) = self._cmd.lchan.read_record(recnr)
                 if (len(data) > 0):
                     recstr = str(data)
                 else:
@@ -787,7 +836,7 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(read_rec_dec_parser)
         def do_read_record_decoded(self, opts):
             """Read + decode a record from a record-oriented EF"""
-            (data, sw) = self._cmd.rs.read_record_dec(opts.record_nr)
+            (data, sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
             self._cmd.poutput_json(data, opts.oneline)
 
         read_recs_parser = argparse.ArgumentParser()
@@ -795,9 +844,9 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(read_recs_parser)
         def do_read_records(self, opts):
             """Read all records from a record-oriented EF"""
-            num_of_rec = self._cmd.rs.selected_file_num_of_rec()
+            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
             for recnr in range(1, 1 + num_of_rec):
-                (data, sw) = self._cmd.rs.read_record(recnr)
+                (data, sw) = self._cmd.lchan.read_record(recnr)
                 if (len(data) > 0):
                     recstr = str(data)
                 else:
@@ -811,11 +860,11 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(read_recs_dec_parser)
         def do_read_records_decoded(self, opts):
             """Read + decode all records from a record-oriented EF"""
-            num_of_rec = self._cmd.rs.selected_file_num_of_rec()
+            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
             # collect all results in list so they are rendered as JSON list when printing
             data_list = []
             for recnr in range(1, 1 + num_of_rec):
-                (data, sw) = self._cmd.rs.read_record_dec(recnr)
+                (data, sw) = self._cmd.lchan.read_record_dec(recnr)
                 data_list.append(data)
             self._cmd.poutput_json(data_list, opts.oneline)
 
@@ -828,7 +877,7 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(upd_rec_parser)
         def do_update_record(self, opts):
             """Update (write) data to a record-oriented EF"""
-            (data, sw) = self._cmd.rs.update_record(opts.record_nr, opts.data)
+            (data, sw) = self._cmd.lchan.update_record(opts.record_nr, opts.data)
             if data:
                 self._cmd.poutput(data)
 
@@ -844,12 +893,12 @@ class LinFixedEF(CardEF):
         def do_update_record_decoded(self, opts):
             """Encode + Update (write) data to a record-oriented EF"""
             if opts.json_path:
-                (data_json, sw) = self._cmd.rs.read_record_dec(opts.record_nr)
+                (data_json, sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
                 js_path_modify(data_json, opts.json_path,
                                json.loads(opts.data))
             else:
                 data_json = json.loads(opts.data)
-            (data, sw) = self._cmd.rs.update_record_dec(
+            (data, sw) = self._cmd.lchan.update_record_dec(
                 opts.record_nr, data_json)
             if data:
                 self._cmd.poutput(data)
@@ -861,7 +910,7 @@ class LinFixedEF(CardEF):
         @cmd2.with_argparser(edit_rec_dec_parser)
         def do_edit_record_decoded(self, opts):
             """Edit the JSON representation of one record in an editor."""
-            (orig_json, sw) = self._cmd.rs.read_record_dec(opts.record_nr)
+            (orig_json, sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
             with tempfile.TemporaryDirectory(prefix='pysim_') as dirname:
                 filename = '%s/file' % dirname
                 # write existing data as JSON to file
@@ -874,13 +923,13 @@ class LinFixedEF(CardEF):
                 if edited_json == orig_json:
                     self._cmd.poutput("Data not modified, skipping write")
                 else:
-                    (data, sw) = self._cmd.rs.update_record_dec(
+                    (data, sw) = self._cmd.lchan.update_record_dec(
                         opts.record_nr, edited_json)
                     if data:
                         self._cmd.poutput_json(data)
 
     def __init__(self, fid: str, sfid: str = None, name: str = None, desc: str = None,
-                 parent: Optional[CardDF] = None, rec_len={1, None}, **kwargs):
+                 parent: Optional[CardDF] = None, rec_len: Size = (1, None), **kwargs):
         """
         Args:
             fid : File Identifier (4 hex digits)
@@ -888,7 +937,7 @@ class LinFixedEF(CardEF):
             name : Brief name of the file, lik EF_ICCID
             desc : Description of the file
             parent : Parent CardFile object within filesystem hierarchy
-            rec_len : set of {minimum_length, recommended_length}
+            rec_len : Tuple of (minimum_length, recommended_length)
         """
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent, **kwargs)
         self.rec_len = rec_len
@@ -1011,7 +1060,7 @@ class CyclicEF(LinFixedEF):
     # we don't really have any special support for those; just recycling LinFixedEF here
 
     def __init__(self, fid: str, sfid: str = None, name: str = None, desc: str = None, parent: CardDF = None,
-                 rec_len={1, None}, **kwargs):
+                 rec_len: Size = (1, None), **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, parent=parent, rec_len=rec_len, **kwargs)
 
 
@@ -1026,7 +1075,7 @@ class TransRecEF(TransparentEF):
     """
 
     def __init__(self, fid: str, rec_len: int, sfid: str = None, name: str = None, desc: str = None,
-                 parent: Optional[CardDF] = None, size={1, None}, **kwargs):
+                 parent: Optional[CardDF] = None, size: Size = (1, None), **kwargs):
         """
         Args:
             fid : File Identifier (4 hex digits)
@@ -1181,12 +1230,12 @@ class BerTlvEF(CardEF):
         @cmd2.with_argparser(retrieve_data_parser)
         def do_retrieve_data(self, opts):
             """Retrieve (Read) data from a BER-TLV EF"""
-            (data, sw) = self._cmd.rs.retrieve_data(opts.tag)
+            (data, sw) = self._cmd.lchan.retrieve_data(opts.tag)
             self._cmd.poutput(data)
 
         def do_retrieve_tags(self, opts):
             """List tags available in a given BER-TLV EF"""
-            tags = self._cmd.rs.retrieve_tags()
+            tags = self._cmd.lchan.retrieve_tags()
             self._cmd.poutput(tags)
 
         set_data_parser = argparse.ArgumentParser()
@@ -1198,7 +1247,7 @@ class BerTlvEF(CardEF):
         @cmd2.with_argparser(set_data_parser)
         def do_set_data(self, opts):
             """Set (Write) data for a given tag in a BER-TLV EF"""
-            (data, sw) = self._cmd.rs.set_data(opts.tag, opts.data)
+            (data, sw) = self._cmd.lchan.set_data(opts.tag, opts.data)
             if data:
                 self._cmd.poutput(data)
 
@@ -1209,12 +1258,12 @@ class BerTlvEF(CardEF):
         @cmd2.with_argparser(del_data_parser)
         def do_delete_data(self, opts):
             """Delete  data for a given tag in a BER-TLV EF"""
-            (data, sw) = self._cmd.rs.set_data(opts.tag, None)
+            (data, sw) = self._cmd.lchan.set_data(opts.tag, None)
             if data:
                 self._cmd.poutput(data)
 
     def __init__(self, fid: str, sfid: str = None, name: str = None, desc: str = None, parent: CardDF = None,
-                 size={1, None}, **kwargs):
+                 size: Size = (1, None), **kwargs):
         """
         Args:
             fid : File Identifier (4 hex digits)
@@ -1230,7 +1279,7 @@ class BerTlvEF(CardEF):
         self.shell_commands = [self.ShellCommands()]
 
 
-class RuntimeState(object):
+class RuntimeState:
     """Represent the runtime state of a session with a card."""
 
     def __init__(self, card, profile: 'CardProfile'):
@@ -1241,10 +1290,10 @@ class RuntimeState(object):
         """
         self.mf = CardMF(profile=profile)
         self.card = card
-        self.selected_file = self.mf  # type: CardDF
         self.profile = profile
-        self.selected_file_fcp = None
-        self.selected_file_fcp_hex = None
+        self.lchan = {}
+        # the basic logical channel always exists
+        self.lchan[0] = RuntimeLchan(0, self)
 
         # make sure the class and selection control bytes, which are specified
         # by the card profile are used
@@ -1303,6 +1352,66 @@ class RuntimeState(object):
                 pass
         return apps_taken
 
+    def reset(self, cmd_app=None) -> Hexstr:
+        """Perform physical card reset and obtain ATR.
+        Args:
+            cmd_app : Command Application State (for unregistering old file commands)
+        """
+        # delete all lchan != 0 (basic lchan)
+        for lchan_nr in self.lchan.keys():
+            if lchan_nr == 0:
+                continue
+            del self.lchan[lchan_nr]
+        atr = i2h(self.card.reset())
+        # select MF to reset internal state and to verify card really works
+        self.lchan[0].select('MF', cmd_app)
+        self.lchan[0].selected_adf = None
+        return atr
+
+    def add_lchan(self, lchan_nr: int) -> 'RuntimeLchan':
+        """Add a logical channel to the runtime state.  You shouldn't call this
+        directly but always go through RuntimeLchan.add_lchan()."""
+        if lchan_nr in self.lchan.keys():
+            raise ValueError('Cannot create already-existing lchan %d' % lchan_nr)
+        self.lchan[lchan_nr] = RuntimeLchan(lchan_nr, self)
+        return self.lchan[lchan_nr]
+
+    def del_lchan(self, lchan_nr: int):
+        if lchan_nr in self.lchan.keys():
+            del self.lchan[lchan_nr]
+            return True
+        else:
+            return False
+
+    def get_lchan_by_cla(self, cla) -> Optional['RuntimeLchan']:
+        lchan_nr = lchan_nr_from_cla(cla)
+        if lchan_nr in self.lchan.keys():
+            return self.lchan[lchan_nr]
+        else:
+            return None
+
+
+class RuntimeLchan:
+    """Represent the runtime state of a logical channel with a card."""
+
+    def __init__(self, lchan_nr: int, rs: RuntimeState):
+        self.lchan_nr = lchan_nr
+        self.rs = rs
+        self.selected_file = self.rs.mf
+        self.selected_adf = None
+        self.selected_file_fcp = None
+        self.selected_file_fcp_hex = None
+
+    def add_lchan(self, lchan_nr: int) -> 'RuntimeLchan':
+        """Add a new logical channel from the current logical channel. Just affects
+        internal state, doesn't actually open a channel with the UICC."""
+        new_lchan = self.rs.add_lchan(lchan_nr)
+        # See TS 102 221 Table 8.3
+        if self.lchan_nr != 0:
+            new_lchan.selected_file = self.get_cwd()
+            new_lchan.selected_adf = self.selected_adf
+        return new_lchan
+
     def selected_file_descriptor_byte(self) -> dict:
         return self.selected_file_fcp['file_descriptor']['file_descriptor_byte']
 
@@ -1318,16 +1427,6 @@ class RuntimeState(object):
     def selected_file_num_of_rec(self) -> Optional[int]:
         return self.selected_file_fcp['file_descriptor'].get('num_of_rec')
 
-    def reset(self, cmd_app=None) -> Hexstr:
-        """Perform physical card reset and obtain ATR.
-        Args:
-            cmd_app : Command Application State (for unregistering old file commands)
-        """
-        atr = i2h(self.card.reset())
-        # select MF to reset internal state and to verify card really works
-        self.select('MF', cmd_app)
-        return atr
-
     def get_cwd(self) -> CardDF:
         """Obtain the current working directory.
 
@@ -1371,7 +1470,7 @@ class RuntimeState(object):
             # card profile.
             if app and hasattr(app, "interpret_sw"):
                 res = app.interpret_sw(sw)
-        return res or self.profile.interpret_sw(sw)
+        return res or self.rs.profile.interpret_sw(sw)
 
     def probe_file(self, fid: str, cmd_app=None):
         """Blindly try to select a file and automatically add a matching file
@@ -1381,7 +1480,7 @@ class RuntimeState(object):
                 "Cannot select unknown file by name %s, only hexadecimal 4 digit FID is allowed" % fid)
 
         try:
-            (data, sw) = self.card._scc.select_file(fid)
+            (data, sw) = self.rs.card._scc.select_file(fid)
         except SwMatchError as swm:
             k = self.interpret_sw(swm.sw_actual)
             if not k:
@@ -1402,7 +1501,7 @@ class RuntimeState(object):
 
         self.selected_file.add_files([f])
         self.selected_file = f
-        return select_resp
+        return select_resp, data
 
     def _select_pre(self, cmd_app):
         # unregister commands of old file
@@ -1433,9 +1532,10 @@ class RuntimeState(object):
         for p in inter_path:
             try:
                 if isinstance(p, CardADF):
-                    (data, sw) = self.card.select_adf_by_aid(p.aid)
+                    (data, sw) = self.rs.card.select_adf_by_aid(p.aid)
+                    self.selected_adf = p
                 else:
-                    (data, sw) = self.card._scc.select_file(p.fid)
+                    (data, sw) = self.rs.card._scc.select_file(p.fid)
                 self.selected_file = p
             except SwMatchError as swm:
                 self._select_post(cmd_app)
@@ -1476,9 +1576,9 @@ class RuntimeState(object):
             f = sels[name]
             try:
                 if isinstance(f, CardADF):
-                    (data, sw) = self.card.select_adf_by_aid(f.aid)
+                    (data, sw) = self.rs.card.select_adf_by_aid(f.aid)
                 else:
-                    (data, sw) = self.card._scc.select_file(f.fid)
+                    (data, sw) = self.rs.card._scc.select_file(f.fid)
                 self.selected_file = f
             except SwMatchError as swm:
                 k = self.interpret_sw(swm.sw_actual)
@@ -1487,7 +1587,8 @@ class RuntimeState(object):
                 raise RuntimeError("%s: %s - %s" % (swm.sw_actual, k[0], k[1]))
             select_resp = f.decode_select_response(data)
         else:
-            select_resp = self.probe_file(name, cmd_app)
+            (select_resp, data) = self.probe_file(name, cmd_app)
+
         # store the raw + decoded FCP for later reference
         self.selected_file_fcp_hex = data
         self.selected_file_fcp = select_resp
@@ -1497,7 +1598,7 @@ class RuntimeState(object):
 
     def status(self):
         """Request STATUS (current selected file FCP) from card."""
-        (data, sw) = self.card._scc.status()
+        (data, sw) = self.rs.card._scc.status()
         return self.selected_file.decode_select_response(data)
 
     def get_file_for_selectable(self, name: str):
@@ -1508,7 +1609,7 @@ class RuntimeState(object):
         """Request ACTIVATE FILE of specified file."""
         sels = self.selected_file.get_selectables()
         f = sels[name]
-        data, sw = self.card._scc.activate_file(f.fid)
+        data, sw = self.rs.card._scc.activate_file(f.fid)
         return data, sw
 
     def read_binary(self, length: int = None, offset: int = 0):
@@ -1522,7 +1623,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, TransparentEF):
             raise TypeError("Only works with TransparentEF")
-        return self.card._scc.read_binary(self.selected_file.fid, length, offset)
+        return self.rs.card._scc.read_binary(self.selected_file.fid, length, offset)
 
     def read_binary_dec(self) -> Tuple[dict, str]:
         """Read [part of] a transparent EF binary data and decode it.
@@ -1546,7 +1647,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, TransparentEF):
             raise TypeError("Only works with TransparentEF")
-        return self.card._scc.update_binary(self.selected_file.fid, data_hex, offset, conserve=self.conserve_write)
+        return self.rs.card._scc.update_binary(self.selected_file.fid, data_hex, offset, conserve=self.rs.conserve_write)
 
     def update_binary_dec(self, data: dict):
         """Update transparent EF from abstract data. Encodes the data to binary and
@@ -1569,7 +1670,7 @@ class RuntimeState(object):
         if not isinstance(self.selected_file, LinFixedEF):
             raise TypeError("Only works with Linear Fixed EF")
         # returns a string of hex nibbles
-        return self.card._scc.read_record(self.selected_file.fid, rec_nr)
+        return self.rs.card._scc.read_record(self.selected_file.fid, rec_nr)
 
     def read_record_dec(self, rec_nr: int = 0) -> Tuple[dict, str]:
         """Read a record and decode it to abstract data.
@@ -1591,7 +1692,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, LinFixedEF):
             raise TypeError("Only works with Linear Fixed EF")
-        return self.card._scc.update_record(self.selected_file.fid, rec_nr, data_hex, conserve=self.conserve_write)
+        return self.rs.card._scc.update_record(self.selected_file.fid, rec_nr, data_hex, conserve=self.rs.conserve_write)
 
     def update_record_dec(self, rec_nr: int, data: dict):
         """Update a record with given abstract data.  Will encode abstract to binary data
@@ -1615,7 +1716,7 @@ class RuntimeState(object):
         if not isinstance(self.selected_file, BerTlvEF):
             raise TypeError("Only works with BER-TLV EF")
         # returns a string of hex nibbles
-        return self.card._scc.retrieve_data(self.selected_file.fid, tag)
+        return self.rs.card._scc.retrieve_data(self.selected_file.fid, tag)
 
     def retrieve_tags(self):
         """Retrieve tags available on BER-TLV EF.
@@ -1625,7 +1726,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, BerTlvEF):
             raise TypeError("Only works with BER-TLV EF")
-        data, sw = self.card._scc.retrieve_data(self.selected_file.fid, 0x5c)
+        data, sw = self.rs.card._scc.retrieve_data(self.selected_file.fid, 0x5c)
         tag, length, value, remainder = bertlv_parse_one(h2b(data))
         return list(value)
 
@@ -1638,7 +1739,7 @@ class RuntimeState(object):
         """
         if not isinstance(self.selected_file, BerTlvEF):
             raise TypeError("Only works with BER-TLV EF")
-        return self.card._scc.set_data(self.selected_file.fid, tag, data_hex, conserve=self.conserve_write)
+        return self.rs.card._scc.set_data(self.selected_file.fid, tag, data_hex, conserve=self.rs.conserve_write)
 
     def unregister_cmds(self, cmd_app=None):
         """Unregister all file specific commands."""
@@ -1647,7 +1748,7 @@ class RuntimeState(object):
                 cmd_app.unregister_command_set(c)
 
 
-class FileData(object):
+class FileData:
     """Represent the runtime, on-card data."""
 
     def __init__(self, fdesc):
@@ -1675,7 +1776,7 @@ def interpret_sw(sw_data: dict, sw: str):
     return None
 
 
-class CardApplication(object):
+class CardApplication:
     """A card application is represented by an ADF (with contained hierarchy) and optionally
        some SW definitions."""
 

pySim/gsm_r.py

@@ -61,7 +61,7 @@ class EF_FN(LinFixedEF):
 
     def __init__(self):
         super().__init__(fid='6ff1', sfid=None, name='EF.EN',
-                         desc='Functional numbers', rec_len={9, 9})
+                         desc='Functional numbers', rec_len=(9, 9))
         self._construct = Struct('functional_number_and_type'/FuncNTypeAdapter(Bytes(8)),
                                  'list_number'/Int8ub)
 
@@ -149,7 +149,7 @@ class EF_CallconfC(TransparentEF):
     """Section 7.3"""
 
     def __init__(self):
-        super().__init__(fid='6ff2', sfid=None, name='EF.CallconfC', size={24, 24},
+        super().__init__(fid='6ff2', sfid=None, name='EF.CallconfC', size=(24, 24),
                          desc='Call Configuration of emergency calls Configuration')
         self._construct = Struct('pl_conf'/PlConfAdapter(Int8ub),
                                  'conf_nr'/BcdAdapter(Bytes(8)),
@@ -166,7 +166,7 @@ class EF_CallconfI(LinFixedEF):
     """Section 7.5"""
 
     def __init__(self):
-        super().__init__(fid='6ff3', sfid=None, name='EF.CallconfI', rec_len={21, 21},
+        super().__init__(fid='6ff3', sfid=None, name='EF.CallconfI', rec_len=(21, 21),
                          desc='Call Configuration of emergency calls Information')
         self._construct = Struct('t_dur'/Int24ub,
                                  't_relcalc'/Int32ub,
@@ -183,9 +183,9 @@ class EF_Shunting(TransparentEF):
 
     def __init__(self):
         super().__init__(fid='6ff4', sfid=None,
-                         name='EF.Shunting', desc='Shunting', size={8, 8})
+                         name='EF.Shunting', desc='Shunting', size=(8, 8))
         self._construct = Struct('common_gid'/Int8ub,
-                                 'shunting_gid'/Bytes(7))
+                                 'shunting_gid'/HexAdapter(Bytes(7)))
 
 
 class EF_GsmrPLMN(LinFixedEF):
@@ -193,7 +193,7 @@ class EF_GsmrPLMN(LinFixedEF):
 
     def __init__(self):
         super().__init__(fid='6ff5', sfid=None, name='EF.GsmrPLMN',
-                         desc='GSM-R network selection', rec_len={9, 9})
+                         desc='GSM-R network selection', rec_len=(9, 9))
         self._construct = Struct('plmn'/BcdAdapter(Bytes(3)),
                                  'class_of_network'/BitStruct('supported'/FlagsEnum(BitsInteger(5), vbs=1, vgcs=2, emlpp=4, fn=8, eirene=16),
                                                               'preference'/BitsInteger(3)),
@@ -207,7 +207,7 @@ class EF_IC(LinFixedEF):
 
     def __init__(self):
         super().__init__(fid='6f8d', sfid=None, name='EF.IC',
-                         desc='International Code', rec_len={7, 7})
+                         desc='International Code', rec_len=(7, 7))
         self._construct = Struct('next_table_type'/NextTableType,
                                  'id_of_next_table'/HexAdapter(Bytes(2)),
                                  'ic_decision_value'/BcdAdapter(Bytes(2)),
@@ -219,7 +219,7 @@ class EF_NW(LinFixedEF):
 
     def __init__(self):
         super().__init__(fid='6f80', sfid=None, name='EF.NW',
-                         desc='Network Name', rec_len={8, 8})
+                         desc='Network Name', rec_len=(8, 8))
         self._construct = GsmString(8)
 
 
@@ -228,7 +228,7 @@ class EF_Switching(LinFixedEF):
 
     def __init__(self, fid, name, desc):
         super().__init__(fid=fid, sfid=None,
-                         name=name, desc=desc, rec_len={6, 6})
+                         name=name, desc=desc, rec_len=(6, 6))
         self._construct = Struct('next_table_type'/NextTableType,
                                  'id_of_next_table'/HexAdapter(Bytes(2)),
                                  'decision_value'/BcdAdapter(Bytes(2)),
@@ -240,7 +240,7 @@ class EF_Predefined(LinFixedEF):
 
     def __init__(self, fid, name, desc):
         super().__init__(fid=fid, sfid=None,
-                         name=name, desc=desc, rec_len={3, 3})
+                         name=name, desc=desc, rec_len=(3, 3))
         # header and other records have different structure. WTF !?!
         self._construct = Struct('next_table_type'/NextTableType,
                                  'id_of_next_table'/HexAdapter(Bytes(2)),
@@ -253,7 +253,7 @@ class EF_DialledVals(TransparentEF):
     """Section 8.6"""
 
     def __init__(self, fid, name, desc):
-        super().__init__(fid=fid, sfid=None, name=name, desc=desc, size={4, 4})
+        super().__init__(fid=fid, sfid=None, name=name, desc=desc, size=(4, 4))
         self._construct = Struct('next_table_type'/NextTableType,
                                  'id_of_next_table'/HexAdapter(Bytes(2)),
                                  'dialed_digits'/BcdAdapter(Bytes(1)))

pySim/gsmtap.py

@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+
+""" Osmocom GSMTAP python implementation.
+GSMTAP is a packet format used for conveying a number of different
+telecom-related protocol traces over UDP.
+"""
+
+#
+# Copyright (C) 2022  Harald Welte <laforge@gnumonks.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import socket
+from typing import List, Dict, Optional
+from construct import Optional as COptional
+from construct import *
+from pySim.construct import *
+
+# The root definition of GSMTAP can be found at
+# https://cgit.osmocom.org/cgit/libosmocore/tree/include/osmocom/core/gsmtap.h
+
+GSMTAP_UDP_PORT = 4729
+
+# GSMTAP_TYPE_*
+gsmtap_type_construct = Enum(Int8ub,
+                             gsm_um = 0x01,
+                             gsm_abis = 0x02,
+                             gsm_um_burst = 0x03,
+                             sim = 0x04,
+                             tetra_i1 = 0x05,
+                             tetra_i1_burst = 0x06,
+                             wimax_burst = 0x07,
+                             gprs_gb_llc = 0x08,
+                             gprs_gb_sndcp = 0x09,
+                             gmr1_um = 0x0a,
+                             umts_rlc_mac = 0x0b,
+                             umts_rrc = 0x0c,
+                             lte_rrc = 0x0d,
+                             lte_mac = 0x0e,
+                             lte_mac_framed = 0x0f,
+                             osmocore_log = 0x10,
+                             qc_diag = 0x11,
+                             lte_nas = 0x12,
+                             e1_t1 = 0x13)
+
+
+# TYPE_UM_BURST
+gsmtap_subtype_burst_construct = Enum(Int8ub,
+                                      unknown = 0x00,
+                                      fcch = 0x01,
+                                      partial_sch = 0x02,
+                                      sch = 0x03,
+                                      cts_sch = 0x04,
+                                      compact_sch = 0x05,
+                                      normal = 0x06,
+                                      dummy = 0x07,
+                                      access = 0x08,
+                                      none = 0x09)
+
+gsmtap_subtype_wimax_burst_construct = Enum(Int8ub,
+                                            cdma_code = 0x10,
+                                            fch = 0x11,
+                                            ffb = 0x12,
+                                            pdu = 0x13,
+                                            hack = 0x14,
+                                            phy_attributes = 0x15)
+
+# GSMTAP_CHANNEL_*
+gsmtap_subtype_um_construct = Enum(Int8ub,
+                                   unknown = 0x00,
+                                   bcch = 0x01,
+                                   ccch = 0x02,
+                                   rach = 0x03,
+                                   agch = 0x04,
+                                   pch = 0x05,
+                                   sdcch = 0x06,
+                                   sdcch4 = 0x07,
+                                   sdcch8 = 0x08,
+                                   facch_f = 0x09,
+                                   facch_h = 0x0a,
+                                   pacch = 0x0b,
+                                   cbch52 = 0x0c,
+                                   pdtch = 0x0d,
+                                   ptcch = 0x0e,
+                                   cbch51 = 0x0f,
+                                   voice_f = 0x10,
+                                   voice_h = 0x11)
+
+
+# GSMTAP_SIM_*
+gsmtap_subtype_sim_construct = Enum(Int8ub,
+                                    apdu = 0x00,
+                                    atr = 0x01,
+                                    pps_req = 0x02,
+                                    pps_rsp = 0x03,
+                                    tpdu_hdr = 0x04,
+                                    tpdu_cmd = 0x05,
+                                    tpdu_rsp = 0x06,
+                                    tpdu_sw = 0x07)
+
+gsmtap_subtype_tetra_construct = Enum(Int8ub,
+                                      bsch = 0x01,
+                                      aach = 0x02,
+                                      sch_hu = 0x03,
+                                      sch_hd = 0x04,
+                                      sch_f = 0x05,
+                                      bnch = 0x06,
+                                      stch = 0x07,
+                                      tch_f = 0x08,
+                                      dmo_sch_s = 0x09,
+                                      dmo_sch_h = 0x0a,
+                                      dmo_sch_f = 0x0b,
+                                      dmo_stch = 0x0c,
+                                      dmo_tch = 0x0d)
+
+gsmtap_subtype_gmr1_construct = Enum(Int8ub,
+                                     unknown = 0x00,
+                                     bcch = 0x01,
+                                     ccch = 0x02,
+                                     pch = 0x03,
+                                     agch = 0x04,
+                                     bach = 0x05,
+                                     rach = 0x06,
+                                     cbch = 0x07,
+                                     sdcch = 0x08,
+                                     tachh = 0x09,
+                                     gbch = 0x0a,
+                                     tch3 = 0x10,
+                                     tch6 = 0x14,
+                                     tch9 = 0x18)
+
+gsmtap_subtype_e1t1_construct = Enum(Int8ub,
+                                     lapd = 0x01,
+                                     fr = 0x02,
+                                     raw = 0x03,
+                                     trau16 = 0x04,
+                                     trau8 = 0x05)
+
+gsmtap_arfcn_construct = BitStruct('pcs'/Flag, 'uplink'/Flag, 'arfcn'/BitsInteger(14))
+
+gsmtap_hdr_construct = Struct('version'/Int8ub,
+                              'hdr_len'/Int8ub,
+                              'type'/gsmtap_type_construct,
+                              'timeslot'/Int8ub,
+                              'arfcn'/gsmtap_arfcn_construct,
+                              'signal_dbm'/Int8sb,
+                              'snr_db'/Int8sb,
+                              'frame_nr'/Int32ub,
+                              'sub_type'/Switch(this.type, {
+                                                'gsm_um': gsmtap_subtype_um_construct,
+                                                'gsm_um_burst': gsmtap_subtype_burst_construct,
+                                                'sim': gsmtap_subtype_sim_construct,
+                                                'tetra_i1': gsmtap_subtype_tetra_construct,
+                                                'tetra_i1_burst': gsmtap_subtype_tetra_construct,
+                                                'wimax_burst': gsmtap_subtype_wimax_burst_construct,
+                                                'gmr1_um': gsmtap_subtype_gmr1_construct,
+                                                'e1_t1': gsmtap_subtype_e1t1_construct,
+                                                }),
+                              'antenna_nr'/Int8ub,
+                              'sub_slot'/Int8ub,
+                              'res'/Int8ub,
+                              'body'/GreedyBytes)
+
+osmocore_log_ts_construct = Struct('sec'/Int32ub, 'usec'/Int32ub)
+osmocore_log_level_construct = Enum(Int8ub, debug=1, info=3, notice=5, error=7, fatal=8)
+gsmtap_osmocore_log_hdr_construct = Struct('ts'/osmocore_log_ts_construct,
+                                           'proc_name'/PaddedString(16, 'ascii'),
+                                           'pid'/Int32ub,
+                                           'level'/osmocore_log_level_construct,
+                                           Bytes(3),
+                                           'subsys'/PaddedString(16, 'ascii'),
+                                           'src_file'/Struct('name'/PaddedString(32, 'ascii'), 'line_nr'/Int32ub))
+
+
+class GsmtapMessage:
+    """Class whose objects represent a single GSMTAP message. Can encode and decode messages."""
+    def __init__(self, encoded = None):
+        self.encoded = encoded
+        self.decoded = None
+
+    def decode(self):
+        self.decoded = parse_construct(gsmtap_hdr_construct, self.encoded)
+        return self.decoded
+
+    def encode(self, decoded):
+        self.encoded = gsmtap_hdr_construct.build(decoded)
+        return self.encoded
+
+class GsmtapSource:
+    def __init__(self, bind_ip:str='127.0.0.1', bind_port:int=4729):
+        self.bind_ip = bind_ip
+        self.bind_port = bind_port
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.sock.bind((self.bind_ip, self.bind_port))
+
+    def read_packet(self) -> GsmtapMessage:
+        data, addr = self.sock.recvfrom(1024)
+        gsmtap_msg = GsmtapMessage(data)
+        gsmtap_msg.decode()
+        if gsmtap_msg.decoded['version'] != 0x02:
+            raise ValueError('Unknown GSMTAP version 0x%02x' % gsmtap_msg.decoded['version'])
+        return gsmtap_msg.decoded, addr

pySim/profile.py

@@ -62,7 +62,7 @@ def match_sim(scc: SimCardCommands) -> bool:
     return _mf_select_test(scc, "a0", "0000")
 
 
-class CardProfile(object):
+class CardProfile:
     """A Card Profile describes a card, it's filesystem hierarchy, an [initial] list of
     applications as well as profile-specific SW and shell commands.  Every card has
     one card profile, but there may be multiple applications within that profile."""

pySim/sysmocom_sja2.py

@@ -85,7 +85,7 @@ class EF_MILENAGE_CFG(TransparentEF):
 
 class EF_0348_KEY(LinFixedEF):
     def __init__(self, fid='6f22', name='EF.0348_KEY', desc='TS 03.48 OTA Keys'):
-        super().__init__(fid, name=name, desc=desc, rec_len={27, 35})
+        super().__init__(fid, name=name, desc=desc, rec_len=(27, 35))
 
     def _decode_record_bin(self, raw_bin_data):
         u = unpack('!BBB', raw_bin_data[0:3])
@@ -103,7 +103,7 @@ class EF_0348_KEY(LinFixedEF):
 
 class EF_0348_COUNT(LinFixedEF):
     def __init__(self, fid='6f23', name='EF.0348_COUNT', desc='TS 03.48 OTA Counters'):
-        super().__init__(fid, name=name, desc=desc, rec_len={7, 7})
+        super().__init__(fid, name=name, desc=desc, rec_len=(7, 7))
 
     def _decode_record_bin(self, raw_bin_data):
         u = unpack('!BB5s', raw_bin_data)
@@ -118,7 +118,7 @@ class EF_SIM_AUTH_COUNTER(TransparentEF):
 
 class EF_GP_COUNT(LinFixedEF):
     def __init__(self, fid='6f26', name='EF.GP_COUNT', desc='GP SCP02 Counters'):
-        super().__init__(fid, name=name, desc=desc, rec_len={5, 5})
+        super().__init__(fid, name=name, desc=desc, rec_len=(5, 5))
 
     def _decode_record_bin(self, raw_bin_data):
         u = unpack('!BBHB', raw_bin_data)
@@ -127,7 +127,7 @@ class EF_GP_COUNT(LinFixedEF):
 
 class EF_GP_DIV_DATA(LinFixedEF):
     def __init__(self, fid='6f27', name='EF.GP_DIV_DATA', desc='GP SCP02 key diversification data'):
-        super().__init__(fid, name=name, desc=desc, rec_len={12, 12})
+        super().__init__(fid, name=name, desc=desc, rec_len=(12, 12))
 
     def _decode_record_bin(self, raw_bin_data):
         u = unpack('!BB8s', raw_bin_data)
@@ -137,18 +137,17 @@ class EF_GP_DIV_DATA(LinFixedEF):
 class EF_SIM_AUTH_KEY(TransparentEF):
     def __init__(self, fid='6f20', name='EF.SIM_AUTH_KEY'):
         super().__init__(fid, name=name, desc='USIM authentication key')
-        CfgByte = BitStruct(Bit[2],
+        CfgByte = BitStruct(Padding(2),
                             'use_sres_deriv_func_2'/Bit,
                             'use_opc_instead_of_op'/Bit,
                             'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/Bytes(16),
-                                 'op' /
-                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op, Bytes(
-                                     16)),
+                                 'key'/HexAdapter(Bytes(16)),
+                                 'op'/ If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16))),
                                  'opc' /
-                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op, Bytes(
-                                     16))
+                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16)))
                                  )
 
 
@@ -193,36 +192,36 @@ class EF_USIM_SQN(TransparentEF):
 class EF_USIM_AUTH_KEY(TransparentEF):
     def __init__(self, fid='af20', name='EF.USIM_AUTH_KEY'):
         super().__init__(fid, name=name, desc='USIM authentication key')
-        CfgByte = BitStruct(Bit, 'only_4bytes_res_in_3g'/Bit,
+        CfgByte = BitStruct(Padding(1), 'only_4bytes_res_in_3g'/Bit,
                             'use_sres_deriv_func_2_in_3g'/Bit,
                             'use_opc_instead_of_op'/Bit,
                             'algorithm'/Enum(Nibble, milenage=4, sha1_aka=5, xor=15))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/Bytes(16),
+                                 'key'/HexAdapter(Bytes(16)),
                                  'op' /
-                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op, Bytes(
-                                     16)),
+                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16))),
                                  'opc' /
-                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op, Bytes(
-                                     16))
+                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16)))
                                  )
 
 
 class EF_USIM_AUTH_KEY_2G(TransparentEF):
     def __init__(self, fid='af22', name='EF.USIM_AUTH_KEY_2G'):
         super().__init__(fid, name=name, desc='USIM authentication key in 2G context')
-        CfgByte = BitStruct(Bit, 'only_4bytes_res_in_3g'/Bit,
+        CfgByte = BitStruct(Padding(1), 'only_4bytes_res_in_3g'/Bit,
                             'use_sres_deriv_func_2_in_3g'/Bit,
                             'use_opc_instead_of_op'/Bit,
                             'algorithm'/Enum(Nibble, milenage=4, comp128v1=1, comp128v2=2, comp128v3=3))
         self._construct = Struct('cfg'/CfgByte,
-                                 'key'/Bytes(16),
+                                 'key'/HexAdapter(Bytes(16)),
                                  'op' /
-                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op, Bytes(
-                                     16)),
+                                 If(this.cfg.algorithm == 'milenage' and not this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16))),
                                  'opc' /
-                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op, Bytes(
-                                     16))
+                                 If(this.cfg.algorithm == 'milenage' and this.cfg.use_opc_instead_of_op,
+                                     HexAdapter(Bytes(16)))
                                  )
 
 
@@ -242,7 +241,7 @@ class EF_GBA_REC_LIST(TransparentEF):
 class EF_GBA_INT_KEY(LinFixedEF):
     def __init__(self, fid='af33', name='EF.GBA_INT_KEY'):
         super().__init__(fid, name=name,
-                         desc='Secret key for GBA key derivation', rec_len={32, 32})
+                         desc='Secret key for GBA key derivation', rec_len=(32, 32))
         self._construct = GreedyBytes
 
 

pySim/tlv.py

@@ -100,7 +100,7 @@ class Transcodable(abc.ABC):
 
     # not an abstractmethod, as it is only required if no _construct exists
     def _to_bytes(self):
-        raise NotImplementedError
+        raise NotImplementedError('%s._to_bytes' % type(self).__name__)
 
     def from_bytes(self, do: bytes):
         """Convert from binary bytes to internal representation. Store the decoded result
@@ -118,7 +118,7 @@ class Transcodable(abc.ABC):
 
     # not an abstractmethod, as it is only required if no _construct exists
     def _from_bytes(self, do: bytes):
-        raise NotImplementedError
+        raise NotImplementedError('%s._from_bytes' % type(self).__name__)
 
 
 class IE(Transcodable, metaclass=TlvMeta):
@@ -236,7 +236,7 @@ class TLV_IE(IE):
             return {}, b''
         (rawtag, remainder) = self.__class__._parse_tag_raw(do)
         if rawtag:
-            if rawtag != self.tag:
+            if rawtag != self._compute_tag():
                 raise ValueError("%s: Encountered tag %s doesn't match our supported tag %s" %
                                  (self, rawtag, self.tag))
             (length, remainder) = self.__class__._parse_len(remainder)

pySim/transport/__init__.py

@@ -9,11 +9,12 @@ from typing import Optional, Tuple
 
 from pySim.exceptions import *
 from pySim.construct import filter_dict
-from pySim.utils import sw_match, b2h, h2b, i2h
+from pySim.utils import sw_match, b2h, h2b, i2h, Hexstr
+from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
 
 #
 # Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
-# Copyright (C) 2021 Harald Welte <laforge@osmocom.org>
+# Copyright (C) 2021-2022 Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -37,13 +38,33 @@ class ApduTracer:
     def trace_response(self, cmd, sw, resp):
         pass
 
+class ProactiveHandler(abc.ABC):
+    """Abstract base class representing the interface of some code that handles
+    the proactive commands, as returned by the card in responses to the FETCH
+    command."""
+    def receive_fetch_raw(self, pcmd: ProactiveCommand, parsed: Hexstr):
+        # try to find a generic handler like handle_SendShortMessage
+        handle_name = 'handle_%s' % type(parsed).__name__
+        if hasattr(self, handle_name):
+            handler = getattr(self, handle_name)
+            return handler(pcmd.decoded)
+        # fall back to common handler
+        return self.receive_fetch(pcmd)
+
+    def receive_fetch(self, pcmd: ProactiveCommand):
+        """Default handler for not otherwise handled proactive commands."""
+        raise NotImplementedError('No handler method for %s' % pcmd.decoded)
+
+
 
 class LinkBase(abc.ABC):
     """Base class for link/transport to card."""
 
-    def __init__(self, sw_interpreter=None, apdu_tracer=None):
+    def __init__(self, sw_interpreter=None, apdu_tracer=None,
+                 proactive_handler: Optional[ProactiveHandler]=None):
         self.sw_interpreter = sw_interpreter
         self.apdu_tracer = apdu_tracer
+        self.proactive_handler = proactive_handler
 
     @abc.abstractmethod
     def _send_apdu_raw(self, pdu: str) -> Tuple[str, str]:
@@ -136,11 +157,57 @@ class LinkBase(abc.ABC):
                         sw   : string (in hex) of status word (ex. "9000")
         """
         rv = self.send_apdu(pdu)
+        last_sw = rv[1]
 
-        if sw == '9000' and sw_match(rv[1], '91xx'):
+        while sw == '9000' and sw_match(last_sw, '91xx'):
+            # It *was* successful after all -- the extra pieces FETCH handled
+            # need not concern the caller.
+            rv = (rv[0], '9000')
             # proactive sim as per TS 102 221 Setion 7.4.2
-            rv = self.send_apdu_checksw('80120000' + rv[1][2:], sw)
-            print("FETCH: %s", rv[0])
+            # TODO: Check SW manually to avoid recursing on the stack (provided this piece of code stays in this place)
+            fetch_rv = self.send_apdu_checksw('80120000' + last_sw[2:], sw)
+            # Setting this in case we later decide not to send a terminal
+            # response immediately unconditionally -- the card may still have
+            # something pending even though the last command was not processed
+            # yet.
+            last_sw = fetch_rv[1]
+            # parse the proactive command
+            pcmd = ProactiveCommand()
+            parsed = pcmd.from_tlv(h2b(fetch_rv[0]))
+            print("FETCH: %s (%s)" % (fetch_rv[0], type(parsed).__name__))
+            result = Result()
+            if self.proactive_handler:
+                # Extension point: If this does return a list of TLV objects,
+                # they could be appended after the Result; if the first is a
+                # Result, that cuold replace the one built here.
+                self.proactive_handler.receive_fetch_raw(pcmd, parsed)
+                result.from_dict({'general_result': 'performed_successfully', 'additional_information': ''})
+            else:
+                result.from_dict({'general_result': 'command_beyond_terminal_capability', 'additional_information': ''})
+
+            # Send response immediately, thus also flushing out any further
+            # proactive commands that the card already wants to send
+            #
+            # Structure as per TS 102 223 V4.4.0 Section 6.8
+
+            # The Command Details are echoed from the command that has been processed.
+            (command_details,) = [c for c in pcmd.decoded.children if isinstance(c, CommandDetails)]
+            # The Device Identities are fixed. (TS 102 223 V4.0.0 Section 6.8.2)
+            device_identities = DeviceIdentities()
+            device_identities.from_dict({'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'})
+
+            # Testing hint: The value of tail does not influence the behavior
+            # of an SJA2 that sent ans SMS, so this is implemented only
+            # following TS 102 223, and not fully tested.
+            tail = command_details.to_tlv() + device_identities.to_tlv() + result.to_tlv()
+            # Testing hint: In contrast to the above, this part is positively
+            # essential to get the SJA2 to provide the later parts of a
+            # multipart SMS in response to an OTA RFM command.
+            terminal_response = '80140000' + b2h(len(tail).to_bytes(1, 'big') + tail)
+
+            terminal_response_rv = self.send_apdu(terminal_response)
+            last_sw = terminal_response_rv[1]
+
         if not sw_match(rv[1], sw):
             raise SwMatchError(rv[1], sw.lower(), self.sw_interpreter)
         return rv

pySim/transport/calypso.py

@@ -26,7 +26,7 @@ from pySim.exceptions import *
 from pySim.utils import h2b, b2h
 
 
-class L1CTLMessage(object):
+class L1CTLMessage:
 
     # Every (encoded) L1CTL message has the following structure:
     #  - msg_length (2 bytes, net order)

pySim/transport/pcsc.py

@@ -34,7 +34,7 @@ class PcscSimLink(LinkBase):
         super().__init__(**kwargs)
         r = readers()
         if reader_number >= len(r):
-            raise ReaderError
+            raise ReaderError('No reader found for number %d' % reader_number)
         self._reader = r[reader_number]
         self._con = self._reader.createConnection()
 

pySim/ts_102_221.py

@@ -80,11 +80,11 @@ ts_102_22x_cmdset = CardCommandSet('TS 102 22x', [
 
 # ETSI TS 102 221 11.1.1.4.2
 class FileSize(BER_TLV_IE, tag=0x80):
-    _construct = GreedyInteger()
+    _construct = GreedyInteger(minlen=2)
 
 # ETSI TS 102 221 11.1.1.4.2
 class TotalFileSize(BER_TLV_IE, tag=0x81):
-    _construct = GreedyInteger()
+    _construct = GreedyInteger(minlen=2)
 
 # ETSI TS 102 221 11.1.1.4.3
 class FileDescriptor(BER_TLV_IE, tag=0x82):
@@ -190,10 +190,11 @@ class ShortFileIdentifier(BER_TLV_IE, tag=0x88):
     # of the TLV value field. In this case, bits b3 to b1 shall be set to 0
     class Shift3RAdapter(Adapter):
         def _decode(self, obj, context, path):
-            return obj >> 3
+            return int.from_bytes(obj, 'big') >> 3
         def _encode(self, obj, context, path):
-            return obj << 3
-    _construct = COptional(Shift3RAdapter(Byte))
+            val = int(obj) << 3
+            return val.to_bytes(1, 'big')
+    _construct = COptional(Shift3RAdapter(Bytes(1)))
 
 # ETSI TS 102 221 11.1.1.4.9
 class LifeCycleStatusInteger(BER_TLV_IE, tag=0x8A):
@@ -587,13 +588,13 @@ class EF_DIR(LinFixedEF):
         pass
 
     def __init__(self, fid='2f00', sfid=0x1e, name='EF.DIR', desc='Application Directory'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len={5, 54})
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(5, 54))
         self._tlv = EF_DIR.ApplicationTemplate
 
 # TS 102 221 Section 13.2
 class EF_ICCID(TransparentEF):
     def __init__(self, fid='2fe2', sfid=0x02, name='EF.ICCID', desc='ICC Identification'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size={10, 10})
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(10, 10))
 
     def _decode_hex(self, raw_hex):
         return {'iccid': dec_iccid(raw_hex)}
@@ -605,7 +606,7 @@ class EF_ICCID(TransparentEF):
 class EF_PL(TransRecEF):
     def __init__(self, fid='2f05', sfid=0x05, name='EF.PL', desc='Preferred Languages'):
         super().__init__(fid, sfid=sfid, name=name,
-                         desc=desc, rec_len=2, size={2, None})
+                         desc=desc, rec_len=2, size=(2, None))
 
     def _decode_record_bin(self, bin_data):
         if bin_data == b'\xff\xff':
@@ -685,19 +686,19 @@ class EF_ARR(LinFixedEF):
         @cmd2.with_argparser(LinFixedEF.ShellCommands.read_rec_dec_parser)
         def do_read_arr_record(self, opts):
             """Read one EF.ARR record in flattened, human-friendly form."""
-            (data, sw) = self._cmd.rs.read_record_dec(opts.record_nr)
-            data = self._cmd.rs.selected_file.flatten(data)
+            (data, sw) = self._cmd.lchan.read_record_dec(opts.record_nr)
+            data = self._cmd.lchan.selected_file.flatten(data)
             self._cmd.poutput_json(data, opts.oneline)
 
         @cmd2.with_argparser(LinFixedEF.ShellCommands.read_recs_dec_parser)
         def do_read_arr_records(self, opts):
             """Read + decode all EF.ARR records in flattened, human-friendly form."""
-            num_of_rec = self._cmd.rs.selected_file_num_of_rec()
+            num_of_rec = self._cmd.lchan.selected_file_num_of_rec()
             # collect all results in list so they are rendered as JSON list when printing
             data_list = []
             for recnr in range(1, 1 + num_of_rec):
-                (data, sw) = self._cmd.rs.read_record_dec(recnr)
-                data = self._cmd.rs.selected_file.flatten(data)
+                (data, sw) = self._cmd.lchan.read_record_dec(recnr)
+                data = self._cmd.lchan.selected_file.flatten(data)
                 data_list.append(data)
             self._cmd.poutput_json(data_list, opts.oneline)
 
@@ -705,7 +706,7 @@ class EF_ARR(LinFixedEF):
 # TS 102 221 Section 13.6
 class EF_UMPC(TransparentEF):
     def __init__(self, fid='2f08', sfid=0x08, name='EF.UMPC', desc='UICC Maximum Power Consumption'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size={5, 5})
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=(5, 5))
         addl_info = FlagsEnum(Byte, req_inc_idle_current=1,
                               support_uicc_suspend=2)
         self._construct = Struct(

pySim/ts_102_222.py

@@ -20,7 +20,6 @@
 from typing import List
 
 import cmd2
-from cmd2 import style, fg, bg
 from cmd2 import CommandSet, with_default_category, with_argparser
 import argparse
 
@@ -52,12 +51,12 @@ class Ts102222Commands(CommandSet):
         if not opts.force_delete:
             self._cmd.perror("Refusing to permanently delete the file, please read the help text.")
             return
-        f = self._cmd.rs.get_file_for_selectable(opts.NAME)
+        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
         (data, sw) = self._cmd.card._scc.delete_file(f.fid)
 
     def complete_delete_file(self, text, line, begidx, endidx) -> List[str]:
         """Command Line tab completion for DELETE FILE"""
-        index_dict = {1: self._cmd.rs.selected_file.get_selectable_names()}
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
         return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
 
     termdf_parser = argparse.ArgumentParser()
@@ -73,12 +72,12 @@ class Ts102222Commands(CommandSet):
         if not opts.force:
             self._cmd.perror("Refusing to terminate the file, please read the help text.")
             return
-        f = self._cmd.rs.get_file_for_selectable(opts.NAME)
+        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
         (data, sw) = self._cmd.card._scc.terminate_df(f.fid)
 
     def complete_terminate_df(self, text, line, begidx, endidx) -> List[str]:
         """Command Line tab completion for TERMINATE DF"""
-        index_dict = {1: self._cmd.rs.selected_file.get_selectable_names()}
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
         return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
 
     @cmd2.with_argparser(termdf_parser)
@@ -89,12 +88,12 @@ class Ts102222Commands(CommandSet):
         if not opts.force:
             self._cmd.perror("Refusing to terminate the file, please read the help text.")
             return
-        f = self._cmd.rs.get_file_for_selectable(opts.NAME)
+        f = self._cmd.lchan.get_file_for_selectable(opts.NAME)
         (data, sw) = self._cmd.card._scc.terminate_ef(f.fid)
 
     def complete_terminate_ef(self, text, line, begidx, endidx) -> List[str]:
         """Command Line tab completion for TERMINATE EF"""
-        index_dict = {1: self._cmd.rs.selected_file.get_selectable_names()}
+        index_dict = {1: self._cmd.lchan.selected_file.get_selectable_names()}
         return self._cmd.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)
 
     tcard_parser = argparse.ArgumentParser()
@@ -139,6 +138,9 @@ class Ts102222Commands(CommandSet):
                 self._cmd.perror("you must specify the --record-length for linear fixed EF")
                 return
             file_descriptor['record_len'] = opts.record_length
+            file_descriptor['num_of_rec'] = opts.file_size // opts.record_length
+            if file_descriptor['num_of_rec'] * file_descriptor['record_len'] != opts.file_size:
+                raise ValueError("File size not evenly divisible by record length")
         elif opts.structure == 'ber_tlv':
             self._cmd.perror("BER-TLV creation not yet fully supported, sorry")
             return
@@ -152,7 +154,7 @@ class Ts102222Commands(CommandSet):
         fcp = FcpTemplate(children=ies)
         (data, sw) = self._cmd.card._scc.create_file(b2h(fcp.to_tlv()))
         # the newly-created file is automatically selected but our runtime state knows nothing of it
-        self._cmd.rs.select_file(self._cmd.rs.selected_file)
+        self._cmd.lchan.select_file(self._cmd.lchan.selected_file)
 
     createdf_parser = argparse.ArgumentParser()
     createdf_parser.add_argument('FILE_ID', type=str, help='File Identifier as 4-character hex string')
@@ -203,4 +205,4 @@ class Ts102222Commands(CommandSet):
         fcp = FcpTemplate(children=ies)
         (data, sw) = self._cmd.card._scc.create_file(b2h(fcp.to_tlv()))
         # the newly-created file is automatically selected but our runtime state knows nothing of it
-        self._cmd.rs.select_file(self._cmd.rs.selected_file)
+        self._cmd.lchan.select_file(self._cmd.lchan.selected_file)

pySim/ts_31_102.py

@@ -10,7 +10,7 @@ Various constants from 3GPP TS 31.102 V16.6.0
 
 #
 # Copyright (C) 2020 Supreeth Herle <herlesupreeth@gmail.com>
-# Copyright (C) 2021 Harald Welte <laforge@osmocom.org>
+# Copyright (C) 2021-2022 Harald Welte <laforge@osmocom.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -33,10 +33,13 @@ from pySim.ts_51_011 import EF_MMSN, EF_MMSICP, EF_MMSUP, EF_MMSUCP, EF_VGCS, EF
 from pySim.ts_51_011 import EF_SMSR, EF_DCK, EF_EXT, EF_CNL, EF_OPL, EF_MBI, EF_MWIS
 from pySim.ts_51_011 import EF_CBMID, EF_CBMIR, EF_ADN, EF_SMS, EF_MSISDN, EF_SMSP, EF_SMSS
 from pySim.ts_51_011 import EF_IMSI, EF_xPLMNwAcT, EF_SPN, EF_CBMI, EF_ACC, EF_PLMNsel
+from pySim.ts_51_011 import EF_Kc, EF_CPBCCH, EF_InvScan
 from pySim.ts_102_221 import EF_ARR
 from pySim.tlv import *
 from pySim.filesystem import *
+from pySim.ts_31_102_telecom import DF_PHONEBOOK, EF_UServiceTable
 from pySim.construct import *
+from pySim.cat import SMS_TPDU, DeviceIdentities, SMSPPDownload
 from construct import Optional as COptional
 from construct import *
 from typing import Tuple
@@ -298,6 +301,10 @@ EF_USIM_ADF_map = {
     'ePDGSelectionEm': '6FF6',
 }
 
+# 3gPP TS 31.102 Section 7.5.2.1
+class SUCI_TlvDataObject(BER_TLV_IE, tag=0xA1):
+    _construct = HexAdapter(GreedyBytes)
+
 ######################################################################
 # ADF.USIM
 ######################################################################
@@ -331,7 +338,7 @@ class EF_5GS3GPPNSC(LinFixedEF):
                                            IdsOfSelectedEpsAlgos]):
         pass
 
-    def __init__(self, fid="4f03", sfid=0x03, name='EF.5GS3GPPNSC', rec_len={57, None},
+    def __init__(self, fid="4f03", sfid=0x03, name='EF.5GS3GPPNSC', rec_len=(57, None),
                  desc='5GS 3GPP Access NAS Security Context', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._tlv = EF_5GS3GPPNSC.FiveGSNasSecurityContext
@@ -347,7 +354,7 @@ class EF_5GAUTHKEYS(TransparentEF):
     class FiveGAuthKeys(TLV_IE_Collection, nested=[K_AUSF, K_SEAF]):
         pass
 
-    def __init__(self, fid='4f05', sfid=0x05, name='EF.5GAUTHKEYS', size={68, None},
+    def __init__(self, fid='4f05', sfid=0x05, name='EF.5GAUTHKEYS', size=(68, None),
                  desc='5G authentication keys', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._tlv = EF_5GAUTHKEYS.FiveGAuthKeys
@@ -381,7 +388,7 @@ class SUCI_CalcInfo(TLV_IE_Collection, nested=[ProtSchemeIdList, HomeNetPubKeyLi
 
 # TS 31.102 4.4.11.8
 class EF_SUCI_Calc_Info(TransparentEF):
-    def __init__(self, fid="4f07", sfid=0x07, name='EF.SUCI_Calc_Info', size={2, None},
+    def __init__(self, fid="4f07", sfid=0x07, name='EF.SUCI_Calc_Info', size=(2, None),
                  desc='SUCI Calc Info', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
 
@@ -504,7 +511,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
 
 
 class EF_LI(TransRecEF):
-    def __init__(self, fid='6f05', sfid=None, name='EF.LI', size={2, None}, rec_len=2,
+    def __init__(self, fid='6f05', sfid=None, name='EF.LI', size=(2, None), rec_len=2,
                  desc='Language Indication'):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
 
@@ -524,7 +531,7 @@ class EF_LI(TransRecEF):
 
 
 class EF_Keys(TransparentEF):
-    def __init__(self, fid='6f08', sfid=0x08, name='EF.Keys', size={33, 33},
+    def __init__(self, fid='6f08', sfid=0x08, name='EF.Keys', size=(33, 33),
                  desc='Ciphering and Integrity Keys'):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         self._construct = Struct(
@@ -532,102 +539,14 @@ class EF_Keys(TransparentEF):
 
 # TS 31.102 Section 4.2.6
 class EF_HPPLMN(TransparentEF):
-    def __init__(self, fid='6f31', sfid=0x12, name='EF.HPPLMN', size={1, 1},
+    def __init__(self, fid='6f31', sfid=0x12, name='EF.HPPLMN', size=(1, 1),
                  desc='Higher Priority PLMN search period'):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         self._construct = Int8ub
 
-# TS 31.102 Section 4.2.8
-class EF_UServiceTable(TransparentEF):
-    def __init__(self, fid, sfid, name, desc, size, table, **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
-        self.table = table
-
-    @staticmethod
-    def _bit_byte_offset_for_service(service: int) -> Tuple[int, int]:
-        i = service - 1
-        byte_offset = i//8
-        bit_offset = (i % 8)
-        return (byte_offset, bit_offset)
-
-    def _decode_bin(self, in_bin):
-        ret = {}
-        for i in range(0, len(in_bin)):
-            byte = in_bin[i]
-            for bitno in range(0, 8):
-                service_nr = i * 8 + bitno + 1
-                ret[service_nr] = {
-                    'activated': True if byte & (1 << bitno) else False
-                }
-                if service_nr in self.table:
-                    ret[service_nr]['description'] = self.table[service_nr]
-        return ret
-
-    def _encode_bin(self, in_json):
-        # compute the required binary size
-        bin_len = 0
-        for srv in in_json.keys():
-            service_nr = int(srv)
-            (byte_offset, bit_offset) = EF_UServiceTable._bit_byte_offset_for_service(
-                service_nr)
-            if byte_offset >= bin_len:
-                bin_len = byte_offset+1
-        # encode the actual data
-        out = bytearray(b'\x00' * bin_len)
-        for srv in in_json.keys():
-            service_nr = int(srv)
-            (byte_offset, bit_offset) = EF_UServiceTable._bit_byte_offset_for_service(
-                service_nr)
-            if in_json[srv]['activated'] == True:
-                bit = 1
-            else:
-                bit = 0
-            out[byte_offset] |= (bit) << bit_offset
-        return out
-
-    def get_active_services(self, cmd):
-        # obtain list of currently active services
-        (service_data, sw) = cmd.rs.read_binary_dec()
-        active_services = []
-        for s in service_data.keys():
-            if service_data[s]['activated']:
-                active_services.append(s)
-        return active_services
-
-    def ust_service_check(self, cmd):
-        """Check consistency between services of this file and files present/activated"""
-        num_problems = 0
-        # obtain list of currently active services
-        active_services = self.get_active_services(cmd)
-        # iterate over all the service-constraints we know of
-        files_by_service = self.parent.files_by_service
-        try:
-            for s in sorted(files_by_service.keys()):
-                active_str = 'active' if s in active_services else 'inactive'
-                cmd.poutput("Checking service No %u (%s)" % (s, active_str))
-                for f in files_by_service[s]:
-                    should_exist = f.should_exist_for_services(active_services)
-                    try:
-                        cmd.rs.select_file(f)
-                        sw = None
-                        exists = True
-                    except SwMatchError as e:
-                        sw = str(e)
-                        exists = False
-                    if exists != should_exist:
-                        num_problems += 1
-                        if exists:
-                            cmd.perror("  ERROR: File %s is selectable but should not!" % f)
-                        else:
-                            cmd.perror("  ERROR: File %s is not selectable (%s) but should!" %  (f, sw))
-        finally:
-            # re-select the EF.UST
-            cmd.rs.select_file(self)
-        return num_problems
-
 class EF_UST(EF_UServiceTable):
     def __init__(self, **kwargs):
-        super().__init__(fid='6f38', sfid=0x04, name='EF.UST', desc='USIM Service Table', size={1,17}, table=EF_UST_map, **kwargs)
+        super().__init__(fid='6f38', sfid=0x04, name='EF.UST', desc='USIM Service Table', size=(1,17), table=EF_UST_map, **kwargs)
         # add those commands to the general commands of a TransparentEF
         self.shell_commands += [self.AddlShellCommands()]
 
@@ -651,7 +570,7 @@ class EF_UST(EF_UServiceTable):
             absent/deactivated.  This performs a consistency check to ensure that no services are activated
             for files that are not - and vice-versa, no files are activated for services that are not.  Error
             messages are printed for every inconsistency found."""
-            selected_file = self._cmd.rs.selected_file
+            selected_file = self._cmd.lchan.selected_file
             num_problems = selected_file.ust_service_check(self._cmd)
             # obtain list of currently active services
             active_services = selected_file.get_active_services(self._cmd)
@@ -683,7 +602,7 @@ class EF_ECC(LinFixedEF):
 
     def __init__(self, fid='6fb7', sfid=0x01, name='EF.ECC',
                  desc='Emergency Call Codes'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len={4, 20})
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(4, 20))
 
     def _decode_record_bin(self, in_bin):
         # mandatory parts
@@ -711,7 +630,7 @@ class EF_ECC(LinFixedEF):
 
 # TS 31.102 Section 4.2.17
 class EF_LOCI(TransparentEF):
-    def __init__(self, fid='6f7e', sfid=0x0b, name='EF.LOCI', desc='Location information', size={11, 11}):
+    def __init__(self, fid='6f7e', sfid=0x0b, name='EF.LOCI', desc='Location information', size=(11, 11)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         Lai = Struct('mcc_mnc'/BcdAdapter(Bytes(3)), 'lac'/HexAdapter(Bytes(2)))
         self._construct = Struct('tmsi'/HexAdapter(Bytes(4)), 'lai'/Lai, 'rfu'/Int8ub, 'lu_status'/Int8ub)
@@ -725,7 +644,7 @@ class EF_AD(TransparentEF):
         maintenance_off_line = 0x02
         cell_test = 0x04
 
-    def __init__(self, fid='6fad', sfid=0x03, name='EF.AD', desc='Administrative Data', size={4, 6}):
+    def __init__(self, fid='6fad', sfid=0x03, name='EF.AD', desc='Administrative Data', size=(4, 6)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         self._construct = BitStruct(
             # Byte 1
@@ -740,17 +659,17 @@ class EF_AD(TransparentEF):
 
 # TS 31.102 Section 4.2.23
 class EF_PSLOCI(TransparentEF):
-    def __init__(self, fid='6f73', sfid=0x0c, name='EF.PSLOCI', desc='PS Location information', size={14, 14}):
+    def __init__(self, fid='6f73', sfid=0x0c, name='EF.PSLOCI', desc='PS Location information', size=(14, 14)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         self._construct = Struct('ptmsi'/HexAdapter(Bytes(4)), 'ptmsi_sig'/HexAdapter(Bytes(3)),
                                  'rai'/HexAdapter(Bytes(6)), 'rau_status'/Int8ub)
 
 # TS 31.102 Section 4.2.33
 class EF_ICI(CyclicEF):
-    def __init__(self, fid='6f80', sfid=0x14, name='EF.ICI', rec_len={28, 48},
+    def __init__(self, fid='6f80', sfid=0x14, name='EF.ICI', rec_len=(28, 48),
                  desc='Incoming Call Information', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
-        self._construct = Struct('alpha_id'/Bytes(this._.total_len-28),
+        self._construct = Struct('alpha_id'/HexAdapter(Bytes(this._.total_len-28)),
                                  'len_of_bcd_contents'/Int8ub,
                                  'ton_npi'/Int8ub,
                                  'call_number'/BcdAdapter(Bytes(10)),
@@ -759,14 +678,14 @@ class EF_ICI(CyclicEF):
                                  'date_and_time'/BcdAdapter(Bytes(7)),
                                  'duration'/Int24ub,
                                  'status'/Byte,
-                                 'link_to_phonebook'/Bytes(3))
+                                 'link_to_phonebook'/HexAdapter(Bytes(3)))
 
 # TS 31.102 Section 4.2.34
 class EF_OCI(CyclicEF):
-    def __init__(self, fid='6f81', sfid=0x15, name='EF.OCI', rec_len={27, 47},
+    def __init__(self, fid='6f81', sfid=0x15, name='EF.OCI', rec_len=(27, 47),
                  desc='Outgoing Call Information', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
-        self._construct = Struct('alpha_id'/Bytes(this._.total_len-27),
+        self._construct = Struct('alpha_id'/HexAdapter(Bytes(this._.total_len-27)),
                                  'len_of_bcd_contents'/Int8ub,
                                  'ton_npi'/Int8ub,
                                  'call_number'/BcdAdapter(Bytes(10)),
@@ -774,11 +693,11 @@ class EF_OCI(CyclicEF):
                                  'ext5_record_id'/Int8ub,
                                  'date_and_time'/BcdAdapter(Bytes(7)),
                                  'duration'/Int24ub,
-                                 'link_to_phonebook'/Bytes(3))
+                                 'link_to_phonebook'/HexAdapter(Bytes(3)))
 
 # TS 31.102 Section 4.2.35
 class EF_ICT(CyclicEF):
-    def __init__(self, fid='6f82', sfid=None, name='EF.ICT', rec_len={3, 3},
+    def __init__(self, fid='6f82', sfid=None, name='EF.ICT', rec_len=(3, 3),
                  desc='Incoming Call Timer', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct('accumulated_call_timer'/Int24ub)
@@ -786,12 +705,12 @@ class EF_ICT(CyclicEF):
 # TS 31.102 Section 4.2.38
 class EF_CCP2(LinFixedEF):
     def __init__(self, fid='6f4f', sfid=0x16, name='EF.CCP2', desc='Capability Configuration Parameters 2', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len={15, None}, **kwargs)
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=(15, None), **kwargs)
 
 # TS 31.102 Section 4.2.47
 class EF_EST(EF_UServiceTable):
     def __init__(self, **kwargs):
-        super().__init__(fid='6f56', sfid=0x05, name='EF.EST', desc='Enabled Services Table', size={1,None}, table=EF_EST_map, **kwargs)
+        super().__init__(fid='6f56', sfid=0x05, name='EF.EST', desc='Enabled Services Table', size=(1,None), table=EF_EST_map, **kwargs)
         # add those commands to the general commands of a TransparentEF
         self.shell_commands += [self.AddlShellCommands()]
 
@@ -810,35 +729,44 @@ class EF_EST(EF_UServiceTable):
 
 # TS 31.102 Section 4.2.48
 class EF_ACL(TransparentEF):
-    def __init__(self, fid='6f57', sfid=None, name='EF.ACL', size={32, None},
+    def __init__(self, fid='6f57', sfid=None, name='EF.ACL', size=(32, None),
                  desc='Access Point Name Control List', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
-        self._construct = Struct('num_of_apns'/Int8ub, 'tlvs'/GreedyBytes)
+        self._construct = Struct('num_of_apns'/Int8ub, 'tlvs'/HexAdapter(GreedyBytes))
 
 # TS 31.102 Section 4.2.51
 class EF_START_HFN(TransparentEF):
-    def __init__(self, fid='6f5b', sfid=0x0f, name='EF.START-HFN', size={6, 6},
+    def __init__(self, fid='6f5b', sfid=0x0f, name='EF.START-HFN', size=(6, 6),
                  desc='Initialisation values for Hyperframe number', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('start_cs'/Int24ub, 'start_ps'/Int24ub)
 
 # TS 31.102 Section 4.2.52
 class EF_THRESHOLD(TransparentEF):
-    def __init__(self, fid='6f5c', sfid=0x10, name='EF.THRESHOLD', size={3, 3},
+    def __init__(self, fid='6f5c', sfid=0x10, name='EF.THRESHOLD', size=(3, 3),
                  desc='Maximum value of START', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('max_start'/Int24ub)
 
+# TS 31.102 (old releases like 3.8.0) Section 4.2.56
+class EF_RPLMNAcT(TransRecEF):
+    def __init__(self, fid='6f65', sfid=None, name='EF.RPLMNAcTD', size=(2, 4), rec_len=2,
+                 desc='RPLMN Last used Access Technology', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
+    def _decode_record_hex(self, in_hex):
+        return dec_act(in_hex)
+    # TODO: Encode
+
 # TS 31.102 Section 4.2.77
 class EF_VGCSCA(TransRecEF):
-    def __init__(self, fid='6fd4', sfid=None, name='EF.VGCSCA', size={2, 100}, rec_len=2,
+    def __init__(self, fid='6fd4', sfid=None, name='EF.VGCSCA', size=(2, 100), rec_len=2,
                  desc='Voice Group Call Service Ciphering Algorithm', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = Struct('alg_v_ki_1'/Int8ub, 'alg_v_ki_2'/Int8ub)
 
 # TS 31.102 Section 4.2.79
 class EF_GBABP(TransparentEF):
-    def __init__(self, fid='6fd6', sfid=None, name='EF.GBABP', size={3, 50},
+    def __init__(self, fid='6fd6', sfid=None, name='EF.GBABP', size=(3, 50),
                  desc='GBA Bootstrapping parameters', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('rand'/LV, 'b_tid'/LV, 'key_lifetime'/LV)
@@ -846,9 +774,9 @@ class EF_GBABP(TransparentEF):
 # TS 31.102 Section 4.2.80
 class EF_MSK(LinFixedEF):
     def __init__(self, fid='6fd7', sfid=None, name='EF.MSK', desc='MBMS Service Key List', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len={20, None}, **kwargs)
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=(20, None), **kwargs)
         msk_ts_constr = Struct('msk_id'/Int32ub, 'timestamp_counter'/Int32ub)
-        self._construct = Struct('key_domain_id'/Bytes(3),
+        self._construct = Struct('key_domain_id'/HexAdapter(Bytes(3)),
                                  'num_msk_id'/Int8ub,
                                  'msk_ids'/msk_ts_constr[this.num_msk_id])
 # TS 31.102 Section 4.2.81
@@ -869,7 +797,7 @@ class EF_MUK(LinFixedEF):
         pass
 
     def __init__(self, fid='6fd8', sfid=None, name='EF.MUK', desc='MBMS User Key', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len={None, None}, **kwargs)
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=(None, None), **kwargs)
         self._tlv = EF_MUK.EF_MUK_Collection
 
 # TS 31.102 Section 4.2.83
@@ -884,12 +812,12 @@ class EF_GBANL(LinFixedEF):
         pass
 
     def __init__(self, fid='6fda', sfid=None, name='EF.GBANL', desc='GBA NAF List', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len={None, None}, **kwargs)
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=(None, None), **kwargs)
         self._tlv = EF_GBANL.EF_GBANL_Collection
 
 # TS 31.102 Section 4.2.85
 class EF_EHPLMNPI(TransparentEF):
-    def __init__(self, fid='6fdb', sfid=None, name='EF.EHPLMNPI', size={1, 1},
+    def __init__(self, fid='6fdb', sfid=None, name='EF.EHPLMNPI', size=(1, 1),
                  desc='Equivalent HPLMN Presentation Indication', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('presentation_ind' /
@@ -899,7 +827,7 @@ class EF_EHPLMNPI(TransparentEF):
 class EF_NAFKCA(LinFixedEF):
     class NAF_KeyCentreAddress(BER_TLV_IE, tag=0x80):
         _construct = HexAdapter(GreedyBytes)
-    def __init__(self, fid='6fdd', sfid=None, name='EF.NAFKCA', rec_len={None, None},
+    def __init__(self, fid='6fdd', sfid=None, name='EF.NAFKCA', rec_len=(None, None),
                  desc='NAF Key Centre Address', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._tlv = EF_NAFKCA.NAF_KeyCentreAddress
@@ -930,19 +858,20 @@ class EF_NCP_IP(LinFixedEF):
     class EF_NCP_IP_Collection(TLV_IE_Collection,
                                nested=[AccessPointName, Login, Password, BearerDescription]):
         pass
-    def __init__(self, fid='6fe2', sfid=None, name='EF.NCP-IP', rec_len={None, None},
+    def __init__(self, fid='6fe2', sfid=None, name='EF.NCP-IP', rec_len=(None, None),
                  desc='Network Connectivity Parameters for USIM IP connections', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._tlv = EF_NCP_IP.EF_NCP_IP_Collection
 
 # TS 31.102 Section 4.2.91
 class EF_EPSLOCI(TransparentEF):
-    def __init__(self, fid='6fe3', sfid=0x1e, name='EF.EPSLOCI', size={18, 18},
-                 desc='EPS Location Information', **kwargs):
+    def __init__(self, fid='6fe3', sfid=0x1e, name='EF.EPSLOCI',
+                 desc='EPS Location Information', size=(18,18), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         upd_status_constr = Enum(
             Byte, updated=0, not_updated=1, roaming_not_allowed=2)
-        self._construct = Struct('guti'/Bytes(12), 'last_visited_registered_tai'/Bytes(5),
+        self._construct = Struct('guti'/HexAdapter(Bytes(12)),
+                                 'last_visited_registered_tai'/HexAdapter(Bytes(5)),
                                  'eps_update_status'/upd_status_constr)
 
 # TS 31.102 Section 4.2.92
@@ -966,14 +895,14 @@ class EF_EPSNSC(LinFixedEF):
                                    nested=[KSI_ASME, K_ASME, UplinkNASCount, DownlinkNASCount,
                                            IDofNASAlgorithms]):
         pass
-    def __init__(self, fid='6fe4', sfid=0x18, name='EF.EPSNSC', rec_len={54, 128},
+    def __init__(self, fid='6fe4', sfid=0x18, name='EF.EPSNSC', rec_len=(54, 128),
                  desc='EPS NAS Security Context', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._tlv = EF_EPSNSC.EPS_NAS_Security_Context
 
 # TS 31.102 Section 4.2.96
 class EF_PWS(TransparentEF):
-    def __init__(self, fid='6fec', sfid=None, name='EF.PWS', desc='Public Warning System', size={1, 1}, **kwargs):
+    def __init__(self, fid='6fec', sfid=None, name='EF.PWS', desc='Public Warning System', size=(1, 1), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         pws_config = FlagsEnum(
             Byte, ignore_pws_in_hplmn_and_equivalent=1, ignore_pws_in_vplmn=2)
@@ -981,7 +910,7 @@ class EF_PWS(TransparentEF):
 
 # TS 31.102 Section 4.2.101
 class EF_IPS(CyclicEF):
-    def __init__(self, fid='6ff1', sfid=None, name='EF.IPS', rec_len={4, 4},
+    def __init__(self, fid='6ff1', sfid=None, name='EF.IPS', rec_len=(4, 4),
                  desc='IMEI(SV) Pairing Status', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct('status'/PaddedString(2, 'ascii'),
@@ -1002,28 +931,46 @@ class EF_ePDGId(TransparentEF):
 
 # TS 31.102 Section 4.2.106
 class EF_FromPreferred(TransparentEF):
-    def __init__(self, fid='6ff7', sfid=None, name='EF.FromPreferred', size={1, 1},
+    def __init__(self, fid='6ff7', sfid=None, name='EF.FromPreferred', size=(1, 1),
                  desc='From Preferred', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = BitStruct('rfu'/BitsRFU(7), 'from_preferred'/Bit)
 
+
+######################################################################
+# DF.GSM-ACCESS
+######################################################################
+
+class DF_GSM_ACCESS(CardDF):
+    def __init__(self, fid='5F3B', name='DF.GSM-ACCESS', desc='GSM Access', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, service=27, **kwargs)
+        files = [
+            EF_Kc(fid='4f20', sfid=0x01, service=27),
+            EF_Kc(fid='4f52', sfid=0x02, name='EF.KcGPRS', desc='GPRS Ciphering key KcGPRS', service=27),
+            EF_CPBCCH(fid='4f63', service=39),
+            EF_InvScan(fid='4f64', service=40),
+        ]
+        self.add_files(files)
+
+
 ######################################################################
 # DF.5GS
 ######################################################################
 
 # TS 31.102 Section 4.4.11.2
 class EF_5GS3GPPLOCI(TransparentEF):
-    def __init__(self, fid='4f01', sfid=0x01, name='EF.5GS3GPPLOCI', size={20, 20},
+    def __init__(self, fid='4f01', sfid=0x01, name='EF.5GS3GPPLOCI', size=(20, 20),
                  desc='5S 3GP location information', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         upd_status_constr = Enum(
             Byte, updated=0, not_updated=1, roaming_not_allowed=2)
-        self._construct = Struct('5g_guti'/Bytes(13), 'last_visited_registered_tai_in_5gs'/Bytes(6),
+        self._construct = Struct('5g_guti'/HexAdapter(Bytes(13)),
+                                 'last_visited_registered_tai_in_5gs'/HexAdapter(Bytes(6)),
                                  '5gs_update_status'/upd_status_constr)
 
 # TS 31.102 Section 4.4.11.7
 class EF_UAC_AIC(TransparentEF):
-    def __init__(self, fid='4f06', sfid=0x06, name='EF.UAC_AIC', size={4, 4},
+    def __init__(self, fid='4f06', sfid=0x06, name='EF.UAC_AIC', size=(4, 4),
                  desc='UAC Access Identities Configuration', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         cfg_constr = FlagsEnum(Byte, multimedia_priority_service=1,
@@ -1033,8 +980,9 @@ class EF_UAC_AIC(TransparentEF):
 # TS 31.102 Section 4.4.11.9
 class EF_OPL5G(LinFixedEF):
     def __init__(self, fid='6f08', sfid=0x08, name='EF.OPL5G', desc='5GS Operator PLMN List', **kwargs):
-        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len={10, None}, **kwargs)
-        Tai = Struct('mcc_mnc'/BcdAdapter(Bytes(3)), 'tac_min'/Bytes(3), 'tac_max'/Bytes(3))
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=(10, None), **kwargs)
+        Tai = Struct('mcc_mnc'/BcdAdapter(Bytes(3)), 'tac_min'/HexAdapter(Bytes(3)),
+                     'tac_max'/HexAdapter(Bytes(3)))
         self._construct = Struct('tai'/Tai, 'pnn_record_id'/Int8ub)
 
 # TS 31.102 Section 4.4.11.10
@@ -1102,12 +1050,12 @@ class DF_HNB(CardDF):
     def __init__(self, fid='5f50', name='DF.HNB', desc='Files for HomeNodeB purpose', **kwargs):
         super().__init__(fid=fid, name=name, desc=desc, **kwargs)
         files = [
-            LinFixedEF('4f01', 0x01, 'EF.ACSGL', 'Allowed CSG Lists', service=86),
-            LinFixedEF('4f02', 0x02, 'EF.CSGTL', 'CSG Types', service=86),
-            LinFixedEF('4f03', 0x03, 'EF.HNBN', 'Home NodeB Name', service=86),
-            LinFixedEF('4f04', 0x04, 'EF.OCSGL', 'Operator CSG Lists', service=90),
-            LinFixedEF('4f05', 0x05, 'EF.OCSGT', 'Operator CSG Type', service=90),
-            LinFixedEF('4f06', 0x06, 'EF.OHNBN', 'Operator Home NodeB Name', service=90),
+            LinFixedEF('4f81', 0x01, 'EF.ACSGL', 'Allowed CSG Lists', service=86),
+            LinFixedEF('4f82', 0x02, 'EF.CSGTL', 'CSG Types', service=86),
+            LinFixedEF('4f83', 0x03, 'EF.HNBN', 'Home NodeB Name', service=86),
+            LinFixedEF('4f84', 0x04, 'EF.OCSGL', 'Operator CSG Lists', service=90),
+            LinFixedEF('4f85', 0x05, 'EF.OCSGT', 'Operator CSG Type', service=90),
+            LinFixedEF('4f86', 0x06, 'EF.OHNBN', 'Operator Home NodeB Name', service=90),
         ]
         self.add_files(files)
 
@@ -1161,7 +1109,7 @@ class DF_USIM_5GS(CardDF):
             EF_OPL5G(service=129),
             EF_SUPI_NAI(service=130),
             TransparentEF('4F0A', 0x0a, 'EF.Routing_Indicator',
-                          'Routing Indicator', size={4, 4}, service=124),
+                          'Routing Indicator', size=(4, 4), service=124),
             TransparentEF('4F0B', 0x0b, 'EF.URSP',
                           'UE Route Selector Policies per PLMN', service=132),
             EF_TN3GPPSNN(service=133),
@@ -1188,16 +1136,16 @@ class ADF_USIM(CardADF):
             EF_ACMmax(service=13),
             EF_UST(),
             CyclicEF('6f39', None, 'EF.ACM',
-                     'Accumulated call meter', rec_len={3, 3}, service=13),
+                     'Accumulated call meter', rec_len=(3, 3), service=13),
             TransparentEF('6f3e', None, 'EF.GID1', 'Group Identifier Level 1', service=17),
             TransparentEF('6f3f', None, 'EF.GID2', 'Group Identifier Level 2', service=18),
             EF_SPN(service=19),
             TransparentEF('6f41', None, 'EF.PUCT',
-                          'Price per unit and currency table', size={5, 5}, service=13),
+                          'Price per unit and currency table', size=(5, 5), service=13),
             EF_CBMI(service=15),
             EF_ACC(sfid=0x06),
             EF_PLMNsel('6f7b', 0x0d, 'EF.FPLMN',
-                       'Forbidden PLMNs', size={12, None}),
+                       'Forbidden PLMNs', size=(12, None)),
             EF_LOCI(),
             EF_AD(),
             EF_CBMID(sfid=0x0e, service=29),
@@ -1234,6 +1182,7 @@ class ADF_USIM(CardADF):
             EF_xPLMNwAcT('6f61', 0x11, 'EF.OPLMNwAcT', 'User controlled PLMN Selector with Access Technology', service=42),
             EF_xPLMNwAcT('6f62', 0x13, 'EF.HPLMNwAcT', 'HPLMN Selector with Access Technology', service=43),
             EF_ARR('6f06', 0x17),
+            EF_RPLMNAcT(),
             TransparentEF('6fc4', None, 'EF.NETPAR', 'Network Parameters'),
             EF_PNN('6fc5', 0x19, service=45),
             EF_OPL(service=46),
@@ -1260,7 +1209,7 @@ class ADF_USIM(CardADF):
             EF_MSK(service=69),
             EF_MUK(service=69),
             EF_GBANL(service=68),
-            EF_PLMNsel('6fd9', 0x1d, 'EF.EHPLMN', 'Equivalent HPLMN', size={12, None}, service=71),
+            EF_PLMNsel('6fd9', 0x1d, 'EF.EHPLMN', 'Equivalent HPLMN', size=(12, None), service=71),
             EF_EHPLMNPI(service=(71, 73)),
             # EF_LRPLMNSI ('6fdc', service=74)
             EF_NAFKCA(service=(68, 76)),
@@ -1269,7 +1218,7 @@ class ADF_USIM(CardADF):
             EF_NCP_IP(service=80),
             EF_EPSLOCI('6fe3', 0x1e, 'EF.EPSLOCI', 'EPS location information', service=85),
             EF_EPSNSC(service=85),
-            TransparentEF('6fe6', None, 'EF.UFC', 'USAT Facility Control', size={1, 16}),
+            TransparentEF('6fe6', None, 'EF.UFC', 'USAT Facility Control', size=(1, 16)),
             TransparentEF('6fe8', None, 'EF.NASCONFIG', 'Non Access Stratum Configuration', service=96),
             # UICC IARI (only in cards that have no ISIM) service=95
             EF_PWS(service=97),
@@ -1282,8 +1231,8 @@ class ADF_USIM(CardADF):
             # FIXME: from EF_ePDGSelection onwards
             EF_FromPreferred(service=114),
             # FIXME: DF_SoLSA service=23
-            # FIXME: DF_PHONEBOOK
-            # FIXME: DF_GSM_ACCESS service=27
+            DF_PHONEBOOK(),
+            DF_GSM_ACCESS(),
             DF_WLAN(service=[59, 60, 61, 62, 63, 66, 81, 82, 83, 84, 88]),
             DF_HNB(service=[86, 90]),
             DF_ProSe(service=101),
@@ -1316,40 +1265,40 @@ class ADF_USIM(CardADF):
         term_prof_parser.add_argument('PROFILE', help='Hexstring of encoded terminal profile')
 
         @cmd2.with_argparser(term_prof_parser)
-        def do_terminal_profile(self, arg):
+        def do_terminal_profile(self, opts):
             """Send a TERMINAL PROFILE command to the card.
             This is used to inform the card about which optional
             features the terminal (modem/phone) supports, particularly
             in the context of SIM Toolkit, Proactive SIM and OTA.  You
             must specify a hex-string with the encoded terminal profile
             you want to send to the card."""
-            (data, sw) = self._cmd.card._scc.terminal_profile(arg)
+            (data, sw) = self._cmd.card._scc.terminal_profile(opts.PROFILE)
             self._cmd.poutput('SW: %s, data: %s' % (sw, data))
 
         envelope_parser = argparse.ArgumentParser()
         envelope_parser.add_argument('PAYLOAD', help='Hexstring of encoded payload to ENVELOPE')
 
         @cmd2.with_argparser(envelope_parser)
-        def do_envelope(self, arg):
+        def do_envelope(self, opts):
             """Send an ENVELOPE command to the card.  This is how a
             variety of information is communicated from the terminal
             (modem/phone) to the card, particularly in the context of
             SIM Toolkit, Proactive SIM and OTA."""
-            (data, sw) = self._cmd.card._scc.envelope(arg)
+            (data, sw) = self._cmd.card._scc.envelope(opts.PAYLOAD)
             self._cmd.poutput('SW: %s, data: %s' % (sw, data))
 
         envelope_sms_parser = argparse.ArgumentParser()
         envelope_sms_parser.add_argument('TPDU', help='Hexstring of encoded SMS TPDU')
 
         @cmd2.with_argparser(envelope_sms_parser)
-        def do_envelope_sms(self, arg):
+        def do_envelope_sms(self, opts):
             """Send an ENVELOPE(SMS-PP-Download) command to the card.
             This emulates a terminal (modem/phone) having received a SMS
             with a PID of 'SMS for the SIM card'.  You can use this
             command in the context of testing OTA related features
             without a modem/phone or a cellular netwokr."""
             tpdu_ie = SMS_TPDU()
-            tpdu_ie.from_bytes(h2b(arg))
+            tpdu_ie.from_bytes(h2b(opts.TPDU))
             dev_ids = DeviceIdentities(
                 decoded={'source_dev_id': 'network', 'dest_dev_id': 'uicc'})
             sms_dl = SMSPPDownload(children=[dev_ids, tpdu_ie])

pySim/ts_31_102_telecom.py

@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# without this, pylint will fail when inner classes are used
+# within the 'nested' kwarg of our TlvMeta metaclass on python 3.7 :(
+# pylint: disable=undefined-variable
+
+"""
+DF_PHONEBOOK, DF_MULTIMEDIA, DF_MCS as specified in 3GPP TS 31.102 V16.6.0
+Needs to be a separate python module to avoid cyclic imports
+"""
+
+#
+# Copyright (C) 2022 Harald Welte <laforge@osmocom.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from pySim.tlv import *
+from pySim.filesystem import *
+from pySim.construct import *
+from construct import Optional as COptional
+from construct import *
+
+# TS 31.102 Section 4.2.8
+class EF_UServiceTable(TransparentEF):
+    def __init__(self, fid, sfid, name, desc, size, table, **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
+        self.table = table
+
+    @staticmethod
+    def _bit_byte_offset_for_service(service: int) -> Tuple[int, int]:
+        i = service - 1
+        byte_offset = i//8
+        bit_offset = (i % 8)
+        return (byte_offset, bit_offset)
+
+    def _decode_bin(self, in_bin):
+        ret = {}
+        for i in range(0, len(in_bin)):
+            byte = in_bin[i]
+            for bitno in range(0, 8):
+                service_nr = i * 8 + bitno + 1
+                ret[service_nr] = {
+                    'activated': True if byte & (1 << bitno) else False
+                }
+                if service_nr in self.table:
+                    ret[service_nr]['description'] = self.table[service_nr]
+        return ret
+
+    def _encode_bin(self, in_json):
+        # compute the required binary size
+        bin_len = 0
+        for srv in in_json.keys():
+            service_nr = int(srv)
+            (byte_offset, bit_offset) = EF_UServiceTable._bit_byte_offset_for_service(
+                service_nr)
+            if byte_offset >= bin_len:
+                bin_len = byte_offset+1
+        # encode the actual data
+        out = bytearray(b'\x00' * bin_len)
+        for srv in in_json.keys():
+            service_nr = int(srv)
+            (byte_offset, bit_offset) = EF_UServiceTable._bit_byte_offset_for_service(
+                service_nr)
+            if in_json[srv]['activated'] == True:
+                bit = 1
+            else:
+                bit = 0
+            out[byte_offset] |= (bit) << bit_offset
+        return out
+
+    def get_active_services(self, cmd):
+        # obtain list of currently active services
+        (service_data, sw) = cmd.lchan.read_binary_dec()
+        active_services = []
+        for s in service_data.keys():
+            if service_data[s]['activated']:
+                active_services.append(s)
+        return active_services
+
+    def ust_service_check(self, cmd):
+        """Check consistency between services of this file and files present/activated"""
+        num_problems = 0
+        # obtain list of currently active services
+        active_services = self.get_active_services(cmd)
+        # iterate over all the service-constraints we know of
+        files_by_service = self.parent.files_by_service
+        try:
+            for s in sorted(files_by_service.keys()):
+                active_str = 'active' if s in active_services else 'inactive'
+                cmd.poutput("Checking service No %u (%s)" % (s, active_str))
+                for f in files_by_service[s]:
+                    should_exist = f.should_exist_for_services(active_services)
+                    try:
+                        cmd.lchan.select_file(f)
+                        sw = None
+                        exists = True
+                    except SwMatchError as e:
+                        sw = str(e)
+                        exists = False
+                    if exists != should_exist:
+                        num_problems += 1
+                        if exists:
+                            cmd.perror("  ERROR: File %s is selectable but should not!" % f)
+                        else:
+                            cmd.perror("  ERROR: File %s is not selectable (%s) but should!" %  (f, sw))
+        finally:
+            # re-select the EF.UST
+            cmd.lchan.select_file(self)
+        return num_problems
+
+
+
+# TS 31.102 Section 4.4.2.1
+class EF_PBR(LinFixedEF):
+    def __init__(self, fid='4F30', name='EF.PBR', desc='Phone Book Reference', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+        #self._tlv = FIXME
+
+# TS 31.102 Section 4.4.2.12.2
+class EF_PSC(TransparentEF):
+    _construct = Struct('synce_counter'/Int32ub)
+    def __init__(self, fid='4F22', name='EF.PSC', desc='Phone Book Synchronization Counter', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+        #self._tlv = FIXME
+
+# TS 31.102 Section 4.4.2.12.3
+class EF_CC(TransparentEF):
+    _construct = Struct('change_counter'/Int16ub)
+    def __init__(self, fid='4F23', name='EF.CC', desc='Change Counter', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+# TS 31.102 Section 4.4.2.12.4
+class EF_PUID(TransparentEF):
+    _construct = Struct('previous_uid'/Int16ub)
+    def __init__(self, fid='4F24', name='EF.PUID', desc='Previous Unique Identifer', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+# TS 31.102 Section 4.4.2
+class DF_PHONEBOOK(CardDF):
+    def __init__(self, fid='5F3A', name='DF.PHONEBOOK', desc='Phonebook', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_PBR(),
+            EF_PSC(),
+            EF_CC(),
+            EF_PUID(),
+            # FIXME: Those 4Fxx entries with unspecified FID...
+            ]
+        self.add_files(files)
+
+
+
+# TS 31.102 Section 4.6.3.1
+class EF_MML(BerTlvEF):
+    def __init__(self, fid='4F47', name='EF.MML', desc='Multimedia Messages List', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+# TS 31.102 Section 4.6.3.2
+class EF_MMDF(BerTlvEF):
+    def __init__(self, fid='4F48', name='EF.MMDF', desc='Multimedia Messages Data File', **kwargs):
+        super().__init__(fid, name=name, desc=desc, **kwargs)
+
+class DF_MULTIMEDIA(CardDF):
+    def __init__(self, fid='5F3B', name='DF.MULTIMEDIA', desc='Multimedia', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_MML(),
+            EF_MMDF(),
+            ]
+        self.add_files(files)
+
+
+# TS 31.102 Section 4.6.4.1
+EF_MST_map = {
+    1: 'MCPTT UE configuration data',
+    2: 'MCPTT User profile data',
+    3: 'MCS Group configuration data',
+    4: 'MCPTT Service configuration data',
+    5: 'MCS UE initial configuration data',
+    6: 'MCData UE configuration data',
+    7: 'MCData user profile data',
+    8: 'MCData service configuration data',
+    9: 'MCVideo UE configuration data',
+    10: 'MCVideo user profile data',
+    11: 'MCVideo service configuration data',
+    }
+
+# TS 31.102 Section 4.6.4.2
+class EF_MCS_CONFIG(BerTlvEF):
+    class McpttUeConfigurationData(BER_TLV_IE, tag=0x80):
+        pass
+    class McpttUserProfileData(BER_TLV_IE, tag=0x81):
+        pass
+    class McsGroupConfigurationData(BER_TLV_IE, tag=0x82):
+        pass
+    class McpttServiceConfigurationData(BER_TLV_IE, tag=0x83):
+        pass
+    class McsUeInitialConfigurationData(BER_TLV_IE, tag=0x84):
+        pass
+    class McdataUeConfigurationData(BER_TLV_IE, tag=0x85):
+        pass
+    class McdataUserProfileData(BER_TLV_IE, tag=0x86):
+        pass
+    class McdataServiceConfigurationData(BER_TLV_IE, tag=0x87):
+        pass
+    class McvideoUeConfigurationData(BER_TLV_IE, tag=0x88):
+        pass
+    class McvideoUserProfileData(BER_TLV_IE, tag=0x89):
+        pass
+    class McvideoServiceConfigurationData(BER_TLV_IE, tag=0x8a):
+        pass
+    class McsConfigDataCollection(TLV_IE_Collection, nested=[McpttUeConfigurationData,
+                                  McpttUserProfileData, McsGroupConfigurationData,
+                                  McpttServiceConfigurationData, McsUeInitialConfigurationData,
+                                  McdataUeConfigurationData, McdataUserProfileData,
+                                  McdataServiceConfigurationData, McvideoUeConfigurationData,
+                                  McvideoUserProfileData, McvideoServiceConfigurationData]):
+        pass
+    def __init__(self, fid='4F02', sfid=0x02, name='EF.MCS_CONFIG', desc='MCS configuration data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_MCS_CONFIG.McsConfigDataCollection
+
+# TS 31.102 Section 4.6.4.1
+class EF_MST(EF_UServiceTable):
+    def __init__(self, fid='4F01', sfid=0x01, name='EF.MST', desc='MCS Service Table', size=(2,2),
+                 table=EF_MST_map, **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, table=table)
+
+class DF_MCS(CardDF):
+    def __init__(self, fid='5F3D', name='DF.MCS', desc='Mission Critical Services', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_MST(),
+            EF_MCS_CONFIG(),
+            ]
+        self.add_files(files)
+
+
+# TS 31.102 Section 4.6.5.2
+EF_VST_map = {
+    1: 'MCPTT UE configuration data',
+    2: 'MCPTT User profile data',
+    3: 'MCS Group configuration data',
+    4: 'MCPTT Service configuration data',
+    5: 'MCS UE initial configuration data',
+    6: 'MCData UE configuration data',
+    7: 'MCData user profile data',
+    8: 'MCData service configuration data',
+    9: 'MCVideo UE configuration data',
+    10: 'MCVideo user profile data',
+    11: 'MCVideo service configuration data',
+    }
+
+# TS 31.102 Section 4.6.5.2
+class EF_VST(EF_UServiceTable):
+    def __init__(self, fid='4F01', sfid=0x01, name='EF.VST', desc='V2X Service Table', size=(2,2),
+                 table=EF_VST_map, **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, size=size, table=table)
+
+# TS 31.102 Section 4.6.5.3
+class EF_V2X_CONFIG(BerTlvEF):
+    class V2xConfigurationData(BER_TLV_IE, tag=0x80):
+        pass
+    class V2xConfigDataCollection(TLV_IE_Collection, nested=[V2xConfigurationData]):
+        pass
+    def __init__(self, fid='4F02', sfid=0x02, name='EF.V2X_CONFIG', desc='V2X configuration data', **kwargs):
+        super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
+        self._tlv = EF_V2X_CONFIG.V2xConfigDataCollection
+
+# TS 31.102 Section 4.6.5
+class DF_V2X(CardDF):
+    def __init__(self, fid='5F3E', name='DF.V2X', desc='Vehicle to X', **kwargs):
+        super().__init__(fid=fid, name=name, desc=desc, **kwargs)
+        files = [
+            EF_VST(),
+            EF_V2X_CONFIG(),
+            ]
+        self.add_files(files)

pySim/ts_31_103.py

@@ -26,7 +26,8 @@ from pySim.filesystem import *
 from pySim.utils import *
 from pySim.tlv import *
 from pySim.ts_51_011 import EF_AD, EF_SMS, EF_SMSS, EF_SMSR, EF_SMSP
-from pySim.ts_31_102 import ADF_USIM, EF_FromPreferred, EF_UServiceTable
+from pySim.ts_31_102 import ADF_USIM, EF_FromPreferred
+from pySim.ts_31_102_telecom import EF_UServiceTable
 import pySim.ts_102_221
 from pySim.ts_102_221 import EF_ARR
 
@@ -107,7 +108,7 @@ class EF_IMPU(LinFixedEF):
 # TS 31.103 Section 4.2.7
 class EF_IST(EF_UServiceTable):
     def __init__(self, **kwargs):
-        super().__init__('6f07', 0x07, 'EF.IST', 'ISIM Service Table', {1, None}, EF_IST_map)
+        super().__init__('6f07', 0x07, 'EF.IST', 'ISIM Service Table', (1, None), EF_IST_map)
         # add those commands to the general commands of a TransparentEF
         self.shell_commands += [self.AddlShellCommands()]
 
@@ -131,7 +132,7 @@ class EF_IST(EF_UServiceTable):
             absent/deactivated.  This performs a consistency check to ensure that no services are activated
             for files that are not - and vice-versa, no files are activated for services that are not.  Error
             messages are printed for every inconsistency found."""
-            selected_file = self._cmd.rs.selected_file
+            selected_file = self._cmd.lchan.selected_file
             num_problems = selected_file.ust_service_check(self._cmd)
             self._cmd.poutput("===> %u service / file inconsistencies detected" % num_problems)
 
@@ -181,7 +182,7 @@ class EF_IMSConfigData(BerTlvEF):
     class ImsConfigData(BER_TLV_IE, tag=0x81):
         _construct = GreedyString
     # pylint: disable=undefined-variable
-    class ImsConfigDataCollection(TLV_IE_Collection, neted=[ImsConfigDataEncoding, ImsConfigData]):
+    class ImsConfigDataCollection(TLV_IE_Collection, nested=[ImsConfigDataEncoding, ImsConfigData]):
         pass
     def __init__(self, fid='6ff8', sfid=None, name='EF.IMSConfigData', desc='IMS Configuration Data', **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, **kwargs)
@@ -248,7 +249,7 @@ class EF_MuDMiDConfigData(BerTlvEF):
     class MudMidConfigData(BER_TLV_IE, tag=0x81):
         _construct = GreedyString
     # pylint: disable=undefined-variable
-    class MudMidConfigDataCollection(TLV_IE_Collection, neted=[MudMidConfigDataEncoding, MudMidConfigData]):
+    class MudMidConfigDataCollection(TLV_IE_Collection, nested=[MudMidConfigDataEncoding, MudMidConfigData]):
         pass
     def __init__(self, fid='6ffe', sfid=None, name='EF.MuDMiDConfigData',
                  desc='MuD and MiD Configuration Data', **kwargs):

pySim/ts_51_011.py

@@ -32,6 +32,7 @@ order to describe the files specified in the relevant ETSI + 3GPP specifications
 from pySim.profile import match_sim
 from pySim.profile import CardProfile
 from pySim.filesystem import *
+from pySim.ts_31_102_telecom import DF_PHONEBOOK, DF_MULTIMEDIA, DF_MCS, DF_V2X
 import enum
 from pySim.construct import *
 from construct import Optional as COptional
@@ -342,7 +343,7 @@ EF_SST_map = {
 # TS 51.011 Section 10.5.1
 class EF_ADN(LinFixedEF):
     def __init__(self, fid='6f3a', sfid=None, name='EF.ADN', desc='Abbreviated Dialing Numbers', **kwargs):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len={14, 30}, **kwargs)
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(14, 30), **kwargs)
         self._construct = Struct('alpha_id'/COptional(GsmStringAdapter(Rpad(Bytes(this._.total_len-14)), codec='ascii')),
                                  'len_of_bcd'/Int8ub,
                                  'ton_npi'/TonNpi,
@@ -353,7 +354,7 @@ class EF_ADN(LinFixedEF):
 # TS 51.011 Section 10.5.5
 class EF_SMS(LinFixedEF):
     def __init__(self, fid='6f3c', sfid=None, name='EF.SMS', desc='Short messages', **kwargs):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len={176, 176}, **kwargs)
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(176, 176), **kwargs)
 
     def _decode_record_bin(self, raw_bin_data):
         def decode_status(status):
@@ -384,7 +385,7 @@ class EF_SMS(LinFixedEF):
 # TS 51.011 Section 10.5.5
 class EF_MSISDN(LinFixedEF):
     def __init__(self, fid='6f40', sfid=None, name='EF.MSISDN', desc='MSISDN', **kwargs):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len={15, 34}, **kwargs)
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(15, 34), **kwargs)
 
     def _decode_record_hex(self, raw_hex_data):
         return {'msisdn': dec_msisdn(raw_hex_data)}
@@ -426,7 +427,7 @@ class EF_SMSP(LinFixedEF):
                 raise ValueError
 
     def __init__(self, fid='6f42', sfid=None, name='EF.SMSP', desc='Short message service parameters', **kwargs):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len={28, None}, **kwargs)
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(28, None), **kwargs)
         ScAddr = Struct('length'/Int8ub, 'ton_npi'/TonNpi, 'call_number'/BcdAdapter(Rpad(Bytes(10))))
         self._construct = Struct('alpha_id'/COptional(GsmStringAdapter(Rpad(Bytes(this._.total_len-28)))),
                                  'parameter_indicators'/InvertAdapter(FlagsEnum(Byte, tp_dest_addr=1, tp_sc_addr=2,
@@ -447,28 +448,28 @@ class EF_SMSS(TransparentEF):
         def _encode(self, obj, context, path):
             return 0 if obj else 1
 
-    def __init__(self, fid='6f43', sfid=None, name='EF.SMSS', desc='SMS status', size={2, 8}, **kwargs):
+    def __init__(self, fid='6f43', sfid=None, name='EF.SMSS', desc='SMS status', size=(2, 8), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct(
             'last_used_tpmr'/Int8ub, 'memory_capacity_exceeded'/self.MemCapAdapter(Int8ub))
 
 # TS 51.011 Section 10.5.8
 class EF_SMSR(LinFixedEF):
-    def __init__(self, fid='6f47', sfid=None, name='EF.SMSR', desc='SMS status reports', rec_len={30, 30}, **kwargs):
+    def __init__(self, fid='6f47', sfid=None, name='EF.SMSR', desc='SMS status reports', rec_len=(30, 30), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
             'sms_record_id'/Int8ub, 'sms_status_report'/HexAdapter(Bytes(29)))
 
 
 class EF_EXT(LinFixedEF):
-    def __init__(self, fid, sfid=None, name='EF.EXT', desc='Extension', rec_len={13, 13}, **kwargs):
+    def __init__(self, fid, sfid=None, name='EF.EXT', desc='Extension', rec_len=(13, 13), **kwargs):
         super().__init__(fid=fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
             'record_type'/Int8ub, 'extension_data'/HexAdapter(Bytes(11)), 'identifier'/Int8ub)
 
 # TS 51.011 Section 10.5.16
 class EF_CMI(LinFixedEF):
-    def __init__(self, fid='6f58', sfid=None, name='EF.CMI', rec_len={2, 21},
+    def __init__(self, fid='6f58', sfid=None, name='EF.CMI', rec_len=(2, 21),
                  desc='Comparison Method Information', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
@@ -483,9 +484,9 @@ class DF_TELECOM(CardDF):
             EF_ADN(fid='6f3b', name='EF.FDN', desc='Fixed dialling numbers'),
             EF_SMS(),
             LinFixedEF(fid='6f3d', name='EF.CCP',
-                       desc='Capability Configuration Parameters', rec_len={14, 14}),
+                       desc='Capability Configuration Parameters', rec_len=(14, 14)),
             LinFixedEF(fid='6f4f', name='EF.ECCP',
-                       desc='Extended Capability Configuration Parameters', rec_len={15, 32}),
+                       desc='Extended Capability Configuration Parameters', rec_len=(15, 32)),
             EF_MSISDN(),
             EF_SMSP(),
             EF_SMSS(),
@@ -497,6 +498,11 @@ class DF_TELECOM(CardDF):
             EF_EXT('6f4e', None, 'EF.EXT4', 'Extension4 (BDN/SSC)'),
             EF_SMSR(),
             EF_CMI(),
+            # not really part of 51.011 but something that TS 31.102 specifies may exist here.
+            DF_PHONEBOOK(),
+            DF_MULTIMEDIA(),
+            DF_MCS(),
+            DF_V2X(),
         ]
         self.add_files(files)
 
@@ -506,7 +512,7 @@ class DF_TELECOM(CardDF):
 
 # TS 51.011 Section 10.3.1
 class EF_LP(TransRecEF):
-    def __init__(self, fid='6f05', sfid=None, name='EF.LP', size={1, None}, rec_len=1,
+    def __init__(self, fid='6f05', sfid=None, name='EF.LP', size=(1, None), rec_len=1,
                  desc='Language Preference'):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
 
@@ -518,7 +524,7 @@ class EF_LP(TransRecEF):
 
 # TS 51.011 Section 10.3.2
 class EF_IMSI(TransparentEF):
-    def __init__(self, fid='6f07', sfid=None, name='EF.IMSI', desc='IMSI', size={9, 9}):
+    def __init__(self, fid='6f07', sfid=None, name='EF.IMSI', desc='IMSI', size=(9, 9)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         # add those commands to the general commands of a TransparentEF
         self.shell_commands += [self.AddlShellCommands(self)]
@@ -539,15 +545,15 @@ class EF_IMSI(TransparentEF):
             """Change the plmn part of the IMSI"""
             plmn = arg.strip()
             if len(plmn) == 5 or len(plmn) == 6:
-                (data, sw) = self._cmd.rs.read_binary_dec()
+                (data, sw) = self._cmd.lchan.read_binary_dec()
                 if sw == '9000' and len(data['imsi'])-len(plmn) == 10:
                     imsi = data['imsi']
                     msin = imsi[len(plmn):]
-                    (data, sw) = self._cmd.rs.update_binary_dec(
+                    (data, sw) = self._cmd.lchan.update_binary_dec(
                         {'imsi': plmn+msin})
                     if sw == '9000' and data:
                         self._cmd.poutput_json(
-                            self._cmd.rs.selected_file.decode_hex(data))
+                            self._cmd.lchan.selected_file.decode_hex(data))
                 else:
                     raise ValueError("PLMN length does not match IMSI length")
             else:
@@ -557,7 +563,7 @@ class EF_IMSI(TransparentEF):
 # TS 51.011 Section 10.3.4
 class EF_PLMNsel(TransRecEF):
     def __init__(self, fid='6f30', sfid=None, name='EF.PLMNsel', desc='PLMN selector',
-                 size={24, None}, rec_len=3, **kwargs):
+                 size=(24, None), rec_len=3, **kwargs):
         super().__init__(fid, name=name, sfid=sfid, desc=desc, size=size, rec_len=rec_len, **kwargs)
 
     def _decode_record_hex(self, in_hex):
@@ -574,7 +580,7 @@ class EF_PLMNsel(TransRecEF):
 
 # TS 51.011 Section 10.3.6
 class EF_ACMmax(TransparentEF):
-    def __init__(self, fid='6f37', sfid=None, name='EF.ACMmax', size={3, 3},
+    def __init__(self, fid='6f37', sfid=None, name='EF.ACMmax', size=(3, 3),
                  desc='ACM maximum value', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('acm_max'/Int24ub)
@@ -632,7 +638,7 @@ class EF_ServiceTable(TransparentEF):
 # TS 51.011 Section 10.3.11
 class EF_SPN(TransparentEF):
     def __init__(self, fid='6f46', sfid=None, name='EF.SPN',
-                 desc='Service Provider Name', size={17, 17}, **kwargs):
+                 desc='Service Provider Name', size=(17, 17), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = BitStruct(
             # Byte 1
@@ -645,7 +651,7 @@ class EF_SPN(TransparentEF):
 
 # TS 51.011 Section 10.3.13
 class EF_CBMI(TransRecEF):
-    def __init__(self, fid='6f45', sfid=None, name='EF.CBMI', size={2, None}, rec_len=2,
+    def __init__(self, fid='6f45', sfid=None, name='EF.CBMI', size=(2, None), rec_len=2,
                  desc='Cell Broadcast message identifier selection', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = GreedyRange(Int16ub)
@@ -653,15 +659,15 @@ class EF_CBMI(TransRecEF):
 # TS 51.011 Section 10.3.15
 class EF_ACC(TransparentEF):
     def __init__(self, fid='6f78', sfid=None, name='EF.ACC',
-                 desc='Access Control Class', size={2, 2}, **kwargs):
+                 desc='Access Control Class', size=(2, 2), **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = HexAdapter(Bytes(2))
 
 # TS 51.011 Section 10.3.16
 class EF_LOCI(TransparentEF):
-    def __init__(self, fid='6f7e', sfid=None, name='EF.LOCI', desc='Location Information', size={11, 11}):
+    def __init__(self, fid='6f7e', sfid=None, name='EF.LOCI', desc='Location Information', size=(11, 11)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-        self._construct = Struct('tmsi'/Bytes(4), 'lai'/Bytes(5), 'tmsi_time'/Int8ub,
+        self._construct = Struct('tmsi'/HexAdapter(Bytes(4)), 'lai'/HexAdapter(Bytes(5)), 'tmsi_time'/Int8ub,
                                  'lu_status'/Enum(Byte, updated=0, not_updated=1, plmn_not_allowed=2,
                                                   location_area_not_allowed=3))
 
@@ -677,7 +683,7 @@ class EF_AD(TransparentEF):
     #OP_MODE_DICT = {int(v) : str(v) for v in EF_AD.OP_MODE}
     #OP_MODE_DICT_REVERSED = {str(v) : int(v) for v in EF_AD.OP_MODE}
 
-    def __init__(self, fid='6fad', sfid=None, name='EF.AD', desc='Administrative Data', size={3, 4}):
+    def __init__(self, fid='6fad', sfid=None, name='EF.AD', desc='Administrative Data', size=(3, 4)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
         self._construct = BitStruct(
             # Byte 1
@@ -698,14 +704,14 @@ class EF_AD(TransparentEF):
 
 # TS 51.011 Section 10.3.20 / 10.3.22
 class EF_VGCS(TransRecEF):
-    def __init__(self, fid='6fb1', sfid=None, name='EF.VGCS', size={4, 200}, rec_len=4,
+    def __init__(self, fid='6fb1', sfid=None, name='EF.VGCS', size=(4, 200), rec_len=4,
                  desc='Voice Group Call Service', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = BcdAdapter(Bytes(4))
 
 # TS 51.011 Section 10.3.21 / 10.3.23
 class EF_VGCSS(TransparentEF):
-    def __init__(self, fid='6fb2', sfid=None, name='EF.VGCSS', size={7, 7},
+    def __init__(self, fid='6fb2', sfid=None, name='EF.VGCSS', size=(7, 7),
                  desc='Voice Group Call Service Status', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = BitStruct(
@@ -713,7 +719,7 @@ class EF_VGCSS(TransparentEF):
 
 # TS 51.011 Section 10.3.24
 class EF_eMLPP(TransparentEF):
-    def __init__(self, fid='6fb5', sfid=None, name='EF.eMLPP', size={2, 2},
+    def __init__(self, fid='6fb5', sfid=None, name='EF.eMLPP', size=(2, 2),
                  desc='enhanced Multi Level Pre-emption and Priority', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         FlagsConstruct = FlagsEnum(
@@ -723,7 +729,7 @@ class EF_eMLPP(TransparentEF):
 
 # TS 51.011 Section 10.3.25
 class EF_AAeM(TransparentEF):
-    def __init__(self, fid='6fb6', sfid=None, name='EF.AAeM', size={1, 1},
+    def __init__(self, fid='6fb6', sfid=None, name='EF.AAeM', size=(1, 1),
                  desc='Automatic Answer for eMLPP Service', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         FlagsConstruct = FlagsEnum(
@@ -732,28 +738,28 @@ class EF_AAeM(TransparentEF):
 
 # TS 51.011 Section 10.3.26
 class EF_CBMID(EF_CBMI):
-    def __init__(self, fid='6f48', sfid=None, name='EF.CBMID', size={2, None}, rec_len=2,
+    def __init__(self, fid='6f48', sfid=None, name='EF.CBMID', size=(2, None), rec_len=2,
                  desc='Cell Broadcast Message Identifier for Data Download', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = GreedyRange(Int16ub)
 
 # TS 51.011 Section 10.3.27
 class EF_ECC(TransRecEF):
-    def __init__(self, fid='6fb7', sfid=None, name='EF.ECC', size={3, 15}, rec_len=3,
+    def __init__(self, fid='6fb7', sfid=None, name='EF.ECC', size=(3, 15), rec_len=3,
                  desc='Emergency Call Codes', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = GreedyRange(BcdAdapter(Bytes(3)))
 
 # TS 51.011 Section 10.3.28
 class EF_CBMIR(TransRecEF):
-    def __init__(self, fid='6f50', sfid=None, name='EF.CBMIR', size={4, None}, rec_len=4,
+    def __init__(self, fid='6f50', sfid=None, name='EF.CBMIR', size=(4, None), rec_len=4,
                  desc='Cell Broadcast message identifier range selection', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = GreedyRange(Struct('lower'/Int16ub, 'upper'/Int16ub))
 
 # TS 51.011 Section 10.3.29
 class EF_DCK(TransparentEF):
-    def __init__(self, fid='6f2c', sfid=None, name='EF.DCK', size={16, 16},
+    def __init__(self, fid='6f2c', sfid=None, name='EF.DCK', size=(16, 16),
                  desc='Depersonalisation Control Keys', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('network'/BcdAdapter(Bytes(4)),
@@ -762,7 +768,7 @@ class EF_DCK(TransparentEF):
                                  'corporate'/BcdAdapter(Bytes(4)))
 # TS 51.011 Section 10.3.30
 class EF_CNL(TransRecEF):
-    def __init__(self, fid='6f32', sfid=None, name='EF.CNL', size={6, None}, rec_len=6,
+    def __init__(self, fid='6f32', sfid=None, name='EF.CNL', size=(6, None), rec_len=6,
                  desc='Co-operative Network List', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
 
@@ -784,7 +790,7 @@ class EF_CNL(TransRecEF):
 
 # TS 51.011 Section 10.3.31
 class EF_NIA(LinFixedEF):
-    def __init__(self, fid='6f51', sfid=None, name='EF.NIA', rec_len={1, 32},
+    def __init__(self, fid='6f51', sfid=None, name='EF.NIA', rec_len=(1, 32),
                  desc='Network\'s Indication of Alerting', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct(
@@ -792,21 +798,21 @@ class EF_NIA(LinFixedEF):
 
 # TS 51.011 Section 10.3.32
 class EF_Kc(TransparentEF):
-    def __init__(self, fid='6f20', sfid=None, name='EF.Kc', desc='Ciphering key Kc', size={9, 9}):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6f20', sfid=None, name='EF.Kc', desc='Ciphering key Kc', size=(9, 9), **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = Struct('kc'/HexAdapter(Bytes(8)), 'cksn'/Int8ub)
 
 # TS 51.011 Section 10.3.33
 class EF_LOCIGPRS(TransparentEF):
-    def __init__(self, fid='6f53', sfid=None, name='EF.LOCIGPRS', desc='GPRS Location Information', size={14, 14}):
+    def __init__(self, fid='6f53', sfid=None, name='EF.LOCIGPRS', desc='GPRS Location Information', size=(14, 14)):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
-        self._construct = Struct('ptmsi'/Bytes(4), 'ptmsi_sig'/Int8ub, 'rai'/Bytes(6),
+        self._construct = Struct('ptmsi'/HexAdapter(Bytes(4)), 'ptmsi_sig'/Int8ub, 'rai'/HexAdapter(Bytes(6)),
                                  'rau_status'/Enum(Byte, updated=0, not_updated=1, plmn_not_allowed=2,
                                                    routing_area_not_allowed=3))
 
 # TS 51.011 Section 10.3.35..37
 class EF_xPLMNwAcT(TransRecEF):
-    def __init__(self, fid, sfid=None, name=None, desc=None, size={40, None}, rec_len=5, **kwargs):
+    def __init__(self, fid, sfid=None, name=None, desc=None, size=(40, None), rec_len=5, **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
 
     def _decode_record_hex(self, in_hex):
@@ -855,16 +861,16 @@ class EF_xPLMNwAcT(TransRecEF):
 
 # TS 51.011 Section 10.3.38
 class EF_CPBCCH(TransRecEF):
-    def __init__(self, fid='6f63', sfid=None, name='EF.CPBCCH', size={2, 14}, rec_len=2,
-                 desc='CPBCCH Information'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len)
+    def __init__(self, fid='6f63', sfid=None, name='EF.CPBCCH', size=(2, 14), rec_len=2,
+                 desc='CPBCCH Information', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, rec_len=rec_len, **kwargs)
         self._construct = Struct('cpbcch'/Int16ub)
 
 # TS 51.011 Section 10.3.39
 class EF_InvScan(TransparentEF):
-    def __init__(self, fid='6f64', sfid=None, name='EF.InvScan', size={1, 1},
-                 desc='IOnvestigation Scan'):
-        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size)
+    def __init__(self, fid='6f64', sfid=None, name='EF.InvScan', size=(1, 1),
+                 desc='IOnvestigation Scan', **kwargs):
+        super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._construct = FlagsEnum(
             Byte, in_limited_service_mode=1, after_successful_plmn_selection=2)
 
@@ -872,11 +878,13 @@ class EF_InvScan(TransparentEF):
 class EF_PNN(LinFixedEF):
     class FullNameForNetwork(BER_TLV_IE, tag=0x43):
         # TS 24.008 10.5.3.5a
-        pass
+        # TODO: proper decode
+        _construct = HexAdapter(GreedyBytes)
 
     class ShortNameForNetwork(BER_TLV_IE, tag=0x45):
         # TS 24.008 10.5.3.5a
-        pass
+        # TODO: proper decode
+        _construct = HexAdapter(GreedyBytes)
 
     class NetworkNameCollection(TLV_IE_Collection, nested=[FullNameForNetwork, ShortNameForNetwork]):
         pass
@@ -887,20 +895,21 @@ class EF_PNN(LinFixedEF):
 
 # TS 51.011 Section 10.3.42
 class EF_OPL(LinFixedEF):
-    def __init__(self, fid='6fc6', sfid=None, name='EF.OPL', rec_len={8, 8}, desc='Operator PLMN List', **kwargs):
+    def __init__(self, fid='6fc6', sfid=None, name='EF.OPL', rec_len=(8, 8), desc='Operator PLMN List', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
-        self._construct = Struct('lai'/Struct('mcc_mnc'/BcdAdapter(Bytes(3)), 'lac_min'/Bytes(2), 'lac_max'/Bytes(2)), 'pnn_record_id'/Int8ub)
+        self._construct = Struct('lai'/Struct('mcc_mnc'/BcdAdapter(Bytes(3)),
+                                 'lac_min'/HexAdapter(Bytes(2)), 'lac_max'/HexAdapter(Bytes(2))), 'pnn_record_id'/Int8ub)
 
 # TS 51.011 Section 10.3.44 + TS 31.102 4.2.62
 class EF_MBI(LinFixedEF):
-    def __init__(self, fid='6fc9', sfid=None, name='EF.MBI', rec_len={4, 5}, desc='Mailbox Identifier', **kwargs):
+    def __init__(self, fid='6fc9', sfid=None, name='EF.MBI', rec_len=(4, 5), desc='Mailbox Identifier', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct('mbi_voicemail'/Int8ub, 'mbi_fax'/Int8ub, 'mbi_email'/Int8ub,
                                  'mbi_other'/Int8ub, 'mbi_videocall'/COptional(Int8ub))
 
 # TS 51.011 Section 10.3.45 + TS 31.102 4.2.63
 class EF_MWIS(LinFixedEF):
-    def __init__(self, fid='6fca', sfid=None, name='EF.MWIS', rec_len={5, 6},
+    def __init__(self, fid='6fca', sfid=None, name='EF.MWIS', rec_len=(5, 6),
                  desc='Message Waiting Indication Status', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._construct = Struct('mwi_status'/FlagsEnum(Byte, voicemail=1, fax=2, email=4, other=8, videomail=16),
@@ -923,10 +932,10 @@ class EF_SPDI(TransparentEF):
 
 # TS 51.011 Section 10.3.51
 class EF_MMSN(LinFixedEF):
-    def __init__(self, fid='6fce', sfid=None, name='EF.MMSN', rec_len={4, 20}, desc='MMS Notification', **kwargs):
+    def __init__(self, fid='6fce', sfid=None, name='EF.MMSN', rec_len=(4, 20), desc='MMS Notification', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
-        self._construct = Struct('mms_status'/Bytes(2), 'mms_implementation'/Bytes(1),
-                                 'mms_notification'/Bytes(this._.total_len-4), 'ext_record_nr'/Byte)
+        self._construct = Struct('mms_status'/HexAdapter(Bytes(2)), 'mms_implementation'/HexAdapter(Bytes(1)),
+                                 'mms_notification'/HexAdapter(Bytes(this._.total_len-4)), 'ext_record_nr'/Byte)
 
 # TS 51.011 Annex K.1
 class MMS_Implementation(BER_TLV_IE, tag=0x80):
@@ -949,7 +958,7 @@ class EF_MMSICP(TransparentEF):
     class MMS_ConnectivityParamters(TLV_IE_Collection,
                                     nested=[MMS_Implementation, MMS_Relay_Server, Interface_to_CN, Gateway]):
         pass
-    def __init__(self, fid='6fd0', sfid=None, name='EF.MMSICP', size={1, None},
+    def __init__(self, fid='6fd0', sfid=None, name='EF.MMSICP', size=(1, None),
                  desc='MMS Issuer Connectivity Parameters', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
         self._tlv = EF_MMSICP.MMS_ConnectivityParamters
@@ -965,14 +974,14 @@ class EF_MMSUP(LinFixedEF):
     class MMS_User_Preferences(TLV_IE_Collection,
                                nested=[MMS_Implementation, MMS_UserPref_ProfileName, MMS_UserPref_Info]):
         pass
-    def __init__(self, fid='6fd1', sfid=None, name='EF.MMSUP', rec_len={1, None},
+    def __init__(self, fid='6fd1', sfid=None, name='EF.MMSUP', rec_len=(1, None),
                  desc='MMS User Preferences', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=rec_len, **kwargs)
         self._tlv = EF_MMSUP.MMS_User_Preferences
 
 # TS 51.011 Section 10.3.55
 class EF_MMSUCP(TransparentEF):
-    def __init__(self, fid='6fd2', sfid=None, name='EF.MMSUCP', size={1, None},
+    def __init__(self, fid='6fd2', sfid=None, name='EF.MMSUCP', size=(1, None),
                  desc='MMS User Connectivity Parameters', **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, size=size, **kwargs)
 
@@ -989,24 +998,24 @@ class DF_GSM(CardDF):
                           'Higher Priority PLMN search period'),
             EF_ACMmax(),
             EF_ServiceTable('6f38', None, 'EF.SST',
-                            'SIM service table', table=EF_SST_map, size={2, 16}),
+                            'SIM service table', table=EF_SST_map, size=(2, 16)),
             CyclicEF('6f39', None, 'EF.ACM',
-                     'Accumulated call meter', rec_len={3, 3}),
+                     'Accumulated call meter', rec_len=(3, 3)),
             TransparentEF('6f3e', None, 'EF.GID1', 'Group Identifier Level 1'),
             TransparentEF('6f3f', None, 'EF.GID2', 'Group Identifier Level 2'),
             EF_SPN(),
             TransparentEF('6f41', None, 'EF.PUCT',
-                          'Price per unit and currency table', size={5, 5}),
+                          'Price per unit and currency table', size=(5, 5)),
             EF_CBMI(),
             TransparentEF('6f7f', None, 'EF.BCCH',
-                          'Broadcast control channels', size={16, 16}),
+                          'Broadcast control channels', size=(16, 16)),
             EF_ACC(),
             EF_PLMNsel('6f7b', None, 'EF.FPLMN',
-                       'Forbidden PLMNs', size={12, 12}),
+                       'Forbidden PLMNs', size=(12, 12)),
             EF_LOCI(),
             EF_AD(),
             TransparentEF('6fa3', None, 'EF.Phase',
-                          'Phase identification', size={1, 1}),
+                          'Phase identification', size=(1, 1)),
             EF_VGCS(),
             EF_VGCSS(),
             EF_VGCS('6fb3', None, 'EF.VBS', 'Voice Broadcast Service'),

pySim/utils.py

@@ -1225,6 +1225,60 @@ def auto_int(x):
     return int(x, 0)
 
 
+def expand_hex(hexstring, length):
+    """Expand a given hexstring to a specified length by replacing "." or ".."
+       with a filler that is derived from the neighboring nibbles respective
+       bytes. Usually this will be the nibble respective byte before "." or
+       "..", execpt when the string begins with "." or "..", then the nibble
+       respective byte after "." or ".." is used.". In case the string cannot
+       be expanded for some reason, the input string is returned unmodified.
+
+    Args:
+            hexstring : hexstring to expand
+            length : desired length of the resulting hexstring.
+    Returns:
+            expanded hexstring
+    """
+
+    # expand digit aligned
+    if hexstring.count(".") == 1:
+        pos = hexstring.index(".")
+        if pos > 0:
+            filler = hexstring[pos - 1]
+        else:
+            filler = hexstring[pos + 1]
+
+        missing = length * 2 - (len(hexstring) - 1)
+        if missing <= 0:
+            return hexstring
+
+        return hexstring.replace(".", filler * missing)
+
+    # expand byte aligned
+    elif hexstring.count("..") == 1:
+        if len(hexstring) % 2:
+            return hexstring
+
+        pos = hexstring.index("..")
+
+        if pos % 2:
+            return hexstring
+
+        if pos > 1:
+            filler = hexstring[pos - 2:pos]
+        else:
+            filler = hexstring[pos + 2:pos+4]
+
+        missing = length * 2 - (len(hexstring) - 2)
+        if missing <= 0:
+            return hexstring
+
+        return hexstring.replace("..", filler * (missing // 2))
+
+    # no change
+    return hexstring
+
+
 class JsonEncoder(json.JSONEncoder):
     """Extend the standard library JSONEncoder with support for more types."""
 

requirements.txt

@@ -3,7 +3,9 @@ pyserial
 pytlv
 cmd2==1.5
 jsonpath-ng
-construct
+construct>=2.9.51
 bidict
 gsm0338
 pyyaml>=5.1
+termcolor
+colorlog

scripts/deactivate-5g.script

@@ -17,6 +17,7 @@ ust_service_deactivate 122
 ust_service_deactivate 123
 ust_service_deactivate 124
 ust_service_deactivate 125
+ust_service_deactivate 126
 ust_service_deactivate 127
 ust_service_deactivate 129
 ust_service_deactivate 130

setup.py

@@ -14,9 +14,11 @@ setup(
         "pytlv",
         "cmd2 >= 1.3.0, < 2.0.0",
         "jsonpath-ng",
-        "construct >= 2.9",
+        "construct >= 2.9.51",
         "bidict",
         "gsm0338",
+        "termcolor",
+        "colorlog"
     ],
     scripts=[
         'pySim-prog.py',

tests/test_apdu.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+
+import unittest
+from pySim.utils import h2b, b2h
+from pySim.construct import filter_dict
+from pySim.apdu import Apdu
+from pySim.apdu.ts_31_102 import UsimAuthenticateEven
+
+class TestApdu(unittest.TestCase):
+    def test_successful(self):
+        apdu = Apdu('00a40400023f00', '9000')
+        self.assertEqual(apdu.successful, True)
+        apdu = Apdu('00a40400023f00', '6733')
+        self.assertEqual(apdu.successful, False)
+
+    def test_successful_method(self):
+        """Test overloading of the success property with a custom method."""
+        class SwApdu(Apdu):
+            def _is_success(self):
+                return False
+        apdu = SwApdu('00a40400023f00', '9000')
+        self.assertEqual(apdu.successful, False)
+
+# TODO: Tests for TS 102 221 / 31.102 ApduCommands
+
+class TestUsimAuth(unittest.TestCase):
+    """Test decoding of the rather complex USIM AUTHENTICATE command."""
+    def test_2g(self):
+        apdu = ('80880080' + '09' + '080001020304050607',
+                '04a0a1a2a308b0b1b2b3b4b5b6b79000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'gsm'},
+                    'body': {'rand': '0001020304050607', 'autn': None}},
+            'rsp': {'body': {'sres': 'a0a1a2a3', 'kc': 'b0b1b2b3b4b5b6b7'}}
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+    def test_3g(self):
+        apdu = ('80880081' + '12' + '080001020304050607081011121314151617',
+                'DB' + '08' + 'a0a1a2a3a4a5a6a7' +
+                       '10' + 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf' +
+                       '10' + 'c0c1c2c3c4c5c6c7c8c9cacbcccdcecf' + '9000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'umts'},
+                    'body': {'rand': '0001020304050607', 'autn': '1011121314151617'}},
+            'rsp': {'body': {'tag': 219,
+                             'body': {
+                                 'res': 'a0a1a2a3a4a5a6a7',
+                                 'ck': 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf',
+                                 'ik': 'c0c1c2c3c4c5c6c7c8c9cacbcccdcecf',
+                                 'kc': None
+                                 }
+                             }
+                    }
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+    def test_3g_sync(self):
+        apdu = ('80880081' + '12' + '080001020304050607081011121314151617',
+                'DC' + '08' + 'a0a1a2a3a4a5a6a7' + '9000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'umts'},
+                    'body': {'rand': '0001020304050607', 'autn': '1011121314151617'}},
+            'rsp': {'body': {'tag': 220, 'body': {'auts': 'a0a1a2a3a4a5a6a7' }}}
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+    def test_vgcs(self):
+        apdu = ('80880082' + '0E' + '04' + '00010203' +
+                             '01' + '10' +
+                             '08' + '2021222324252627',
+                'DB' + '10' + 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf' + '9000')
+        res = {
+            'cmd': {'p1': 0, 'p2': {'scope': 'df_adf_specific', 'authentication_context': 'vgcs_vbs'},
+                    'body': { 'vk_id': '10', 'vservice_id': '00010203', 'vstk_rand': '2021222324252627'}},
+            'rsp': {'body': {'vstk': 'b0b1b2b3b4b5b6b7b8b9babbbcbdbebf'}}
+            }
+        u = UsimAuthenticateEven(apdu[0], apdu[1])
+        d = filter_dict(u.to_dict())
+        self.assertEqual(d, res)
+
+
+
+if __name__ == "__main__":
+	unittest.main()