pySim-shell.py: make it work with cmd2 >= v2.4.0

In v2.3.0 both cmd2.{fg,bg} have been deprecated in favour of cmd2.{Fg,Bg}.
In v2.4.0 both cmd2.{fg,bg} have been removed.

See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md

Change-Id: I7ca95e85fc45ba66fd9ba6bea1fec2bae0e892c0

README.md

@@ -23,10 +23,10 @@ Git Repository
 
 You can clone from the official Osmocom  git repository using
 ```
-git clone git://git.osmocom.org/pysim.git
+git clone https://gitea.osmocom.org/sim-card/pysim.git
 ```
 
-There is a cgit interface at <https://git.osmocom.org/pysim>
+There is a web interface at <https://gitea.osmocom.org/sim-card/pysim>.
 
 
 Installation
@@ -35,18 +35,26 @@ Installation
 Please install the following dependencies:
 
  - pyscard
- - serial
+ - pyserial
  - pytlv
  - cmd2 >= 1.3.0 but < 2.0.0
  - jsonpath-ng
- - construct
+ - construct >= 2.9.51
  - bidict
  - gsm0338
+ - pyyaml >= 5.1
+ - termcolor
+ - colorlog
 
 Example for Debian:
-```
+```sh
-apt-get install python3-pyscard python3-serial python3-pip python3-yaml python3-termcolor python3-colorlog
+sudo apt-get install --no-install-recommends \
-pip3 install -r requirements.txt
+	pcscd libpcsclite-dev \
+	python3 \
+	python3-setuptools \
+	python3-pyscard \
+	python3-pip
+pip3 install --user -r requirements.txt
 ```
 
 After installing all dependencies, the pySim applications ``pySim-read.py``, ``pySim-prog.py`` and ``pySim-shell.py`` may be started directly from the cloned repository.

contrib/jenkins.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -xe
 # jenkins build helper script for pysim.  This is how we build on jenkins.osmocom.org
 #
 # environment variables:
@@ -6,8 +6,6 @@
 # * PUBLISH: upload manuals after building if set to "1" (ignored without WITH_MANUALS = "1")
 #
 
-set -e
-
 if [ ! -d "./pysim-testdata/" ] ; then
 	echo "###############################################"
 	echo "Please call from pySim-prog top directory"
@@ -17,15 +15,7 @@ fi
 
 virtualenv -p python3 venv --system-site-packages
 . venv/bin/activate
-pip install pytlv
+pip install -r requirements.txt
-pip install 'pyyaml>=5.1'
-pip install cmd2==1.5
-pip install jsonpath-ng
-pip install construct
-pip install bidict
-pip install gsm0338
-pip install termcolor
-pip install colorlog
 
 # Execute automatically discovered unit tests first
 python -m unittest discover -v -s tests/
@@ -36,8 +26,8 @@ python -m unittest discover -v -s tests/
 # Ignore E0401: import-error
 #   pySim/utils.py:276: E0401: Unable to import 'Crypto.Cipher' (import-error)
 #   pySim/utils.py:277: E0401: Unable to import 'Crypto.Util.strxor' (import-error)
-pip install pylint
+pip install pylint==2.14.5 # FIXME: 2.15 is crashing, see OS#5668
-python -m pylint --errors-only \
+python -m pylint -j0 --errors-only \
 	--disable E1102 \
 	--disable E0401 \
 	--enable W0301 \

pySim-shell.py

@@ -23,7 +23,7 @@ import json
 import traceback
 
 import cmd2
-from cmd2 import style, fg
+from cmd2 import style, Fg
 from cmd2 import CommandSet, with_default_category, with_argparser
 import argparse
 
@@ -134,8 +134,8 @@ class PysimApp(cmd2.Cmd):
 
     def __init__(self, card, rs, sl, ch, script=None):
         super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
-                         use_ipython=True, auto_load_commands=False, startup_script=script)
+                         auto_load_commands=False, startup_script=script)
-        self.intro = style('Welcome to pySim-shell!', fg=fg.red)
+        self.intro = style('Welcome to pySim-shell!', fg=Fg.RED)
         self.default_category = 'pySim-shell built-in commands'
         self.card = None
         self.rs = None
@@ -145,16 +145,16 @@ class PysimApp(cmd2.Cmd):
         self.ch = ch
 
         self.numeric_path = False
-        self.add_settable(cmd2.Settable('numeric_path', bool, 'Print File IDs instead of names',
+        self.add_settable(cmd2.Settable('numeric_path', bool, 'Print File IDs instead of names', self,
                                         onchange_cb=self._onchange_numeric_path))
         self.conserve_write = True
-        self.add_settable(cmd2.Settable('conserve_write', bool, 'Read and compare before write',
+        self.add_settable(cmd2.Settable('conserve_write', bool, 'Read and compare before write', self,
                                         onchange_cb=self._onchange_conserve_write))
         self.json_pretty_print = True
         self.add_settable(cmd2.Settable('json_pretty_print',
-                          bool, 'Pretty-Print JSON output'))
+                          bool, 'Pretty-Print JSON output', self))
         self.apdu_trace = False
-        self.add_settable(cmd2.Settable('apdu_trace', bool, 'Trace and display APDUs exchanged with card',
+        self.add_settable(cmd2.Settable('apdu_trace', bool, 'Trace and display APDUs exchanged with card', self,
                                         onchange_cb=self._onchange_apdu_trace))
 
         self.equip(card, rs)
@@ -287,23 +287,23 @@ class PysimApp(cmd2.Cmd):
             sys.stderr = self._stderr_backup
 
     def _show_failure_sign(self):
-        self.poutput(style("  +-------------+", fg=fg.bright_red))
+        self.poutput(style("  +-------------+", fg=Fg.LIGHT_RED))
-        self.poutput(style("  +   ##   ##   +", fg=fg.bright_red))
+        self.poutput(style("  +   ##   ##   +", fg=Fg.LIGHT_RED))
-        self.poutput(style("  +    ## ##    +", fg=fg.bright_red))
+        self.poutput(style("  +    ## ##    +", fg=Fg.LIGHT_RED))
-        self.poutput(style("  +     ###     +", fg=fg.bright_red))
+        self.poutput(style("  +     ###     +", fg=Fg.LIGHT_RED))
-        self.poutput(style("  +    ## ##    +", fg=fg.bright_red))
+        self.poutput(style("  +    ## ##    +", fg=Fg.LIGHT_RED))
-        self.poutput(style("  +   ##   ##   +", fg=fg.bright_red))
+        self.poutput(style("  +   ##   ##   +", fg=Fg.LIGHT_RED))
-        self.poutput(style("  +-------------+", fg=fg.bright_red))
+        self.poutput(style("  +-------------+", fg=Fg.LIGHT_RED))
         self.poutput("")
 
     def _show_success_sign(self):
-        self.poutput(style("  +-------------+", fg=fg.bright_green))
+        self.poutput(style("  +-------------+", fg=Fg.LIGHT_GREEN))
-        self.poutput(style("  +          ## +", fg=fg.bright_green))
+        self.poutput(style("  +          ## +", fg=Fg.LIGHT_GREEN))
-        self.poutput(style("  +         ##  +", fg=fg.bright_green))
+        self.poutput(style("  +         ##  +", fg=Fg.LIGHT_GREEN))
-        self.poutput(style("  +  #    ##    +", fg=fg.bright_green))
+        self.poutput(style("  +  #    ##    +", fg=Fg.LIGHT_GREEN))
-        self.poutput(style("  +   ## #      +", fg=fg.bright_green))
+        self.poutput(style("  +   ## #      +", fg=Fg.LIGHT_GREEN))
-        self.poutput(style("  +    ##       +", fg=fg.bright_green))
+        self.poutput(style("  +    ##       +", fg=Fg.LIGHT_GREEN))
-        self.poutput(style("  +-------------+", fg=fg.bright_green))
+        self.poutput(style("  +-------------+", fg=Fg.LIGHT_GREEN))
         self.poutput("")
 
     def _process_card(self, first, script_path):

pySim/construct.py

@@ -2,7 +2,7 @@ from construct.lib.containers import Container, ListContainer
 from construct.core import EnumIntegerString
 import typing
 from construct import *
-from construct.core import evaluate, bytes2integer, integer2bytes, BitwisableString
+from construct.core import evaluate, BitwisableString
 from construct.lib import integertypes
 from pySim.utils import b2h, h2b, swap_nibbles
 import gsm0338
@@ -219,7 +219,7 @@ class GreedyInteger(Construct):
         if evaluate(self.swapped, context):
             data = swapbytes(data)
         try:
-            return bytes2integer(data, self.signed)
+            return int.from_bytes(data, byteorder='big', signed=self.signed)
         except ValueError as e:
             raise IntegerError(str(e), path=path)
 
@@ -248,7 +248,7 @@ class GreedyInteger(Construct):
             raise IntegerError(f"value {obj} is not an integer", path=path)
         length = self.__bytes_required(obj, self.minlen)
         try:
-            data = integer2bytes(obj, length, self.signed)
+            data = obj.to_bytes(length, byteorder='big', signed=self.signed)
         except ValueError as e:
             raise IntegerError(str(e), path=path)
         if evaluate(self.swapped, context):

pySim/transport/__init__.py

@@ -10,7 +10,7 @@ from typing import Optional, Tuple
 from pySim.exceptions import *
 from pySim.construct import filter_dict
 from pySim.utils import sw_match, b2h, h2b, i2h, Hexstr
-from pySim.cat import ProactiveCommand
+from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result
 
 #
 # Copyright (C) 2009-2010  Sylvain Munaut <tnt@246tNt.com>
@@ -42,10 +42,7 @@ class ProactiveHandler(abc.ABC):
     """Abstract base class representing the interface of some code that handles
     the proactive commands, as returned by the card in responses to the FETCH
     command."""
-    def receive_fetch_raw(self, payload: Hexstr):
+    def receive_fetch_raw(self, pcmd: ProactiveCommand, parsed: Hexstr):
-        # parse the proactive command
-        pcmd = ProactiveCommand()
-        parsed = pcmd.from_tlv(h2b(payload))
         # try to find a generic handler like handle_SendShortMessage
         handle_name = 'handle_%s' % type(parsed).__name__
         if hasattr(self, handle_name):
@@ -160,13 +157,57 @@ class LinkBase(abc.ABC):
                         sw   : string (in hex) of status word (ex. "9000")
         """
         rv = self.send_apdu(pdu)
+        last_sw = rv[1]
 
-        while sw == '9000' and sw_match(rv[1], '91xx'):
+        while sw == '9000' and sw_match(last_sw, '91xx'):
+            # It *was* successful after all -- the extra pieces FETCH handled
+            # need not concern the caller.
+            rv = (rv[0], '9000')
             # proactive sim as per TS 102 221 Setion 7.4.2
-            rv = self.send_apdu_checksw('80120000' + rv[1][2:], sw)
+            # TODO: Check SW manually to avoid recursing on the stack (provided this piece of code stays in this place)
-            print("FETCH: %s" % rv[0])
+            fetch_rv = self.send_apdu_checksw('80120000' + last_sw[2:], sw)
+            # Setting this in case we later decide not to send a terminal
+            # response immediately unconditionally -- the card may still have
+            # something pending even though the last command was not processed
+            # yet.
+            last_sw = fetch_rv[1]
+            # parse the proactive command
+            pcmd = ProactiveCommand()
+            parsed = pcmd.from_tlv(h2b(fetch_rv[0]))
+            print("FETCH: %s (%s)" % (fetch_rv[0], type(parsed).__name__))
+            result = Result()
             if self.proactive_handler:
-                self.proactive_handler.receive_fetch_raw(rv[0])
+                # Extension point: If this does return a list of TLV objects,
+                # they could be appended after the Result; if the first is a
+                # Result, that cuold replace the one built here.
+                self.proactive_handler.receive_fetch_raw(pcmd, parsed)
+                result.from_dict({'general_result': 'performed_successfully', 'additional_information': ''})
+            else:
+                result.from_dict({'general_result': 'command_beyond_terminal_capability', 'additional_information': ''})
+
+            # Send response immediately, thus also flushing out any further
+            # proactive commands that the card already wants to send
+            #
+            # Structure as per TS 102 223 V4.4.0 Section 6.8
+
+            # The Command Details are echoed from the command that has been processed.
+            (command_details,) = [c for c in pcmd.decoded.children if isinstance(c, CommandDetails)]
+            # The Device Identities are fixed. (TS 102 223 V4.0.0 Section 6.8.2)
+            device_identities = DeviceIdentities()
+            device_identities.from_dict({'source_dev_id': 'terminal', 'dest_dev_id': 'uicc'})
+
+            # Testing hint: The value of tail does not influence the behavior
+            # of an SJA2 that sent ans SMS, so this is implemented only
+            # following TS 102 223, and not fully tested.
+            tail = command_details.to_tlv() + device_identities.to_tlv() + result.to_tlv()
+            # Testing hint: In contrast to the above, this part is positively
+            # essential to get the SJA2 to provide the later parts of a
+            # multipart SMS in response to an OTA RFM command.
+            terminal_response = '80140000' + b2h(len(tail).to_bytes(1, 'big') + tail)
+
+            terminal_response_rv = self.send_apdu(terminal_response)
+            last_sw = terminal_response_rv[1]
+
         if not sw_match(rv[1], sw):
             raise SwMatchError(rv[1], sw.lower(), self.sw_interpreter)
         return rv

requirements.txt

@@ -3,7 +3,7 @@ pyserial
 pytlv
 cmd2==1.5
 jsonpath-ng
-construct
+construct>=2.9.51
 bidict
 gsm0338
 pyyaml>=5.1

setup.py

@@ -14,7 +14,7 @@ setup(
         "pytlv",
         "cmd2 >= 1.3.0, < 2.0.0",
         "jsonpath-ng",
-        "construct >= 2.9",
+        "construct >= 2.9.51",
         "bidict",
         "gsm0338",
         "termcolor",