Allow saving unknown files with raw data

When a file has no custom encoder, the decode path returns {'raw': ...}
but the encode path raised NotImplementedError. Add a fallback in all
encoder methods to return the raw data directly when present in the
abstract_data dict, allowing unknown files to round-trip.

pySim-shell.py

@@ -24,21 +24,21 @@ import traceback
 import re
 import cmd2
 from packaging import version
-from cmd2 import style
-
 import logging
 from pySim.log import PySimLogger
 from osmocom.utils import auto_uint8
 
-# cmd2 >= 2.3.0 has deprecated the bg/fg in favor of Bg/Fg :(
+# cmd2 >= 3.0 replaced Fg + style() with Color + stylize()
-if version.parse(cmd2.__version__) < version.parse("2.3.0"):
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
-    from cmd2 import fg, bg # pylint: disable=no-name-in-module
+    from cmd2 import Color, stylize # pylint: disable=no-name-in-module
-    RED = fg.red
+    RED = Color.RED
-    YELLOW = fg.yellow
+    YELLOW = Color.YELLOW
-    LIGHT_RED = fg.bright_red
+    LIGHT_RED = Color.BRIGHT_RED
-    LIGHT_GREEN = fg.bright_green
+    LIGHT_GREEN = Color.BRIGHT_GREEN
+    def style(text, fg=None, bg=None, bold=False): # pylint: disable=function-redefined
+        return stylize(text, fg) if fg else text
 else:
-    from cmd2 import Fg, Bg # pylint: disable=no-name-in-module
+    from cmd2 import style, Fg # pylint: disable=no-name-in-module
     RED = Fg.RED
     YELLOW = Fg.YELLOW
     LIGHT_RED = Fg.LIGHT_RED
@@ -76,43 +76,19 @@ from pySim.app import init_card
 
 log = PySimLogger.get(Path(__file__).stem)
 
-class Cmd2Compat(cmd2.Cmd):
+class PysimApp(cmd2.Cmd):
-    """Backwards-compatibility wrapper around cmd2.Cmd to support older and newer
-    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
-    def run_editor(self, file_path: Optional[str] = None) -> None:
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            return self._run_editor(file_path) # pylint: disable=no-member
-        else:
-            return super().run_editor(file_path) # pylint: disable=no-member
-
-class Settable2Compat(cmd2.Settable):
-    """Backwards-compatibility wrapper around cmd2.Settable to support older and newer
-    releases. See https://github.com/python-cmd2/cmd2/blob/master/CHANGELOG.md"""
-    def __init__(self, name, val_type, description, settable_object, **kwargs):
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            super().__init__(name, val_type, description, **kwargs) # pylint: disable=no-value-for-parameter
-        else:
-            super().__init__(name, val_type, description, settable_object, **kwargs) # pylint: disable=too-many-function-args
-
-class PysimApp(Cmd2Compat):
     CUSTOM_CATEGORY = 'pySim Commands'
     BANNER = """Welcome to pySim-shell!
 (C) 2021-2023 by Harald Welte, sysmocom - s.f.m.c. GmbH and contributors
 Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/shell.html """
 
     def __init__(self, verbose, card, rs, sl, ch, script=None):
-        if version.parse(cmd2.__version__) < version.parse("2.0.0"):
-            kwargs = {'use_ipython': True}
-        else:
-            kwargs = {'include_ipy': True}
-
         self.verbose = verbose
         PySimLogger.setup(self.poutput, {logging.WARN: YELLOW})
         self._onchange_verbose('verbose', False, self.verbose)
 
-        # pylint: disable=unexpected-keyword-arg
         super().__init__(persistent_history_file='~/.pysim_shell_history', allow_cli_args=False,
-                         auto_load_commands=False, startup_script=script, **kwargs)
+                         auto_load_commands=False, startup_script=script, include_ipy=True)
         self.intro = style(self.BANNER, fg=RED)
         self.default_category = 'pySim-shell built-in commands'
         self.card = None
@@ -128,18 +104,24 @@ Online manual available at https://downloads.osmocom.org/docs/pysim/master/html/
         self.apdu_trace = False
         self.apdu_strict = False
 
-        self.add_settable(Settable2Compat('numeric_path', bool, 'Print File IDs instead of names', self,
+        self.add_settable(cmd2.Settable('numeric_path', bool,
-                                          onchange_cb=self._onchange_numeric_path))
+                                        'Print File IDs instead of names',
-        self.add_settable(Settable2Compat('conserve_write', bool, 'Read and compare before write', self,
+                                        self, onchange_cb=self._onchange_numeric_path))
-                                          onchange_cb=self._onchange_conserve_write))
+        self.add_settable(cmd2.Settable('conserve_write', bool,
-        self.add_settable(Settable2Compat('json_pretty_print', bool, 'Pretty-Print JSON output', self))
+                                        'Read and compare before write',
-        self.add_settable(Settable2Compat('apdu_trace', bool, 'Trace and display APDUs exchanged with card', self,
+                                        self, onchange_cb=self._onchange_conserve_write))
-                                          onchange_cb=self._onchange_apdu_trace))
+        self.add_settable(cmd2.Settable('json_pretty_print', bool,
-        self.add_settable(Settable2Compat('apdu_strict', bool,
+                                        'Pretty-Print JSON output',
-                                          'Strictly apply APDU format according to ISO/IEC 7816-3, table 12', self))
+                                        self))
-        self.add_settable(Settable2Compat('verbose', bool,
+        self.add_settable(cmd2.Settable('apdu_trace', bool,
-                                          'Enable/disable verbose logging', self,
+                                        'Trace and display APDUs exchanged with card',
-                                          onchange_cb=self._onchange_verbose))
+                                        self, onchange_cb=self._onchange_apdu_trace))
+        self.add_settable(cmd2.Settable('apdu_strict', bool,
+                                        'Strictly apply APDU format according to ISO/IEC 7816-3, table 12',
+                                        self))
+        self.add_settable(cmd2.Settable('verbose', bool,
+                                        'Enable/disable verbose logging',
+                                        self, onchange_cb=self._onchange_verbose))
         self.equip(card, rs)
 
     def equip(self, card, rs):

pySim/filesystem.py

@@ -863,6 +863,8 @@ class TransparentEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -892,6 +894,8 @@ class TransparentEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1166,6 +1170,8 @@ class LinFixedEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1195,6 +1201,8 @@ class LinFixedEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1386,6 +1394,8 @@ class TransRecEF(TransparentEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1415,6 +1425,8 @@ class TransRecEF(TransparentEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 

pySim/log.py

@@ -24,7 +24,15 @@
 #
 
 import logging
-from cmd2 import style
+import cmd2
+from packaging import version
+
+if version.parse(cmd2.__version__) >= version.parse("3.0.0"):
+    from cmd2 import stylize as _stylize # pylint: disable=no-name-in-module
+    def _style(text, fg=None): # pylint: disable=function-redefined
+        return _stylize(text, fg) if fg else text
+else: # cmd2>=2.6.2
+    from cmd2 import style as _style # pylint: disable=no-name-in-module
 
 class _PySimLogHandler(logging.Handler):
     def __init__(self, log_callback):
@@ -121,7 +129,7 @@ class PySimLogger:
                 if isinstance(color, str):
                     PySimLogger.print_callback(color + formatted_message + "\033[0m")
                 else:
-                    PySimLogger.print_callback(style(formatted_message, fg = color))
+                    PySimLogger.print_callback(_style(formatted_message, fg = color))
             else:
                 PySimLogger.print_callback(formatted_message)
 

requirements.txt

@@ -1,7 +1,7 @@
 pyscard
 pyserial
 pytlv
-cmd2>=2.6.2,<3.0
+cmd2>=2.6.2,<4.0
 jsonpath-ng
 construct>=2.10.70
 bidict

setup.py

@@ -21,7 +21,7 @@ setup(
         "pyscard",
         "pyserial",
         "pytlv",
-        "cmd2 >= 1.5.0, < 3.0",
+        "cmd2 >= 2.6.2, < 4.0",
         "jsonpath-ng",
         "construct >= 2.10.70",
         "bidict",