Allow saving unknown files with raw data

When a file has no custom encoder, the decode path returns {'raw': ...}
but the encode path raised NotImplementedError. Add a fallback in all
encoder methods to return the raw data directly when present in the
abstract_data dict, allowing unknown files to round-trip.

pySim/ara_m.py

@@ -300,51 +300,6 @@ class ADF_ARAM(CardADF):
                                'major': v_major, 'minor': v_minor, 'patch': v_patch}}])
         return ADF_ARAM.xceive_apdu_tlv(scc, '80cadf21', cmd_do, ResponseAramConfigDO)
 
-    @staticmethod
-    def store_ref_ar_do(scc, aid:Hexstr, aid_empty:bool, device_app_id:Hexstr, pkg_ref:str,
-                        apdu_filter:Hexstr, apdu_never:bool, apdu_always:bool,
-                        nfc_always:bool, nfc_never:bool, android_permissions:Hexstr):
-        # REF
-        ref_do_content = []
-        if aid is not None:
-            ref_do_content += [{'aid_ref_do': aid}]
-        elif aid_empty:
-            ref_do_content += [{'aid_ref_empty_do': None}]
-        ref_do_content += [{'dev_app_id_ref_do': device_app_id}]
-        if pkg_ref:
-            ref_do_content += [{'pkg_ref_do': {'package_name_string': pkg_ref}}]
-        # AR
-        ar_do_content = []
-        if apdu_never:
-            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
-        elif apdu_always:
-            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
-        elif apdu_filter:
-            if len(apdu_filter) % 16:
-                raise ValueError(f'Invalid non-modulo-16 length of APDU filter: {len(apdu_filter)}')
-            offset = 0
-            apdu_filter_list = []
-            while offset < len(apdu_filter):
-                apdu_filter_list += [{'header': apdu_filter[offset:offset+8],
-                                      'mask': apdu_filter[offset+8:offset+16]}]
-                offset += 16 # Move offset to the beginning of the next apdu_filter object
-            ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter_list}}]
-        if nfc_never:
-            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
-        elif nfc_always:
-            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
-        if android_permissions:
-            ar_do_content += [{'perm_ar_do': {'permissions': android_permissions}}]
-        d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
-        csrado = CommandStoreRefArDO()
-        csrado.from_val_dict(d)
-        return ADF_ARAM.store_data(scc, csrado)
-
-    @staticmethod
-    def aram_delete_all(scc):
-        deldo = CommandDelete()
-        return ADF_ARAM.store_data(scc, deldo)
-
     @with_default_category('Application-Specific Commands')
     class AddlShellCommands(CommandSet):
         def do_aram_get_all(self, _opts):
@@ -389,15 +344,48 @@ class ADF_ARAM(CardADF):
         @cmd2.with_argparser(store_ref_ar_do_parse)
         def do_aram_store_ref_ar_do(self, opts):
             """Perform STORE DATA [Command-Store-REF-AR-DO] to store a (new) access rule."""
-            res_do = ADF_ARAM.store_ref_ar_do(self._cmd.lchan.scc, opts.aid, opts.aid_empty, opts.device_app_id,
+            # REF
-                                              opts.pkg_ref, opts.apdu_filter, opts.apdu_never, opts.apdu_always,
+            ref_do_content = []
-                                              opts.nfc_always, opts.nfc_never, opts.android_permissions)
+            if opts.aid is not None:
+                ref_do_content += [{'aid_ref_do': opts.aid}]
+            elif opts.aid_empty:
+                ref_do_content += [{'aid_ref_empty_do': None}]
+            ref_do_content += [{'dev_app_id_ref_do': opts.device_app_id}]
+            if opts.pkg_ref:
+                ref_do_content += [{'pkg_ref_do': {'package_name_string': opts.pkg_ref}}]
+            # AR
+            ar_do_content = []
+            if opts.apdu_never:
+                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
+            elif opts.apdu_always:
+                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
+            elif opts.apdu_filter:
+                if len(opts.apdu_filter) % 16:
+                    raise ValueError(f'Invalid non-modulo-16 length of APDU filter: {len(opts.apdu_filter)}')
+                offset = 0
+                apdu_filter = []
+                while offset < len(opts.apdu_filter):
+                    apdu_filter += [{'header': opts.apdu_filter[offset:offset+8],
+                                     'mask': opts.apdu_filter[offset+8:offset+16]}]
+                    offset += 16 # Move offset to the beginning of the next apdu_filter object
+                ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter}}]
+            if opts.nfc_always:
+                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
+            elif opts.nfc_never:
+                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
+            if opts.android_permissions:
+                ar_do_content += [{'perm_ar_do': {'permissions': opts.android_permissions}}]
+            d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
+            csrado = CommandStoreRefArDO()
+            csrado.from_val_dict(d)
+            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, csrado)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
         def do_aram_delete_all(self, _opts):
             """Perform STORE DATA [Command-Delete[all]] to delete all access rules."""
-            res_do = ADF_ARAM.aram_delete_all(self._cmd.lchan.scc)
+            deldo = CommandDelete()
+            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, deldo)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
@@ -406,6 +394,7 @@ class ADF_ARAM(CardADF):
             (Proprietary feature that is specific to sysmocom's fork of Bertrand Martel’s ARA-M implementation.)"""
             self._cmd.lchan.scc.send_apdu_checksw('80e2900001A1', '9000')
 
+
 # SEAC v1.1 Section 4.1.2.2 + 5.1.2.2
 sw_aram = {
     'ARA-M': {

pySim/esim/saip/personalization.py

@@ -21,11 +21,9 @@ import io
 import re
 from typing import List, Tuple, Generator, Optional
 
-from construct.core import StreamError
 from osmocom.tlv import camel_to_snake
 from osmocom.utils import hexstr
 from pySim.utils import enc_iccid, dec_iccid, enc_imsi, dec_imsi, h2b, b2h, rpad, sanitize_iccid
-from pySim.ts_31_102 import EF_AD
 from pySim.ts_51_011 import EF_SMSP
 from pySim.esim.saip import param_source
 from pySim.esim.saip import ProfileElement, ProfileElementSD, ProfileElementSequence
@@ -662,72 +660,6 @@ class SmspTpScAddr(ConfigurableParameter):
             yield { cls.name: cls.tuple_to_str((international, digits)) }
 
 
-class MncLen(EnumParam):
-    """MNC length.  Sets only the MNC length field in EF.AD (Administrative Data).
-    Accepted values: integer 2 or 3, digit strings '2' or '3', or enum names 'MNC2'/'MNC3'.
-    """
-    name = 'MNC-LEN'
-    example_input = '2'
-    default_source = param_source.ConstantSource
-
-    class Values(enum.IntEnum):
-        MNC2 = 2
-        MNC3 = 3
-
-    @classmethod
-    def validate_val(cls, val):
-        if isinstance(val, str) and val.isdigit():
-            val = int(val)
-        return super().validate_val(val)
-
-    @classmethod
-    def _get_f_ad(cls, pe: ProfileElement):
-        if not hasattr(pe, 'files'):
-            return None
-        f_ad = pe.files.get('ef-ad', None)
-        if f_ad and f_ad.body:
-            return f_ad
-        return None
-
-    @classmethod
-    def _decode_f_ad(cls, f_ad):
-        try:
-            ef_ad_dec = EF_AD().decode_bin(f_ad.body)
-        except StreamError:
-            return None
-        if 'mnc_len' not in ef_ad_dec:
-            return None
-        return ef_ad_dec
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val: int):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ad = cls._get_f_ad(pe)
-            if f_ad is None:
-                continue
-            # decode existing values
-            ef_ad_dec = cls._decode_f_ad(f_ad)
-            if ef_ad_dec is None:
-                continue
-            # change mnc_len
-            ef_ad_dec['mnc_len'] = val
-            # re-encode into the File body
-            f_ad.body = EF_AD().encode_bin(ef_ad_dec)
-            pe.file2pe(f_ad)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ad = cls._get_f_ad(pe)
-            if f_ad is None:
-                continue
-            ef_ad_dec = cls._decode_f_ad(f_ad)
-            if ef_ad_dec is None:
-                continue
-            mnc_len = ef_ad_dec.get('mnc_len')
-            yield { cls.name: str(mnc_len) }
-
-
 class SdKey(BinaryParam):
     """Configurable Security Domain (SD) Key.  Value is presented as bytes.
        Non-abstract implementations are generated in SdKey.generate_sd_key_classes"""

pySim/filesystem.py

@@ -863,6 +863,8 @@ class TransparentEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -892,6 +894,8 @@ class TransparentEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1166,6 +1170,8 @@ class LinFixedEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1195,6 +1201,8 @@ class LinFixedEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1386,6 +1394,8 @@ class TransRecEF(TransparentEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1415,6 +1425,8 @@ class TransRecEF(TransparentEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 

tests/unittests/test_configurable_parameters.py

@@ -267,15 +267,6 @@ class ConfigurableParameterTest(unittest.TestCase):
                                  '11111111111111111111111111111111'
                                  '22222222222222222222222222222222'),
 
-            Paramtest(param_cls=p13n.MncLen,
-                      val='2',
-                      expect_clean_val=2,
-                      expect_val='2'),
-            Paramtest(param_cls=p13n.MncLen,
-                      val=3,
-                      expect_clean_val=3,
-                      expect_val='3'),
-
             ]
 
         for sdkey_cls in (

tests/unittests/xo/test_configurable_parameters

@@ -163,14 +163,6 @@ ok: TS48v5_SAIP2.1A_NoBERTLV.der MilenageXoringConstants(val=	b'\xaa\xaa\xaa\xaa
 	clean_val=	b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11""""""""""""""""':bytes
 	read_back_val=	{'MilenageXOR': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc1111111111111111111111111111111122222222222222222222222222222222'}:{hexstr}
 
-ok: TS48v5_SAIP2.1A_NoBERTLV.der MncLen(val=	'2':str)
-	clean_val=	2:int
-	read_back_val=	{'MNC-LEN': '2'}:{str}
-
-ok: TS48v5_SAIP2.1A_NoBERTLV.der MncLen(val=	3:int)
-	clean_val=	3:int
-	read_back_val=	{'MNC-LEN': '3'}:{str}
-
 ok: TS48v5_SAIP2.1A_NoBERTLV.der SdKeyScp02Kvn20AesDek(val=	'01020304050607080910111213141516':str)
 	clean_val=	b'\x01\x02\x03\x04\x05\x06\x07\x08\t\x10\x11\x12\x13\x14\x15\x16':bytes
 	read_back_val=	{'SCP02-KVN20-AES-DEK': '01020304050607080910111213141516'}:{hexstr}
@@ -863,14 +855,6 @@ ok: TS48v5_SAIP2.3_BERTLV_SUCI.der MilenageXoringConstants(val=	b'\xaa\xaa\xaa\x
 	clean_val=	b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11""""""""""""""""':bytes
 	read_back_val=	{'MilenageXOR': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc1111111111111111111111111111111122222222222222222222222222222222'}:{hexstr}
 
-ok: TS48v5_SAIP2.3_BERTLV_SUCI.der MncLen(val=	'2':str)
-	clean_val=	2:int
-	read_back_val=	{'MNC-LEN': '2'}:{str}
-
-ok: TS48v5_SAIP2.3_BERTLV_SUCI.der MncLen(val=	3:int)
-	clean_val=	3:int
-	read_back_val=	{'MNC-LEN': '3'}:{str}
-
 ok: TS48v5_SAIP2.3_BERTLV_SUCI.der SdKeyScp02Kvn20AesDek(val=	'01020304050607080910111213141516':str)
 	clean_val=	b'\x01\x02\x03\x04\x05\x06\x07\x08\t\x10\x11\x12\x13\x14\x15\x16':bytes
 	read_back_val=	{'SCP02-KVN20-AES-DEK': '01020304050607080910111213141516'}:{hexstr}
@@ -1563,14 +1547,6 @@ ok: TS48v5_SAIP2.1B_NoBERTLV.der MilenageXoringConstants(val=	b'\xaa\xaa\xaa\xaa
 	clean_val=	b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11""""""""""""""""':bytes
 	read_back_val=	{'MilenageXOR': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc1111111111111111111111111111111122222222222222222222222222222222'}:{hexstr}
 
-ok: TS48v5_SAIP2.1B_NoBERTLV.der MncLen(val=	'2':str)
-	clean_val=	2:int
-	read_back_val=	{'MNC-LEN': '2'}:{str}
-
-ok: TS48v5_SAIP2.1B_NoBERTLV.der MncLen(val=	3:int)
-	clean_val=	3:int
-	read_back_val=	{'MNC-LEN': '3'}:{str}
-
 ok: TS48v5_SAIP2.1B_NoBERTLV.der SdKeyScp02Kvn20AesDek(val=	'01020304050607080910111213141516':str)
 	clean_val=	b'\x01\x02\x03\x04\x05\x06\x07\x08\t\x10\x11\x12\x13\x14\x15\x16':bytes
 	read_back_val=	{'SCP02-KVN20-AES-DEK': '01020304050607080910111213141516'}:{hexstr}
@@ -2263,14 +2239,6 @@ ok: TS48v5_SAIP2.3_NoBERTLV.der MilenageXoringConstants(val=	b'\xaa\xaa\xaa\xaa\
 	clean_val=	b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xbb\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11""""""""""""""""':bytes
 	read_back_val=	{'MilenageXOR': 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccc1111111111111111111111111111111122222222222222222222222222222222'}:{hexstr}
 
-ok: TS48v5_SAIP2.3_NoBERTLV.der MncLen(val=	'2':str)
-	clean_val=	2:int
-	read_back_val=	{'MNC-LEN': '2'}:{str}
-
-ok: TS48v5_SAIP2.3_NoBERTLV.der MncLen(val=	3:int)
-	clean_val=	3:int
-	read_back_val=	{'MNC-LEN': '3'}:{str}
-
 ok: TS48v5_SAIP2.3_NoBERTLV.der SdKeyScp02Kvn20AesDek(val=	'01020304050607080910111213141516':str)
 	clean_val=	b'\x01\x02\x03\x04\x05\x06\x07\x08\t\x10\x11\x12\x13\x14\x15\x16':bytes
 	read_back_val=	{'SCP02-KVN20-AES-DEK': '01020304050607080910111213141516'}:{hexstr}