Allow saving unknown files with raw data

When a file has no custom encoder, the decode path returns {'raw': ...}
but the encode path raised NotImplementedError. Add a fallback in all
encoder methods to return the raw data directly when present in the
abstract_data dict, allowing unknown files to round-trip.

pySim-trace.py

@@ -117,7 +117,7 @@ class Tracer:
             try:
                 apdu = self.source.read()
                 apdu_counter = apdu_counter + 1
-            except (StopIteration, KeyboardInterrupt):
+            except StopIteration:
                 print("%i APDUs parsed, stop iteration." % apdu_counter)
                 return 0
 

pySim/ara_m.py

@@ -300,51 +300,6 @@ class ADF_ARAM(CardADF):
                                'major': v_major, 'minor': v_minor, 'patch': v_patch}}])
         return ADF_ARAM.xceive_apdu_tlv(scc, '80cadf21', cmd_do, ResponseAramConfigDO)
 
-    @staticmethod
-    def store_ref_ar_do(scc, aid:Hexstr, aid_empty:bool, device_app_id:Hexstr, pkg_ref:str,
-                        apdu_filter:Hexstr, apdu_never:bool, apdu_always:bool,
-                        nfc_always:bool, nfc_never:bool, android_permissions:Hexstr):
-        # REF
-        ref_do_content = []
-        if aid is not None:
-            ref_do_content += [{'aid_ref_do': aid}]
-        elif aid_empty:
-            ref_do_content += [{'aid_ref_empty_do': None}]
-        ref_do_content += [{'dev_app_id_ref_do': device_app_id}]
-        if pkg_ref:
-            ref_do_content += [{'pkg_ref_do': {'package_name_string': pkg_ref}}]
-        # AR
-        ar_do_content = []
-        if apdu_never:
-            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
-        elif apdu_always:
-            ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
-        elif apdu_filter:
-            if len(apdu_filter) % 16:
-                raise ValueError(f'Invalid non-modulo-16 length of APDU filter: {len(apdu_filter)}')
-            offset = 0
-            apdu_filter_list = []
-            while offset < len(apdu_filter):
-                apdu_filter_list += [{'header': apdu_filter[offset:offset+8],
-                                      'mask': apdu_filter[offset+8:offset+16]}]
-                offset += 16 # Move offset to the beginning of the next apdu_filter object
-            ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter_list}}]
-        if nfc_never:
-            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
-        elif nfc_always:
-            ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
-        if android_permissions:
-            ar_do_content += [{'perm_ar_do': {'permissions': android_permissions}}]
-        d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
-        csrado = CommandStoreRefArDO()
-        csrado.from_val_dict(d)
-        return ADF_ARAM.store_data(scc, csrado)
-
-    @staticmethod
-    def aram_delete_all(scc):
-        deldo = CommandDelete()
-        return ADF_ARAM.store_data(scc, deldo)
-
     @with_default_category('Application-Specific Commands')
     class AddlShellCommands(CommandSet):
         def do_aram_get_all(self, _opts):
@@ -389,15 +344,48 @@ class ADF_ARAM(CardADF):
         @cmd2.with_argparser(store_ref_ar_do_parse)
         def do_aram_store_ref_ar_do(self, opts):
             """Perform STORE DATA [Command-Store-REF-AR-DO] to store a (new) access rule."""
-            res_do = ADF_ARAM.store_ref_ar_do(self._cmd.lchan.scc, opts.aid, opts.aid_empty, opts.device_app_id,
-                                              opts.pkg_ref, opts.apdu_filter, opts.apdu_never, opts.apdu_always,
-                                              opts.nfc_always, opts.nfc_never, opts.android_permissions)
+            # REF
+            ref_do_content = []
+            if opts.aid is not None:
+                ref_do_content += [{'aid_ref_do': opts.aid}]
+            elif opts.aid_empty:
+                ref_do_content += [{'aid_ref_empty_do': None}]
+            ref_do_content += [{'dev_app_id_ref_do': opts.device_app_id}]
+            if opts.pkg_ref:
+                ref_do_content += [{'pkg_ref_do': {'package_name_string': opts.pkg_ref}}]
+            # AR
+            ar_do_content = []
+            if opts.apdu_never:
+                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'never'}}]
+            elif opts.apdu_always:
+                ar_do_content += [{'apdu_ar_do': {'generic_access_rule': 'always'}}]
+            elif opts.apdu_filter:
+                if len(opts.apdu_filter) % 16:
+                    raise ValueError(f'Invalid non-modulo-16 length of APDU filter: {len(opts.apdu_filter)}')
+                offset = 0
+                apdu_filter = []
+                while offset < len(opts.apdu_filter):
+                    apdu_filter += [{'header': opts.apdu_filter[offset:offset+8],
+                                     'mask': opts.apdu_filter[offset+8:offset+16]}]
+                    offset += 16 # Move offset to the beginning of the next apdu_filter object
+                ar_do_content += [{'apdu_ar_do': {'apdu_filter': apdu_filter}}]
+            if opts.nfc_always:
+                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'always'}}]
+            elif opts.nfc_never:
+                ar_do_content += [{'nfc_ar_do': {'nfc_event_access_rule': 'never'}}]
+            if opts.android_permissions:
+                ar_do_content += [{'perm_ar_do': {'permissions': opts.android_permissions}}]
+            d = [{'ref_ar_do': [{'ref_do': ref_do_content}, {'ar_do': ar_do_content}]}]
+            csrado = CommandStoreRefArDO()
+            csrado.from_val_dict(d)
+            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, csrado)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
         def do_aram_delete_all(self, _opts):
             """Perform STORE DATA [Command-Delete[all]] to delete all access rules."""
-            res_do = ADF_ARAM.aram_delete_all(self._cmd.lchan.scc)
+            deldo = CommandDelete()
+            res_do = ADF_ARAM.store_data(self._cmd.lchan.scc, deldo)
             if res_do:
                 self._cmd.poutput_json(res_do.to_dict())
 
@@ -406,6 +394,7 @@ class ADF_ARAM(CardADF):
             (Proprietary feature that is specific to sysmocom's fork of Bertrand Martel’s ARA-M implementation.)"""
             self._cmd.lchan.scc.send_apdu_checksw('80e2900001A1', '9000')
 
+
 # SEAC v1.1 Section 4.1.2.2 + 5.1.2.2
 sw_aram = {
     'ARA-M': {

pySim/esim/saip/__init__.py

@@ -34,7 +34,7 @@ from pySim import ts_102_222
 from pySim.utils import dec_imsi
 from pySim.ts_102_221 import FileDescriptor
 from pySim.filesystem import CardADF, Path
-from pySim.ts_31_102 import ADF_USIM, EF_UST, EF_SUCI_Calc_Info
+from pySim.ts_31_102 import ADF_USIM
 from pySim.ts_31_103 import ADF_ISIM
 from pySim.esim import compile_asn1_subdir
 from pySim.esim.saip import templates
@@ -1726,52 +1726,7 @@ class ProfileElementSequence:
         if 'BT' in ftype_list:
             svc_set.add('ber-tlv')
         # FIXME:dfLinked files (scan all files, check for non-empty Fcp.linkPath presence of DFs)
-
-        # 5G:
-        # - When SUCI is:
-        #   - enabled (EF.UST 124 = true)
-        #   AND
-        #   - calculated in the USIM (EF.UST 125 = true),
-        #   then eUICC-Mandatory-services needs 'get-identity'.
-        # - 'get-identity' implies that the eUICC must support ONE OF profile-A OR profile-B.
-        #   (One might assume from this that, when SUCI-CalcInfo for USIM in DF.SAIP contains both key types, then no
-        #   profile-A or B services need to be requested explicitly. However, the correct logic is:)
-        # - Iff the SUCI-CalcInfo for USIM (DF.SAIP) contains a key of profile-A ("identifier": 1),
-        #   then eUICC-Mandatory-services needs 'profile-a-x25519'.
-        # - Same: profile-B ("identifier": 2) needs 'profile-b-p256'.
-        # - (When SUCI is calculated in the UE, then the eUICC does not need to provide any of these services.)
-        suci_in_usim_enabled = False
-        try:
-            f_ust = self.get_pe_for_type("usim").files["ef-ust"]
-            ust = EF_UST().decode_bin(f_ust.body)
-            suci_in_usim_enabled = ust[124]['activated'] and ust[125]['activated']
-        except (KeyError, AttributeError):
-            pass
-        if suci_in_usim_enabled:
-            svc_set.add('get-identity')
-            # now check for profile-a and profile-b presence
-            suci_calcinfo_has_profile_a = False
-            suci_calcinfo_has_profile_b = False
-            try:
-                f_sucici = self.get_pe_for_type("df-saip").files["ef-suci-calc-info-usim"]
-                sucici = EF_SUCI_Calc_Info().decode_bin(f_sucici.body) or {}
-                for prot_scheme in sucici['prot_scheme_id_list']:
-                    if not isinstance(prot_scheme, dict):
-                        continue
-                    ps_id = prot_scheme["identifier"]
-                    if ps_id == 1:
-                        suci_calcinfo_has_profile_a = True
-                    elif ps_id == 2:
-                        suci_calcinfo_has_profile_b = True
-            except (KeyError, AttributeError):
-                pass
-            if suci_calcinfo_has_profile_a:
-                # The profile has a profile-A key, so require that
-                svc_set.add('profile-a-x25519')
-            if suci_calcinfo_has_profile_b:
-                # The profile has a profile-B key, so require that
-                svc_set.add('profile-b-p256')
-
+        # TODO: 5G related bits (derive from EF.UST or file presence?)
         hdr_pe = self.get_pe_for_type('header')
         # patch in the 'manual' services from the existing list:
         for old_svc in hdr_pe.decoded['eUICC-Mandatory-services'].keys():

pySim/esim/saip/batch.py

@@ -123,8 +123,6 @@ class BatchPersonalization:
                 except Exception as e:
                     raise ValueError(f'{p.param_cls.get_name()} fed by {p.src.name}: {e}') from e
 
-            pes.rebuild_mandatory_services()
-
             yield pes
 
 

pySim/esim/saip/personalization.py

@@ -21,11 +21,9 @@ import io
 import re
 from typing import List, Tuple, Generator, Optional
 
-from construct.core import StreamError
 from osmocom.tlv import camel_to_snake
 from osmocom.utils import hexstr
 from pySim.utils import enc_iccid, dec_iccid, enc_imsi, dec_imsi, h2b, b2h, rpad, sanitize_iccid
-from pySim.ts_31_102 import EF_AD
 from pySim.ts_51_011 import EF_SMSP
 from pySim.esim.saip import param_source
 from pySim.esim.saip import ProfileElement, ProfileElementSD, ProfileElementSequence
@@ -662,72 +660,6 @@ class SmspTpScAddr(ConfigurableParameter):
             yield { cls.name: cls.tuple_to_str((international, digits)) }
 
 
-class MncLen(EnumParam):
-    """MNC length.  Sets only the MNC length field in EF.AD (Administrative Data).
-    Accepted values: integer 2 or 3, digit strings '2' or '3', or enum names 'MNC2'/'MNC3'.
-    """
-    name = 'MNC-LEN'
-    example_input = '2'
-    default_source = param_source.ConstantSource
-
-    class Values(enum.IntEnum):
-        MNC2 = 2
-        MNC3 = 3
-
-    @classmethod
-    def validate_val(cls, val):
-        if isinstance(val, str) and val.isdigit():
-            val = int(val)
-        return super().validate_val(val)
-
-    @classmethod
-    def _get_f_ad(cls, pe: ProfileElement):
-        if not hasattr(pe, 'files'):
-            return None
-        f_ad = pe.files.get('ef-ad', None)
-        if f_ad and f_ad.body:
-            return f_ad
-        return None
-
-    @classmethod
-    def _decode_f_ad(cls, f_ad):
-        try:
-            ef_ad_dec = EF_AD().decode_bin(f_ad.body)
-        except StreamError:
-            return None
-        if 'mnc_len' not in ef_ad_dec:
-            return None
-        return ef_ad_dec
-
-    @classmethod
-    def apply_val(cls, pes: ProfileElementSequence, val: int):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ad = cls._get_f_ad(pe)
-            if f_ad is None:
-                continue
-            # decode existing values
-            ef_ad_dec = cls._decode_f_ad(f_ad)
-            if ef_ad_dec is None:
-                continue
-            # change mnc_len
-            ef_ad_dec['mnc_len'] = val
-            # re-encode into the File body
-            f_ad.body = EF_AD().encode_bin(ef_ad_dec)
-            pe.file2pe(f_ad)
-
-    @classmethod
-    def get_values_from_pes(cls, pes: ProfileElementSequence):
-        for pe in pes.get_pes_for_type('usim'):
-            f_ad = cls._get_f_ad(pe)
-            if f_ad is None:
-                continue
-            ef_ad_dec = cls._decode_f_ad(f_ad)
-            if ef_ad_dec is None:
-                continue
-            mnc_len = ef_ad_dec.get('mnc_len')
-            yield { cls.name: str(mnc_len) }
-
-
 class SdKey(BinaryParam):
     """Configurable Security Domain (SD) Key.  Value is presented as bytes.
        Non-abstract implementations are generated in SdKey.generate_sd_key_classes"""
@@ -873,30 +805,6 @@ class SdKey(BinaryParam):
 
         SdKey.all_implementations = []
 
-        transitional_name_mapping = {
-            'SCP02-KVN20-AES-DEK': 'SCP02-20-AES-DEK',
-            'SCP02-KVN20-AES-ENC': 'SCP02-20-AES-ENC',
-            'SCP02-KVN20-AES-MAC': 'SCP02-20-AES-MAC',
-            'SCP02-KVN21-AES-DEK': 'SCP02-21-AES-DEK',
-            'SCP02-KVN21-AES-ENC': 'SCP02-21-AES-ENC',
-            'SCP02-KVN21-AES-MAC': 'SCP02-21-AES-MAC',
-            'SCP02-KVN22-AES-DEK': 'SCP02-22-AES-DEK',
-            'SCP02-KVN22-AES-ENC': 'SCP02-22-AES-ENC',
-            'SCP02-KVN22-AES-MAC': 'SCP02-22-AES-MAC',
-            'SCP02-KVNff-AES-DEK': 'SCP02-ff-AES-DEK',
-            'SCP02-KVNff-AES-ENC': 'SCP02-ff-AES-ENC',
-            'SCP02-KVNff-AES-MAC': 'SCP02-ff-AES-MAC',
-            'SCP03-KVN30-AES-DEK': 'SCP03-30-AES-DEK',
-            'SCP03-KVN30-AES-ENC': 'SCP03-30-AES-ENC',
-            'SCP03-KVN30-AES-MAC': 'SCP03-30-AES-MAC',
-            'SCP03-KVN31-AES-DEK': 'SCP03-31-AES-DEK',
-            'SCP03-KVN31-AES-ENC': 'SCP03-31-AES-ENC',
-            'SCP03-KVN31-AES-MAC': 'SCP03-31-AES-MAC',
-            'SCP03-KVN32-AES-DEK': 'SCP03-32-AES-DEK',
-            'SCP03-KVN32-AES-ENC': 'SCP03-32-AES-ENC',
-            'SCP03-KVN32-AES-MAC': 'SCP03-32-AES-MAC',
-        }
-
         def camel(s):
             return s[:1].upper() + s[1:].lower()
 
@@ -928,8 +836,6 @@ class SdKey(BinaryParam):
 
             max_key_len = attrs.get('allow_len')[-1]
 
-            cls_label = transitional_name_mapping.get(cls_label, cls_label)
-
             attrs.update({
                 'name' : cls_label,
                 'kvn': kvn,
@@ -1182,7 +1088,7 @@ class MilenageRotationConstants(BinaryParam, AlgoConfig):
 
 class MilenageXoringConstants(BinaryParam, AlgoConfig):
     """XOR-ing constants c1,c2,c3,c4,c5 of Milenage, 128bit each. See 3GPP TS 35.206 Sections 2.3 + 5.3.
-    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concatenation
+    Provided as octet-string concatenation of all 5 constants. The default value by 3GPP is the concetenation
     of::
 
      00000000000000000000000000000000

pySim/filesystem.py

@@ -863,6 +863,8 @@ class TransparentEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -892,6 +894,8 @@ class TransparentEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1166,6 +1170,8 @@ class LinFixedEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1195,6 +1201,8 @@ class LinFixedEF(CardEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1386,6 +1394,8 @@ class TransRecEF(TransparentEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return b2h(t.to_tlv())
+        if 'raw' in abstract_data:
+            return abstract_data['raw']
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
@@ -1415,6 +1425,8 @@ class TransRecEF(TransparentEF):
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
             return t.to_tlv()
+        if 'raw' in abstract_data:
+            return h2b(abstract_data['raw'])
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 

pySim/ts_31_102.py

@@ -285,14 +285,6 @@ class EF_SUCI_Calc_Info(TransparentEF):
                                 {"hnet_pubkey_identifier": 11, "hnet_pubkey":
                                  h2b("d1bc365f4997d17ce4374e72181431cbfeba9e1b98d7618f79d48561b144672a")}]} ),
     ]
-    _test_decode = [
-        ( 'A000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
-          {"prot_scheme_id_list": [],
-           "hnet_pubkey_list": []} ),
-        ( 'A000A100FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF',
-          {"prot_scheme_id_list": [],
-           "hnet_pubkey_list": []} ),
-    ]
     # 3GPP TS 31.102 Section 4.4.11.8
     class ProtSchemeIdList(BER_TLV_IE, tag=0xa0):
         # FIXME: 3GPP TS 24.501 Protection Scheme Identifier
@@ -335,7 +327,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         """conversion method to generate list of {hnet_pubkey_identifier, hnet_pubkey} dicts
         from flat [{hnet_pubkey_identifier: }, {net_pubkey: }, ...] list"""
         out = []
-        while l:
+        while len(l):
             a = l.pop(0)
             b = l.pop(0)
             z = {**a, **b}
@@ -397,7 +389,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
         # remaining data holds Home Network Public Key Data Object
         hpkl = EF_SUCI_Calc_Info.HnetPubkeyList()
         hpkl.from_tlv(in_bytes[pos:])
-        hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'] or [])
+        hnet_pubkey_list = self._compact_pubkey_list(hpkl.to_dict()['hnet_pubkey_list'])
 
         return {
             'prot_scheme_id_list': prot_scheme_id_list,

tests/unittests/test_configurable_parameters.py

@@ -55,6 +55,8 @@ class ConfigurableParameterTest(unittest.TestCase):
         upp_fnames = (
                 'TS48v5_SAIP2.1A_NoBERTLV.der',
                 'TS48v5_SAIP2.3_BERTLV_SUCI.der',
+                'TS48v5_SAIP2.1B_NoBERTLV.der',
+                'TS48v5_SAIP2.3_NoBERTLV.der',
                )
 
         class Paramtest:
@@ -265,15 +267,6 @@ class ConfigurableParameterTest(unittest.TestCase):
                                  '11111111111111111111111111111111'
                                  '22222222222222222222222222222222'),
 
-            Paramtest(param_cls=p13n.MncLen,
-                      val='2',
-                      expect_clean_val=2,
-                      expect_val='2'),
-            Paramtest(param_cls=p13n.MncLen,
-                      val=3,
-                      expect_clean_val=3,
-                      expect_val='3'),
-
             ]
 
         for sdkey_cls in (