personalization: implement reading back values from a PES

Implement get_values_from_pes(), the reverse direction of apply_val():
read back and return values from a ProfileElementSequence. Implement for
all ConfigurableParameter subclasses.

Future: SdKey.get_values_from_pes() is reading pe.decoded[], which works
fine, but I07dfc378705eba1318e9e8652796cbde106c6a52 will change this
implementation to use the higher level ProfileElementSD members.

Implementation detail:

Implement get_values_from_pes() as classmethod that returns a generator.
Subclasses should yield all occurences of their parameter in a given
PES.

For example, the ICCID can appear in multiple places.
Iccid.get_values_from_pes() yields all of the individual values. A set()
of the results quickly tells whether the PES is consistent.

Rationales for reading back values:

This allows auditing an eSIM profile, particularly for producing an
output.csv from a batch personalization (that generated lots of random
key material which now needs to be fed to an HLR...).

Reading back from a binary result is more reliable than storing the
values that were fed into a personalization.
By auditing final DER results with this code, I discovered:
- "oh, there already was some key material in my UPP template."
- "all IMSIs ended up the same, forgot to set up the parameter."
- the SdKey.apply() implementations currently don't work, see
  I07dfc378705eba1318e9e8652796cbde106c6a52 for a fix.

Change-Id: I234fc4317f0bdc1a486f0cee4fa432c1dce9b463

pySim/esim/saip/personalization.py

@@ -17,14 +17,19 @@
 
 import abc
 import io
+import os
 import copy
-from typing import List, Tuple, Generator
+from typing import List, Tuple, Generator, Optional
 
 from osmocom.tlv import camel_to_snake
-from pySim.utils import enc_iccid, enc_imsi, h2b, rpad, sanitize_iccid, all_subclasses_of
+from osmocom.utils import hexstr
+from pySim.utils import enc_iccid, dec_iccid, enc_imsi, dec_imsi, h2b, b2h, rpad, sanitize_iccid, all_subclasses_of
 from pySim.esim.saip import ProfileElement, ProfileElementSequence
 from pySim.esim.saip import param_source
 
+def unrpad(s: hexstr, c='f') -> hexstr:
+    return hexstr(s.rstrip(c))
+
 def remove_unwanted_tuples_from_list(l: List[Tuple], unwanted_keys: List[str]) -> List[Tuple]:
     """In a list of tuples, remove all tuples whose first part equals 'unwanted_key'."""
     return list(filter(lambda x: x[0] not in unwanted_keys, l))
@@ -36,6 +41,22 @@ def file_replace_content(file: List[Tuple], new_content: bytes):
     file.append(('fillFileContent', new_content))
     return file
 
+def file_tuples_content_as_bytes(l: List[Tuple]) -> Optional[bytes]:
+    """linearize a list of fillFileContent / fillFileOffset tuples into a stream of bytes."""
+    stream = io.BytesIO()
+    for k, v in l:
+        if k == 'doNotCreate':
+            return None
+        if k == 'fileDescriptor':
+            pass
+        elif k == 'fillFileOffset':
+            stream.seek(v, os.SEEK_CUR)
+        elif k == 'fillFileContent':
+            stream.write(v)
+        else:
+            return ValueError("Unknown key '%s' in tuple list" % k)
+    return stream.getvalue()
+
 class ConfigurableParameter:
     r"""Base class representing a part of the eSIM profile that is configurable during the
     personalization process (with dynamic data from elsewhere).
@@ -194,6 +215,30 @@ class ConfigurableParameter:
         Write the given val in the right format in all the right places in pes."""
         pass
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence) -> Generator:
+        '''This is what subclasses implement: yield all values from a decoded profile package.
+           Find all values in the pes, and yield them decoded to a valid cls.input_value format.
+           Should be a generator function, i.e. use 'yield' instead of 'return'.
+
+           Usage example:
+
+             cls = esim.saip.personalization.Iccid
+             # use a set() to get a list of unique values from all results
+             vals = set( cls.get_values_from_pes(pes) )
+             if len(vals) != 1:
+                 raise ValueError(f'{cls.name}: need exactly one value, got {vals}')
+             # the set contains a single value, return it
+             return vals.pop()
+
+           Implementation example:
+
+             for pe in pes:
+                if my_condition(pe):
+                    yield b2h(my_bin_value_from(pe))
+           '''
+        pass
+
     @classmethod
     def get_len_range(cls):
         """considering all of min_len, max_len and allow_len, get a tuple of the resulting (min, max) of permitted
@@ -260,6 +305,17 @@ class DecimalHexParam(DecimalParam):
         # a DecimalHexParam subclass expects the apply_val() input to be a bytes instance ready for the pes
         return h2b(val)
 
+    @classmethod
+    def decimal_hex_to_str(cls, val):
+        'useful for get_values_from_pes() implementations of subclasses'
+        if isinstance(val, bytes):
+            val = b2h(val)
+        assert isinstance(val, hexstr)
+        if cls.rpad is not None:
+            c = cls.rpad_char or 'f'
+            val = unrpad(val, c)
+        return val.to_bytes().decode('ascii')
+
 
 class BinaryParam(ConfigurableParameter):
     allow_types = (str, io.BytesIO, bytes, bytearray)
@@ -305,6 +361,17 @@ class Iccid(DecimalParam):
         # patch MF/EF.ICCID
         file_replace_content(pes.get_pe_for_type('mf').decoded['ef-iccid'], h2b(enc_iccid(val)))
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        padded = b2h(pes.get_pe_for_type('header').decoded['iccid'])
+        iccid = unrpad(padded)
+        yield iccid
+
+        for pe in pes.get_pes_for_type('mf'):
+            iccid_pe = pe.decoded.get('ef-iccid', None)
+            if iccid_pe:
+                yield dec_iccid(b2h(file_tuples_content_as_bytes(iccid_pe)))
+
 class Imsi(DecimalParam):
     """Configurable IMSI. Expects value to be a string of digits. Automatically sets the ACC to
     the last digit of the IMSI."""
@@ -326,6 +393,13 @@ class Imsi(DecimalParam):
             file_replace_content(pe.decoded['ef-acc'], acc.to_bytes(2, 'big'))
         # TODO: DF.GSM_ACCESS if not linked?
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('usim'):
+            imsi_pe = pe.decoded.get('ef-imsi', None)
+            if imsi_pe:
+                yield dec_imsi(b2h(file_tuples_content_as_bytes(imsi_pe)))
+
 
 class SdKey(BinaryParam):
     """Configurable Security Domain (SD) Key.  Value is presented as bytes."""
@@ -359,6 +433,14 @@ class SdKey(BinaryParam):
         for pe in pes.get_pes_for_type('securityDomain'):
             cls._apply_sd(pe, value)
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('securityDomain'):
+            for key in pe.decoded['keyList']:
+                if key['keyIdentifier'][0] == cls.key_id and key['keyVersionNumber'][0] == cls.kvn:
+                    if len(key['keyComponents']) >= 1:
+                        yield b2h(key['keyComponents'][0]['keyData'])
+
 class SdKeyScp80_01(SdKey):
     kvn = 0x01
     key_type = 0x88 # AES key type
@@ -495,6 +577,14 @@ class Puk(DecimalHexParam):
         raise ValueError("input template UPP has unexpected structure:"
                          f" cannot find pukCode with keyReference={cls.keyReference}")
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        mf_pes = pes.pes_by_naa['mf'][0]
+        for pukCodes in obtain_all_pe_from_pelist(mf_pes, 'pukCodes'):
+            for pukCode in pukCodes.decoded['pukCodes']:
+                if pukCode['keyReference'] == cls.keyReference:
+                    yield cls.decimal_hex_to_str(pukCode['pukValue'])
+
 class Puk1(Puk):
     is_abstract = False
     name = 'PUK1'
@@ -532,6 +622,20 @@ class Pin(DecimalHexParam):
             raise ValueError('input template UPP has unexpected structure:'
                              + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference}')
 
+    @classmethod
+    def _read_all_pinvalues_from_pe(cls, pe: ProfileElement):
+        for pinCodes in obtain_all_pe_from_pelist(pe, 'pinCodes'):
+            if pinCodes.decoded['pinCodes'][0] != 'pinconfig':
+                continue
+
+            for pinCode in pinCodes.decoded['pinCodes'][1]:
+                if pinCode['keyReference'] == cls.keyReference:
+                     yield cls.decimal_hex_to_str(pinCode['pinValue'])
+
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        yield from cls._read_all_pinvalues_from_pe(pes.pes_by_naa['mf'][0])
+
 class Pin1(Pin):
     is_abstract = False
     name = 'PIN1'
@@ -555,6 +659,14 @@ class Pin2(Pin1):
                     raise ValueError('input template UPP has unexpected structure:'
                             + f' {cls.get_name()} cannot find pinCode with keyReference={cls.keyReference} in {naa=}')
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for naa in pes.pes_by_naa:
+            if naa not in ['usim','isim','csim','telecom']:
+                continue
+            for pe in pes.pes_by_naa[naa]:
+                yield from cls._read_all_pinvalues_from_pe(pe)
+
 class Adm1(Pin):
     is_abstract = False
     name = 'ADM1'
@@ -581,6 +693,14 @@ class AlgoConfig(ConfigurableParameter):
             raise ValueError('input template UPP has unexpected structure:'
                              f' {cls.__name__} cannot find algoParameter with key={cls.algo_config_key}')
 
+    @classmethod
+    def get_values_from_pes(cls, pes: ProfileElementSequence):
+        for pe in pes.get_pes_for_type('akaParameter'):
+            algoConfiguration = pe.decoded['algoConfiguration']
+            if algoConfiguration[0] != 'algoParameter':
+                continue
+            yield algoConfiguration[1][cls.algo_config_key]
+
 
 class AlgorithmID(DecimalParam, AlgoConfig):
     is_abstract = False