filesystem: pass total_len to construct of when encoding file contents

In our construct models we frequently use a context parameter "total_len",
we also pass this parameter to construct when we decode files, but we
do not pass it when we generate files. This is a problem, because when
total_len is used in the construct model, this parameter must be known
also when decoding the file.

Let's make sure that the total_len is properly determined and and passed
to construct (via pyosmocom).

Related: OS#5714
Change-Id: I1b7a51594fbc5d9fe01132c39354a2fa88d53f9b

pySim/filesystem.py

@@ -743,7 +743,26 @@ class TransparentEF(CardEF):
             return t.to_dict()
         return {'raw': raw_bin_data.hex()}
 
-    def encode_bin(self, abstract_data: dict) -> bytearray:
+    def __get_size(self, total_len: Optional[int] = None) -> Optional[int]:
+        """Get the size (total length) of the file"""
+
+        # Caller has provided the actual total length of the file, this should be the default case
+        if total_len is not None:
+            return total_len
+
+        if self.size is None:
+            return None
+
+        # Alternatively use the recommended size from the specification
+        if self.size[1] is not None:
+            return self.size[1]
+        # In case no recommended size is specified, use the minimum size
+        if self.size[0] is not None:
+            return self.size[0]
+
+        return None
+
+    def encode_bin(self, abstract_data: dict, total_len: Optional[int] = None) -> bytearray:
         """Encode abstract representation into raw (binary) data.
 
         A derived class would typically provide an _encode_bin() or _encode_hex() method
@@ -752,17 +771,18 @@ class TransparentEF(CardEF):
 
         Args:
             abstract_data : dict representing the decoded data
+            total_len : expected total length of the encoded data (file size)
         Returns:
             binary encoded data
         """
         method = getattr(self, '_encode_bin', None)
         if callable(method):
-            return method(abstract_data)
+            return method(abstract_data, total_len = self.__get_size(total_len))
         method = getattr(self, '_encode_hex', None)
         if callable(method):
-            return h2b(method(abstract_data))
+            return h2b(method(abstract_data, total_len = self.__get_size(total_len)))
         if self._construct:
-            return build_construct(self._construct, abstract_data)
+            return build_construct(self._construct, abstract_data, {'total_len' : self.__get_size(total_len)})
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -770,7 +790,7 @@ class TransparentEF(CardEF):
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
-    def encode_hex(self, abstract_data: dict) -> str:
+    def encode_hex(self, abstract_data: dict, total_len: Optional[int] = None) -> str:
         """Encode abstract representation into raw (hex string) data.
 
         A derived class would typically provide an _encode_bin() or _encode_hex() method
@@ -779,18 +799,19 @@ class TransparentEF(CardEF):
 
         Args:
             abstract_data : dict representing the decoded data
+            total_len : expected total length of the encoded data (file size)
         Returns:
             hex string encoded data
         """
         method = getattr(self, '_encode_hex', None)
         if callable(method):
-            return method(abstract_data)
+            return method(abstract_data, total_len = self.__get_size(total_len))
         method = getattr(self, '_encode_bin', None)
         if callable(method):
-            raw_bin_data = method(abstract_data)
+            raw_bin_data = method(abstract_data, total_len = self.__get_size(total_len))
             return b2h(raw_bin_data)
         if self._construct:
-            return b2h(build_construct(self._construct, abstract_data))
+            return b2h(build_construct(self._construct, abstract_data, {'total_len':self.__get_size(total_len)}))
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -1030,7 +1051,26 @@ class LinFixedEF(CardEF):
             return t.to_dict()
         return {'raw': raw_hex_data}
 
-    def encode_record_hex(self, abstract_data: dict, record_nr: int) -> str:
+    def __get_rec_len(self, total_len: Optional[int] = None) -> Optional[int]:
+        """Get the length (total length) of the file record"""
+
+        # Caller has provided the actual total length of the record, this should be the default case
+        if total_len is not None:
+            return total_len
+
+        if self.rec_len is None:
+            return None
+
+        # Alternatively use the recommended length from the specification
+        if self.rec_len[1] is not None:
+            return self.rec_len[1]
+        # In case no recommended length is specified, use the minimum length
+        if self.rec_len[0] is not None:
+            return self.rec_len[0]
+
+        return None
+
+    def encode_record_hex(self, abstract_data: dict, record_nr: int, total_len: Optional[int] = None) -> str:
         """Encode abstract representation into raw (hex string) data.
 
         A derived class would typically provide an _encode_record_bin() or _encode_record_hex()
@@ -1040,18 +1080,19 @@ class LinFixedEF(CardEF):
         Args:
             abstract_data : dict representing the decoded data
             record_nr : record number (1 for first record, ...)
+            total_len : expected total length of the encoded data (record length)
         Returns:
             hex string encoded data
         """
         method = getattr(self, '_encode_record_hex', None)
         if callable(method):
-            return method(abstract_data, record_nr=record_nr)
+            return method(abstract_data, record_nr=record_nr, total_len = self.__get_rec_len(total_len))
         method = getattr(self, '_encode_record_bin', None)
         if callable(method):
-            raw_bin_data = method(abstract_data, record_nr=record_nr)
+            raw_bin_data = method(abstract_data, record_nr=record_nr, total_len = self.__get_rec_len(total_len))
             return b2h(raw_bin_data)
         if self._construct:
-            return b2h(build_construct(self._construct, abstract_data))
+            return b2h(build_construct(self._construct, abstract_data, {'total_len':self.__get_rec_len(total_len)}))
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -1059,7 +1100,7 @@ class LinFixedEF(CardEF):
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
-    def encode_record_bin(self, abstract_data: dict, record_nr : int) -> bytearray:
+    def encode_record_bin(self, abstract_data: dict, record_nr : int, total_len: Optional[int] = None) -> bytearray:
         """Encode abstract representation into raw (binary) data.
 
         A derived class would typically provide an _encode_record_bin() or _encode_record_hex()
@@ -1069,17 +1110,18 @@ class LinFixedEF(CardEF):
         Args:
             abstract_data : dict representing the decoded data
             record_nr : record number (1 for first record, ...)
+            total_len : expected total length of the encoded data (record length)
         Returns:
             binary encoded data
         """
         method = getattr(self, '_encode_record_bin', None)
         if callable(method):
-            return method(abstract_data, record_nr=record_nr)
+            return method(abstract_data, record_nr=record_nr, total_len = self.__get_rec_len(total_len))
         method = getattr(self, '_encode_record_hex', None)
         if callable(method):
-            return h2b(method(abstract_data, record_nr=record_nr))
+            return h2b(method(abstract_data, record_nr=record_nr, total_len = self.__get_rec_len(total_len)))
         if self._construct:
-            return build_construct(self._construct, abstract_data)
+            return build_construct(self._construct, abstract_data, {'total_len':self.__get_rec_len(total_len)})
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -1224,7 +1266,20 @@ class TransRecEF(TransparentEF):
             return t.to_dict()
         return {'raw': raw_hex_data}
 
-    def encode_record_hex(self, abstract_data: dict) -> str:
+    def __get_rec_len(self, total_len: Optional[int] = None) -> Optional[int]:
+        """Get the length (total length) of the file record"""
+
+        # Caller has provided the actual total length of the record, this should be the default case
+        if total_len is not None:
+            return total_len
+
+        # Alternatively use the record length from the specification
+        if self.rec_len:
+            return self.rec_len
+
+        return None
+
+    def encode_record_hex(self, abstract_data: dict, total_len: Optional[int] = None) -> str:
         """Encode abstract representation into raw (hex string) data.
 
         A derived class would typically provide an _encode_record_bin() or _encode_record_hex()
@@ -1233,17 +1288,19 @@ class TransRecEF(TransparentEF):
 
         Args:
             abstract_data : dict representing the decoded data
+            total_len : expected total length of the encoded data (record length)
         Returns:
             hex string encoded data
         """
         method = getattr(self, '_encode_record_hex', None)
         if callable(method):
-            return method(abstract_data)
+            return method(abstract_data, total_len = self.__get_rec_len(total_len))
         method = getattr(self, '_encode_record_bin', None)
         if callable(method):
-            return b2h(method(abstract_data))
+            return b2h(method(abstract_data, total_len = self.__get_rec_len(total_len)))
         if self._construct:
-            return b2h(filter_dict(build_construct(self._construct, abstract_data)))
+            return b2h(filter_dict(build_construct(self._construct, abstract_data,
+                                                   {'total_len':self.__get_rec_len(total_len)})))
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -1251,7 +1308,7 @@ class TransRecEF(TransparentEF):
         raise NotImplementedError(
             "%s encoder not yet implemented. Patches welcome." % self)
 
-    def encode_record_bin(self, abstract_data: dict) -> bytearray:
+    def encode_record_bin(self, abstract_data: dict, total_len: Optional[int] = None) -> bytearray:
         """Encode abstract representation into raw (binary) data.
 
         A derived class would typically provide an _encode_record_bin() or _encode_record_hex()
@@ -1260,17 +1317,19 @@ class TransRecEF(TransparentEF):
 
         Args:
             abstract_data : dict representing the decoded data
+            total_len : expected total length of the encoded data (record length)
         Returns:
             binary encoded data
         """
         method = getattr(self, '_encode_record_bin', None)
         if callable(method):
-            return method(abstract_data)
+            return method(abstract_data, total_len = self.__get_rec_len(total_len))
         method = getattr(self, '_encode_record_hex', None)
         if callable(method):
-            return h2b(method(abstract_data))
+            return h2b(method(abstract_data, total_len = self.__get_rec_len(total_len)))
         if self._construct:
-            return filter_dict(build_construct(self._construct, abstract_data))
+            return filter_dict(build_construct(self._construct, abstract_data,
+                                               {'total_len':self.__get_rec_len(total_len)}))
         if self._tlv:
             t = self._tlv() if inspect.isclass(self._tlv) else self._tlv
             t.from_dict(abstract_data)
@@ -1283,8 +1342,8 @@ class TransRecEF(TransparentEF):
                   for i in range(0, len(raw_bin_data), self.rec_len)]
         return [self.decode_record_bin(x) for x in chunks]
 
-    def _encode_bin(self, abstract_data) -> bytes:
-        chunks = [self.encode_record_bin(x) for x in abstract_data]
+    def _encode_bin(self, abstract_data, **kwargs) -> bytes:
+        chunks = [self.encode_record_bin(x, total_len = kwargs.get('total_len', None)) for x in abstract_data]
         # FIXME: pad to file size
         return b''.join(chunks)
 

pySim/gsm_r.py

@@ -290,7 +290,7 @@ class EF_Predefined(LinFixedEF):
         else:
             return parse_construct(self.construct_others, raw_bin_data)
 
-    def _encode_record_bin(self, abstract_data : dict, record_nr : int) -> bytearray:
+    def _encode_record_bin(self, abstract_data : dict, record_nr : int, **kwargs) -> bytearray:
         r = None
         if record_nr == 1:
             r = self.construct_first.build(abstract_data)

pySim/runtime.py

@@ -524,7 +524,7 @@ class RuntimeLchan:
         Args:
             data : abstract data which is to be encoded and written
         """
-        data_hex = self.selected_file.encode_hex(data)
+        data_hex = self.selected_file.encode_hex(data, self.selected_file_size())
         return self.update_binary(data_hex)
 
     def read_record(self, rec_nr: int = 0):
@@ -574,7 +574,7 @@ class RuntimeLchan:
             rec_nr : Record number to read
             data_hex : Abstract data to be written
         """
-        data_hex = self.selected_file.encode_record_hex(data, rec_nr)
+        data_hex = self.selected_file.encode_record_hex(data, rec_nr, self.selected_file_record_len())
         return self.update_record(rec_nr, data_hex)
 
     def retrieve_data(self, tag: int = 0):

pySim/sysmocom_sja2.py

@@ -252,7 +252,7 @@ class EF_USIM_AUTH_KEY(TransparentEF):
         else:
             return parse_construct(self._construct, raw_bin_data)
 
-    def _encode_bin(self, abstract_data: dict) -> bytearray:
+    def _encode_bin(self, abstract_data: dict, **kwargs) -> bytearray:
         if abstract_data['cfg']['algorithm'] == 'tuak':
             return build_construct(self._constr_tuak, abstract_data)
         else:

pySim/ts_31_102.py

@@ -344,7 +344,7 @@ class EF_SUCI_Calc_Info(TransparentEF):
                 out.append({k: v})
         return out
 
-    def _encode_hex(self, in_json):
+    def _encode_hex(self, in_json, **kwargs):
         out_bytes = self._encode_prot_scheme_id_list(
             in_json['prot_scheme_id_list'])
         d = self._expand_pubkey_list(in_json['hnet_pubkey_list'])
@@ -396,8 +396,8 @@ class EF_SUCI_Calc_Info(TransparentEF):
             'hnet_pubkey_list':    hnet_pubkey_list
         }
 
-    def _encode_bin(self, in_json):
-        return h2b(self._encode_hex(in_json))
+    def _encode_bin(self, in_json, **kwargs):
+        return h2b(self._encode_hex(in_json, **kwargs))
 
 
 class EF_LI(TransRecEF):

pySim/ts_31_102_telecom.py

@@ -59,7 +59,7 @@ class EF_UServiceTable(TransparentEF):
                     ret[service_nr]['description'] = self.table[service_nr]
         return ret
 
-    def _encode_bin(self, in_json):
+    def _encode_bin(self, in_json, **kwargs):
         # compute the required binary size
         bin_len = 0
         for srv in in_json.keys():

pySim/ts_51_011.py

@@ -358,7 +358,7 @@ class EF_IMSI(TransparentEF):
     def _decode_hex(self, raw_hex):
         return {'imsi': dec_imsi(raw_hex)}
 
-    def _encode_hex(self, abstract):
+    def _encode_hex(self, abstract, **kwargs):
         return enc_imsi(abstract['imsi'])
 
     @with_default_category('File-Specific Commands')
@@ -446,7 +446,7 @@ class EF_ServiceTable(TransparentEF):
             }
         return ret
 
-    def _encode_bin(self, in_json):
+    def _encode_bin(self, in_json, **kwargs):
         # compute the required binary size
         bin_len = 0
         for srv in in_json.keys():
@@ -969,7 +969,7 @@ class EF_ICCID(TransparentEF):
     def _decode_hex(self, raw_hex):
         return {'iccid': dec_iccid(raw_hex)}
 
-    def _encode_hex(self, abstract):
+    def _encode_hex(self, abstract, **kwargs):
         return enc_iccid(abstract['iccid'])
 
 # TS 102 221 Section 13.3 / TS 31.101 Secction 13 / TS 51.011 Section 10.1.2