support UCS-2 characters in EF.MMSUP, EF.ADN, EF.SPN, EF.PNN, EF.ECC

Now that we have support for the UCS-2 encoding as per TS 102 221 Annex A,
we can start to make use of it from various file constructs.

As some specs say "Either 7-bit GSM or UCS-2" we also introduce
a related automatic GsmOrUcs2Adapter and GsmOrUcs2String class.

Change-Id: I4eb8aea0a13260a143e2c60fca73c3c4312fd3b2

pySim/construct.py

@@ -48,6 +48,29 @@ class Utf8Adapter(Adapter):
     def _encode(self, obj, context, path):
         return codecs.encode(obj, "utf-8")
 
+class GsmOrUcs2Adapter(Adapter):
+    """Try to encode into a GSM 03.38 string; if that fails, fall back to UCS-2 as described
+    in TS 102 221 Annex A."""
+    def _decode(self, obj, context, path):
+        # In case the string contains only 0xff bytes we interpret it as an empty string
+        if obj == b'\xff' * len(obj):
+                return ""
+        # one of the magic bytes of TS 102 221 Annex A
+        if obj[0] in [0x80, 0x81, 0x82]:
+            ad = Ucs2Adapter(GreedyBytes)
+        else:
+            ad = GsmString(GreedyBytes)
+        return ad._decode(obj, context, path)
+
+    def _encode(self, obj, context, path):
+        # first try GSM 03.38; then fall back to TS 102 221 Annex A UCS-2
+        try:
+            ad = GsmString(GreedyBytes)
+            return ad._encode(obj, context, path)
+        except:
+            ad = Ucs2Adapter(GreedyBytes)
+            return ad._encode(obj, context, path)
+
 class Ucs2Adapter(Adapter):
     """convert a bytes() type that contains UCS2 encoded characters encoded as defined in TS 102 221
     Annex A to normal python string representation (and back)."""
@@ -447,6 +470,20 @@ def GsmString(n):
     '''
     return GsmStringAdapter(Rpad(Bytes(n), pattern=b'\xff'), codec='gsm03.38')
 
+def GsmOrUcs2String(n):
+    '''
+    GSM 03.38 or UCS-2 (TS 102 221 Annex A) encoded byte string of fixed length n.
+    Encoder appends padding bytes (b'\\xff') to maintain
+    length. Decoder removes those trailing bytes.
+
+    Exceptions are raised for invalid characters
+    and length excess.
+
+    Parameters:
+        n (Integer): Fixed length of the encoded byte string
+    '''
+    return GsmOrUcs2Adapter(Rpad(Bytes(n), pattern=b'\xff'))
+
 class GreedyInteger(Construct):
     """A variable-length integer implementation, think of combining GrredyBytes with BytesInteger."""
     def __init__(self, signed=False, swapped=False, minlen=0):

pySim/ts_31_102.py

@@ -529,7 +529,7 @@ class EF_ECC(LinFixedEF):
     cc_construct = BcdAdapter(Rpad(Bytes(3)))
     category_construct = FlagsEnum(Byte, police=1, ambulance=2, fire_brigade=3, marine_guard=4,
                                    mountain_rescue=5, manual_ecall=6, automatic_ecall=7)
-    alpha_construct = GsmStringAdapter(Rpad(GreedyBytes))
+    alpha_construct = GsmOrUcs2Adapter(Rpad(GreedyBytes))
 
     def __init__(self, fid='6fb7', sfid=0x01, name='EF.ECC',
                  desc='Emergency Call Codes'):

pySim/ts_51_011.py

@@ -145,7 +145,7 @@ class EF_ADN(LinFixedEF):
     def __init__(self, fid='6f3a', sfid=None, name='EF.ADN', desc='Abbreviated Dialing Numbers', ext=1, **kwargs):
         super().__init__(fid, sfid=sfid, name=name, desc=desc, rec_len=(14, 30), **kwargs)
         ext_name = 'ext%u_record_id' % ext
-        self._construct = Struct('alpha_id'/COptional(GsmStringAdapter(Rpad(Bytes(this._.total_len-14)), codec='ascii')),
+        self._construct = Struct('alpha_id'/COptional(GsmOrUcs2Adapter(Rpad(Bytes(this._.total_len-14)))),
                                  'len_of_bcd'/Int8ub,
                                  'ton_npi'/TonNpi,
                                  'dialing_nr'/ExtendedBcdAdapter(BcdAdapter(Rpad(Bytes(10)))),
@@ -514,7 +514,7 @@ class EF_SPN(TransparentEF):
             'hide_in_oplmn'/Flag,
             'show_in_hplmn'/Flag,
             # Bytes 2..17
-            'spn'/Bytewise(GsmString(16))
+            'spn'/Bytewise(GsmOrUcs2String(16))
         )
 
 # TS 51.011 Section 10.3.13
@@ -929,7 +929,7 @@ class EF_MMSICP(TransparentEF):
 # TS 51.011 Section 10.3.54
 class EF_MMSUP(LinFixedEF):
     class MMS_UserPref_ProfileName(BER_TLV_IE, tag=0x81):
-        pass
+        _construct = GsmOrUcs2Adapter(GreedyBytes)
 
     class MMS_UserPref_Info(BER_TLV_IE, tag=0x82):
         pass