fix: decode the Response Scripting template and the compact listings (v3.6.5)

Explore still missed the F0414C46416101 package on a card whose responses
wrap the R-APDU in the TS 102 226 5.2.2 Response Scripting template
(`AB <len> 80 <count> 23 <len> <R-APDU>`): `_decode_por` parsed that as a
compact response, so the frontend got `last_status_word` 81d0/7680 and data
starting `80 01 01 23 ...` instead of the listing.

- server: `_parse_response_scripting()` (AB definite / AF 80 ... 00 00
  indefinite) extracts the executed-command count and the last R-APDU's SW
  and data; `_decode_por` exposes it as `response_type: scripting` in the
  same `decoded` shape as compact, so the RAM explore paging and the RAM
  install `por_sw` see the real 9000/6310.
- frontend: the compact listing walk is deterministic on the AID length
  (`len AID life ver`; P1=10 adds `module_count (len module_AID)*`).
  `_parseRawAppEntry` no longer guesses `rawLen-1` for >8-byte AIDs (16-byte
  A113 applet AIDs were truncated), and `_parseRawElfEntry` no longer scans
  for `0x10` (a length/AID byte equal to 0x10 derailed the walk: a live page
  parsed to 1 entry with no F0414C46416101).
- tests: exact trace fixtures - the ELF page lists F0414C46416101 (11
  entries), the P1=10 page attaches its F0414C4641610101 module, the app page
  keeps the 16-byte A113 AIDs; Python covers the scripting template
  (definite/indefinite) against the live `AB 12` ISD and listing vectors.

610 frontend / 495 Python green; version 3.6.5; sw simple-v278.

docs/api.md

@@ -300,7 +300,10 @@ or more proactive SEND SHORT MESSAGE commands.  The server captures those
 SMS-SUBMIT TPDUs, reassembles the concatenated segments and returns the
 decoded response in `por` (as if it had arrived in the ENVELOPE), so callers
 see a normal `por.response_status == "por_ok"` with the remote status word
-and response data.
+and response data.  Responses wrapped in the TS 102 226 5.2.2 Response
+Scripting template (`AB`/`AF`: executed-count TLV `80` + R-APDU TLV `23`) are
+decoded the same way (`response_type: "scripting"`), with the R-APDU's own
+status word and data.
 
 **Request body:**
 ```json

frontend/index.html

@@ -1665,7 +1665,7 @@
 // ===== Version =====
 // Single source of truth for the PWA version: shown in the header and used
 // by the server version check in pysimConnect().
-const SIMPLE_VERSION = '3.6.4';
+const SIMPLE_VERSION = '3.6.5';
 document.getElementById('app-version').textContent = 'v' + SIMPLE_VERSION;
 
 // ===== Tab switching =====
@@ -8461,42 +8461,48 @@ function _parseE3Entry(hex) {
 
 // Raw format (P2=00): ISD uses pure AID length, Apps use combined length (AID+lifecycle).
 // Heuristic: length > 8 means combined (AID = length-1 bytes, lifecycle inside length).
+// Raw format (P2=00): consecutive <aid_len><AID><lifecycle><privileges>.
+// The length byte is the AID length (16-byte A113/D276 AIDs included - the old
+// "rawLen-1" guess truncated exactly those).
 function _parseRawAppEntry(hex, i) {
-	const rawLen = parseInt(hex.substr(i, 2), 16);
+	const aidLen = parseInt(hex.substr(i, 2), 16);
-	if (rawLen < 1 || i + 2 + rawLen * 2 + 2 > hex.length) return null;
+	if (aidLen < 5 || aidLen > 16 || i + 2 + aidLen * 2 + 4 > hex.length) return null;
-	const aidLen = rawLen > 8 ? rawLen - 1 : rawLen;
 	const aid = hex.substr(i + 2, aidLen * 2).toUpperCase();
-	let j = i + 2 + rawLen * 2;
+	let j = i + 2 + aidLen * 2;
 	const lifecycle = hex.substr(j, 2).toUpperCase(); j += 2;
 	const privileges = hex.substr(j, 2).toUpperCase(); j += 2;
 	return { aid, lifecycle, privileges, next: j };
 }
 
-// Raw format (P2=00): ELF header then trailing bytes with module entries.
+// Raw format (P2=00) ELF entry: <aid_len><AID><lifecycle><version>; the P1=10
-// P1=20: <ELF_header> <00> <module_entries separated by 00>
+// (ELF + modules) listing appends module entries (<len><AID>[, lifecycle]).
-// P1=10: <ELF_header> <version(2B)> <SD_AID> <module_entries...>
+// The walk is deterministic on the AID length - the old "scan for 0x10"
-// In both, a module entry starts with 0x10 (len=16) followed by 15-byte AID + lifecycle.
+// heuristic derailed whenever a length byte or an AID byte happened to be
-// Scan for 0x10 markers, advance past each full entry to avoid AID-internal false positives.
+// 0x10 (which silently dropped entries like F0414C46416101).
-function _parseRawElfEntry(hex, i) {
+function _parseRawElfEntry(hex, i, withModules) {
 	const aidLen = parseInt(hex.substr(i, 2), 16);
-	if (aidLen < 1 || i + 2 + aidLen * 2 + 2 > hex.length) return null;
+	if (aidLen < 5 || aidLen > 16 || i + 2 + aidLen * 2 + 4 > hex.length) return null;
 	const aid = hex.substr(i + 2, aidLen * 2).toUpperCase();
 	let j = i + 2 + aidLen * 2;
 	const lifecycle = hex.substr(j, 2).toUpperCase(); j += 2;
-	// Scan: when 0x10 found, validate 15-byte AID follows, then advance past full entry
+	const version = hex.substr(j + 2, 2).toUpperCase(); j += 2;
+	const out = { aid, lifecycle, version, next: j };
+	if (!withModules) return out;
+	// P1=10 (ELF + modules) entry: ...<lifecycle><version><module_count>
+	// followed by <module_len><module_AID> entries.  Verified against two
+	// live traces (a F0414C46416101 ELF with its F0414C4641610101 module).
+	if (j + 2 > hex.length) return out;
+	const modCount = parseInt(hex.substr(j, 2), 16); j += 2;
 	const moduleAids = [];
-	const seen = new Set();
+	for (let k = 0; k < modCount && j + 4 <= hex.length; k++) {
-	while (j + 32 <= hex.length) {
+		const len = parseInt(hex.substr(j, 2), 16);
-		const tag = parseInt(hex.substr(j, 2), 16);
+		if (len < 5 || len > 16 || j + 2 + len * 2 > hex.length) break;
-		if (tag === 0x10) {
+		moduleAids.push(hex.substr(j + 2, len * 2).toUpperCase());
-			const modAid = hex.substr(j + 2, 30).toUpperCase();
+		j += 2 + len * 2;
-			if (!seen.has(modAid)) { seen.add(modAid); moduleAids.push(modAid); }
-			j += 32; // skip marker(1) + AID(15), continue past lifecycle+appCount
-		} else {
-			j += 2;
-		}
 	}
-	return { aid, lifecycle, moduleAids: moduleAids.length ? moduleAids : undefined, next: j };
+	if (moduleAids.length) out.moduleAids = moduleAids;
+	out.next = j;
+	return out;
 }
 
 function ramParseAppStatus(hex) {
@@ -8523,7 +8529,7 @@ function ramParseAppStatus(hex) {
 	return out;
 }
 
-function ramParseElfStatus(hex) {
+function ramParseElfStatus(hex, withModules) {
 	const s = (hex || '').toUpperCase();
 	if (!s) return [];
 	// TLV format (P2=02): E3 templates with structured tags
@@ -8535,14 +8541,15 @@ function ramParseElfStatus(hex) {
 			return r;
 		}).filter(r => r.aid);
 	}
-	// Raw format (P2=00 fallback): consecutive <aid_len><AID><lifecycle>
+	// Raw format (P2=00 fallback): consecutive <aid_len><AID><lifecycle><version>
+	// entries; P1=10 pages carry the module list (withModules).  Resync one byte
+	// at a time instead of stopping at the first unparsable byte.
 	const out = [];
 	let i = 0;
-	while (i + 6 <= s.length) {
+	while (i + 4 <= s.length) {
-		const r = _parseRawElfEntry(s, i);
+		const r = _parseRawElfEntry(s, i, withModules);
-		if (!r) break;
+		if (r) { out.push({ type: 'elf', ...r }); i = r.next; continue; }
-		out.push({ type: 'elf', ...r });
+		i += 2;
-		i = r.next;
 	}
 	return out;
 }
@@ -8807,7 +8814,7 @@ async function ramExplore(sp) {
 	await paginate('80', isd, ramParseAppStatus, t('ISD'));
 	await paginate('40', apps, ramParseAppStatus, t('Apps'));
 	await paginate('20', elfs, ramParseElfStatus, t('ELFs'));
-	await paginate('10', modules, ramParseElfStatus, t('ELF Modules'));
+	await paginate('10', modules, (hex) => ramParseElfStatus(hex, true), t('ELF Modules'));
 
 	ramMergeElfData(elfs, modules);
 	ramSaveCntr(cntr);

frontend/sw.js

@@ -1,4 +1,4 @@
-const CACHE = 'simple-v277';
+const CACHE = 'simple-v278';
 const URLS = [
   'index.html',
   'help.html',

frontend/tests/ram.test.js

@@ -23,6 +23,7 @@ function extractFunc(src, name) {
 
 // Extract chain builder functions and dependencies
 const FNS = ['berLenStr', 'buildApdu', 'escHtml', 'esc', 'chainInit', 'chainRamBuildRowHex', 'ramFmtLifecycle', 'ramFmtPrivileges', 'ramRenderExploreHtml', 'ramStepLine', 'ramGetStatusApdu',
+	'_parseRawElfEntry', '_parseRawAppEntry', 'ramParseElfStatus', 'ramParseAppStatus', 'parseTLV', '_parseE3Entry',
 	'ramCardIdxAfterRemove', 'ramClearResults', 'ramHideProgress', 'ramOpChanged', 'ramRender', 'ramApplyCard', 'ramExecute',
 	'jcAidNorm', 'jcAidName', 'jcAidSuffix', 'jcAidHtml'];
 let code = '';
@@ -379,3 +380,37 @@ test('ramCardIdxAfterRemove keeps the remembered index aligned', () => {
 	assert.strictEqual(ramCardIdxAfterRemove(0, 2), 0);
 	assert.strictEqual(ramCardIdxAfterRemove(null, 1), null);
 });
+
+test('ramParseElfStatus lists the compact ELF and module listings (F0414C46416101)', () => {
+	// Exact bytes from a live RAM Explore: the P1=20 ELF page and the P1=10
+	// (ELF+modules) page.  The old 0x10-scan walk dropped everything after the
+	// first entry; the deterministic AID walk lists them all.
+	const elfPage = '10A0000000090005FFFFFFFF8911000000010010A0000000871005FFFFFFFF8913100000010010A0000000871005FFFFFFFF8914100000010010A0000000090005FFFFFFFF8912000000010010A0000000871005FFFFFFFF8913200000010010A0000000090005FFFFFFFF8913000000010010A0000000090005FFFFFFFF8911010000010010D2760001180002FF49100A89AA060F00010010A1130001180001FFFFFFFF89A1003900010010A1130001180002FFF7100E8904000200010007F0414C464161010100';
+	const elfs = ramParseElfStatus(elfPage);
+	const f041 = elfs.find(r => r.aid === 'F0414C46416101');
+	assert.ok(f041, JSON.stringify(elfs.map(r => r.aid)));
+	assert.strictEqual(f041.lifecycle, '01');
+	assert.ok(elfs.some(r => r.aid === 'A1130001180002FFF7100E8904000200'), 'A113 ELF missing');
+	assert.ok(elfs.some(r => r.aid === 'A0000000090005FFFFFFFF8912000000'), 'uicc.toolkit ELF missing');
+
+	const modulesPage = '10A1130001180001FFFFFFFF89A100390001000110A1130001180001FFFFFFFF89A100390810A1130001180002FFF7100E890400020001000210A1130001180002FFF7100E890400020810A1130001180002FFF7100E89494D450807F0414C4641610101000108F0414C4641610101';
+	const mods = ramParseElfStatus(modulesPage, true);
+	const f041Row = mods.find(r => r.aid === 'F0414C46416101');
+	assert.ok(f041Row, JSON.stringify(mods.map(r => r.aid)));
+	assert.deepStrictEqual(f041Row.moduleAids, ['F0414C4641610101']);
+});
+
+test('ramParseAppStatus keeps 16-byte AIDs (no rawLen-1 truncation)', () => {
+	const page = '08D276000005AA3F010704'
+		+ '0FD276000005AA060200000000B000000700'
+		+ '0FD276000005AA060200000000B00001070010A1130001180001FFFFFFFF89A10039080700'
+		+ '10A1130001180002FFF7100E8904000208070010A1130001180002FFF7100E89494D45080700';
+	const apps = ramParseAppStatus(page);
+	assert.deepStrictEqual(apps.map(r => r.aid), [
+		'D276000005AA3F01', 'D276000005AA060200000000B00000',
+		'D276000005AA060200000000B00001',
+		'A1130001180001FFFFFFFF89A1003908', 'A1130001180002FFF7100E8904000208',
+		'A1130001180002FFF7100E89494D4508',
+	]);
+	assert.strictEqual(apps[3].lifecycle, '07');
+});

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pysim-simple-server"
-version = "3.6.4"
+version = "3.6.5"
 description = "HTTP REST server wrapping pysim for the SIMple PWA"
 requires-python = ">=3.8"
 # pysim is a git-only dependency installed explicitly by setup.bat/setup.sh.

pysim_simple_server/server.py

@@ -31,7 +31,7 @@ from osmocom.tlv import BER_TLV_IE
 from osmocom.utils import rpad
 
 
-VERSION = '3.6.4'
+VERSION = '3.6.5'
 
 MAX_ENVELOPE_SEGMENTS = 5  # max SMS segments for outgoing C-APDU in ENVELOPE
 
@@ -1171,6 +1171,46 @@ def _ram_step_result(step_name, last_sw, por, por_hex, bytes_, segments):
     return step, error
 
 
+def _parse_response_scripting(data):
+    """Parse a Response Scripting template (TS 102 226 5.2.2, tables
+    5.10/5.10a): `AB <len>` (definite) or `AF 80 ... 00 00` (indefinite),
+    containing the executed-command-count TLV `80` and one or more R-APDU
+    TLVs `23` (COMPREHENSION-TLV; the last two bytes are SW1 SW2).
+
+    Returns (count, sw, rapdu_data_hex) from the last R-APDU, or None when the
+    data is not a scripting template."""
+    if not data:
+        return None
+    if data[0] == 0xAF:
+        if len(data) < 4 or data[1] != 0x80 or data[-2:] != b'\x00\x00':
+            return None
+        body = data[2:-2]
+    elif data[0] == 0xAB:
+        ln, voff = _ber_len_at(data, 1)
+        if ln <= 0 or voff + ln > len(data):
+            return None
+        body = data[voff:voff + ln]
+    else:
+        return None
+    count = None
+    last = None
+    off = 0
+    while off < len(body) - 1:
+        tag = body[off]
+        ln, voff = _ber_len_at(body, off + 1)
+        if ln < 0 or voff + ln > len(body):
+            break
+        val = body[voff:voff + ln]
+        if tag == 0x80 and val:
+            count = int.from_bytes(val, 'big')
+        elif tag == 0x23 and len(val) >= 2:
+            last = (val[-2:].hex().upper(), val[:-2].hex().upper())
+        off = voff + ln
+    if last is None:
+        return None
+    return (count, last[0], last[1])
+
+
 def _sms_submit_por(submit_handler):
     """Response packet carried by an actual-response SMS-SUBMIT, in the
     DELIVER-style form `_decode_por` expects.
@@ -1215,53 +1255,23 @@ def _decode_por(spi1, spi2, kic, kid, cntr_hex, kic_key_hex, kid_key_hex, respon
         'raw': response_hex,
     }
     
-    # Try ExpandedRemoteResponse first (TS 102 226 §5.2.2)
+    # TS 102 226 5.2.2 Response Scripting template (AB/AF): cards wrap the
+    # R-APDU(s) of the executed remote command(s) this way instead of the
+    # plain compact response.  The R-APDU's own SW and data are the useful
+    # result (a bare CompactRemoteResp parse would read `AB` as the command
+    # count and produce garbage).
     if res.response_status == 'por_ok' and len(res['secured_data']):
-        expanded_response_data = ''
+        scripted = _parse_response_scripting(bytes(res['secured_data']))
-        try:
+        if scripted is not None:
-            from construct import Struct, Int8ub, Bytes, GreedyBytes, Optional, Array, this
+            count, sw, data_hex = scripted
-            ExpandedRemoteResponse = Struct(
+            out['response_type'] = 'scripting'
-                'response_count'/Int8ub,
+            out['decoded'] = {
-                'responses'/Array(this.response_count, Struct(
+                'number_of_commands': count,
-                    'command_number'/Int8ub,
+                'last_status_word': sw,
-                    'status_word'/Bytes(2),
+                'last_response_data': data_hex,
-                    'response_data'/GreedyBytes,
+            }
-                    'error_details'/Optional(Struct(
+        elif dec is not None:
-                        'error_code'/Int8ub,
-                        'error_info'/GreedyBytes
-                    )),
-                    'chaining_context'/Optional(Struct(
-                        'script_id'/Bytes(4),
-                        'is_first'/Int8ub,
-                        'is_last'/Int8ub,
-                    ))
-                ))
-            )
-            expanded = ExpandedRemoteResponse.parse(res['secured_data'])
-            out['response_type'] = 'expanded'
-            out['response_count'] = expanded.response_count
-            out['responses'] = []
-            for resp in expanded.responses:
-                response_data = {
-                    'command_number': resp.command_number,
-                    'status_word': resp.status_word.hex().upper(),
-                    'response_data': b2h(resp.response_data).upper() if resp.response_data else '',
-                }
-                if resp.error_details:
-                    response_data['error_code'] = resp.error_details.error_code
-                    response_data['error_info'] = b2h(resp.error_details.error_info).upper()
-                if resp.chaining_context:
-                    response_data['script_id'] = resp.chaining_context.script_id.hex().upper()
-                    response_data['is_first'] = resp.chaining_context.is_first == 0x01
-                    response_data['is_last'] = resp.chaining_context.is_last == 0x01
-                out['responses'].append(response_data)
-            if expanded.response_count > 0 and expanded.responses[0].response_data:
-                expanded_response_data = b2h(expanded.responses[0].response_data).upper()
-        except Exception:
-            pass
-        if dec is not None:
             out['response_type'] = 'compact'
-            # Use compact parser's last_response_data; expanded parser gives wrong results for compact format
             out['decoded'] = {
                 'number_of_commands': dec.number_of_commands,
                 'last_status_word': str(dec.last_status_word),

tests/test_ota_helpers.py

@@ -32,6 +32,7 @@ from pysim_simple_server.server import (
     _parse_setup_menu_items,
     _calc_ud_offset,
     _find_sms_tpdu,
+    _parse_response_scripting,
     _parse_sms_concat,
     _por_remote_sw,
     _ram_next_cntr,
@@ -1381,3 +1382,46 @@ class SmsSubmitCaptureTest(unittest.TestCase):
         handler.submit_ud_hex = None
         self.assertEqual(_sms_submit_por(handler), '')
 
+
+
+class ResponseScriptingTest(unittest.TestCase):
+    """TS 102 226 5.2.2 Response Scripting template (AB definite / AF
+    indefinite): the card wraps the R-APDU(s) of the executed remote
+    commands.  Real vectors from the live RAM Explore traces - a bare
+    compact parse would read `AB` as the command count and lose the data."""
+
+    def test_definite_template_from_a_live_response(self):
+        # AB 12: count 80 01 01, R-APDU 23 0D 08A0000000030000000F809000
+        pkt = '027100001F0A00000000000002AA0000AB12800101230D08A0000000030000000F8090009000'
+        out = _decode_por('00', '00', '01', '01', '0', '00' * 16, '00' * 16, pkt)
+        self.assertEqual(out['response_type'], 'scripting')
+        self.assertEqual(out['decoded']['number_of_commands'], 1)
+        self.assertEqual(out['decoded']['last_status_word'], '9000')
+        self.assertEqual(out['decoded']['last_response_data'], '08A0000000030000000F80')
+
+    def test_elf_listing_page_from_a_live_response(self):
+        # the F0414C46416101 ELF page (assembled SMS-SUBMIT UD, AB wrapper)
+        ud = ('00E90A000000000000030600000263100BD276000005AAFFCAFE0001010007'
+              'F0414C4641610101' + '00')
+        # build a valid scripting template around the listing tail instead of
+        # trusting the truncated sample above
+        rapdu = bytes.fromhex('10A1130001180002FFF7100E8904000200' '0100' '07F0414C46416101' '0100' + '9000')
+        tmpl = bytes([0xAB, 0x80]) if False else None
+        body = bytes([0x80, 0x01, 0x01, 0x23, len(rapdu)]) + rapdu
+        data = bytes([0xAB, len(body)]) + body
+        scripted = _parse_response_scripting(data)
+        self.assertIsNotNone(scripted)
+        count, sw, listing = scripted
+        self.assertEqual((count, sw), (1, '9000'))
+        self.assertTrue(listing.startswith('10A1130001'), listing[:20])
+        self.assertIn('F0414C46416101', listing)
+
+    def test_indefinite_template_and_plain_data(self):
+        body = bytes([0x80, 0x01, 0x02, 0x23, 0x04, 0xAA, 0xBB, 0x90, 0x00])
+        data = bytes([0xAF, 0x80]) + body + b'\x00\x00'
+        count, sw, listing = _parse_response_scripting(data)
+        self.assertEqual((count, sw, listing), (2, '9000', 'AABB'))
+        # a compact response is not a scripting template
+        self.assertIsNone(_parse_response_scripting(bytes.fromhex('027100000263100BD2')))
+        self.assertIsNone(_parse_response_scripting(b''))
+