Merge simtrace-analyser-server into simtrace2_pysniff.server

Add simtrace2-pysniff-server CLI — HTTP API for simtrace-analyser PWA.
Two capture modes: GSMTAP UDP listener (compatible with simtrace2-sniff)
or direct SIMtrace2 hardware capture. SQLite storage with sessions +
messages tables. REST API with CORS.

server/ sub-package: database.py, capture.py, server.py, __main__.py

README updated with server usage section.
pyproject.toml: simtrace2-pysniff-server script entry added.

README.md

@@ -34,7 +34,11 @@ pip install -e .
 
 ## Running
 
-Two equivalent ways to start the sniffer:
+Two tools are provided:
+
+### Sniffer CLI (`simtrace2-pysniff`)
+
+Captures SIMtrace2 traffic and writes it to stdout, GSMTAP (Wireshark), or PCAP files.
 
 ### Startup script (Linux only)
 
@@ -49,12 +53,30 @@ launches the Python module.
 ./sniff-start.sh --pcap trace.pcap --format atr-time
 ```
 
-Options can also be set via environment variables:
+OPTIONS can also be set via environment variables:
 
 ```sh
 FORMAT=timestamp GSMTAP=127.0.0.1 ./sniff-start.sh
 ```
 
+### Analysis server (`simtrace2-pysniff-server`)
+
+Stores captured APDU traffic in SQLite and serves it over HTTP for the
+[simtrace-analyser](https://github.com/anttro/simtrace-analyser) PWA.
+Two capture modes:
+
+```sh
+# Listen for GSMTAP from simtrace2-pysniff (or original simtrace2-sniff):
+simtrace2-pysniff-server --capture gsmtap
+
+# Capture directly from SIMtrace2 hardware (no external tool needed):
+simtrace2-pysniff-server --capture direct
+```
+
+The PWA then connects to `http://127.0.0.1:8081` for live viewing and
+session management.  See the [simtrace-analyser](https://github.com/anttro/simtrace-analyser)
+repo for the browser interface.
+
 ### Python module (cross-platform)
 
 ```sh

pyproject.toml

@@ -15,6 +15,7 @@ dependencies = [
 
 [project.scripts]
 simtrace2-pysniff = "simtrace2_pysniff.__main__:main"
+simtrace2-pysniff-server = "simtrace2_pysniff.server.__main__:main"
 
 [tool.setuptools.packages.find]
 include = ["simtrace2_pysniff*"]

simtrace2_pysniff/server/__init__.py

@@ -0,0 +1 @@
+"""simtrace2-pysniff server sub-package."""

simtrace2_pysniff/server/__main__.py

@@ -0,0 +1,63 @@
+"""simtrace2-pysniff-server — CLI entry point.
+
+Usage:
+    simtrace2-pysniff-server --capture gsmtap [--gsmtap-port 4729] [--port 8081]
+    simtrace2-pysniff-server --capture direct [--port 8081]
+"""
+
+import argparse
+import sys
+from http.server import HTTPServer
+
+from .server import RequestHandler
+from .database import Database, DEFAULT_DB_PATH
+from .capture import CaptureManager, GsmtapListener, DirectSniffer
+
+
+def main():
+    p = argparse.ArgumentParser(
+        prog='simtrace2-pysniff-server',
+        description='HTTP API server for SIMtrace2 APDU capture and analysis')
+    p.add_argument('--port', type=int, default=8081,
+                   help='HTTP server port (default: 8081)')
+    p.add_argument('--db', default=DEFAULT_DB_PATH,
+                   help=f'SQLite database path (default: {DEFAULT_DB_PATH})')
+    p.add_argument('--capture', choices=['gsmtap', 'direct'], default='gsmtap',
+                   help='Capture mode: gsmtap (listen UDP 4729) or direct (SIMtrace2 USB)')
+    p.add_argument('--gsmtap-port', type=int, default=4729,
+                   help='UDP port for GSMTAP listener (default: 4729)')
+    args = p.parse_args()
+
+    db = Database(args.db)
+
+    if args.capture == 'gsmtap':
+        backend = GsmtapListener(bind_port=args.gsmtap_port)
+    else:
+        backend = DirectSniffer()
+
+    capture = CaptureManager(backend, db)
+
+    RequestHandler.db = db
+    RequestHandler.capture = capture
+
+    server = HTTPServer(('127.0.0.1', args.port), RequestHandler)
+
+    print(f'simtrace2-pysniff-server — http://127.0.0.1:{args.port}', file=sys.stderr)
+    print(f'  capture mode: {args.capture}', file=sys.stderr)
+    if args.capture == 'gsmtap':
+        print(f'  GSMTAP port:  {args.gsmtap_port}', file=sys.stderr)
+    print(f'  database:     {args.db}', file=sys.stderr)
+    print(file=sys.stderr)
+
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        if capture.active:
+            capture.stop_session()
+        server.server_close()
+
+
+if __name__ == '__main__':
+    main()

simtrace2_pysniff/server/capture.py

@@ -0,0 +1,111 @@
+"""Capture backends: GSMTAP listener and direct SIMtrace2 sniffer."""
+
+import time
+import threading
+
+from ..gsmtap import GsmtapReceiver, GSMTAP_SIM_ATR, GSMTAP_SIM_APDU
+
+
+class GsmtapListener:
+    def __init__(self, bind_port=4729):
+        self._receiver = GsmtapReceiver(bind_port=bind_port)
+        self._running = False
+
+    def start(self):
+        self._running = True
+
+    def stop(self):
+        self._running = False
+        self._receiver.close()
+
+    def iter_messages(self):
+        while self._running:
+            sub_type, data = self._receiver.read_packet()
+            if sub_type is None:
+                continue
+            msg_type = 'atr' if sub_type == GSMTAP_SIM_ATR else 'tpdu'
+            yield msg_type, data
+
+
+class DirectSniffer:
+    def __init__(self):
+        from ..device import SniffSession
+        self._session = SniffSession(
+            reconnect=True,
+            reconnect_delay_min=1.0,
+            reconnect_delay_max=10.0,
+            inactivity_timeout=0.0,
+        )
+        self._running = False
+
+    def start(self):
+        self._running = True
+
+    def stop(self):
+        self._running = False
+        self._session.close()
+
+    def iter_messages(self):
+        for msg in self._session.iter_messages():
+            if not self._running:
+                break
+            yield msg.type, msg.data
+
+
+class CaptureManager:
+    def __init__(self, backend, db):
+        self._backend = backend
+        self._db = db
+        self._session_id = None
+        self._thread = None
+        self._start_time = 0.0
+        self._latest_msg_id = 0
+
+    @property
+    def active(self):
+        return self._session_id is not None
+
+    @property
+    def session_id(self):
+        return self._session_id
+
+    @property
+    def start_time(self):
+        return self._start_time
+
+    @property
+    def latest_msg_id(self):
+        return self._latest_msg_id
+
+    def start_session(self):
+        if self._session_id is not None:
+            self.stop_session()
+
+        mode = 'gsmtap' if isinstance(self._backend, GsmtapListener) else 'direct'
+        self._session_id = self._db.create_session(mode)
+        self._start_time = time.monotonic()
+        self._latest_msg_id = 0
+        self._backend.start()
+
+        self._thread = threading.Thread(target=self._capture_loop, daemon=True)
+        self._thread.start()
+
+        return self._session_id
+
+    def stop_session(self):
+        if self._session_id is None:
+            return
+        self._backend.stop()
+        self._db.close_session(self._session_id)
+        sid = self._session_id
+        self._session_id = None
+        self._start_time = 0.0
+        return sid
+
+    def _capture_loop(self):
+        for msg_type, data in self._backend.iter_messages():
+            if self._session_id is None:
+                break
+            elapsed = round(time.monotonic() - self._start_time, 3)
+            mid = self._db.insert_message(self._session_id, elapsed, msg_type, data)
+            self._latest_msg_id = mid

simtrace2_pysniff/server/database.py

@@ -0,0 +1,177 @@
+"""SQLite database for session and APDU storage."""
+
+import os
+import sqlite3
+import time
+from contextlib import contextmanager
+
+DEFAULT_DB_DIR = os.path.expanduser('~/.simtrace-analyser')
+DEFAULT_DB_PATH = os.path.join(DEFAULT_DB_DIR, 'sessions.db')
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS sessions (
+    id       INTEGER PRIMARY KEY AUTOINCREMENT,
+    name     TEXT,
+    started  TEXT NOT NULL,
+    ended    TEXT,
+    mode     TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS messages (
+    id         INTEGER PRIMARY KEY AUTOINCREMENT,
+    session_id INTEGER NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+    elapsed    REAL NOT NULL,
+    type       TEXT NOT NULL,
+    data       BLOB NOT NULL,
+    flags      INTEGER DEFAULT 0
+);
+
+CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id);
+CREATE INDEX IF NOT EXISTS idx_messages_type ON messages(type);
+CREATE INDEX IF NOT EXISTS idx_messages_elapsed ON messages(session_id, elapsed);
+"""
+
+
+def _iso_now():
+    return time.strftime('%Y-%m-%dT%H:%M:%S.', time.localtime()) + \
+        f'{time.time() % 1 * 1000:03.0f}'
+
+
+class Database:
+    def __init__(self, db_path=None):
+        if db_path is None:
+            db_path = DEFAULT_DB_PATH
+        os.makedirs(os.path.dirname(db_path), exist_ok=True)
+        self._conn = sqlite3.connect(db_path, check_same_thread=False)
+        self._conn.execute('PRAGMA journal_mode=WAL')
+        self._conn.execute('PRAGMA foreign_keys=ON')
+        self._conn.executescript(_SCHEMA)
+        self._conn.commit()
+
+    # --- Sessions ---
+
+    def create_session(self, mode):
+        cur = self._conn.execute(
+            'INSERT INTO sessions (started, mode) VALUES (?, ?)',
+            (_iso_now(), mode))
+        self._conn.commit()
+        return cur.lastrowid
+
+    def close_session(self, session_id):
+        self._conn.execute(
+            'UPDATE sessions SET ended=? WHERE id=? AND ended IS NULL',
+            (_iso_now(), session_id))
+        self._conn.commit()
+
+    def rename_session(self, session_id, name):
+        self._conn.execute(
+            'UPDATE sessions SET name=? WHERE id=?',
+            (name, session_id))
+        self._conn.commit()
+
+    def delete_session(self, session_id):
+        self._conn.execute('DELETE FROM messages WHERE session_id=?', (session_id,))
+        self._conn.execute('DELETE FROM sessions WHERE id=?', (session_id,))
+        self._conn.commit()
+
+    def get_session(self, session_id):
+        row = self._conn.execute(
+            'SELECT id, name, started, ended, mode FROM sessions WHERE id=?',
+            (session_id,)).fetchone()
+        if row is None:
+            return None
+        return self._row_to_session(row)
+
+    def list_sessions(self):
+        rows = self._conn.execute("""
+            SELECT s.id, s.name, s.started, s.ended, s.mode,
+                   COUNT(m.id) AS msg_count
+            FROM sessions s LEFT JOIN messages m ON m.session_id = s.id
+            GROUP BY s.id
+            ORDER BY s.started DESC
+        """).fetchall()
+        return [self._row_to_session(r) for r in rows]
+
+    def get_active_session(self):
+        row = self._conn.execute(
+            'SELECT id, name, started, ended, mode FROM sessions WHERE ended IS NULL ORDER BY id DESC LIMIT 1'
+        ).fetchone()
+        if row is None:
+            return None
+        return self._row_to_session(row)
+
+    def _row_to_session(self, row):
+        d = {'id': row[0], 'name': row[1], 'started': row[2], 'ended': row[3], 'mode': row[4]}
+        if len(row) > 5:
+            d['message_count'] = row[5]
+        return d
+
+    # --- Messages ---
+
+    def insert_message(self, session_id, elapsed, msg_type, data, flags=0):
+        cur = self._conn.execute(
+            'INSERT INTO messages (session_id, elapsed, type, data, flags) VALUES (?,?,?,?,?)',
+            (session_id, elapsed, msg_type, data, flags))
+        self._conn.commit()
+        return cur.lastrowid
+
+    def get_message(self, msg_id):
+        row = self._conn.execute(
+            'SELECT id, session_id, elapsed, type, data, flags FROM messages WHERE id=?',
+            (msg_id,)).fetchone()
+        if row is None:
+            return None
+        return self._row_to_message(row)
+
+    def get_messages(self, session_id, offset=0, limit=200):
+        rows = self._conn.execute(
+            'SELECT id, session_id, elapsed, type, data, flags FROM messages '
+            'WHERE session_id=? ORDER BY elapsed, id LIMIT ? OFFSET ?',
+            (session_id, limit, offset)).fetchall()
+        return [self._row_to_message(r) for r in rows]
+
+    def get_messages_after(self, session_id, after_id):
+        rows = self._conn.execute(
+            'SELECT id, session_id, elapsed, type, data, flags FROM messages '
+            'WHERE session_id=? AND id > ? ORDER BY elapsed, id',
+            (session_id, after_id)).fetchall()
+        return [self._row_to_message(r) for r in rows]
+
+    def count_messages(self, session_id):
+        row = self._conn.execute(
+            'SELECT COUNT(*) FROM messages WHERE session_id=?',
+            (session_id,)).fetchone()
+        return row[0]
+
+    def search_messages(self, session_id, query):
+        hex_q = ''.join(c for c in query if c in '0123456789abcdefABCDEF')
+        if not hex_q:
+            return []
+        rows = self._conn.execute(
+            "SELECT id, session_id, elapsed, type, data, flags FROM messages "
+            "WHERE session_id=? AND hex(data) LIKE ? ORDER BY elapsed, id",
+            (session_id, f'%{hex_q.lower()}%')).fetchall()
+        return [self._row_to_message(r) for r in rows]
+
+    def filter_messages(self, session_id, msg_type):
+        rows = self._conn.execute(
+            'SELECT id, session_id, elapsed, type, data, flags FROM messages '
+            'WHERE session_id=? AND type=? ORDER BY elapsed, id',
+            (session_id, msg_type)).fetchall()
+        return [self._row_to_message(r) for r in rows]
+
+    def get_type_counts(self, session_id):
+        rows = self._conn.execute(
+            'SELECT type, COUNT(*) FROM messages WHERE session_id=? GROUP BY type ORDER BY COUNT(*) DESC',
+            (session_id,)).fetchall()
+        return [{'type': r[0], 'count': r[1]} for r in rows]
+
+    def _row_to_message(self, row):
+        return {
+            'id': row[0],
+            'session_id': row[1],
+            'elapsed': row[2],
+            'type': row[3],
+            'data': row[4].hex(),
+            'flags': row[5],
+        }

simtrace2_pysniff/server/server.py

@@ -0,0 +1,216 @@
+"""HTTP API server for simtrace-analyser PWA."""
+
+import json
+import os
+import sys
+import threading
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from urllib.parse import urlparse, parse_qs
+
+from .database import Database
+from .capture import CaptureManager, GsmtapListener, DirectSniffer
+
+
+class RequestHandler(BaseHTTPRequestHandler):
+    db: Database = None
+    capture: CaptureManager = None
+
+    def log_message(self, fmt, *args):
+        print(f'[{self.log_date_time_string()}] {args[0]}', file=sys.stderr)
+
+    def _send_json(self, data, status=200):
+        body = json.dumps(data, ensure_ascii=False).encode('utf-8')
+        self.send_response(status)
+        self.send_header('Content-Type', 'application/json')
+        self.send_header('Content-Length', len(body))
+        self.send_header('Access-Control-Allow-Origin', '*')
+        self.send_header('Access-Control-Allow-Methods', 'GET, POST, PATCH, DELETE, OPTIONS')
+        self.send_header('Access-Control-Allow-Headers', 'Content-Type')
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _send_error(self, status, message):
+        self._send_json({'error': message}, status)
+
+    def _read_json(self):
+        length = int(self.headers.get('Content-Length', 0))
+        if length == 0:
+            return {}
+        return json.loads(self.rfile.read(length))
+
+    def _parse_path(self):
+        parsed = urlparse(self.path)
+        path = parsed.path.rstrip('/')
+        return path, parse_qs(parsed.query)
+
+    # --- CORS preflight ---
+    def do_OPTIONS(self):
+        self.send_response(204)
+        self.send_header('Access-Control-Allow-Origin', '*')
+        self.send_header('Access-Control-Allow-Methods', 'GET, POST, PATCH, DELETE, OPTIONS')
+        self.send_header('Access-Control-Allow-Headers', 'Content-Type')
+        self.end_headers()
+
+    # --- GET ---
+    def do_GET(self):
+        path, params = self._parse_path()
+
+        if path == '/api/status':
+            self._handle_status()
+        elif path == '/api/sessions':
+            self._handle_list_sessions()
+        elif path.startswith('/api/sessions/'):
+            session_id = int(path.split('/')[-1])
+            self._handle_get_session(session_id, params)
+        elif path == '/api/capture/latest':
+            self._handle_capture_latest(params)
+        elif path == '/api/capture/status':
+            self._handle_capture_status()
+        elif path.startswith('/api/apdu/search'):
+            session_id = int(params.get('session_id', [0])[0])
+            query = params.get('q', [''])[0]
+            self._handle_search(session_id, query)
+        elif path.startswith('/api/apdu/filter'):
+            session_id = int(params.get('session_id', [0])[0])
+            msg_type = params.get('type', [''])[0]
+            self._handle_filter(session_id, msg_type)
+        elif path.startswith('/api/apdu/'):
+            msg_id = int(path.split('/')[-1])
+            self._handle_get_apdu(msg_id)
+        else:
+            self._send_error(404, 'Not found')
+
+    # --- POST ---
+    def do_POST(self):
+        path, _ = self._parse_path()
+
+        if path == '/api/capture/start':
+            self._handle_capture_start()
+        elif path == '/api/capture/stop':
+            self._handle_capture_stop()
+        else:
+            self._send_error(404, 'Not found')
+
+    # --- PATCH ---
+    def do_PATCH(self):
+        path, _ = self._parse_path()
+
+        if path.startswith('/api/sessions/') and path.endswith('/name'):
+            session_id = int(path.split('/')[-2])
+            body = self._read_json()
+            self._handle_rename_session(session_id, body.get('name', ''))
+        else:
+            self._send_error(404, 'Not found')
+
+    # --- DELETE ---
+    def do_DELETE(self):
+        path, _ = self._parse_path()
+
+        if path.startswith('/api/sessions/'):
+            session_id = int(path.split('/')[-1])
+            self._handle_delete_session(session_id)
+        else:
+            self._send_error(404, 'Not found')
+
+    # --- Handlers ---
+
+    def _handle_status(self):
+        active_session = self.db.get_active_session()
+        self._send_json({
+            'server': 'simtrace-analyser-server',
+            'capture_active': self.capture.active,
+            'session_id': self.capture.session_id,
+            'mode': active_session['mode'] if active_session else None,
+            'messages_count': self.db.count_messages(self.capture.session_id) if self.capture.session_id else 0,
+        })
+
+    def _handle_list_sessions(self):
+        sessions = self.db.list_sessions()
+        self._send_json({'sessions': sessions})
+
+    def _handle_get_session(self, session_id, params):
+        session = self.db.get_session(session_id)
+        if session is None:
+            self._send_error(404, 'Session not found')
+            return
+        offset = int(params.get('offset', [0])[0])
+        limit = int(params.get('limit', [200])[0])
+        messages = self.db.get_messages(session_id, offset=offset, limit=limit)
+        total = self.db.count_messages(session_id)
+        type_counts = self.db.get_type_counts(session_id)
+        self._send_json({
+            'session': session,
+            'messages': messages,
+            'total': total,
+            'type_counts': type_counts,
+        })
+
+    def _handle_capture_latest(self, params):
+        after_id = int(params.get('after', [0])[0])
+        if not self.capture.active:
+            self._send_json({'messages': [], 'next_after': after_id, 'active': False})
+            return
+        msg_id = self.capture.latest_msg_id
+        messages = self.db.get_messages_after(self.capture.session_id, after_id)
+        self._send_json({
+            'messages': messages,
+            'next_after': msg_id,
+            'active': True,
+            'session_id': self.capture.session_id,
+        })
+
+    def _handle_capture_status(self):
+        self._send_json({
+            'active': self.capture.active,
+            'session_id': self.capture.session_id,
+        })
+
+    def _handle_capture_start(self):
+        if self.capture.active:
+            self.capture.stop_session()
+        session_id = self.capture.start_session()
+        started = self.db.get_session(session_id)['started']
+        self._send_json({'session_id': session_id, 'started': started})
+
+    def _handle_capture_stop(self):
+        if not self.capture.active:
+            self._send_error(400, 'No active capture')
+            return
+        session_id = self.capture.stop_session()
+        session = self.db.get_session(session_id)
+        self._send_json({
+            'session_id': session_id,
+            'ended': session['ended'],
+            'messages_count': self.db.count_messages(session_id),
+        })
+
+    def _handle_rename_session(self, session_id, name):
+        self.db.rename_session(session_id, name)
+        self._send_json({'ok': True})
+
+    def _handle_delete_session(self, session_id):
+        if self.capture.session_id == session_id:
+            self.capture.stop_session()
+        self.db.delete_session(session_id)
+        self._send_json({'ok': True})
+
+    def _handle_search(self, session_id, query):
+        if not query:
+            self._send_json({'messages': []})
+            return
+        messages = self.db.search_messages(session_id, query)
+        self._send_json({'messages': messages})
+
+    def _handle_filter(self, session_id, msg_type):
+        if not msg_type:
+            self._send_json({'messages': []})
+            return
+        messages = self.db.filter_messages(session_id, msg_type)
+        self._send_json({'messages': messages})
+
+    def _handle_get_apdu(self, msg_id):
+        msg = self.db.get_message(msg_id)
+        if msg is None:
+            self._send_error(404, 'Message not found')
+            return
+        self._send_json(msg)